web-annotation-discovery-webextension/src/storage

import { RpcClient } from 'webextension-rpc';
import { contentScriptRpcServer } from '../content_script';
import { asArray, asSingleValue, completeAnnotationStub } from 'web-annotation-utils';
import type { WebAnnotation, ZeroOrMore } from 'web-annotation-utils';
import { Annotation } from './Annotation';
import { db } from './db';

export type AnnotationSourceType = 'container' | 'embeddedJsonld';
export type AnnotationSourceAuthType = 'HttpBasicAuth';

export class IAnnotationSource {
  constructor(
    public _id: number,
    public url: string,
    public type: AnnotationSourceType,
    public active?: boolean,
    public title?: string,
    public writable?: boolean,
    public useForNewAnnotations?: boolean,
    public needsAuth?: boolean,
    public lastUpdate?: Date,
    public lastModified?: Date,
  ) {}
}

/**
 * The information needed to subscribe to a source.
 */
export type AnnotationSourceDescriptor = Pick<
  IAnnotationSource,
  'url' | 'title' | 'type'
>;

export class AnnotationSource {
  static sourceUpdatePeriod = 10 * 60;

  constructor(public data: IAnnotationSource) {}

  protected async save() {
    await db.annotationSources.put({
      ...this.data,
      lastModified: new Date(),
    });
  }

  async delete() {
    await this.deleteAnnotationsLocally();
    await db.annotationSources.delete(this.data._id);
    console.log(`Deleted source ${this.data.url}`);
  }

  protected async deleteAnnotationsLocally() {
    const count = await db.annotations
      .where('source')
      .equals(this.data._id)
      .delete();
    console.log(`Deleted ${count} annotations for source ${this.data.url}`);
  }

  /**
   * Reload all annotations from this source.
   * @param force Also refresh if the source is not active.
   */
  async refresh(force = false) {
    if (!(this.data.active || force)) return;

    if (this.data.writable) {
      await this.uploadDirtyAnnotations();
    }

    const webAnnotations = await this.fetchAllAnnotations();

    // Delete all existing items from this source, to avoid duplicates/zombies.
    // TODO Make this a little smarter.
    await this.deleteAnnotationsLocally();

    // Insert annotations.
    await Promise.all(
      webAnnotations.map(async (webAnnotation) => {
        await Annotation.new({
          annotation: webAnnotation,
          source: this.data._id,
        });
      }),
    );
    console.log(
      `Inserted ${webAnnotations.length} annotations for source ${this.data.url}`,
    );

    // Update source metadata.
    this.data.lastUpdate = new Date();
    await this.save();
  }

  protected async fetchAllAnnotations(): Promise<WebAnnotation[]> {
    if (this.data.type === 'container')
      return await getAllAnnotationsFromContainerSource(this.data.url);
    if (this.data.type === 'embeddedJsonld')
      return await getAnnotationsFromEmbeddedJsonld(this.data.url);
    throw new Error(
      `Getting annotations from source of type '${this.data.type}' is not yet implemented.`,
    );
  }

  protected async createAnnotation(annotationStub: Partial<WebAnnotation>) {
    const webAnnotation = completeAnnotationStub(annotationStub);
    const annotation = await Annotation.new({
      annotation: webAnnotation,
      source: this.data._id,
      dirty: true,
    });
    try {
      const createdAnnotation = await this.postAnnotation(
        annotation.data.annotation,
      );
      await annotation.setDirty(false, { annotation: createdAnnotation });
      return annotation;
    } catch (error: any) {
      throw new Error(
        `Error while uploading created annotation: ${error.message}`,
      );
    }
  }

  async uploadAnnotation(annotation: WebAnnotation) {
    if (!annotation.id) await this.postAnnotation(annotation);
    else await this.putAnnotation(annotation);
  }

  protected async postAnnotation(annotation: WebAnnotation) {
    const webAnnotation: Omit<WebAnnotation, 'id'> & {
      id?: WebAnnotation['id'];
    } = { ...annotation };
    delete webAnnotation.id;
    // This one was not yet uploaded. POST it.
    const response = await fetch(this.data.url, {
      method: 'POST',
      credentials: 'include',
      headers: {
        'Content-Type':
          'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
        Accept:
          'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
      },
      body: JSON.stringify(webAnnotation),
    });
    if (response.status !== 201) {
      if (response.status === 401) {
        this.data.needsAuth = true;
      }
      throw new Error(
        `Could not POST the annotation, got: ${response.status} ${response.statusText} (expected: 201 Created)`,
      );
    }
    const location = response.headers.get('Location');
    if (!location)
      throw new Error(
        'Server did not provide Location header for created annotation.',
      );
    const locationUrl = new URL(location, response.url).href;
    const contentLocation = response.headers.get('Content-Location');
    const contentLocationUrl =
      contentLocation && new URL(contentLocation, response.url).href;

    // Replace the local annotation with the one from the server, to update its id (and possibly other properties).
    let createdAnnotation: WebAnnotation;
    if (contentLocationUrl === locationUrl) {
      // Easy: the server responded with the annotation itself.
      createdAnnotation = await response.json();
    } else {
      // If we did not receive it, then we fetch it.
      createdAnnotation = await resolveSingle(locationUrl);
    }
    // TODO better validation.
    if (!createdAnnotation.target) {
      throw new Error('Server returned something else than an annotation.');
    }
    return createdAnnotation;
  }

  protected async putAnnotation(annotation: WebAnnotation) {
    // This annotation exists already. PUT it.
    const annotationUrl = annotation.id;
    if (!annotationUrl.startsWith(this.data.url)) {
      throw new Error(
        `Annotation to be updated is not part of this collection.`,
      );
    }
    const response = await fetch(annotationUrl, {
      method: 'PUT',
      credentials: 'include',
      headers: {
        'Content-Type':
          'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
        Accept:
          'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
      },
      body: JSON.stringify(annotation),
    });
    if (!response.ok) {
      if (response.status === 401) {
        this.data.needsAuth = true;
      }
      throw new Error(
        `Could not POST the annotation, got: ${response.status} ${response.statusText}`,
      );
    }
  }

  async uploadDirtyAnnotations() {
    const dirtyAnnotations = await db.annotations
      .where('source')
      .equals(this.data._id)
      .filter(({ dirty }) => !!dirty)
      .toArray();

    // PUT/POST each one individually.
    await Promise.all(
      dirtyAnnotations
        .map((annotation) => new Annotation(annotation))
        .map(async (annotation) => {
          if (annotation.data.toDelete) await annotation.delete();
          else await this.uploadAnnotation(annotation.data.annotation);
        }),
    );
  }

  async deleteAnnotationRemotely(annotation: WebAnnotation) {
    const annotationUrl = annotation.id;
    if (!annotationUrl.startsWith(this.data.url)) {
      throw new Error(
        `Annotation to be deleted is not part of this collection.`,
      );
    }
    const response = await fetch(annotationUrl, {
      method: 'DELETE',
      credentials: 'include',
      headers: {
        'Content-Type':
          'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
        Accept:
          'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
      },
    });
    if (!response.ok) {
      if (response.status === 401) {
        this.data.needsAuth = true;
      }
      throw new Error(
        `Could not delete the annotation, got: ${response.status} ${response.statusText}`,
      );
    }
  }

  async testWritable() {
    const createdAnnotation = await this.postAnnotation(
      completeAnnotationStub({
        target: 'http://example.com/page1',
        bodyValue: 'Test annotation, should have been deleted directly',
      }),
    );
    this.data.writable = true;
    await this.save();
    await this.deleteAnnotationRemotely(createdAnnotation);
  }

  async useForNewAnnotations(value: boolean) {
    this.data.useForNewAnnotations = value;
    await this.save();
  }

  static async new(data: Omit<IAnnotationSource, '_id'>) {
    // @ts-ignore: _id is not needed in put()
    const source: IAnnotationSource = {
      ...data,
      lastModified: new Date(),
    };
    const key = (await db.annotationSources.put(
      source,
    )) as IAnnotationSource['_id'];
    return new this({ ...data, _id: key });
  }

  static async addSource(
    sourceDescriptor: AnnotationSourceDescriptor,
    active?: boolean,
  ) {
    if (await this.exists(sourceDescriptor)) return;

    // Auto-update from containers, but not from pages with embedded annotations.
    if (active === undefined) {
      active = sourceDescriptor.type === 'container';
    }

    const source = await AnnotationSource.new({
      url: sourceDescriptor.url,
      title: sourceDescriptor.title,
      type: sourceDescriptor.type,
      active,
    });

    await source.refresh(true);
    return source;
  }

  static async exists(sourceDescriptor: AnnotationSourceDescriptor) {
    const source = await db.annotationSources
      .filter((source) => source.url === sourceDescriptor.url)
      .first();
    return source !== undefined;
  }

  static async get(id: IAnnotationSource['_id']) {
    const source = await db.annotationSources.get(id);
    if (!source) throw new Error(`No annotation source exists with id ${id}.`);
    return new AnnotationSource(source);
  }

  static async getByUrl(url: string) {
    const source = await db.annotationSources
      .filter((source) => source.url === url)
      .first();
    if (!source)
      throw new Error(`No annotation source exists with url ${url}.`);
    return new AnnotationSource(source);
  }

  static async getAll() {
    const sources = await db.annotationSources.toArray();
    return sources.map((source) => new AnnotationSource(source));
  }

  static async getActiveSources() {
    // How to do this nicely in Dexie?
    const sources = await db.annotationSources.toArray();
    const activeSources = sources.filter(({ active }) => active);
    return activeSources.map((source) => new AnnotationSource(source));
  }

  static async getPossiblyWritableSources() {
    const sources = await this.getActiveSources();
    return sources.filter((source) => source.data.type === 'container');
  }

  static async getSourcesNeedingUpdate() {
    const cutoffDate = new Date(Date.now() - this.sourceUpdatePeriod * 1000);
    // Using filters; the where() clause cannot get items with lastUpdate===undefined
    const sources = await db.annotationSources
      // .where('lastUpdate')
      // .belowOrEqual(cutoffDate)
      // .and((annotationSource) => annotationSource.active)
      .filter(({ lastUpdate }) => !lastUpdate || lastUpdate < cutoffDate)
      .filter(({ active }) => !!active)
      .toArray();
    return sources.map((source) => new AnnotationSource(source));
  }

  static async createAnnotation(
    annotationStub: Partial<WebAnnotation>,
    sourceId?: AnnotationSource['data']['_id'],
  ) {
    let sourceObjs;
    if (sourceId) {
      sourceObjs = [await AnnotationSource.get(sourceId)];
    } else {
      sourceObjs = await AnnotationSource.getPossiblyWritableSources();
      if (sourceObjs.length === 0)
        throw new Error(
          'Please first subscribe to the annotation collection where you want to store your annotations.',
        );
      sourceObjs = sourceObjs.filter(
        (sourceObj) => sourceObj.data.useForNewAnnotations,
      );
      if (sourceObjs.length === 0)
        throw new Error(
          'Please select (in the extension’s popup menu) in which collection to store your annotations.',
        );
    }
    // There should only be one source marked with useForNewAnnotations.
    const createdAnnotation = await sourceObjs[0].createAnnotation(
      annotationStub,
    );
    return createdAnnotation;
  }
}

async function getAllAnnotationsFromContainerSource(
  sourceUrl: string,
): Promise<WebAnnotation[]> {
  console.log(`Fetching annotations from ${sourceUrl}`);
  const annotationSourceData = await resolveSingle(sourceUrl);

  // Check what type of source we got.
  const nodeTypes = asArray(annotationSourceData.type);

  // If the source is a single annotation, import that one.
  if (nodeTypes.includes('Annotation')) {
    return [annotationSourceData];
  }

  // If the source is an annotation container/collection, import all its items.
  if (nodeTypes.includes('AnnotationCollection')) {
    // Read the first page of annotations.
    let page = await resolveSingle(annotationSourceData.first);
    const annotations: WebAnnotation[] = asArray(page.items);

    // Fetch the subsequent pages, if any.
    while ((page = await resolveSingle(page.next))) {
      annotations.push(...asArray(page.items));
    }

    return annotations;
  }

  throw new Error(
    `Annotation source is neither AnnotationCollection nor Annotation.`,
  );
}

/**
 * Given the value of an `@id`-property, get the ‘actual’ value:
 * - if the node is nested, return value as-is.
 * - if the value is a string (= a URL), fetch and return its data.
 * - if there is no value, return `undefined`.
 * - if there are multiple values, process only the first.
 *
 * TODO Consider using json-ld tools:
 * - https://github.com/LDflex/LDflex/
 * - https://github.com/assemblee-virtuelle/LDP-navigator
 */
async function resolveSingle(
  valuesOrIds: ZeroOrMore<string> | object,
): Promise<undefined | any> {
  const valueOrId = asSingleValue(valuesOrIds);

  // If it’s a value (or undefined), we are done.
  if (typeof valueOrId !== 'string') return valueOrId;

  // It’s an id, i.e. a URL. (TODO use correct base for relative URLs)
  const response = await fetch(valueOrId, {
    headers: {
      Accept: 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"',
      Prefer:
        'return=representation;include="http://www.w3.org/ns/oa#PreferContainedDescriptions"',
    },
    cache: 'no-cache',
  });

  let data;
  try {
    data = await response.json();
  } catch (error) {
    throw new Error(`Received invalid JSON from URL <${valueOrId}>, ${error}`);
  }
  if (typeof data !== 'object')
    throw new Error('Response is valid JSON but not an object.');
  return data;
}

/**
 * Extract the annotations embedded in a page, via the content script.
 * Only works if the page is opened. (though we could fetch&parse the html ourselves)
 */
async function getAnnotationsFromEmbeddedJsonld(
  url: string,
): Promise<WebAnnotation[]> {
  const tabs = await browser.tabs.query({});
  const sourceTab = tabs.find((tab) => tab.url?.startsWith(url.split('#')[0]));
  if (sourceTab) {
    const contentScriptRpc = new RpcClient<typeof contentScriptRpcServer>({
      tabId: sourceTab.id,
    });
    const annotations = await contentScriptRpc.func(
      'discoverAnnotationsEmbeddedAsJSONLD',
    )();
    return annotations;
  } else {
    throw new Error(
      `To refresh annotations extracted from a web page, first open that page.`,
    );
  }
}