import { RpcClient } from 'webextension-rpc'; import { contentScriptRpcServer } from '../content_script'; import { asArray, asSingleValue, completeAnnotationStub } from 'web-annotation-utils'; import type { WebAnnotation, ZeroOrMore } from 'web-annotation-utils'; import { Annotation } from './Annotation'; import { db } from './db'; export type AnnotationSourceType = 'container' | 'embeddedJsonld'; export type AnnotationSourceAuthType = 'HttpBasicAuth'; export class IAnnotationSource { constructor( public _id: number, public url: string, public type: AnnotationSourceType, public active?: boolean, public title?: string, public writable?: boolean, public useForNewAnnotations?: boolean, public needsAuth?: boolean, public lastUpdate?: Date, public lastModified?: Date, ) {} } /** * The information needed to subscribe to a source. */ export type AnnotationSourceDescriptor = Pick< IAnnotationSource, 'url' | 'title' | 'type' >; export class AnnotationSource { static sourceUpdatePeriod = 10 * 60; constructor(public data: IAnnotationSource) {} protected async save() { await db.annotationSources.put({ ...this.data, lastModified: new Date(), }); } async delete() { await this.deleteAnnotationsLocally(); await db.annotationSources.delete(this.data._id); console.log(`Deleted source ${this.data.url}`); } protected async deleteAnnotationsLocally() { const count = await db.annotations .where('source') .equals(this.data._id) .delete(); console.log(`Deleted ${count} annotations for source ${this.data.url}`); } /** * Reload all annotations from this source. * @param force Also refresh if the source is not active. */ async refresh(force = false) { if (!(this.data.active || force)) return; if (this.data.writable) { await this.uploadDirtyAnnotations(); } const webAnnotations = await this.fetchAllAnnotations(); // Delete all existing items from this source, to avoid duplicates/zombies. // TODO Make this a little smarter. await this.deleteAnnotationsLocally(); // Insert annotations. await Promise.all( webAnnotations.map(async (webAnnotation) => { await Annotation.new({ annotation: webAnnotation, source: this.data._id, }); }), ); console.log( `Inserted ${webAnnotations.length} annotations for source ${this.data.url}`, ); // Update source metadata. this.data.lastUpdate = new Date(); await this.save(); } protected async fetchAllAnnotations(): Promise { if (this.data.type === 'container') return await getAllAnnotationsFromContainerSource(this.data.url); if (this.data.type === 'embeddedJsonld') return await getAnnotationsFromEmbeddedJsonld(this.data.url); throw new Error( `Getting annotations from source of type '${this.data.type}' is not yet implemented.`, ); } protected async createAnnotation(annotationStub: Partial) { const webAnnotation = completeAnnotationStub(annotationStub); const annotation = await Annotation.new({ annotation: webAnnotation, source: this.data._id, dirty: true, }); try { const createdAnnotation = await this.postAnnotation( annotation.data.annotation, ); await annotation.setDirty(false, { annotation: createdAnnotation }); return annotation; } catch (error: any) { throw new Error( `Error while uploading created annotation: ${error.message}`, ); } } async uploadAnnotation(annotation: WebAnnotation) { if (!annotation.id) await this.postAnnotation(annotation); else await this.putAnnotation(annotation); } protected async postAnnotation(annotation: WebAnnotation) { const webAnnotation: Omit & { id?: WebAnnotation['id']; } = { ...annotation }; delete webAnnotation.id; // This one was not yet uploaded. POST it. const response = await fetch(this.data.url, { method: 'POST', credentials: 'include', headers: { 'Content-Type': 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', Accept: 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', }, body: JSON.stringify(webAnnotation), }); if (response.status !== 201) { if (response.status === 401) { this.data.needsAuth = true; } throw new Error( `Could not POST the annotation, got: ${response.status} ${response.statusText} (expected: 201 Created)`, ); } const location = response.headers.get('Location'); if (!location) throw new Error( 'Server did not provide Location header for created annotation.', ); const locationUrl = new URL(location, response.url).href; const contentLocation = response.headers.get('Content-Location'); const contentLocationUrl = contentLocation && new URL(contentLocation, response.url).href; // Replace the local annotation with the one from the server, to update its id (and possibly other properties). let createdAnnotation: WebAnnotation; if (contentLocationUrl === locationUrl) { // Easy: the server responded with the annotation itself. createdAnnotation = await response.json(); } else { // If we did not receive it, then we fetch it. createdAnnotation = await resolveSingle(locationUrl); } // TODO better validation. if (!createdAnnotation.target) { throw new Error('Server returned something else than an annotation.'); } return createdAnnotation; } protected async putAnnotation(annotation: WebAnnotation) { // This annotation exists already. PUT it. const annotationUrl = annotation.id; if (!annotationUrl.startsWith(this.data.url)) { throw new Error( `Annotation to be updated is not part of this collection.`, ); } const response = await fetch(annotationUrl, { method: 'PUT', credentials: 'include', headers: { 'Content-Type': 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', Accept: 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', }, body: JSON.stringify(annotation), }); if (!response.ok) { if (response.status === 401) { this.data.needsAuth = true; } throw new Error( `Could not POST the annotation, got: ${response.status} ${response.statusText}`, ); } } async uploadDirtyAnnotations() { const dirtyAnnotations = await db.annotations .where('source') .equals(this.data._id) .filter(({ dirty }) => !!dirty) .toArray(); // PUT/POST each one individually. await Promise.all( dirtyAnnotations .map((annotation) => new Annotation(annotation)) .map(async (annotation) => { if (annotation.data.toDelete) await annotation.delete(); else await this.uploadAnnotation(annotation.data.annotation); }), ); } async deleteAnnotationRemotely(annotation: WebAnnotation) { const annotationUrl = annotation.id; if (!annotationUrl.startsWith(this.data.url)) { throw new Error( `Annotation to be deleted is not part of this collection.`, ); } const response = await fetch(annotationUrl, { method: 'DELETE', credentials: 'include', headers: { 'Content-Type': 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', Accept: 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', }, }); if (!response.ok) { if (response.status === 401) { this.data.needsAuth = true; } throw new Error( `Could not delete the annotation, got: ${response.status} ${response.statusText}`, ); } } async testWritable() { const createdAnnotation = await this.postAnnotation( completeAnnotationStub({ target: 'http://example.com/page1', bodyValue: 'Test annotation, should have been deleted directly', }), ); this.data.writable = true; await this.save(); await this.deleteAnnotationRemotely(createdAnnotation); } async useForNewAnnotations(value: boolean) { this.data.useForNewAnnotations = value; await this.save(); } static async new(data: Omit) { // @ts-ignore: _id is not needed in put() const source: IAnnotationSource = { ...data, lastModified: new Date(), }; const key = (await db.annotationSources.put( source, )) as IAnnotationSource['_id']; return new this({ ...data, _id: key }); } static async addSource( sourceDescriptor: AnnotationSourceDescriptor, active?: boolean, ) { if (await this.exists(sourceDescriptor)) return; // Auto-update from containers, but not from pages with embedded annotations. if (active === undefined) { active = sourceDescriptor.type === 'container'; } const source = await AnnotationSource.new({ url: sourceDescriptor.url, title: sourceDescriptor.title, type: sourceDescriptor.type, active, }); await source.refresh(true); return source; } static async exists(sourceDescriptor: AnnotationSourceDescriptor) { const source = await db.annotationSources .filter((source) => source.url === sourceDescriptor.url) .first(); return source !== undefined; } static async get(id: IAnnotationSource['_id']) { const source = await db.annotationSources.get(id); if (!source) throw new Error(`No annotation source exists with id ${id}.`); return new AnnotationSource(source); } static async getByUrl(url: string) { const source = await db.annotationSources .filter((source) => source.url === url) .first(); if (!source) throw new Error(`No annotation source exists with url ${url}.`); return new AnnotationSource(source); } static async getAll() { const sources = await db.annotationSources.toArray(); return sources.map((source) => new AnnotationSource(source)); } static async getActiveSources() { // How to do this nicely in Dexie? const sources = await db.annotationSources.toArray(); const activeSources = sources.filter(({ active }) => active); return activeSources.map((source) => new AnnotationSource(source)); } static async getPossiblyWritableSources() { const sources = await this.getActiveSources(); return sources.filter((source) => source.data.type === 'container'); } static async getSourcesNeedingUpdate() { const cutoffDate = new Date(Date.now() - this.sourceUpdatePeriod * 1000); // Using filters; the where() clause cannot get items with lastUpdate===undefined const sources = await db.annotationSources // .where('lastUpdate') // .belowOrEqual(cutoffDate) // .and((annotationSource) => annotationSource.active) .filter(({ lastUpdate }) => !lastUpdate || lastUpdate < cutoffDate) .filter(({ active }) => !!active) .toArray(); return sources.map((source) => new AnnotationSource(source)); } static async createAnnotation( annotationStub: Partial, sourceId?: AnnotationSource['data']['_id'], ) { let sourceObjs; if (sourceId) { sourceObjs = [await AnnotationSource.get(sourceId)]; } else { sourceObjs = await AnnotationSource.getPossiblyWritableSources(); if (sourceObjs.length === 0) throw new Error( 'Please first subscribe to the annotation collection where you want to store your annotations.', ); sourceObjs = sourceObjs.filter( (sourceObj) => sourceObj.data.useForNewAnnotations, ); if (sourceObjs.length === 0) throw new Error( 'Please select (in the extension’s popup menu) in which collection to store your annotations.', ); } // There should only be one source marked with useForNewAnnotations. const createdAnnotation = await sourceObjs[0].createAnnotation( annotationStub, ); return createdAnnotation; } } async function getAllAnnotationsFromContainerSource( sourceUrl: string, ): Promise { console.log(`Fetching annotations from ${sourceUrl}`); const annotationSourceData = await resolveSingle(sourceUrl); // Check what type of source we got. const nodeTypes = asArray(annotationSourceData.type); // If the source is a single annotation, import that one. if (nodeTypes.includes('Annotation')) { return [annotationSourceData]; } // If the source is an annotation container/collection, import all its items. if (nodeTypes.includes('AnnotationCollection')) { // Read the first page of annotations. let page = await resolveSingle(annotationSourceData.first); const annotations: WebAnnotation[] = asArray(page.items); // Fetch the subsequent pages, if any. while ((page = await resolveSingle(page.next))) { annotations.push(...asArray(page.items)); } return annotations; } throw new Error( `Annotation source is neither AnnotationCollection nor Annotation.`, ); } /** * Given the value of an `@id`-property, get the ‘actual’ value: * - if the node is nested, return value as-is. * - if the value is a string (= a URL), fetch and return its data. * - if there is no value, return `undefined`. * - if there are multiple values, process only the first. * * TODO Consider using json-ld tools: * - https://github.com/LDflex/LDflex/ * - https://github.com/assemblee-virtuelle/LDP-navigator */ async function resolveSingle( valuesOrIds: ZeroOrMore | object, ): Promise { const valueOrId = asSingleValue(valuesOrIds); // If it’s a value (or undefined), we are done. if (typeof valueOrId !== 'string') return valueOrId; // It’s an id, i.e. a URL. (TODO use correct base for relative URLs) const response = await fetch(valueOrId, { headers: { Accept: 'application/ld+json;profile="http://www.w3.org/ns/anno.jsonld"', Prefer: 'return=representation;include="http://www.w3.org/ns/oa#PreferContainedDescriptions"', }, cache: 'no-cache', }); let data; try { data = await response.json(); } catch (error) { throw new Error(`Received invalid JSON from URL <${valueOrId}>, ${error}`); } if (typeof data !== 'object') throw new Error('Response is valid JSON but not an object.'); return data; } /** * Extract the annotations embedded in a page, via the content script. * Only works if the page is opened. (though we could fetch&parse the html ourselves) */ async function getAnnotationsFromEmbeddedJsonld( url: string, ): Promise { const tabs = await browser.tabs.query({}); const sourceTab = tabs.find((tab) => tab.url?.startsWith(url.split('#')[0])); if (sourceTab) { const contentScriptRpc = new RpcClient({ tabId: sourceTab.id, }); const annotations = await contentScriptRpc.func( 'discoverAnnotationsEmbeddedAsJSONLD', )(); return annotations; } else { throw new Error( `To refresh annotations extracted from a web page, first open that page.`, ); } }