refactor: use IndexedDB to store note link relation data

addon/chrome/content/preferences.xhtml

@@ -12,11 +12,6 @@
       native="true"
       preference="__prefsPrefix__.openNote.takeover"
     />
-    <checkbox
-      data-l10n-id="basic-related-takeover"
-      native="true"
-      preference="__prefsPrefix__.related.takeover"
-    />
   </groupbox>
   <groupbox>
     <label><html:h2 data-l10n-id="editor-title"></html:h2></label>

addon/prefs.js

@@ -29,4 +29,3 @@ pref("__prefsPrefix__.workspace.outline.expandLevel", 2);
 pref("__prefsPrefix__.workspace.outline.keepLinks", true);
 
 pref("__prefsPrefix__.openNote.takeover", true);
-pref("__prefsPrefix__.related.takeover", false);

package.json

@@ -34,6 +34,7 @@
   "homepage": "https://github.com/windingwind/zotero-better-notes#readme",
   "dependencies": {
     "asciidoctor": "^3.0.2",
+    "dexie": "^4.0.4",
     "diff": "^5.1.0",
     "hast-util-to-html": "^9.0.0",
     "hast-util-to-mdast": "^8.4.1",

src/addon.ts

@@ -73,6 +73,9 @@ class Addon {
         data: Record<string, any>;
       };
     };
+    relation: {
+      worker?: Worker;
+    };
     readonly prompt?: Prompt;
   } = {
     alive: true,
@@ -115,6 +118,7 @@ class Addon {
         data: {},
       },
     },
+    relation: {},
     get prompt() {
       return ToolkitGlobal.getInstance().prompt.instance;
     },

src/api.ts

@@ -65,7 +65,11 @@ import {
   getNoteTreeFlattened,
   getLinesInNote,
 } from "./utils/note";
-import { updateRelatedNotes, getRelatedNoteIds } from "./utils/related";
+import {
+  getNoteLinkInboundRelation,
+  getNoteLinkOutboundRelation,
+  updateNoteLinkRelation,
+} from "./utils/relation";
 
 const workspace = {};
 
@@ -145,8 +149,9 @@ const note = {
 };
 
 const related = {
-  updateRelatedNotes,
-  getRelatedNoteIds,
+  getNoteLinkInboundRelation,
+  getNoteLinkOutboundRelation,
+  updateNoteLinkRelation,
 };
 
 export default {

src/elements/outlinePane.ts

@@ -169,9 +169,7 @@ export class OutlinePane extends PluginCEBase {
     if (event === "modify" && type === "item") {
       if ((ids as number[]).includes(this.item.id)) {
         this.updateOutline();
-        if (getPref("related.takeover")) {
-          this._addon.api.related.updateRelatedNotes(this.item.id);
-        }
+        this._addon.api.related.updateNoteLinkRelation(this.item.id);
       }
     }
   }

src/elements/related.ts

@@ -78,10 +78,7 @@ export class NoteRelatedBox extends RelatedBox {
           row.append(note);
         }
 
-        if (
-          this.editable &&
-          (!relatedItem.isNote() || !getPref("related.takeover"))
-        ) {
+        if (this.editable) {
           const remove = document.createXULElement("toolbarbutton");
           remove.addEventListener("command", () => this._handleRemove(id));
           remove.className = "zotero-clicky zotero-clicky-minus";

src/elements/workspace.ts

@@ -58,6 +58,8 @@ export class Workspace extends PluginCEBase {
   }
 
   set item(val) {
+    if (!val) return;
+    this._addon.api.related.updateNoteLinkRelation(val.id);
     this._item = val;
     this._outline.item = val;
     this._context.item = val;
@@ -83,6 +85,7 @@ export class Workspace extends PluginCEBase {
     this._loadPersist();
 
     this.resizeOb = new ResizeObserver(() => {
+      if (!this.editor) return;
       this._addon.api.editor.scroll(
         this.editor,
         this._addon.api.editor.getLineAtCursor(this.editor),

src/extras/relationWorker.ts

@@ -0,0 +1,108 @@
+import Dexie from "dexie";
+
+const db = new Dexie("BN_Two_Way_Relation") as Dexie & {
+  link: Dexie.Table<LinkModel>;
+};
+
+db.version(1).stores({
+  link: "++id, fromLibID, fromKey, toLibID, toKey, fromLine, toLine, toSection, url",
+});
+
+console.log("Using Dexie v" + Dexie.semVer, db);
+
+postMessage({
+  type: "ready",
+});
+
+async function addLink(model: LinkModel) {
+  await db.link.add(model);
+}
+
+async function bulkAddLink(models: LinkModel[]) {
+  await db.link.bulkAdd(models);
+}
+
+async function rebuildLinkForNote(
+  fromLibID: number,
+  fromKey: string,
+  links: LinkModel[],
+) {
+  console.log("rebuildLinkForNote", fromLibID, fromKey, links);
+  await db.link
+    .where({ fromLibID, fromKey })
+    .delete()
+    .then((deleteCount) => {
+      console.log("Deleted " + deleteCount + " objects");
+      bulkAddLink(links);
+    });
+}
+
+async function getOutboundLinks(fromLibID: number, fromKey: string) {
+  console.log("getOutboundLinks", fromLibID, fromKey);
+  return db.link.where({ fromLibID, fromKey }).toArray();
+}
+
+async function getInboundLinks(toLibID: number, toKey: string) {
+  console.log("getInboundLinks", toLibID, toKey);
+  return db.link.where({ toLibID, toKey }).toArray();
+}
+
+interface LinkModel {
+  fromLibID: number;
+  fromKey: string;
+  toLibID: number;
+  toKey: string;
+  fromLine: number;
+  toLine: number | null;
+  toSection: string | null;
+  url: string;
+}
+
+// Handle messages from the main thread and send responses back for await
+onmessage = async (event) => {
+  const { type, jobID, data } = event.data;
+  console.log("Worker received message", type, jobID, data);
+  switch (type) {
+    case "addLink":
+      postMessage({
+        type,
+        jobID,
+        result: await addLink(data),
+      });
+      break;
+    case "bulkAddLink":
+      postMessage({
+        type,
+        jobID,
+        result: await bulkAddLink(data),
+      });
+      break;
+    case "rebuildLinkForNote":
+      postMessage({
+        type,
+        jobID,
+        result: await rebuildLinkForNote(
+          data.fromLibID,
+          data.fromKey,
+          data.links,
+        ),
+      });
+      break;
+    case "getOutboundLinks":
+      postMessage({
+        type,
+        jobID,
+        result: await getOutboundLinks(data.fromLibID, data.fromKey),
+      });
+      break;
+    case "getInboundLinks":
+      postMessage({
+        type,
+        jobID,
+        result: await getInboundLinks(data.toLibID, data.toKey),
+      });
+      break;
+    default:
+      break;
+  }
+};

src/hooks.ts

@@ -35,10 +35,7 @@ import { createZToolkit } from "./utils/ztoolkit";
 import { waitUtilAsync } from "./utils/wait";
 import { initSyncList } from "./modules/sync/api";
 import { patchViewItems } from "./modules/viewItems";
-import {
-  onUpdateRelated,
-  promptRelatedPermission,
-} from "./modules/relatedNotes";
+import { onUpdateRelated } from "./modules/relatedNotes";
 import { getFocusedWindow } from "./utils/window";
 import { registerNoteRelation } from "./modules/workspace/relation";
 import { getPref } from "./utils/prefs";
@@ -69,8 +66,6 @@ async function onStartup() {
 
   setSyncing();
 
-  promptRelatedPermission();
-
   await onMainWindowLoad(window);
 }
 
@@ -130,7 +125,7 @@ function onNotify(
         skipActive: true,
         reason: "item-modify",
       });
-      addon.hooks.onUpdateRelated(modifiedNotes, { skipActive: true });
+      addon.hooks.onUpdateRelated(modifiedNotes);
       onUpdateNoteTabsTitle(modifiedNotes);
     }
   } else {

src/modules/editor/toolbar.ts

@@ -174,7 +174,7 @@ async function getMenuData(editor: Zotero.EditorInstance) {
           id: makeId("settings-updateRelatedNotes"),
           text: getString("editor-toolbar-settings-updateRelatedNotes"),
           callback: (e) => {
-            addon.api.related.updateRelatedNotes(e.editor._item.id);
+            addon.api.related.updateNoteLinkRelation(e.editor._item.id);
           },
         },
       ]),

src/modules/relatedNotes.ts

@@ -1,49 +1,7 @@
-import { getPref, setPref } from "../utils/prefs";
+export { onUpdateRelated };
 
-export { onUpdateRelated, promptRelatedPermission };
-
-function onUpdateRelated(
-  items: Zotero.Item[] = [],
-  { skipActive } = {
-    skipActive: true,
-  },
-) {
-  if (!getPref("related.takeover")) {
-    return;
-  }
-  if (skipActive) {
-    // Skip active note editors' targets
-    const activeNoteIds = Zotero.Notes._editorInstances
-      .filter(
-        (editor) =>
-          !Components.utils.isDeadWrapper(editor._iframeWindow) &&
-          editor._iframeWindow.document.hasFocus(),
-      )
-      .map((editor) => editor._item.id);
-    const filteredItems = items.filter(
-      (item) => !activeNoteIds.includes(item.id),
-    );
-    items = filteredItems;
-  }
+function onUpdateRelated(items: Zotero.Item[] = []) {
   for (const item of items) {
-    addon.api.related.updateRelatedNotes(item.id);
-  }
-}
-
-function promptRelatedPermission() {
-  if (getPref("related.takeover")) {
-    return;
-  }
-  const result = Zotero.Prompt.confirm({
-    title: "Permission Request",
-    text: `Better Notes want to take over (add and remove) related field of your notes.
-If you refuse, you can still use Better Notes, but most of the linking features will not work.
-You can change this permission in settings later.`,
-    button0: "Allow",
-    button1: "Refuse",
-  });
-
-  if (result === 0) {
-    setPref("related.takeover", true);
+    addon.api.related.updateNoteLinkRelation(item.id);
   }
 }

src/modules/sync/infoWindow.ts

@@ -44,12 +44,17 @@ export async function showSyncInfo(noteId: number) {
     })
     .addButton(getString("syncInfo.unSync"), "unSync", {
       callback: async (ev) => {
-        const { detectedIDSet } =
-          await addon.api.related.getRelatedNoteIds(noteId);
-        for (const itemId of Array.from(detectedIDSet)) {
-          addon.api.sync.removeSyncNote(itemId);
+        const outLink =
+          await addon.api.related.getNoteLinkOutboundRelation(noteId);
+        for (const linkData of outLink) {
+          const noteItem = await Zotero.Items.getByLibraryAndKeyAsync(
+            linkData.toLibID,
+            linkData.toKey,
+          );
+          if (!noteItem) continue;
+          addon.api.sync.removeSyncNote(noteItem.id);
         }
-        showHint(`Cancel sync of ${detectedIDSet.size} notes.`);
+        showHint(`Cancel sync of ${outLink.length} notes.`);
       },
     })
     .addButton(getString("syncInfo.reveal"), "reveal", {

src/modules/workspace/relation.ts

@@ -78,48 +78,53 @@ async function refresh(body: HTMLElement, item: Zotero.Item) {
 
 async function getRelationData(note: Zotero.Item) {
   if (!note) return;
-  const currentContent = note.getNote();
-  const currentLink = addon.api.convert.note2link(note);
-  const currentTitle = slice(note.getNoteTitle(), 15);
-  const { detectedIDSet, currentIDSet } =
-    await addon.api.related.getRelatedNoteIds(note.id);
-  if (!areSetsEqual(detectedIDSet, currentIDSet)) {
-    await addon.api.related.updateRelatedNotes(note.id);
-  }
-  const items = Zotero.Items.get(Array.from(detectedIDSet));
+  const inLink = await addon.api.related.getNoteLinkInboundRelation(note.id);
+  const outLink = await addon.api.related.getNoteLinkOutboundRelation(note.id);
 
-  const nodes = [];
   const links = [];
-  for (const item of items) {
-    const compareContent = item.getNote();
-    const compareLink = addon.api.convert.note2link(item);
-    const compareTitle = slice(item.getNoteTitle(), 15);
+  const noteSet: Set<number> = new Set();
 
-    if (currentLink && compareContent.includes(currentLink)) {
-      links.push({
-        source: item.id,
-        target: note.id,
-        value: 1,
-      });
-    }
-    if (compareLink && currentContent.includes(compareLink)) {
-      links.push({
-        source: note.id,
-        target: item.id,
-        value: 1,
-      });
-    }
-
-    nodes.push({
-      id: item.id,
-      title: compareTitle,
-      group: 2,
+  for (const linkData of inLink) {
+    const noteItem = await Zotero.Items.getByLibraryAndKeyAsync(
+      linkData.fromLibID,
+      linkData.fromKey,
+    );
+    if (!noteItem) continue;
+    noteSet.add(noteItem.id);
+    links.push({
+      source: noteItem.id,
+      target: note.id,
+      value: 1,
     });
   }
 
+  for (const linkData of outLink) {
+    const noteItem = await Zotero.Items.getByLibraryAndKeyAsync(
+      linkData.toLibID,
+      linkData.toKey,
+    );
+    if (!noteItem) continue;
+    noteSet.add(noteItem.id);
+    links.push({
+      source: note.id,
+      target: noteItem.id,
+      value: 1,
+    });
+  }
+
+  noteSet.delete(note.id);
+  const nodes = Array.from(noteSet).map((id) => {
+    const item = Zotero.Items.get(id);
+    return {
+      id: item.id,
+      title: slice(item.getNoteTitle(), 15),
+      group: 2,
+    };
+  });
+
   nodes.push({
     id: note.id,
-    title: currentTitle,
+    title: slice(note.getNoteTitle(), 15),
     group: 1,
   });
 

src/utils/related.ts

@@ -1,87 +0,0 @@
-import { getNoteLink, getNoteLinkParams } from "./link";
-
-export { getRelatedNoteIds, updateRelatedNotes };
-
-async function updateRelatedNotes(noteID: number) {
-  const noteItem = Zotero.Items.get(noteID);
-  if (!noteItem) {
-    ztoolkit.log(`updateRelatedNotes: ${noteID} is not a note.`);
-    return;
-  }
-  const { detectedIDSet, currentIDSet } = await getRelatedNoteIds(noteID);
-
-  await Zotero.DB.executeTransaction(async () => {
-    const saveParams = {
-      skipDateModifiedUpdate: true,
-      skipSelect: true,
-      notifierData: {
-        skipBN: true,
-      },
-    };
-    for (const toAddNote of Zotero.Items.get(Array.from(detectedIDSet))) {
-      if (currentIDSet.has(toAddNote.id)) {
-        // Remove existing notes from current dict for later process
-        currentIDSet.delete(toAddNote.id);
-        continue;
-      }
-      toAddNote.addRelatedItem(noteItem);
-      noteItem.addRelatedItem(toAddNote);
-      toAddNote.save(saveParams);
-      currentIDSet.delete(toAddNote.id);
-    }
-    for (const toRemoveNote of Zotero.Items.get(Array.from(currentIDSet))) {
-      // Remove related notes that are not in the new list
-      toRemoveNote.removeRelatedItem(noteItem);
-      noteItem.removeRelatedItem(toRemoveNote);
-      toRemoveNote.save(saveParams);
-    }
-    noteItem.save(saveParams);
-  });
-}
-
-async function getRelatedNoteIds(noteId: number) {
-  let detectedIDs: number[] = [];
-  const note = Zotero.Items.get(noteId);
-  const linkMatches = note.getNote().match(/zotero:\/\/note\/\w+\/\w+\//g);
-  const currentIDs: number[] = [];
-
-  if (linkMatches) {
-    const subNoteIds = (
-      await Promise.all(
-        linkMatches.map(async (link) => getNoteLinkParams(link).noteItem),
-      )
-    )
-      .filter((item) => item && item.isNote())
-      .map((item) => (item as Zotero.Item).id);
-    detectedIDs = detectedIDs.concat(subNoteIds);
-  }
-
-  const currentNoteLink = getNoteLink(note);
-  if (currentNoteLink) {
-    // Get current related items
-    for (const relItemKey of note.relatedItems) {
-      try {
-        const relItem = (await Zotero.Items.getByLibraryAndKeyAsync(
-          note.libraryID,
-          relItemKey,
-        )) as Zotero.Item;
-
-        // If the related item is a note and contains the current note link
-        // Add it to the related note list
-        if (relItem.isNote()) {
-          if (relItem.getNote().includes(currentNoteLink)) {
-            detectedIDs.push(relItem.id);
-          }
-          currentIDs.push(relItem.id);
-        }
-      } catch (e) {
-        ztoolkit.log(e);
-      }
-    }
-  }
-
-  const detectedIDSet = new Set(detectedIDs);
-  detectedIDSet.delete(noteId);
-  const currentIDSet = new Set(currentIDs);
-  return { detectedIDSet, currentIDSet };
-}

src/utils/relation.ts

@@ -0,0 +1,128 @@
+import { config } from "../../package.json";
+import { getNoteLinkParams } from "./link";
+
+export {
+  updateNoteLinkRelation,
+  getNoteLinkInboundRelation,
+  getNoteLinkOutboundRelation,
+};
+
+async function getRelationWorker() {
+  if (addon.data.relation.worker) {
+    return addon.data.relation.worker;
+  }
+  const deferred = Zotero.Promise.defer();
+  const worker = new Worker(
+    `chrome://${config.addonRef}/content/scripts/relationWorker.js`,
+  );
+  addon.data.relation.worker = worker;
+  worker.addEventListener(
+    "message",
+    (e) => {
+      if (e.data === "ready") {
+        ztoolkit.log("Relation worker is ready.");
+        deferred.resolve();
+      }
+    },
+    { once: true },
+  );
+  await deferred.promise;
+  return worker;
+}
+
+async function executeRelationWorker(data: {
+  type: string;
+  data: any;
+}): Promise<any> {
+  const worker = await getRelationWorker();
+  const deferred = Zotero.Promise.defer();
+  const jobID = Zotero.Utilities.randomString(8);
+  let retData;
+  ztoolkit.log("executeRelationWorker", data, jobID);
+  worker.addEventListener(
+    "message",
+    (e) => {
+      if (e.data.jobID === jobID) {
+        retData = e.data;
+        deferred.resolve();
+      }
+    },
+    { once: true },
+  );
+  worker.postMessage({ ...data, jobID });
+  await Promise.race([deferred.promise, Zotero.Promise.delay(5000)]);
+  if (!retData) {
+    throw new Error(`Worker timeout: ${data.type}, ${jobID}`);
+  }
+  ztoolkit.log("executeRelationWorker return", retData);
+  return (retData as { result: any }).result;
+}
+
+async function updateNoteLinkRelation(noteID: number) {
+  const note = Zotero.Items.get(noteID);
+  const fromLibID = note.libraryID;
+  const fromKey = note.key;
+  const lines = addon.api.note.getLinesInNote(note);
+  const linkToData: LinkModel[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    const linkMatches = lines[i].match(/zotero:\/\/note\/\w+\/\w+\//g);
+    if (!linkMatches) {
+      continue;
+    }
+    for (const link of linkMatches) {
+      const { noteItem, libraryID, noteKey, lineIndex, sectionName } =
+        getNoteLinkParams(link);
+      if (noteItem && noteItem.isNote()) {
+        linkToData.push({
+          fromLibID,
+          fromKey,
+          toLibID: libraryID,
+          toKey: noteKey!,
+          fromLine: i,
+          toLine: lineIndex ?? null,
+          toSection: sectionName ?? null,
+          url: link,
+        });
+      }
+    }
+  }
+  await executeRelationWorker({
+    type: "rebuildLinkForNote",
+    data: { fromLibID, fromKey, links: linkToData },
+  });
+}
+
+async function getNoteLinkOutboundRelation(
+  noteID: number,
+): Promise<LinkModel[]> {
+  const note = Zotero.Items.get(noteID);
+  const fromLibID = note.libraryID;
+  const fromKey = note.key;
+  return executeRelationWorker({
+    type: "getOutboundLinks",
+    data: { fromLibID, fromKey },
+  });
+}
+
+async function getNoteLinkInboundRelation(
+  noteID: number,
+): Promise<LinkModel[]> {
+  const note = Zotero.Items.get(noteID);
+  const toLibID = note.libraryID;
+  const toKey = note.key;
+  return executeRelationWorker({
+    type: "getInboundLinks",
+    data: { toLibID, toKey },
+  });
+}
+
+interface LinkModel {
+  fromLibID: number;
+  fromKey: string;
+  toLibID: number;
+  toKey: string;
+  fromLine: number;
+  toLine: number | null;
+  toSection: string | null;
+  url: string;
+}