add: parsing worker

fix: #1166

package-lock.json

@@ -35,7 +35,7 @@
         "unist-util-visit": "^5.0.0",
         "unist-util-visit-parents": "^6.0.1",
         "yamljs": "^0.3.0",
-        "zotero-plugin-toolkit": "^4.0.6"
+        "zotero-plugin-toolkit": "^4.0.7"
       },
       "devDependencies": {
         "@esbuild-plugins/node-globals-polyfill": "^0.2.3",
@@ -13588,9 +13588,9 @@
       }
     },
     "node_modules/zotero-plugin-toolkit": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/zotero-plugin-toolkit/-/zotero-plugin-toolkit-4.0.6.tgz",
-      "integrity": "sha512-juxIrSrUYTxk+efQJAH7OpfQHSboErMd0Ygu2eZs/xKA1177XlCYhe0sdxlTxI8Y/4UGtRwLicThUddaOfArMQ==",
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/zotero-plugin-toolkit/-/zotero-plugin-toolkit-4.0.7.tgz",
+      "integrity": "sha512-9bAXXJTgH5SkyauGft50EerudZYGYzN/T3kPgi7udB8lScVyO+HkYERxJqnFuvk2gHzV+stqeaKaX9a0ODTxEQ==",
       "dependencies": {
         "zotero-types": "^2.2.0"
       },

package.json

@@ -56,7 +56,7 @@
     "unist-util-visit": "^5.0.0",
     "unist-util-visit-parents": "^6.0.1",
     "yamljs": "^0.3.0",
-    "zotero-plugin-toolkit": "^4.0.6"
+    "zotero-plugin-toolkit": "^4.0.7"
   },
   "devDependencies": {
     "@esbuild-plugins/node-globals-polyfill": "^0.2.3",

src/addon.ts

@@ -5,6 +5,8 @@ import { SyncDataType } from "./modules/sync/managerWindow";
 import hooks from "./hooks";
 import api from "./api";
 import { createZToolkit } from "./utils/ztoolkit";
+import { MessageHelper } from "zotero-plugin-toolkit/dist/helpers/message";
+import type { handlers } from "./extras/parsingWorker";
 
 class Addon {
   public data: {
@@ -70,6 +72,9 @@ class Addon {
     relation: {
       worker?: Worker;
     };
+    parsing: {
+      server?: MessageHelper<typeof handlers>;
+    };
     imageCache: Record<number, string>;
     hint: {
       silent: boolean;
@@ -117,6 +122,7 @@ class Addon {
       },
     },
     relation: {},
+    parsing: {},
     imageCache: {},
     hint: {
       silent: false,

src/elements/linkCreator/outlinePicker.ts

@@ -131,7 +131,9 @@ export class OutlinePicker extends PluginCEBase {
     if (!this.item) {
       return;
     }
-    this.noteOutline = this._addon.api.note.getNoteTreeFlattened(this.item);
+    this.noteOutline = await this._addon.api.note.getNoteTreeFlattened(
+      this.item,
+    );
     // Fake a cursor position
     if (typeof this.lineIndex === "number") {
       // @ts-ignore - formatValues is not in the types

src/elements/workspace/outlinePane.ts

@@ -211,12 +211,13 @@ export class OutlinePane extends PluginCEBase {
       "message",
       this.messageHandler,
     );
+    const nodes = await this._addon.api.note.getNoteTreeFlattened(this.item, {
+      keepLink: !!getPref("workspace.outline.keepLinks"),
+    });
     this._outlineContainer.contentWindow?.postMessage(
       {
         type: "setMindMapData",
-        nodes: this._addon.api.note.getNoteTreeFlattened(this.item, {
-          keepLink: !!getPref("workspace.outline.keepLinks"),
-        }),
+        nodes,
         expandLevel: getPref("workspace.outline.expandLevel"),
       },
       "*",
@@ -316,13 +317,13 @@ export class OutlinePane extends PluginCEBase {
       }
       case "moveNode": {
         if (!this.item) return;
-        const tree = this._addon.api.note.getNoteTree(this.item);
-        const fromNode = this._addon.api.note.getNoteTreeNodeById(
+        const tree = await this._addon.api.note.getNoteTree(this.item);
+        const fromNode = await this._addon.api.note.getNoteTreeNodeById(
           this.item,
           ev.data.fromID,
           tree,
         );
-        const toNode = this._addon.api.note.getNoteTreeNodeById(
+        const toNode = await this._addon.api.note.getNoteTreeNodeById(
           this.item,
           ev.data.toID,
           tree,

src/extras/parsingWorker.ts

@@ -0,0 +1,114 @@
+import { MessageHelper } from "zotero-plugin-toolkit";
+
+export { handlers };
+
+function parseHTMLLines(html: string): string[] {
+  const randomString: string = `${Math.random()}`;
+  console.time(`parseHTMLLines-${randomString}`);
+
+  // Remove container with one of the attrs named data-schema-version if exists
+  if (html.includes("data-schema-version")) {
+    html = html.replace(/<div[^>]*data-schema-version[^>]*>/, "");
+    html = html.replace(/<\/div>/, "");
+  }
+  const noteLines = html.split("\n").filter((e) => e);
+
+  // A cache for temporarily stored lines
+  let previousLineCache = [];
+  let nextLineCache = [];
+
+  const forceInline = ["table", "blockquote", "pre", "ol", "ul"];
+  const selfInline: string[] = [];
+  const forceInlineStack = [];
+  let forceInlineFlag = false;
+  let selfInlineFlag = false;
+
+  const parsedLines = [];
+  for (const line of noteLines) {
+    // restore self inline flag
+    selfInlineFlag = false;
+
+    // For force inline tags, set flag to append lines to current line
+    for (const tag of forceInline) {
+      const startReg = `<${tag}`;
+      const isStart = line.includes(startReg);
+      const endReg = `</${tag}>`;
+      const isEnd = line.includes(endReg);
+      if (isStart && !isEnd) {
+        forceInlineStack.push(tag);
+        // console.log("push", tag, line, forceInlineStack);
+        forceInlineFlag = true;
+        break;
+      }
+      if (isEnd && !isStart) {
+        forceInlineStack.pop();
+        // console.log("pop", tag, line, forceInlineStack);
+        // Exit force inline mode if the stack is empty
+        if (forceInlineStack.length === 0) {
+          forceInlineFlag = false;
+        }
+        break;
+      }
+    }
+
+    if (forceInlineFlag) {
+      nextLineCache.push(line);
+    } else {
+      // For self inline tags, cache start as previous line and end as next line
+      for (const tag of selfInline) {
+        const isStart = line.includes(`<${tag}`);
+        const isEnd = line.includes(`</${tag}>`);
+        if (isStart && !isEnd) {
+          selfInlineFlag = true;
+          nextLineCache.push(line);
+          break;
+        }
+        if (!isStart && isEnd) {
+          selfInlineFlag = true;
+          previousLineCache.push(line);
+          break;
+        }
+      }
+
+      if (!selfInlineFlag) {
+        // Append cache to previous line
+        if (previousLineCache.length) {
+          parsedLines[parsedLines.length - 1] += `\n${previousLineCache.join(
+            "\n",
+          )}`;
+          previousLineCache = [];
+        }
+        let nextLine = "";
+        // Append cache to next line
+        if (nextLineCache.length) {
+          nextLine = nextLineCache.join("\n");
+          nextLineCache = [];
+        }
+        if (nextLine) {
+          nextLine += "\n";
+        }
+        nextLine += `${line}`;
+        parsedLines.push(nextLine);
+      }
+    }
+  }
+  console.timeEnd(`parseHTMLLines-${randomString}`);
+
+  return parsedLines;
+}
+
+const funcs = {
+  parseHTMLLines,
+};
+
+const handlers = MessageHelper.wrapHandlers(funcs);
+
+const messageServer = new MessageHelper({
+  canBeDestroyed: true,
+  dev: true,
+  name: "parsingWorker",
+  target: self,
+  handlers,
+});
+
+messageServer.start();

src/hooks.ts

@@ -46,6 +46,7 @@ import { closeRelationWorker } from "./utils/relation";
 import { registerNoteLinkSection } from "./modules/workspace/link";
 import { showUserGuide } from "./modules/userGuide";
 import { refreshTemplatesInNote } from "./modules/template/refresh";
+import { closeParsingServer } from "./utils/parsing";
 
 async function onStartup() {
   await Promise.all([
@@ -109,6 +110,7 @@ async function onMainWindowUnload(win: Window): Promise<void> {
 
 function onShutdown(): void {
   closeRelationWorker();
+  closeParsingServer();
   ztoolkit.unregisterAll();
   // Remove addon object
   addon.data.alive = false;

src/modules/editor/toolbar.ts

@@ -71,7 +71,7 @@ async function getMenuData(editor: Zotero.EditorInstance) {
   const noteItem = editor._item;
 
   const currentLine = getLineAtCursor(editor);
-  const currentSection = getSectionAtCursor(editor) || "";
+  const currentSection = (await getSectionAtCursor(editor)) || "";
   const settingsMenuData: PopupData[] = [
     {
       id: makeId("settings-openAsTab"),

src/modules/export/api.ts

@@ -217,7 +217,7 @@ async function embedLinkedNotes(noteItem: Zotero.Item): Promise<string> {
   const globalCitationData = getNoteCitationData(noteItem as Zotero.Item);
 
   const newLines: string[] = [];
-  const noteLines = getLinesInNote(noteItem);
+  const noteLines = await getLinesInNote(noteItem);
   for (const i in noteLines) {
     newLines.push(noteLines[i]);
     const doc = parser.parseFromString(noteLines[i], "text/html");

src/modules/export/freemind.ts

@@ -15,7 +15,7 @@ async function note2mm(
   noteItem: Zotero.Item,
   options: { withContent?: boolean } = { withContent: true },
 ) {
-  const root = getNoteTree(noteItem, false);
+  const root = await getNoteTree(noteItem, false);
   const textNodeForEach = (e: Node, callbackfn: (e: any) => void) => {
     if (e.nodeType === Zotero.getMainWindow().document.TEXT_NODE) {
       callbackfn(e);
@@ -32,7 +32,7 @@ async function note2mm(
     textNodeForEach(doc.body, (e: Text) => {
       e.data = htmlEscape(doc, e.data);
     });
-    lines = parseHTMLLines(doc.body.innerHTML).map((line) =>
+    lines = (await parseHTMLLines(doc.body.innerHTML)).map((line) =>
       htmlUnescape(line),
     );
   }

src/modules/template/refresh.ts

@@ -4,7 +4,7 @@ import { htmlUnescape } from "../../utils/str";
 export { refreshTemplatesInNote };
 
 async function refreshTemplatesInNote(editor: Zotero.EditorInstance) {
-  const lines = addon.api.note.getLinesInNote(editor._item);
+  const lines = await addon.api.note.getLinesInNote(editor._item);
   let startIndex = -1;
   const matchedIndexPairs: { from: number; to: number }[] = [];
 

src/utils/convert.ts

@@ -212,13 +212,15 @@ async function link2html(
     let lineIndex = linkParams.lineIndex;
 
     if (typeof linkParams.sectionName === "string") {
-      const sectionTree = addon.api.note.getNoteTreeFlattened(item);
+      const sectionTree = await addon.api.note.getNoteTreeFlattened(item);
       const sectionNode = sectionTree.find(
         (node) => node.model.name.trim() === linkParams.sectionName!.trim(),
       );
       lineIndex = sectionNode?.model.lineIndex;
     }
-    html = addon.api.note.getLinesInNote(item).slice(lineIndex).join("\n");
+    html = (await addon.api.note.getLinesInNote(item))
+      .slice(lineIndex)
+      .join("\n");
   } else {
     html = addon.api.sync.getNoteStatus(linkParams.noteItem.id)?.content || "";
   }

src/utils/editor.ts

@@ -136,9 +136,12 @@ function scroll(editor: Zotero.EditorInstance, lineIndex: number) {
   core.view.dom.parentElement?.scrollTo(0, offset);
 }
 
-function scrollToSection(editor: Zotero.EditorInstance, sectionName: string) {
+async function scrollToSection(
+  editor: Zotero.EditorInstance,
+  sectionName: string,
+) {
   const item = editor._item;
-  const sectionTree = getNoteTreeFlattened(item);
+  const sectionTree = await getNoteTreeFlattened(item);
   const sectionNode = sectionTree.find(
     (node) => node.model.name.trim() === sectionName.trim(),
   );
@@ -199,11 +202,13 @@ function getLineAtCursor(editor: Zotero.EditorInstance) {
   return i;
 }
 
-function getSectionAtCursor(editor: Zotero.EditorInstance): string | undefined {
+async function getSectionAtCursor(
+  editor: Zotero.EditorInstance,
+): Promise<string | undefined> {
   const lineIndex = getLineAtCursor(editor);
   if (lineIndex < 0) return undefined;
   const item = editor._item;
-  const sectionTree = getNoteTreeFlattened(item);
+  const sectionTree = await getNoteTreeFlattened(item);
   let sectionNode;
   for (let i = 0; i < sectionTree.length; i++) {
     if (

src/utils/note.ts

@@ -2,8 +2,8 @@ import TreeModel = require("tree-model");
 import katex = require("katex");
 import { getEditorInstance, getPositionAtLine, insert } from "./editor";
 import { formatPath, getItemDataURL } from "./str";
-import { showHint } from "./hint";
 import { config } from "../../package.json";
+import { getParsingServer } from "./parsing";
 
 export {
   renderNoteHTML,
@@ -18,111 +18,17 @@ export {
   importImageToNote,
 };
 
-function parseHTMLLines(html: string): string[] {
-  // Remove container with one of the attrs named data-schema-version if exists
-  if (html.includes("data-schema-version")) {
-    html = html.replace(/<div[^>]*data-schema-version[^>]*>/, "");
-    html = html.replace(/<\/div>/, "");
-  }
-  const noteLines = html.split("\n").filter((e) => e);
-
-  // A cache for temporarily stored lines
-  let previousLineCache = [];
-  let nextLineCache = [];
-
-  const forceInline = ["table", "blockquote", "pre", "ol", "ul"];
-  const selfInline: string[] = [];
-  const forceInlineStack = [];
-  let forceInlineFlag = false;
-  let selfInlineFlag = false;
-
-  const parsedLines = [];
-  for (const line of noteLines) {
-    // restore self inline flag
-    selfInlineFlag = false;
-
-    // For force inline tags, set flag to append lines to current line
-    for (const tag of forceInline) {
-      const startReg = `<${tag}`;
-      const isStart = line.includes(startReg);
-      const endReg = `</${tag}>`;
-      const isEnd = line.includes(endReg);
-      if (isStart && !isEnd) {
-        forceInlineStack.push(tag);
-        ztoolkit.log("push", tag, line, forceInlineStack);
-        forceInlineFlag = true;
-        break;
-      }
-      if (isEnd && !isStart) {
-        forceInlineStack.pop();
-        ztoolkit.log("pop", tag, line, forceInlineStack);
-        // Exit force inline mode if the stack is empty
-        if (forceInlineStack.length === 0) {
-          forceInlineFlag = false;
-        }
-        break;
-      }
-    }
-
-    if (forceInlineFlag) {
-      nextLineCache.push(line);
-    } else {
-      // For self inline tags, cache start as previous line and end as next line
-      for (const tag of selfInline) {
-        const isStart = line.includes(`<${tag}`);
-        const isEnd = line.includes(`</${tag}>`);
-        if (isStart && !isEnd) {
-          selfInlineFlag = true;
-          nextLineCache.push(line);
-          break;
-        }
-        if (!isStart && isEnd) {
-          selfInlineFlag = true;
-          previousLineCache.push(line);
-          break;
-        }
-      }
-
-      if (!selfInlineFlag) {
-        // Append cache to previous line
-        if (previousLineCache.length) {
-          parsedLines[parsedLines.length - 1] += `\n${previousLineCache.join(
-            "\n",
-          )}`;
-          previousLineCache = [];
-        }
-        let nextLine = "";
-        // Append cache to next line
-        if (nextLineCache.length) {
-          nextLine = nextLineCache.join("\n");
-          nextLineCache = [];
-        }
-        if (nextLine) {
-          nextLine += "\n";
-        }
-        nextLine += `${line}`;
-        parsedLines.push(nextLine);
-      }
-    }
-  }
-  return parsedLines;
+async function parseHTMLLines(html: string) {
+  const server = await getParsingServer();
+  return await server.exec("parseHTMLLines", [html]);
 }
 
-function getLinesInNote(note: Zotero.Item): string[];
-
 async function getLinesInNote(
-  note: Zotero.Item,
-  options: {
-    convertToHTML?: true;
-  },
-): Promise<string[]>;
-
-function getLinesInNote(
   note: Zotero.Item,
   options?: {
     convertToHTML?: boolean;
   },
-): string[] | Promise<string[]> {
+): Promise<string[]> {
   if (!note) {
     return [];
   }
@@ -170,7 +76,7 @@ async function addLineToNote(
   if (!note || !html) {
     return;
   }
-  const noteLines = getLinesInNote(note);
+  const noteLines = await getLinesInNote(note);
   if (lineIndex < 0 || lineIndex >= noteLines.length) {
     lineIndex = noteLines.length;
   }
@@ -292,11 +198,14 @@ async function renderNoteHTML(
   return doc.body.innerHTML;
 }
 
-function getNoteTree(
+async function getNoteTree(
   note: Zotero.Item,
   parseLink: boolean = true,
-): TreeModel.Node<NoteNodeData> {
-  const noteLines = getLinesInNote(note);
+): Promise<TreeModel.Node<NoteNodeData>> {
+  const timeLabel = `getNoteTree-${note.id}-${Math.random()}`;
+  Zotero.getMainWindow().console.time(timeLabel);
+
+  const noteLines = await getLinesInNote(note);
   const parser = new DOMParser();
   const tree = new TreeModel();
   const root = tree.parse({
@@ -353,6 +262,8 @@ function getNoteTree(
           if (node.model.endIndex > parseInt(i) - 1) {
             node.model.endIndex = parseInt(i) - 1;
           }
+          Zotero.getMainWindow().console.timeEnd(timeLabel);
+
           return true;
         });
         previousNode.model.endIndex = parseInt(i) - 1;
@@ -360,21 +271,22 @@ function getNoteTree(
       lastNode = currentNode;
     }
   }
+  Zotero.getMainWindow().console.timeEnd(timeLabel);
   return root;
 }
 
-function getNoteTreeFlattened(
+async function getNoteTreeFlattened(
   note: Zotero.Item,
   options: {
     keepRoot?: boolean;
     keepLink?: boolean;
     customFilter?: (node: TreeModel.Node<NoteNodeData>) => boolean;
   } = { keepRoot: false, keepLink: false },
-): TreeModel.Node<NoteNodeData>[] {
+): Promise<TreeModel.Node<NoteNodeData>[]> {
   if (!note) {
     return [];
   }
-  return getNoteTree(note).all(
+  return (await getNoteTree(note)).all(
     (node) =>
       (options.keepRoot || node.model.lineIndex >= 0) &&
       (options.keepLink || node.model.level <= 6) &&
@@ -382,23 +294,23 @@ function getNoteTreeFlattened(
   );
 }
 
-function getNoteTreeNodeById(
+async function getNoteTreeNodeById(
   note: Zotero.Item,
   id: number,
   root: TreeModel.Node<NoteNodeData> | undefined = undefined,
 ) {
-  root = root || getNoteTree(note);
+  root = root || (await getNoteTree(note));
   return root.first(function (node) {
     return node.model.id === id;
   });
 }
 
-function getNoteTreeNodesByLevel(
+async function getNoteTreeNodesByLevel(
   note: Zotero.Item,
   level: number,
   root: TreeModel.Node<NoteNodeData> | undefined = undefined,
 ) {
-  root = root || getNoteTree(note);
+  root = root || (await getNoteTree(note));
   return root.all(function (node) {
     return node.model.level === level;
   });
@@ -534,7 +446,7 @@ async function importImageToNote(
   }
 
   if (!blob) {
-    showHint("Failed to import image.");
+    ztoolkit.log("Failed to import image.");
     return;
   }
 

src/utils/parsing.ts

@@ -0,0 +1,33 @@
+import { MessageHelper } from "zotero-plugin-toolkit";
+import { config } from "../../package.json";
+import type { handlers } from "../extras/parsingWorker";
+
+function closeParsingServer() {
+  if (addon.data.parsing.server) {
+    addon.data.parsing.server.destroy();
+    addon.data.parsing.server = undefined;
+  }
+}
+
+async function getParsingServer() {
+  if (addon.data.parsing.server) {
+    return addon.data.parsing.server;
+  }
+  const worker = new ChromeWorker(
+    `chrome://${config.addonRef}/content/scripts/parsingWorker.js`,
+    { name: "parsingWorker" },
+  );
+  const server = new MessageHelper<typeof handlers>({
+    canBeDestroyed: false,
+    dev: true,
+    name: "parsingWorkerMain",
+    target: worker,
+    handlers: {},
+  });
+  server.start();
+  await server.exec("_ping");
+  addon.data.parsing.server = server;
+  return server;
+}
+
+export { getParsingServer, closeParsingServer };

src/utils/relation.ts

@@ -73,7 +73,7 @@ async function updateNoteLinkRelation(noteID: number) {
   const affectedNoteIDs = new Set([noteID]);
   const fromLibID = note.libraryID;
   const fromKey = note.key;
-  const lines = addon.api.note.getLinesInNote(note);
+  const lines = await addon.api.note.getLinesInNote(note);
   const linkToData: LinkModel[] = [];
   for (let i = 0; i < lines.length; i++) {
     const linkMatches = lines[i].match(/href="zotero:\/\/note\/[^"]+"/g);