blue
/
Zotero-TLDR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update code

This commit is contained in:
shenyutao 2023-08-18 18:31:10 +08:00
parent 7fd2de3554
commit 24c229f736
7 changed files with 248 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
addon/chrome/content/preferences.xhtml
View File

@ -1 +0,0 @@

2
addon/prefs.js
View File

@ -1 +1 @@
/* eslint-disable no-undef */
/* eslint-disable no-undef */

41
src/hooks.ts
View File

@ -1,7 +1,4 @@
import {
RegisterFactory,
UIFactory,
} from "./modules/Common";
import { RegisterFactory, UIFactory } from "./modules/Common";
import { config } from "../package.json";
import { getString, initLocale } from "./utils/locale";
import { registerPrefsScripts } from "./modules/preferenceScript";
@ -17,7 +14,7 @@ async function onStartup() {
]);
initLocale();
await DataStorage.instance(TLDRFieldKey).getAsync(); // 加载TLDR数据
await DataStorage.instance(TLDRFieldKey).getAsync(); // 加载TLDR数据
RegisterFactory.registerPrefs();
@ -92,9 +89,11 @@ function onLoad() {
(async () => {
let needFetchItems: Zotero.Item[] = [];
for (const lib of Zotero.Libraries.getAll()) {
needFetchItems = needFetchItems.concat((await Zotero.Items.getAll(lib.id)).filter((item: Zotero.Item) => {
return item.isRegularItem() && !item.isCollection();
}));
needFetchItems = needFetchItems.concat(
(await Zotero.Items.getAll(lib.id)).filter((item: Zotero.Item) => {
return item.isRegularItem() && !item.isCollection();
}),
);
}
onUpdateItems(needFetchItems, false);
})();
@ -102,7 +101,7 @@ function onLoad() {
function noNotifyDeleteItem(ids: (string | number)[]) {
DataStorage.instance(TLDRFieldKey).modify((data: any) => {
ids.forEach(id => {
ids.forEach((id) => {
delete data[id];
});
return data;
@ -125,11 +124,17 @@ function onNotifyAddItems(ids: (string | number)[]) {
function onUpdateItems(items: Zotero.Item[], forceFetch: boolean = false) {
items = items.filter((item: Zotero.Item) => {
if (!item.getField('title')) { return false; }
if (!forceFetch) { return DataStorage.instance(TLDRFieldKey).get()[item.id] === undefined; }
if (!item.getField("title")) {
return false;
}
if (!forceFetch) {
return DataStorage.instance(TLDRFieldKey).get()[item.id] === undefined;
}
return true;
});
if (items.length <= 0) { return; }
if (items.length <= 0) {
return;
}
const newPopWin = (closeOnClick = false) => {
return new ztoolkit.ProgressWindow(config.addonName, {
closeOnClick: closeOnClick,
@ -138,7 +143,7 @@ function onUpdateItems(items: Zotero.Item[], forceFetch: boolean = false) {
type: "default",
progress: 0,
});
}
};
const popupWin = newPopWin().show(-1);
(async function () {
const count = items.length;
@ -146,12 +151,16 @@ function onUpdateItems(items: Zotero.Item[], forceFetch: boolean = false) {
const succeedItems: Zotero.Item[] = [];
await (async function () {
for (const [index, item] of items.entries()) {
await new TLDRFetcher(item).fetchTLDR() ? succeedItems.push(item) : failedItems.push(item);
(await new TLDRFetcher(item).fetchTLDR())
? succeedItems.push(item)
: failedItems.push(item);
await Zotero.Promise.delay(50);
ztoolkit.ItemBox.refresh();
popupWin.changeLine({
progress: index * 100 / count,
text: `Waiting: ${count - index - 1}, succeed: ${succeedItems.length}, failed: ${failedItems.length}`,
progress: (index * 100) / count,
text: `Waiting: ${count - index - 1}, succeed: ${
succeedItems.length
}, failed: ${failedItems.length}`,
});
}
})();

19
src/modules/Common.ts
View File

@ -11,7 +11,7 @@ export class RegisterFactory {
event: string,
type: string,
ids: number[] | string[],
extraData: { [key: string]: any }
extraData: { [key: string]: any },
) => {
if (!addon?.data.alive) {
this.unregisterNotifier(notifierID);
@ -22,9 +22,7 @@ export class RegisterFactory {
};
// Register the callback in Zotero as an item observer
const notifierID = Zotero.Notifier.registerObserver(callback, [
"item",
]);
const notifierID = Zotero.Notifier.registerObserver(callback, ["item"]);
// Unregister callback when the window closes (important to avoid a memory leak)
window.addEventListener(
@ -32,7 +30,7 @@ export class RegisterFactory {
(e: Event) => {
this.unregisterNotifier(notifierID);
},
false
false,
);
}
@ -40,7 +38,6 @@ export class RegisterFactory {
Zotero.Notifier.unregisterObserver(notifierID);
}
// 注册首选项配置
static registerPrefs() {
// const prefOptions = {
@ -79,9 +76,13 @@ export class UIFactory {
tag: "menuitem",
id: "zotero-collectionmenu-tldr",
label: getString("menucollection-updatetldrlabel"),
commandListener: (ev) => addon.hooks.onUpdateItems(ZoteroPane.getSelectedCollection()?.getChildItems() ?? [], false),
commandListener: (ev) =>
addon.hooks.onUpdateItems(
ZoteroPane.getSelectedCollection()?.getChildItems() ?? [],
false,
),
icon: menuIcon,
})
});
}
// tldr行
@ -120,4 +121,4 @@ export class UIFactory {
},
);
}
}
}

125
src/modules/dataStorage.ts
View File

@ -1,80 +1,89 @@
import { config } from "../../package.json";
export class Data {
private filePath: string
private inited = false;
private _data: any
private filePath: string;
private inited = false;
private _data: any;
constructor(filePath: string) {
this.filePath = filePath;
}
constructor(filePath: string) {
this.filePath = filePath;
}
async getAsync() {
await this.initDataIfNeed();
return this.data;
}
async getAsync() {
await this.initDataIfNeed();
return this.data;
}
get() {
return this.data;
}
get() {
return this.data;
}
async modify(action: (data: any) => Promise<any>) {
await this.initDataIfNeed();
const data = await this.data;
const newData = await action(data);
try {
await IOUtils.writeJSON(this.filePath, newData, {mode: 'overwrite', compress: false});
this.data = newData;
return newData;
} catch (error) {
return data;
}
async modify(action: (data: any) => Promise<any>) {
await this.initDataIfNeed();
const data = await this.data;
const newData = await action(data);
try {
await IOUtils.writeJSON(this.filePath, newData, {
mode: "overwrite",
compress: false,
});
this.data = newData;
return newData;
} catch (error) {
return data;
}
}
async delete() {
try {
await IOUtils.remove(this.filePath);
this.data = {};
return true;
} catch (error) {
return false;
}
async delete() {
try {
await IOUtils.remove(this.filePath);
this.data = {};
return true;
} catch (error) {
return false;
}
}
private get data() {
return this._data;
}
private get data() {
return this._data;
}
private set data(value: any) {
this._data = value;
}
private set data(value: any) {
this._data = value;
}
private async initDataIfNeed() {
if (this.inited) { return; }
try {
this.data = await IOUtils.readJSON(this.filePath, {decompress: false});
} catch (error) {
this.data = {};
}
private async initDataIfNeed() {
if (this.inited) {
return;
}
try {
this.data = await IOUtils.readJSON(this.filePath, { decompress: false });
} catch (error) {
this.data = {};
}
}
}
export class DataStorage {
private readonly dataDir = PathUtils.join(PathUtils.profileDir, 'extensions', config.addonName);
private dataMap: { [key: string]: Data } = {};
private readonly dataDir = PathUtils.join(
PathUtils.profileDir,
"extensions",
config.addonName,
);
private dataMap: { [key: string]: Data } = {};
private static shared = new DataStorage();
private static shared = new DataStorage();
static instance(dataType: string) {
const path = PathUtils.join(this.shared.dataDir, dataType);
if (this.shared.dataMap[dataType] === undefined) {
const data = new Data(path);
this.shared.dataMap[dataType] = data;
return data;
} else {
return this.shared.dataMap[dataType];
}
static instance(dataType: string) {
const path = PathUtils.join(this.shared.dataDir, dataType);
if (this.shared.dataMap[dataType] === undefined) {
const data = new Data(path);
this.shared.dataMap[dataType] = data;
return data;
} else {
return this.shared.dataMap[dataType];
}
}
private constructor() { }
private constructor() {}
}

3
src/modules/preferenceScript.ts
View File

@ -14,5 +14,4 @@ export async function registerPrefsScripts(_window: Window) {
bindPrefEvents();
}
function bindPrefEvents() {
}
function bindPrefEvents() {}

280
src/modules/tldrFetcher.ts
View File

@ -1,157 +1,165 @@
import { DataStorage } from "./dataStorage";
type SemanticScholarItemInfo = {
title?: string,
abstract?: string,
tldr?: string,
title?: string;
abstract?: string;
tldr?: string;
};
export const TLDRFieldKey = 'TLDR';
export const TLDRUnrelated = 'tldr-unrelated' // semantic scholar 找到了该item，但是该item没有tldr
export const TLDRItemNotFound = 'tldr-itemnotfound' // semantic scholar 找不到该item，
export const TLDRFieldKey = "TLDR";
export const TLDRUnrelated = "tldr-unrelated"; // semantic scholar 找到了该item，但是该item没有tldr
export const TLDRItemNotFound = "tldr-itemnotfound"; // semantic scholar 找不到该item，
export class TLDRFetcher {
private readonly zoteroItem: Zotero.Item;
private readonly title?: string;
private readonly abstract?: string;
private readonly zoteroItem: Zotero.Item;
private readonly title?: string;
private readonly abstract?: string;
constructor(item: Zotero.Item) {
this.zoteroItem = item;
if (item.isRegularItem() && !item.isCollection()) {
this.title = item.getField('title') as string;
this.abstract = item.getField('abstractNote') as string;
}
constructor(item: Zotero.Item) {
this.zoteroItem = item;
if (item.isRegularItem() && !item.isCollection()) {
this.title = item.getField("title") as string;
this.abstract = item.getField("abstractNote") as string;
}
}
async fetchTLDR() {
if (!this.title || this.title.length <= 0) { return false; }
try {
const infos = await this.fetchRelevanceItemInfos(this.title);
for (const info of infos) {
let match = false;
if (info.title && this.title && this.checkLCS(info.title, this.title)) {
match = true;
} else if (info.abstract && this.abstract && this.checkLCS(info.abstract, this.abstract)) {
match = true;
}
if (match) {
const result = info.tldr ?? TLDRUnrelated;
DataStorage.instance(TLDRFieldKey).modify((data: any) => {
data[this.zoteroItem.id] = result;
return data;
});
return true;
}
}
DataStorage.instance(TLDRFieldKey).modify((data: any) => {
data[this.zoteroItem.id] = TLDRItemNotFound;
return data;
});
return false;
} catch (error) {
Zotero.log(`post semantic scholar request error: ${error}`);
return false;
}
async fetchTLDR() {
if (!this.title || this.title.length <= 0) {
return false;
}
try {
const infos = await this.fetchRelevanceItemInfos(this.title);
for (const info of infos) {
let match = false;
if (info.title && this.title && this.checkLCS(info.title, this.title)) {
match = true;
} else if (
info.abstract &&
this.abstract &&
this.checkLCS(info.abstract, this.abstract)
) {
match = true;
}
if (match) {
const result = info.tldr ?? TLDRUnrelated;
DataStorage.instance(TLDRFieldKey).modify((data: any) => {
data[this.zoteroItem.id] = result;
return data;
});
return true;
}
}
DataStorage.instance(TLDRFieldKey).modify((data: any) => {
data[this.zoteroItem.id] = TLDRItemNotFound;
return data;
});
return false;
} catch (error) {
Zotero.log(`post semantic scholar request error: ${error}`);
return false;
}
}
private async fetchRelevanceItemInfos(title: string): Promise<SemanticScholarItemInfo[]> {
const semanticScholarURL = 'https://www.semanticscholar.org/api/1/search';
const params = {
queryString: title,
page: 1,
pageSize: 10,
sort: 'relevance',
authors: [],
coAuthors: [],
venues: [],
performTitleMatch: true,
requireViewablePdf: false,
includeTldrs: true,
private async fetchRelevanceItemInfos(
title: string,
): Promise<SemanticScholarItemInfo[]> {
const semanticScholarURL = "https://www.semanticscholar.org/api/1/search";
const params = {
queryString: title,
page: 1,
pageSize: 10,
sort: "relevance",
authors: [],
coAuthors: [],
venues: [],
performTitleMatch: true,
requireViewablePdf: false,
includeTldrs: true,
};
const resp = await Zotero.HTTP.request("POST", semanticScholarURL, {
headers: { "Content-Type": "application/json" },
body: JSON.stringify(params),
});
if (resp.status === 200) {
const results = JSON.parse(resp.response).results;
return results.map((item: any) => {
const result = {
title: item.title.text,
abstract: item.paperAbstract.text,
tldr: undefined,
};
const resp = await Zotero.HTTP.request("POST", semanticScholarURL, {
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(params),
});
if (resp.status === 200) {
const results = JSON.parse(resp.response).results;
return results.map((item: any) => {
const result = {
title: item.title.text,
abstract: item.paperAbstract.text,
tldr: undefined,
};
if (item.tldr) {
result.tldr = item.tldr.text;
}
return result;
});
if (item.tldr) {
result.tldr = item.tldr.text;
}
return [];
return result;
});
}
return [];
}
private checkLCS(pattern: string, content: string): boolean {
const LCS = StringMatchUtils.longestCommonSubsequence(pattern, content);
return LCS.length >= Math.max(pattern.length, content.length) * 0.9
}
private checkLCS(pattern: string, content: string): boolean {
const LCS = StringMatchUtils.longestCommonSubsequence(pattern, content);
return LCS.length >= Math.max(pattern.length, content.length) * 0.9;
}
}
class StringMatchUtils {
static longestCommonSubsequence(text1: string, text2: string): string {
const m = text1.length;
const n = text2.length;
const dp: number[][] = new Array(m + 1);
for (let i = 0; i <= m; i++) {
dp[i] = new Array(n + 1).fill(0);
}
for (let i = 1; i <= m; i++) {
for (let j = 1; j <= n; j++) {
if (text1[i - 1] === text2[j - 1]) {
dp[i][j] = dp[i - 1][j - 1] + 1;
} else {
dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
}
}
}
let i = m, j = n;
const lcs: string[] = [];
while (i > 0 && j > 0) {
if (text1[i - 1] === text2[j - 1]) {
lcs.unshift(text1[i - 1]);
i--;
j--;
} else if (dp[i - 1][j] > dp[i][j - 1]) {
i--;
} else {
j--;
}
}
return lcs.join('');
static longestCommonSubsequence(text1: string, text2: string): string {
const m = text1.length;
const n = text2.length;
const dp: number[][] = new Array(m + 1);
for (let i = 0; i <= m; i++) {
dp[i] = new Array(n + 1).fill(0);
}
// static minWindow(s: string, t: string): [number, number] | null {
// const m = s.length, n = t.length
// let start = -1, minLen = Number.MAX_SAFE_INTEGER, i = 0, j = 0, end;
// while (i < m) {
// if (s[i] == t[j]) {
// if (++j == n) {
// end = i + 1;
// while (--j >= 0) {
// while (s[i--] != t[j]);
// }
// ++i; ++j;
// if (end - i < minLen) {
// minLen = end - i;
// start = i;
// }
// }
// }
// ++i;
// }
// return start == -1 ? null : [start, minLen];
// }
}
for (let i = 1; i <= m; i++) {
for (let j = 1; j <= n; j++) {
if (text1[i - 1] === text2[j - 1]) {
dp[i][j] = dp[i - 1][j - 1] + 1;
} else {
dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
}
}
}
let i = m,
j = n;
const lcs: string[] = [];
while (i > 0 && j > 0) {
if (text1[i - 1] === text2[j - 1]) {
lcs.unshift(text1[i - 1]);
i--;
j--;
} else if (dp[i - 1][j] > dp[i][j - 1]) {
i--;
} else {
j--;
}
}
return lcs.join("");
}
// static minWindow(s: string, t: string): [number, number] | null {
// const m = s.length, n = t.length
// let start = -1, minLen = Number.MAX_SAFE_INTEGER, i = 0, j = 0, end;
// while (i < m) {
// if (s[i] == t[j]) {
// if (++j == n) {
// end = i + 1;
// while (--j >= 0) {
// while (s[i--] != t[j]);
// }
// ++i; ++j;
// if (end - i < minLen) {
// minLen = end - i;
// start = i;
// }
// }
// }
// ++i;
// }
// return start == -1 ? null : [start, minLen];
// }
}