Skip to content

Commit 33408eb

Browse files
HahaBillHahaBill
committed
code cleanup + adding word2vec worker
1 parent f3dd5d4 commit 33408eb

8 files changed

+117
-83
lines changed

‎.DS_Store

0 Bytes
Binary file not shown.

‎package-lock.json

+15-6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "joplin-plugin-ai-summarization",
3-
"version": "0.1.0",
3+
"version": "0.1.2",
44
"scripts": {
55
"dist": "webpack --env joplin-plugin-config=buildMain && webpack --env joplin-plugin-config=buildExtraScripts && webpack --env joplin-plugin-config=createArchive",
66
"prepare": "npm run dist",
@@ -23,7 +23,7 @@
2323
"fs-extra": "^10.1.0",
2424
"glob": "^8.0.3",
2525
"prettier": "3.3.3",
26-
"pyodide": "^0.26.1",
26+
"pyodide": "^0.26.2",
2727
"tar": "^6.1.11",
2828
"ts-loader": "^9.3.1",
2929
"typescript": "^4.8.2",
@@ -66,6 +66,7 @@
6666
"styled-components": "^6.1.11",
6767
"summarybot": "^0.0.7",
6868
"svd-js": "^1.1.1",
69+
"webpack-node-externals": "^3.0.0",
6970
"word2vec": "^1.1.5"
7071
}
7172
}

‎src/index.ts

-28
Original file line numberDiff line numberDiff line change
@@ -14,35 +14,7 @@ joplin.plugins.register({
1414
try {
1515
const dataDir = await joplin.plugins.installationDir();
1616

17-
const transformersWorker = new Worker(
18-
`${await joplin.plugins.installationDir()}/workers/transformersWorker.js`,
19-
);
20-
transformersWorker.onmessage = (event) => {
21-
if ("response" in event.data) {
22-
alert(JSON.stringify(event.data.response));
23-
}
24-
};
25-
26-
joplin.commands.register({
27-
name: "classifySelection",
28-
label: "Run sentiment analysis on selected text",
29-
iconName: "fas fa-robot",
30-
execute: async () => {
31-
const selectedText = await joplin.commands.execute("selectedText");
32-
if (!selectedText) {
33-
alert("No text selected!");
34-
return;
35-
}
36-
37-
transformersWorker.postMessage({
38-
type: "classify",
39-
text: selectedText,
40-
});
41-
},
42-
});
43-
4417
// Dialogs
45-
4618
const dialogButtons = [
4719
{
4820
id: 'ok',

‎src/init/initWorkers.ts

+27-21
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,44 @@
11
import joplin from "api";
2-
import { ToolbarButtonLocation } from "api/types";
3-
import { SummarisationPanel } from "src/ui/panel";
42

3+
const path = require("path");
4+
const logger = require("electron-log");
55

6-
async function initWorkers(panelInstance: SummarisationPanel) {
7-
await initTransformersWorker(panelInstance);
6+
async function initWorkers() {
7+
await initTransformersWorker();
88
}
99

10-
async function initTransformersWorker(panelInstance: SummarisationPanel) {
11-
const transformersWorker = new Worker(`${await joplin.plugins.installationDir()}/workers/transformersWorker.js`);
10+
async function initPyodideWorker() {
11+
logger.info('Initializing Pyodide worker');
12+
const pyodideWorker = new Worker(`${await joplin.plugins.installationDir()}/workers/pyodideWorker.js`);
13+
}
1214

15+
async function initWord2VecWorker() {
16+
logger.info('Initializing word2vec worker');
17+
const word2vecWorker = new Worker(`${await joplin.plugins.installationDir()}/workers/word2vecWorker.js`);
1318
joplin.commands.register({
14-
name: 'classifySelection',
15-
label: 'Run sentiment analysis on selected text',
16-
iconName: 'fas fa-robot',
17-
execute: async () => {
18-
const selectedText = await joplin.commands.execute('selectedText');
19-
if (!selectedText) {
20-
alert('No text selected!');
19+
name: 'word2vec:vectorize',
20+
label: 'Create vectors from sentences',
21+
execute: async(text, type) => {
22+
if (!text) {
23+
alert('No sentences found!');
2124
return;
2225
}
2326

24-
transformersWorker.postMessage({
25-
type: 'classify',
26-
text: selectedText,
27-
});
28-
},
27+
word2vecWorker.postMessage({ type: type, file: path.join(await joplin.plugins.dataDir(), "lib", "word2vec", "vectors.txt"), text: text });
28+
word2vecWorker.onmessage = async (event) => {
29+
const model = event.data.model;
30+
};
31+
}
2932
});
33+
}
34+
35+
async function initTransformersWorker() {
36+
logger.info('Initializing Transformers.js worker');
37+
const transformersWorker = new Worker(`${await joplin.plugins.installationDir()}/workers/transformersWorker.js`);
3038

3139
joplin.commands.register({
3240
name: 'transformers:summarise',
33-
label: 'Run summary prediction on the ',
34-
iconName: 'fas fa-robot',
41+
label: 'Run summary prediction on note content',
3542
execute: async (text: string) => {
3643
if (!text) {
3744
alert('No text detected!');
@@ -52,7 +59,6 @@ async function initTransformersWorker(panelInstance: SummarisationPanel) {
5259
}
5360
};
5461
});
55-
5662
return result;
5763
},
5864
});

‎src/manifest.json

+19-19
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
{
2-
"manifest_version": 1,
3-
"id": "org.joplinapp.plugins.AISummarisation",
4-
"app_min_version": "2.13",
5-
"version": "0.1.0",
6-
"name": "Summarise your notes and notebooks!",
7-
"description": "Allows users to summarize their notes and notebooks with AI",
8-
"author": "HahaBill",
9-
"homepage_url": "https://discourse.joplinapp.org/t/about-the-summarize-with-ai-category/37955",
10-
"repository_url": "https://github.com/joplin/plugin-ai-summarisation",
11-
"keywords": [
12-
"web-development",
13-
"nlp-machine-learning",
14-
"artificial-intelligence",
15-
"summarisation"
16-
],
17-
"categories": [],
18-
"screenshots": [],
19-
"icons": {}
20-
}
2+
"manifest_version": 1,
3+
"id": "org.joplinapp.plugins.AISummarisation",
4+
"app_min_version": "2.13",
5+
"version": "0.1.2",
6+
"name": "Summarise your notes and notebooks!",
7+
"description": "Allows users to summarize their notes and notebooks with AI",
8+
"author": "HahaBill",
9+
"homepage_url": "https://discourse.joplinapp.org/t/about-the-summarize-with-ai-category/37955",
10+
"repository_url": "https://github.com/joplin/plugin-ai-summarisation",
11+
"keywords": [
12+
"web-development",
13+
"nlp-machine-learning",
14+
"artificial-intelligence",
15+
"summarisation"
16+
],
17+
"categories": [],
18+
"screenshots": [],
19+
"icons": {}
20+
}

‎src/workers/transformersWorker.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ env.backends.onnx.wasm.wasmPaths = "../ai/onnx-dist/";
1212

1313
let pipe: (text: string, config: any) => Promise<unknown>;
1414

15-
const classify = async (text: string) => {
15+
const summarise = async (text: string) => {
1616
pipe ??= await pipeline("summarization", "Xenova/flan-t5-small");
1717
let min_length = 250;
1818
if (text.length <= 5000) {
@@ -30,13 +30,9 @@ const classify = async (text: string) => {
3030
self.addEventListener("message", async (event) => {
3131
console.debug("got message", event.data);
3232

33-
if (event.data.type === "classify") {
33+
if (event.data.type === "summarise") {
3434
postMessage({
35-
response: await classify(event.data.text),
35+
generatedSummary: await summarise(event.data.text),
3636
});
37-
} else {
38-
postMessage({
39-
generatedSummary: await classify(event.data.text),
40-
});
4137
}
4238
});

‎src/workers/word2vecWorker.ts

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import w2v from 'word2vec';
2+
3+
self.addEventListener("message", async (event) => {
4+
console.debug("Got message in Word2Vec worker");
5+
const { type, file, text } = event.data;
6+
7+
if (type === 'loadModel') {
8+
w2v.loadModel(file, (error, model) => {
9+
if (error) {
10+
postMessage({ error: error.message });
11+
} else {
12+
postMessage({ model });
13+
}
14+
});
15+
}
16+
17+
if (type === 'vectorize') {
18+
console.debug(`Vectorization initialized: ${file}`);
19+
const regexPattern = /\[[^\]]*\]|\bhttps?:\/\/\S+\b/g;
20+
const cleanedText = text.replace(regexPattern, '');
21+
cleanedText.trim();
22+
23+
const params = {
24+
size: 200,
25+
window: 5,
26+
sample: 1e-3,
27+
hs: 0,
28+
negative: 5,
29+
threads: 12,
30+
iter: 10,
31+
minCount: 5,
32+
alpha: 0.025,
33+
cbow: 1,
34+
debug: 2,
35+
binary: 0,
36+
silent: false
37+
};
38+
39+
w2v.word2vec(cleanedText, file, params, (code) => {
40+
if (code !== 0) {
41+
console.error('Error during Word2Vec training');
42+
} else {
43+
console.log(`Word2Vec model training complete. Vectors saved to ${file}`);
44+
}
45+
});
46+
}
47+
});
48+
49+
50+

0 commit comments

Comments
 (0)