From f3e475487467861bb0021eb451443003fcccc511 Mon Sep 17 00:00:00 2001 From: devland Date: Tue, 25 Jun 2024 16:18:38 +0200 Subject: [PATCH 1/3] add worker_threads optimization for setup --- cli.js | 126 ++++++++++++++++++++++++++++++++++++++----------------- logic.js | 38 +++++++++++------ 2 files changed, 113 insertions(+), 51 deletions(-) diff --git a/cli.js b/cli.js index 1f59c655..c0da6962 100644 --- a/cli.js +++ b/cli.js @@ -3,6 +3,7 @@ const fs = require('fs'); const logic = require('./logic.js'); const config = require('./config.js'); const utils = require('./assets/utils/cli.js'); +const { Worker } = require('worker_threads'); let marked = require('marked'); const markedTerminal = require('marked-terminal'); marked.setOptions({ @@ -172,51 +173,100 @@ const cli = { } }, // computes & installs dependencies for h5p library - setup: async function(library, version, download) { - const isUrl = ['http', 'git@'].includes(library.slice(0, 4)) ? true : false; - const url = library; - const missingOptionals = {}; - try { - if (isUrl) { - const entry = await this.register(url); - library = logic.machineToShort(Object.keys(entry)[0]); - } + setup: function(library, version, download) { + return new Promise(async (resolve, reject) => { + const isUrl = ['http', 'git@'].includes(library.slice(0, 4)) ? true : false; + const url = library; + const missingOptionals = {}; + const benchStart = new Date(); + let preparing = true; let toSkip = []; - const action = parseInt(download) ? 'download' : 'clone'; - const latest = version ? false : true; - let result = await logic.computeDependencies(library, 'view', 1, version); - for (let item in result) { - // setup editor dependencies for every view dependency - if (!result[item].id) { - handleMissingOptionals(missingOptionals, result, item); + const toDo = { + pending: 0, + done: 0 + } + const handleWorkerDone = (result) => { + if (result) { + toSkip = result; } - else { - toSkip = await logic.getWithDependencies(action, item, 'edit', 1, latest, toSkip); + toDo.done++; + console.log(`> ${toDo.done}/${toDo.pending} done`); + if (!preparing && toDo.done === toDo.pending) { + console.log(`> done setting up ${library} (${new Date() - benchStart} ms)`); + resolve(); } } - result = await logic.computeDependencies(library, 'edit', 1, version); - for (let item in result) { - if (!result[item].id) { - handleMissingOptionals(missingOptionals, result, item); - } + const runWorker = (data) => { + console.log('<<< worker started'); + const worker = new Worker(`${require.main.path}/logic.js` , { workerData : data }); + worker.on('message', (result) => { + console.log('<<< worker done'); + handleWorkerDone(); + }); + worker.on('error', (error) => { + console.log('<<< worker error'); + console.error(error); + handleWorkerDone(); + }); } - toSkip = []; - console.log(`> ${action} ${library} library "view" dependencies into "${config.folders.libraries}" folder`); - toSkip = await logic.getWithDependencies(action, library, 'view', 1, latest, toSkip); - console.log(`> ${action} ${library} library "edit" dependencies into "${config.folders.libraries}" folder`); - toSkip = await logic.getWithDependencies(action, library, 'edit', 1, latest, toSkip); - if (Object.keys(missingOptionals).length) { - console.log('!!! missing optional libraries'); - for (let item in missingOptionals) { - console.log(`${item} (${missingOptionals[item].optional ? 'optional' : 'required'}) required by ${missingOptionals[item].parent}`); + try { + if (isUrl) { + const entry = await this.register(url); + library = logic.machineToShort(Object.keys(entry)[0]); + } + const action = parseInt(download) ? 'download' : 'clone'; + const latest = version ? false : true; + const viewDeps = await logic.computeDependencies(library, 'view', 1, version); + const editDeps = await logic.computeDependencies(library, 'edit', 1, version); + for (let item in viewDeps) { + // setup editor dependencies for every view dependency + if (!viewDeps[item].id) { + handleMissingOptionals(missingOptionals, viewDeps, item); + } + else { + toDo.pending++; + //logic.getWithDependencies(action, item, 'edit', 1, latest, toSkip).then(handleToDo); + runWorker({ + function: 'getWithDependencies', + arguments: [action, item, 'edit', 1, latest, toSkip] + }); + } + } + for (let item in editDeps) { + if (!editDeps[item].id) { + handleMissingOptionals(missingOptionals, editDeps, item); + } + } + toSkip = []; + console.log(`> ${action} ${library} library "view" dependencies into "${config.folders.libraries}" folder`); + toDo.pending++; + //logic.getWithDependencies(action, library, 'view', 1, latest, toSkip).then(handleToDo); + runWorker({ + function: 'getWithDependencies', + arguments: [action, library, 'view', 1, latest, toSkip] + }); + console.log(`> ${action} ${library} library "edit" dependencies into "${config.folders.libraries}" folder`); + toDo.pending++; + //logic.getWithDependencies(action, library, 'edit', 1, latest, toSkip).then(handleToDo); + runWorker({ + function: 'getWithDependencies', + arguments: [action, library, 'edit', 1, latest, toSkip] + }); + if (Object.keys(missingOptionals).length) { + console.log('!!! missing optional libraries'); + for (let item in missingOptionals) { + console.log(`${item} (${missingOptionals[item].optional ? 'optional' : 'required'}) required by ${missingOptionals[item].parent}`); + } } + console.log('<<< done preparing'); + preparing = false; } - console.log(`> done setting up ${library}`); - } - catch (error) { - console.log('> error'); - console.log(error); - } + catch (error) { + console.log('> error'); + console.log(error); + reject(error); + } + }); }, // updates local library registry entry register: async (input) => { diff --git a/logic.js b/logic.js index 2ad2d916..4c0eb37e 100644 --- a/logic.js +++ b/logic.js @@ -106,7 +106,7 @@ const getFileList = (folder) => { } module.exports = { // imports content type from zip archive file in the .h5p format - import: (folder, archive) => { + import: function (folder, archive) { const target = `${config.folders.temp}/${folder}`; new admZip(archive).extractAllTo(target); fs.renameSync(`${target}/content`, `content/${folder}`); @@ -115,7 +115,7 @@ module.exports = { return folder; }, // creates zip archive export file in the .h5p format - export: (library, folder) => { + export: function (library, folder) { const libsFile = `${config.folders.cache}/${library}.json`; const editLibsFile = `${config.folders.cache}/${library}_edit.json`; const target = `${config.folders.temp}/${folder}`; @@ -151,7 +151,7 @@ module.exports = { }, /* retrieves list of h5p librarie ignoreCache - if true cache file is overwritten with online data */ - getRegistry: async (ignoreCache) => { + getRegistry: async function (ignoreCache) { const registryFile = `${config.folders.cache}/${config.registry}`; let list; if (!ignoreCache && fs.existsSync(registryFile)) { @@ -192,7 +192,7 @@ module.exports = { saveToCache - if true list is saved to cache folder version - optional version to compute; defaults to 'master' folder - optional local library folder to use instead of git repo; use "" to ignore */ - computeDependencies: async (library, mode, saveToCache, version, folder) => { + computeDependencies: async function (library, mode, saveToCache, version, folder) { console.log(`> ${library} deps ${mode}`); version = version || 'master'; let level = -1; @@ -369,7 +369,7 @@ module.exports = { return output; }, // list tags for library using git - tags: (org, repo, mainBranch = 'master') => { + tags: function (org, repo, mainBranch = 'master') { const library = getRepoFile(fromTemplate(config.urls.library.clone, { org, repo }), 'library.json', mainBranch, true); const label = `${repo}_${mainBranch}`; const folder = `${config.folders.temp}/${label}`; @@ -394,7 +394,7 @@ module.exports = { return output; }, // download & unzip repository - download: async (org, repo, version, target) => { + download: async function (org, repo, version, target) { const blob = (await superAgent.get(fromTemplate(config.urls.library.zip, { org, repo, version })))._body; const zipFile = `${config.folders.temp}/temp.zip`; fs.writeFileSync(zipFile, blob); @@ -403,7 +403,7 @@ module.exports = { fs.renameSync(`${config.folders.libraries}/${repo}-master`, target); }, // clone repository using git - clone: (org, repo, branch, target) => { + clone: function (org, repo, branch, target) { return execSync(`git clone ${fromTemplate(config.urls.library.clone, {org, repo})} ${target} --branch ${branch}`, { cwd: config.folders.libraries }).toString(); }, /* clones/downloads dependencies to libraries folder using git and runs relevant npm commands @@ -411,7 +411,7 @@ module.exports = { useCache - if true cached dependency list is used latest - if true master branch libraries are used; otherwise the versions found in the cached deps list are used toSkip - optional array of libraries to skip; after a library is parsed by the function it's auto-added to the array so it's skipped for efficiency */ - getWithDependencies: async (action, library, mode, useCache, latest, toSkip = []) => { + getWithDependencies: async function (action, library, mode, useCache, latest, toSkip = []) { let list; const doneFile = `${config.folders.cache}/${library}${mode == 'edit' ? '_edit' : ''}.json`; if (useCache && fs.existsSync(doneFile)) { @@ -475,7 +475,7 @@ module.exports = { }, /* checks if dependency lists are cached and dependencies are installed for a given library; returns a report with boolean statuses; the overall status is reflected under the "ok" attribute;*/ - verifySetup: async (library) => { + verifySetup: async function (library) { const registry = await module.exports.getRegistry(); const viewList = `${config.folders.cache}/${library}.json`; const editList = `${config.folders.cache}/${library}_edit.json`; @@ -509,7 +509,7 @@ module.exports = { return output; }, // generates h5p.json file with info describing the library in the specified folder - generateInfo: (folder, library) => { + generateInfo: function (folder, library) { const target = `content/${folder}`; const lib = JSON.parse(fs.readFileSync(`${config.folders.cache}/${library}.json`, 'utf-8'))[library]; const viewDepsFile = `${config.folders.cache}/${library}.json`; @@ -545,7 +545,7 @@ module.exports = { fs.writeFileSync(`${target}/h5p.json`, JSON.stringify(info)); }, // upgrades content via current main library upgrades.js scripts - upgrade: (folder, library) => { + upgrade: function (folder, library) { const lib = JSON.parse(fs.readFileSync(`${config.folders.cache}/${library}.json`, 'utf-8'))[library]; const info = JSON.parse(fs.readFileSync(`content/${folder}/h5p.json`, 'utf-8')); const extraAttrs = [ @@ -606,11 +606,11 @@ module.exports = { fs.writeFileSync(contentFile, JSON.stringify(content)); module.exports.generateInfo(folder, library); }, - machineToShort: (machineName) => { + machineToShort: function (machineName) { machineName = machineName.replace('H5PEditor', 'H5P-Editor'); return machineName.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase().replace('.', '-'); }, - registryEntryFromRepoUrl: function(gitUrl) { + registryEntryFromRepoUrl: function (gitUrl) { let { host, org, repoName } = parseGitUrl(gitUrl); const list = getRepoFile(gitUrl, 'library.json', 'master', true); const shortName = this.machineToShort(list.machineName); @@ -701,3 +701,15 @@ const parseSemanticLibraries = (entries) => { } return output; } + +const { workerData, parentPort } = require('worker_threads'); +if (parentPort && workerData) { + console.log('<<< worker working...'); + const run = async () => { + const result = await module.exports[workerData.function].apply(null, workerData.arguments); + console.log('ding'); + parentPort.postMessage(result); + console.log('dong'); + } + run(); +} From 7e8228f3fccc8003786b613aa8ac912b5ba00945 Mon Sep 17 00:00:00 2001 From: devland Date: Wed, 26 Jun 2024 11:49:12 +0200 Subject: [PATCH 2/3] add worker exit event handler and remove unused toSkip logic --- cli.js | 20 +++++++------------- logic.js | 4 ++-- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/cli.js b/cli.js index c0da6962..66718c51 100644 --- a/cli.js +++ b/cli.js @@ -180,15 +180,11 @@ const cli = { const missingOptionals = {}; const benchStart = new Date(); let preparing = true; - let toSkip = []; const toDo = { pending: 0, done: 0 } - const handleWorkerDone = (result) => { - if (result) { - toSkip = result; - } + const handleWorkerDone = () => { toDo.done++; console.log(`> ${toDo.done}/${toDo.pending} done`); if (!preparing && toDo.done === toDo.pending) { @@ -201,11 +197,13 @@ const cli = { const worker = new Worker(`${require.main.path}/logic.js` , { workerData : data }); worker.on('message', (result) => { console.log('<<< worker done'); - handleWorkerDone(); }); worker.on('error', (error) => { console.log('<<< worker error'); console.error(error); + }); + worker.on('exit', () => { + console.log('<<< worker exit'); handleWorkerDone(); }); } @@ -225,10 +223,9 @@ const cli = { } else { toDo.pending++; - //logic.getWithDependencies(action, item, 'edit', 1, latest, toSkip).then(handleToDo); runWorker({ function: 'getWithDependencies', - arguments: [action, item, 'edit', 1, latest, toSkip] + arguments: [action, item, 'edit', 1, latest] }); } } @@ -237,20 +234,17 @@ const cli = { handleMissingOptionals(missingOptionals, editDeps, item); } } - toSkip = []; console.log(`> ${action} ${library} library "view" dependencies into "${config.folders.libraries}" folder`); toDo.pending++; - //logic.getWithDependencies(action, library, 'view', 1, latest, toSkip).then(handleToDo); runWorker({ function: 'getWithDependencies', - arguments: [action, library, 'view', 1, latest, toSkip] + arguments: [action, library, 'view', 1, latest] }); console.log(`> ${action} ${library} library "edit" dependencies into "${config.folders.libraries}" folder`); toDo.pending++; - //logic.getWithDependencies(action, library, 'edit', 1, latest, toSkip).then(handleToDo); runWorker({ function: 'getWithDependencies', - arguments: [action, library, 'edit', 1, latest, toSkip] + arguments: [action, library, 'edit', 1, latest] }); if (Object.keys(missingOptionals).length) { console.log('!!! missing optional libraries'); diff --git a/logic.js b/logic.js index 4c0eb37e..65dd32d5 100644 --- a/logic.js +++ b/logic.js @@ -422,10 +422,10 @@ module.exports = { list = await module.exports.computeDependencies(library, mode, 1); } for (let item in list) { - if (toSkip.indexOf(item) != -1) { + if (toSkip?.indexOf(item) != -1) { continue; } - toSkip.push(item); + toSkip?.push(item); if (!list[item].id) { if (list[item].optional) { console.log(`> skipping optional unregistered ${item} library`); From 2389a8731a6da9b0445d2730e90d04578ff174da Mon Sep 17 00:00:00 2001 From: devland Date: Wed, 26 Jun 2024 12:04:28 +0200 Subject: [PATCH 3/3] remove debug logs --- logic.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/logic.js b/logic.js index 65dd32d5..5f64b3bd 100644 --- a/logic.js +++ b/logic.js @@ -707,9 +707,7 @@ if (parentPort && workerData) { console.log('<<< worker working...'); const run = async () => { const result = await module.exports[workerData.function].apply(null, workerData.arguments); - console.log('ding'); parentPort.postMessage(result); - console.log('dong'); } run(); }