Chore: destroy worker thread

Build/build-reject-domainset.js

@@ -12,6 +12,7 @@ const threads = isCI ? cpuCount : cpuCount / 2;
   /** @type Set<string> */
   const domainSets = new Set();
 
+  console.log('Downloading hosts file...');
   console.time('* Download and process Hosts');
 
   // Parse from remote hosts & domain lists
@@ -230,7 +231,10 @@ const threads = isCI ? cpuCount : cpuCount / 2;
 
   const piscina = new Piscina({
     filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
-    workerData: [...domainSets]
+    workerData: [...domainSets],
+    idleTimeout: 50,
+    minThreads: threads,
+    maxThreads: threads
   });
 
   console.log(`Launching ${threads} threads...`)
@@ -244,19 +248,18 @@ const threads = isCI ? cpuCount : cpuCount / 2;
       return result;
     }, []);
 
-  (await Promise.all(
-    Array.from(domainSets)
-      .reduce((result, element, index) => {
-        const chunk = index % threads;
-        result[chunk] ??= [];
-
-        result[chunk].push(element);
-        return result;
-      }, [])
-      .map(chunk => piscina.run(
-        { chunk }
-      ))
-  )).forEach((result, taskIndex) => {
+  (
+    await Promise.all(
+      Array.from(domainSets)
+        .reduce((result, element, index) => {
+          const chunk = index % threads;
+          result[chunk] ??= [];
+          result[chunk].push(element);
+          return result;
+        }, [])
+        .map(chunk => piscina.run({ chunk }, { name: 'dedupe' }))
+    )
+  ).forEach((result, taskIndex) => {
     const chunk = tasksArray[taskIndex];
     result.forEach((value, index) => {
       if (value === 1) {
@@ -268,10 +271,17 @@ const threads = isCI ? cpuCount : cpuCount / 2;
   console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
   console.log(`Deduped ${previousSize - domainSets.size} rules!`);
 
-  await fsPromises.writeFile(
-    pathResolve(__dirname, '../List/domainset/reject.conf'),
-    `${[...domainSets].join('\n')}\n`,
-    { encoding: 'utf-8' });
+  await Promise.all([
+    fsPromises.writeFile(
+      pathResolve(__dirname, '../List/domainset/reject.conf'),
+      `${[...domainSets].join('\n')}\n`,
+      { encoding: 'utf-8' }
+    ),
+    piscina.destroy()
+  ]);
 
   console.timeEnd('Total Time - build-reject-domain-set');
+  if (piscina.queueSize === 0) {
+    process.exit(0);
+  }
 })();

Build/lib/parse-filter.js

@@ -55,6 +55,8 @@ async function processDomainLists (domainListsUrl) {
  * @param {string | URL} hostsUrl
  */
 async function processHosts (hostsUrl, includeAllSubDomain = false) {
+  console.time(`   - processHosts: ${hostsUrl}`);
+
   if (typeof hostsUrl === 'string') {
     hostsUrl = new URL(hostsUrl);
   }
@@ -87,6 +89,8 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
     }
   });
 
+  console.timeEnd(`   - processHosts: ${hostsUrl}`);
+
   return [...domainSets];
 }
 
@@ -98,10 +102,6 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
 async function processFilterRules (filterRulesUrl, fallbackUrls) {
   console.time(`   - processFilterRules: ${filterRulesUrl}`);
 
-  if (typeof filterRulesUrl === 'string') {
-    filterRulesUrl = new URL(filterRulesUrl);
-  }
-
   /** @type Set<string> */
   const whitelistDomainSets = new Set();
   /** @type Set<string> */

Build/worker/build-reject-domainset-worker.js

@@ -1,16 +1,15 @@
-const { workerData, move } = require('piscina');
-
+const Piscina = require('piscina');
 // pre check if fullset domain is starts with a "."
 // This avoid calling chatCodeAt repeatedly
 
 // workerData is an array of string. Sort it by length, short first:
-const fullsetDomainStartsWithADot = workerData.filter(domain => (
+const fullsetDomainStartsWithADot = Piscina.workerData.filter(domain => (
   domain.charCodeAt(0) === 46
   && !canExcludeFromDedupe(domain)
 ));
 const totalLen = fullsetDomainStartsWithADot.length;
 
-module.exports = ({ chunk }) => {
+module.exports.dedupe = ({ chunk }) => {
   const chunkLength = chunk.length;
   const outputToBeRemoved = new Int8Array(chunkLength);
 
@@ -61,11 +60,11 @@ module.exports = ({ chunk }) => {
     }
   }
 
-  return move(outputToBeRemoved);
+  return outputToBeRemoved;
 };
 
 // duckdns.org domain will not overlap and doesn't need dedupe
-function canExcludeFromDedupe(domain) {
+function canExcludeFromDedupe (domain) {
   if (domain.length === 23 && domain.endsWith('.duckdns.org')) {
     return true;
   }

List/non_ip/reject.conf

@@ -35,7 +35,6 @@ DOMAIN-KEYWORD,49329d48d6c.
 DOMAIN-KEYWORD,vsvevvcca
 DOMAIN-KEYWORD,envci.
 DOMAIN-KEYWORD,aarsenvs.
-DOMAIN-SUFFIX,duckdns.org
 
 # --- End of Blacklist Section