Perf: faster reject domainset build

Build/build-reject-domainset.js

@@ -352,11 +352,11 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     )
   ).forEach((result, taskIndex) => {
     const chunk = tasksArray[taskIndex];
-    result.forEach((value, index) => {
-      if (value === 1) {
-        domainSets.delete(chunk[index])
+    for (let i = 0, len = result.length; i < len; i++) {
+      if (result[i]) {
+        domainSets.delete(chunk[i]);
       }
-    })
+    }
   });
 
   console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);

Build/lib/parse-filter.js

@@ -227,25 +227,11 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
 function preprocessFullDomainSetBeforeUsedAsWorkerData (data) {
   return data.filter(domain => (
     domain.charCodeAt(0) === 46
-    && !canExcludeFromDedupe(domain)
   ));
 }
 
-// duckdns.org domain will not overlap and doesn't need dedupe
-function canExcludeFromDedupe (domain) {
-  if (
-    // starts with a dot
-    domain.charCodeAt(0) === 46
-    && domain.length === 23
-    && domain.endsWith('.duckdns.org')
-  ) {
-    return true;
-  }
-  return false;
-}
 
 module.exports.processDomainLists = processDomainLists;
 module.exports.processHosts = processHosts;
 module.exports.processFilterRules = processFilterRules;
 module.exports.preprocessFullDomainSetBeforeUsedAsWorkerData = preprocessFullDomainSetBeforeUsedAsWorkerData;
-module.exports.canExcludeFromDedupe = canExcludeFromDedupe;

Build/worker/build-reject-domainset-worker.js

@@ -1,5 +1,4 @@
 const Piscina = require('piscina');
-const { canExcludeFromDedupe } = require('../lib/parse-filter')
 
 const fullsetDomainStartsWithADot = Piscina.workerData
 const totalLen = fullsetDomainStartsWithADot.length;
@@ -11,13 +10,9 @@ module.exports.dedupe = ({ chunk }) => {
   for (let i = 0; i < chunkLength; i++) {
     const domainFromInput = chunk[i];
 
-    if (canExcludeFromDedupe(domainFromInput)) {
-      continue;
-    }
-
     for (let j = 0; j < totalLen; j++) {
       const domainFromFullSet = fullsetDomainStartsWithADot[j];
-      // domainFromFullSet is now startsWith a "."
+      // domainFromFullSet is always startsWith "."
 
       if (domainFromFullSet === domainFromInput) continue;