Perf: replace sha256 w/ xxhash

Build/lib/get-phishing-domains.ts

@@ -10,8 +10,7 @@ import picocolors from 'picocolors';
 import createKeywordFilter from './aho-corasick';
 import { createCacheKey, deserializeArray, fsFetchCache, serializeArray } from './cache-filesystem';
 import { fastStringArrayJoin } from './misc';
-
-import { sha256 } from 'hash-wasm';
+import { stringHash } from './string-hash';
 
 const BLACK_TLD = new Set([
   'accountant', 'autos',
@@ -114,16 +113,17 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
     return domainArr;
   });
 
+  const cacheHash = span.traceChildSync('get hash', () => stringHash(fastStringArrayJoin(domainArr, '|')));
+
   return span.traceChildAsync(
     'process phishing domain set',
-    () => processPhihsingDomains(domainArr)
+    () => processPhihsingDomains(domainArr, cacheHash)
   );
 });
 
-async function processPhihsingDomains(domainArr: string[]) {
-  const hash = await sha256(fastStringArrayJoin(domainArr, '|'));
+async function processPhihsingDomains(domainArr: string[], cacheHash = '') {
   return fsFetchCache.apply(
-    cacheKey('processPhihsingDomains|' + hash),
+    cacheKey('processPhihsingDomains|' + cacheHash),
     () => {
       const domainCountMap: Record<string, number> = {};
       const domainScoreMap: Record<string, number> = {};

Build/lib/text-line-transform-stream.ts

@@ -48,17 +48,19 @@ export class TextLineStream extends TransformStream<string, string> {
             }
           }
 
-          if (lfIndex !== -1) {
-            let crOrLfIndex = lfIndex;
-            if (chunk[lfIndex - 1] === '\r') {
-              crOrLfIndex--;
-            }
-            controller.enqueue(chunk.slice(chunkIndex, crOrLfIndex));
-            chunkIndex = lfIndex + 1;
-            continue;
+          if (lfIndex === -1) {
+            // we can no longer find a line break in the chunk, break the current loop
+            break;
           }
 
-          break;
+          // enqueue current line, and loop again to find next line
+          let crOrLfIndex = lfIndex;
+          if (chunk[lfIndex - 1] === '\r') {
+            crOrLfIndex--;
+          }
+          controller.enqueue(chunk.slice(chunkIndex, crOrLfIndex));
+          chunkIndex = lfIndex + 1;
+          continue;
         }
 
         __buf = chunk.slice(chunkIndex);

package.json

@@ -31,7 +31,6 @@
     "fast-cidr-tools": "^0.3.1",
     "fdir": "^6.4.0",
     "foxact": "^0.2.38",
-    "hash-wasm": "^4.11.0",
     "json-stringify-pretty-compact": "^3.0.0",
     "mnemonist": "^0.39.8",
     "picocolors": "^1.1.0",

pnpm-lock.yaml

@@ -44,9 +44,6 @@ importers:
       foxact:
         specifier: ^0.2.38
         version: 0.2.38
-      hash-wasm:
-        specifier: ^4.11.0
-        version: 4.11.0
       json-stringify-pretty-compact:
         specifier: ^3.0.0
         version: 3.0.0
@@ -188,10 +185,6 @@ packages:
     resolution: {integrity: sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
 
-  '@eslint/js@9.11.1':
-    resolution: {integrity: sha512-/qu+TWz8WwPWc7/HcIJKi+c+MOm46GdVaSlTTQcaqaL53+GsoA6MxWp5PtTx48qbSP7ylM1Kn7nhvkugfJvRSA==}
-    engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
-
   '@eslint/js@9.12.0':
     resolution: {integrity: sha512-eohesHH8WFRUprDNyEREgqP6beG6htMeUYeCpkEgBCieCMme5r9zFWjzAJp//9S+Kub4rqE+jXe9Cp1a7IYIIA==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
@@ -706,15 +699,6 @@ packages:
       supports-color:
         optional: true
 
-  debug@4.3.6:
-    resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==}
-    engines: {node: '>=6.0'}
-    peerDependencies:
-      supports-color: '*'
-    peerDependenciesMeta:
-      supports-color:
-        optional: true
-
   debug@4.3.7:
     resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==}
     engines: {node: '>=6.0'}
@@ -1030,9 +1014,6 @@ packages:
     resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
     engines: {node: '>=8'}
 
-  hash-wasm@4.11.0:
-    resolution: {integrity: sha512-HVusNXlVqHe0fzIzdQOGolnFN6mX/fqcrSAOcTBXdvzrXVHwTz11vXeKRmkR5gTuwVpvHZEIyKoePDvuAR+XwQ==}
-
   hasown@2.0.2:
     resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
     engines: {node: '>= 0.4'}
@@ -1186,9 +1167,6 @@ packages:
     engines: {node: '>= 14.0.0'}
     hasBin: true
 
-  ms@2.1.2:
-    resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
-
   ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
 
@@ -1624,7 +1602,7 @@ snapshots:
   '@eslint/config-array@0.18.0':
     dependencies:
       '@eslint/object-schema': 2.1.4
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       minimatch: 3.1.2
     transitivePeerDependencies:
       - supports-color
@@ -1634,7 +1612,7 @@ snapshots:
   '@eslint/eslintrc@3.1.0':
     dependencies:
       ajv: 6.12.6
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       espree: 10.2.0
       globals: 14.0.0
       ignore: 5.3.2
@@ -1645,8 +1623,6 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  '@eslint/js@9.11.1': {}
-
   '@eslint/js@9.12.0': {}
 
   '@eslint/object-schema@2.1.4': {}
@@ -1777,7 +1753,7 @@ snapshots:
       '@swc-node/sourcemap-support': 0.5.1
       '@swc/core': 1.7.28
       colorette: 2.0.20
-      debug: 4.3.6(supports-color@8.1.1)
+      debug: 4.3.7(supports-color@8.1.1)
       oxc-resolver: 1.10.2
       pirates: 4.0.6
       tslib: 2.7.0
@@ -1925,7 +1901,7 @@ snapshots:
       '@typescript-eslint/types': 8.7.0
       '@typescript-eslint/typescript-estree': 8.7.0(typescript@5.6.2)
       '@typescript-eslint/visitor-keys': 8.7.0
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       eslint: 9.12.0
     optionalDependencies:
       typescript: 5.6.2
@@ -1941,7 +1917,7 @@ snapshots:
     dependencies:
       '@typescript-eslint/typescript-estree': 8.7.0(typescript@5.6.2)
       '@typescript-eslint/utils': 8.7.0(eslint@9.12.0)(typescript@5.6.2)
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       ts-api-utils: 1.3.0(typescript@5.6.2)
     optionalDependencies:
       typescript: 5.6.2
@@ -1955,7 +1931,7 @@ snapshots:
     dependencies:
       '@typescript-eslint/types': 8.7.0
       '@typescript-eslint/visitor-keys': 8.7.0
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       fast-glob: 3.3.2
       is-glob: 4.0.3
       minimatch: 9.0.5
@@ -2166,16 +2142,12 @@ snapshots:
     dependencies:
       ms: 2.1.3
 
-  debug@4.3.6(supports-color@8.1.1):
+  debug@4.3.7(supports-color@8.1.1):
     dependencies:
-      ms: 2.1.2
+      ms: 2.1.3
     optionalDependencies:
       supports-color: 8.1.1
 
-  debug@4.3.7:
-    dependencies:
-      ms: 2.1.3
-
   decamelize@4.0.0: {}
 
   decompress-response@6.0.0:
@@ -2224,7 +2196,7 @@ snapshots:
     dependencies:
       '@eslint-community/eslint-plugin-eslint-comments': 4.4.0(eslint@9.12.0)
       '@eslint-sukka/shared': 6.6.1(eslint@9.12.0)(typescript@5.6.2)
-      '@eslint/js': 9.11.1
+      '@eslint/js': 9.12.0
       '@stylistic/eslint-plugin-js': 2.8.0(eslint@9.12.0)
       '@stylistic/eslint-plugin-plus': 2.8.0(eslint@9.12.0)
       '@stylistic/eslint-plugin-ts': 2.8.0(eslint@9.12.0)(typescript@5.6.2)
@@ -2283,7 +2255,7 @@ snapshots:
   eslint-plugin-import-x@4.3.0(eslint@9.12.0)(typescript@5.6.2):
     dependencies:
       '@typescript-eslint/utils': 8.7.0(eslint@9.12.0)(typescript@5.6.2)
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       doctrine: 3.0.0
       eslint: 9.12.0
       eslint-import-resolver-node: 0.3.9
@@ -2380,7 +2352,7 @@ snapshots:
       ajv: 6.12.6
       chalk: 4.1.2
       cross-spawn: 7.0.3
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       escape-string-regexp: 4.0.0
       eslint-scope: 8.1.0
       eslint-visitor-keys: 4.1.0
@@ -2529,8 +2501,6 @@ snapshots:
 
   has-flag@4.0.0: {}
 
-  hash-wasm@4.11.0: {}
-
   hasown@2.0.2:
     dependencies:
       function-bind: 1.1.2
@@ -2662,7 +2632,7 @@ snapshots:
       ansi-colors: 4.1.3
       browser-stdout: 1.3.1
       chokidar: 3.6.0
-      debug: 4.3.6(supports-color@8.1.1)
+      debug: 4.3.7(supports-color@8.1.1)
       diff: 5.2.0
       escape-string-regexp: 4.0.0
       find-up: 5.0.0
@@ -2680,8 +2650,6 @@ snapshots:
       yargs-parser: 20.2.9
       yargs-unparser: 2.0.0
 
-  ms@2.1.2: {}
-
   ms@2.1.3: {}
 
   napi-build-utils@1.0.2: {}