Perf: run reverse chn cidr in worker_threads

Build/build-chn-cidr.ts

@@ -16,7 +16,7 @@ const PROBE_CHN_CIDR_V4 = [
   '120.78.92.171'
 ];
 
-export const getChnCidrPromise = createMemoizedPromise(cachedOnlyFail(
+export const getChnCidrPromise = createMemoizedPromise(cachedOnlyFail<[], [string[], string[]]>(
   async function getChnCidr() {
     const [_cidr4, cidr6] = await Promise.all([
       fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt', true).then(Array.fromAsync<string>),

Build/build-internal-reverse-chn-cidr.ts

@@ -1,31 +1,61 @@
 import path from 'node:path';
 import { task } from './trace';
 
-import { exclude, merge } from 'fast-cidr-tools';
 import { getChnCidrPromise } from './build-chn-cidr';
-import { NON_CN_CIDR_INCLUDED_IN_CHNROUTE, RESERVED_IPV4_CIDR } from './constants/cidr';
+// import { RESERVED_IPV4_CIDR, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
 
 import fs from 'node:fs';
 import { OUTPUT_INTERNAL_DIR } from './constants/dir';
 import { asyncWriteToStream } from 'foxts/async-write-to-stream';
 import { mkdirp } from './lib/misc';
-import { appendArrayInPlace } from './lib/append-array-in-place';
+// import { appendArrayInPlace } from './lib/append-array-in-place';
+import Worktank from 'worktank';
 
-export const buildInternalReverseChnCIDR = task(require.main === module, __filename)(async () => {
-  const [cidr] = await getChnCidrPromise();
+const pool = new Worktank({
+  name: 'build-internal-reverse-chn-cidr',
+  size: 1,
+  timeout: 10000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects
+  warmup: true,
+  autoterminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
+  env: {},
+  methods: { // An object mapping function names to functions objects to serialize and deserialize into each worker thread, only functions that don't depend on their closure can be serialized
+    // eslint-disable-next-line object-shorthand -- workertank
+    getreversedCidr: async function (cidr: string[], importMetaUrl: string): Promise<string[]> {
+      // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
+      const { default: module } = await import('node:module');
+      const __require = module.createRequire(importMetaUrl);
+      const { exclude, merge } = __require('fast-cidr-tools');
+      const { RESERVED_IPV4_CIDR, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } = __require('./constants/cidr');
+      const { appendArrayInPlace } = __require('./lib/append-array-in-place');
 
-  const reversedCidr = merge(
-    appendArrayInPlace(
-      exclude(
-        ['0.0.0.0/0'],
-        RESERVED_IPV4_CIDR.concat(cidr),
+      return merge(
+        appendArrayInPlace(
+          exclude(
+            ['0.0.0.0/0'],
+            RESERVED_IPV4_CIDR.concat(cidr),
+            true
+          ),
+          // https://github.com/misakaio/chnroutes2/issues/25
+          NON_CN_CIDR_INCLUDED_IN_CHNROUTE
+        ),
         true
-      ),
-      // https://github.com/misakaio/chnroutes2/issues/25
-      NON_CN_CIDR_INCLUDED_IN_CHNROUTE
-    ),
-    true
-  );
+      );
+    }
+  }
+});
+
+export const buildInternalReverseChnCIDR = task(require.main === module, __filename)(async (span) => {
+  const [cidr] = await span.traceChildPromise('download chnroutes2', getChnCidrPromise());
+
+  const reversedCidr = await span.traceChildAsync('build reversed chn cidr', async () => {
+    const reversedCidr = await pool.exec(
+      'getreversedCidr',
+      [cidr, import.meta.url]
+    );
+    pool.terminate();
+
+    return reversedCidr;
+  });
 
   const outputFile = path.join(OUTPUT_INTERNAL_DIR, 'reversed-chn-cidr.txt');
   await mkdirp(OUTPUT_INTERNAL_DIR);

Build/trace/index.ts

@@ -1,3 +1,4 @@
+import { noop } from 'foxts/noop';
 import { basename, extname } from 'node:path';
 import process from 'node:process';
 import picocolors from 'picocolors';
@@ -98,8 +99,12 @@ export function createSpan(name: string, parentTraceResult?: TraceResult): Span
 export const dummySpan = createSpan('');
 
 export function task(importMetaMain: boolean, importMetaPath: string) {
-  return <T>(fn: (span: Span) => Promise<T>, customName?: string) => {
+  return <T>(fn: (span: Span, onCleanup: (cb: () => Promise<void> | void) => void) => Promise<T>, customName?: string) => {
     const taskName = customName ?? basename(importMetaPath, extname(importMetaPath));
+    let cleanup: () => Promise<void> | void = noop;
+    const onCleanup = (cb: () => void) => {
+      cleanup = cb;
+    };
 
     const dummySpan = createSpan(taskName);
     if (importMetaMain) {
@@ -112,7 +117,7 @@ export function task(importMetaMain: boolean, importMetaPath: string) {
         process.exit(1);
       });
 
-      dummySpan.traceChildAsync('dummy', fn).finally(() => {
+      dummySpan.traceChildAsync('dummy', (childSpan) => fn(childSpan, onCleanup)).finally(() => {
         dummySpan.stop();
         printTraceResult(dummySpan.traceResult);
         whyIsNodeRunning();
@@ -121,9 +126,9 @@ export function task(importMetaMain: boolean, importMetaPath: string) {
 
     return async (span?: Span) => {
       if (span) {
-        return span.traceChildAsync(taskName, fn);
+        return span.traceChildAsync(taskName, (childSpan) => fn(childSpan, onCleanup).finally(() => cleanup()));
       }
-      return fn(dummySpan);
+      return fn(dummySpan, onCleanup).finally(() => cleanup());
     };
   };
 }

package.json

@@ -44,6 +44,7 @@
     "undici-cache-store-better-sqlite3": "^0.1.1",
     "whoiser": "^1.18.0",
     "why-is-node-running": "^3.2.1",
+    "worktank": "^2.7.3",
     "xbits": "^0.2.0",
     "yaml": "^2.7.0"
   },

pnpm-lock.yaml

@@ -83,6 +83,9 @@ importers:
       why-is-node-running:
         specifier: ^3.2.1
         version: 3.2.1
+      worktank:
+        specifier: ^2.7.3
+        version: 2.7.3
       xbits:
         specifier: ^0.2.0
         version: 0.2.0
@@ -1549,6 +1552,9 @@ packages:
     resolution: {integrity: sha512-Azwzvl90HaF0aCz1JrDdXQykFakSSNPaPoiZ9fm5qJIMHioDZEi7OAdRwSm6rSoPtY3Qutnm3L7ogmg3dc+wbQ==}
     engines: {node: ^18.17.0 || >=20.5.0}
 
+  promise-make-naked@2.1.2:
+    resolution: {integrity: sha512-y7s8ZuHIG56JYspB24be9GFkXA1zXL85Ur9u1DKrW/tvyUoPxWgBjnalK6Nc6l7wHBcAW0c3PO07+XOsWTRuhg==}
+
   promise-retry@2.0.1:
     resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==}
     engines: {node: '>=10'}
@@ -1844,6 +1850,9 @@ packages:
   util-deprecate@1.0.2:
     resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
 
+  webworker-shim@1.1.0:
+    resolution: {integrity: sha512-LhPJDED3cM0+K9w4JjIio+RYPqvr712b3lyM+JjMR/rSD9elrSYHrrwE9qu3inPSSf60xqePgFgEwuAg17AuhA==}
+
   which@2.0.2:
     resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
     engines: {node: '>= 8'}
@@ -1865,6 +1874,9 @@ packages:
   workerpool@6.5.1:
     resolution: {integrity: sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==}
 
+  worktank@2.7.3:
+    resolution: {integrity: sha512-M0fesnpttBPdvNYBdzRvLDsacN0na9RYWFxwmM/x1+/6mufjduv9/9vBObK8EXDqxRMX/SOYJabpo0UCYYBUdQ==}
+
   wrap-ansi@7.0.0:
     resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
     engines: {node: '>=10'}
@@ -3456,6 +3468,8 @@ snapshots:
 
   proc-log@5.0.0: {}
 
+  promise-make-naked@2.1.2: {}
+
   promise-retry@2.0.1:
     dependencies:
       err-code: 2.0.3
@@ -3758,6 +3772,8 @@ snapshots:
 
   util-deprecate@1.0.2: {}
 
+  webworker-shim@1.1.0: {}
+
   which@2.0.2:
     dependencies:
       isexe: 2.0.0
@@ -3772,6 +3788,11 @@ snapshots:
 
   workerpool@6.5.1: {}
 
+  worktank@2.7.3:
+    dependencies:
+      promise-make-naked: 2.1.2
+      webworker-shim: 1.1.0
+
   wrap-ansi@7.0.0:
     dependencies:
       ansi-styles: 4.3.0