Chore: minor changes

Build/build-chn-cidr.ts

@@ -11,12 +11,12 @@ const EXCLUDE_CIDRS = [
 ];
 
 export const buildChnCidr = task(__filename, async () => {
-  const [{ exclude: excludeCidrs }, cidr] = await Promise.all([
+  const [{ exclude }, cidr] = await Promise.all([
     import('cidr-tools-wasm'),
     processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'))
   ]);
 
-  const filteredCidr = excludeCidrs(cidr, EXCLUDE_CIDRS, true);
+  const filteredCidr = exclude(cidr, EXCLUDE_CIDRS, true);
 
   const description = [
     'License: CC BY-SA 2.0',

Build/build-speedtest-domainset.ts

@@ -6,18 +6,29 @@ import domainSorter from './lib/stable-sort-domain';
 import { Sema } from 'async-sema';
 import * as tldts from 'tldts';
 import { task } from './lib/trace-runner';
-const s = new Sema(2);
+const s = new Sema(3);
+
+const latestTopUserAgentsPromise = fetch('https://unpkg.com/top-user-agents@latest/index.json')
+  .then(res => res.json() as Promise<string[]>);
 
 const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> => {
-  await s.acquire();
+  const [topUserAgents] = await Promise.all([
+    latestTopUserAgentsPromise,
+    s.acquire()
+  ]);
+
+  const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
 
   try {
+    const key = `fetch speedtest endpoints: ${keyword}`;
+    console.time(key);
+
     const res = await fetch(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, {
       headers: {
         dnt: '1',
         Referer: 'https://www.speedtest.net/',
         accept: 'application/json, text/plain, */*',
-        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
+        'User-Agent': randomUserAgent,
         'Accept-Language': 'en-US,en;q=0.9',
         'Sec-Ch-Ua': '"Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"',
         'Sec-Ch-Ua-Mobile': '?0',
@@ -28,12 +39,14 @@ const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> =>
       }
     });
     if (!res.ok) {
-      const text = await res.text();
-      throw new Error(text);
+      throw new Error(res.statusText + '\n' + await res.text());
     }
 
     const json = await res.json() as { url: string; }[];
     s.release();
+
+    console.timeEnd(key);
+
     return json.map(({ url }) => tldts.getHostname(url, { detectIp: false }));
   } catch (e) {
     s.release();

Build/download-previous-build.ts

@@ -75,8 +75,6 @@ export const downloadPreviousBuild = task(__filename, async () => {
       );
     }
   }));
-
-  // return fsp.unlink(extractedPath).catch(() => { });
 });
 
 export const downloadPublicSuffixList = task(__filename, async () => {

Build/index.ts

@@ -16,39 +16,11 @@ import { buildRedirectModule } from './build-redirect-module';
 import { validate } from './validate-domainset';
 
 import { buildPublicHtml } from './build-public';
-
-import { Worker } from 'jest-worker';
-
-type WithWorker<T> = import('jest-worker').Worker & { __sukka_worker_name: string } & T
-
-const requireWorker = <T>(path: string, exposedMethods?: (keyof T & string)[]): WithWorker<T> => {
-  const _worker = new Worker(
-    import.meta.require.resolve(path),
-    {
-      numWorkers: 1,
-      maxRetries: 0,
-      enableWorkerThreads: true,
-      exposedMethods
-    }
-  ) as WithWorker<T>;
-  _worker.getStderr().pipe(process.stderr);
-  _worker.getStdout().pipe(process.stdout);
-  _worker.__sukka_worker_name = path;
-  return _worker;
-};
-
-const endWorker = async <T>(worker: WithWorker<T>) => {
-  const { forceExited } = await worker.end();
-  if (forceExited && worker.__sukka_worker_name) {
-    console.log(worker.__sukka_worker_name, 'forceExited');
-  }
-};
+import { TaskResult } from './lib/trace-runner';
 
 (async () => {
-  const buildInternalReverseChnCIDRWorker: WithWorker<typeof import('./build-internal-reverse-chn-cidr')> = requireWorker('./build-internal-reverse-chn-cidr', ['buildInternalReverseChnCIDR']);
+  const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url));
   try {
-    const { buildInternalReverseChnCIDR } = buildInternalReverseChnCIDRWorker;
-
     const downloadPreviousBuildPromise = downloadPreviousBuild();
     const downloadPublicSuffixListPromise = downloadPublicSuffixList();
     const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon());
@@ -75,7 +47,15 @@ const endWorker = async <T>(worker: WithWorker<T>) => {
       buildCommonPromise,
       buildCdnConfPromise
     ]).then(() => buildInternalCDNDomains());
-    const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR();
+
+    const buildInternalReverseChnCIDRPromise = new Promise<TaskResult>(resolve => {
+      buildInternalReverseChnCIDRWorker.postMessage(null);
+      buildInternalReverseChnCIDRWorker.onmessage = (e: MessageEvent<TaskResult>) => {
+        buildInternalReverseChnCIDRWorker.terminate();
+        resolve(e.data);
+      };
+    });
+
     const buildInternalChnDomainsPromise = buildInternalChnDomains();
     const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset());
 
@@ -109,9 +89,7 @@ const endWorker = async <T>(worker: WithWorker<T>) => {
 
     printStats(stats);
   } catch (e) {
-    console.error(e)
-  } finally {
-    await endWorker(buildInternalReverseChnCIDRWorker)
+    console.error(e);
   }
 })();
 

Build/lib/fetch-remote-text-by-line.ts

@@ -56,7 +56,6 @@ export async function* createReadlineInterfaceFromResponse(resp: Response): Asyn
   }
 }
 
-export async function fetchRemoteTextAndCreateReadlineInterface(url: string | URL, opt?: RequestInit): Promise<AsyncGenerator<string>> {
-  const resp = await fetchWithRetry(url, opt);
-  return createReadlineInterfaceFromResponse(resp);
+export function fetchRemoteTextAndCreateReadlineInterface(url: string | URL, opt?: RequestInit): Promise<AsyncGenerator<string>> {
+  return fetchWithRetry(url, opt).then(res => createReadlineInterfaceFromResponse(res));
 }

Build/lib/parse-filter.ts

@@ -68,7 +68,7 @@ export async function processDomainLists(domainListsUrl: string | URL) {
 }
 
 export async function processHosts(hostsUrl: string | URL, includeAllSubDomain = false) {
-  console.time(`   - processHosts: ${hostsUrl}`);
+  console.time(`- processHosts: ${hostsUrl}`);
 
   if (typeof hostsUrl === 'string') {
     hostsUrl = new URL(hostsUrl);

Build/lib/trace-runner.ts

@@ -18,6 +18,12 @@ const traceAsync = async <T>(prefix: string, fn: () => Promise<T>): Promise<T> =
 };
 export { traceAsync };
 
+export interface TaskResult {
+  readonly start: number;
+  readonly end: number;
+  readonly taskName: string;
+}
+
 const task = <T>(__filename: string, fn: () => Promise<T>, customname: string | null = null) => {
   const taskName = customname ?? path.basename(__filename, path.extname(__filename));
   return async () => {
@@ -27,7 +33,7 @@ const task = <T>(__filename: string, fn: () => Promise<T>, customname: string |
     const end = performance.now();
     console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`);
 
-    return { start, end, taskName } as const;
+    return { start, end, taskName } as TaskResult;
   };
 };
 export { task };

Build/workers/build-internal-reverse-chn-cidr-worker.ts

@@ -0,0 +1,8 @@
+declare const self: Worker;
+
+import { buildInternalReverseChnCIDR } from '../build-internal-reverse-chn-cidr';
+
+self.onmessage = async () => {
+  const stat = await buildInternalReverseChnCIDR();
+  postMessage(stat);
+};

package.json

@@ -19,10 +19,9 @@
     "@vercel/fetch-retry": "^5.1.3",
     "async-sema": "^3.1.1",
     "ci-info": "^4.0.0",
-    "cidr-tools-wasm": "^0.0.11",
+    "cidr-tools-wasm": "^0.0.13",
     "eslint": "^8.53.0",
     "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js",
-    "jest-worker": "^29.7.0",
     "mnemonist": "^0.39.5",
     "path-scurry": "^1.10.1",
     "picocolors": "^1.0.0",