Refactor: use `jest-worker`

Build/build-cdn-download-conf.ts → Build/build-cdn-download-conf.worker.ts

@@ -1,12 +1,12 @@
 import path from 'node:path';
-import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
+import { readFileIntoProcessedArray, fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { appendArrayInPlace } from 'foxts/append-array-in-place';
 import { SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput } from './lib/rules/domainset';
 import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
-import Worktank from 'worktank';
+import { HostnameTrie } from './lib/trie';
 import { $$fetch } from './lib/fetch-retry';
 import { fastUri } from 'fast-uri';
 
@@ -14,25 +14,17 @@ const cdnDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'do
 const downloadDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf'));
 const steamDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf'));
 
-const pool = new Worktank({
-  pool: {
-    name: 'extract-s3-from-publicssuffix',
-    size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
-  },
-  worker: {
-    autoAbort: 10000,
-    autoTerminate: 20000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
-    autoInstantiate: true,
-    methods: {
-      // eslint-disable-next-line object-shorthand -- workertank
-      getS3OSSDomains: async function (__filename: string): Promise<string[]> {
-        // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
-        const { default: module } = await import('node:module');
-        const __require = module.createRequire(__filename);
-
-        const { HostnameTrie } = __require('./lib/trie') as typeof import('./lib/trie');
-        const { fetchRemoteTextByLine } = __require('./lib/fetch-text-by-line') as typeof import('./lib/fetch-text-by-line');
-
+export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => {
+  const [
+    S3OSSDomains,
+    IPFSDomains,
+    cdnDomainsList,
+    downloadDomainSet,
+    steamDomainSet
+  ] = await Promise.all([
+    span.traceChildAsync(
+      'download public suffix list for s3',
+      async () => {
         const trie = new HostnameTrie();
 
         for await (const line of await fetchRemoteTextByLine('https://publicsuffix.org/list/public_suffix_list.dat', true)) {
@@ -70,24 +62,6 @@ const pool = new Worktank({
 
         return S3OSSDomains;
       }
-    }
-  }
-});
-
-export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => {
-  const [
-    S3OSSDomains,
-    IPFSDomains,
-    cdnDomainsList,
-    downloadDomainSet,
-    steamDomainSet
-  ] = await Promise.all([
-    span.traceChildAsync(
-      'download public suffix list for s3',
-      () => pool.exec(
-        'getS3OSSDomains',
-        [__filename]
-      ).finally(() => pool.terminate())
     ),
     span.traceChildAsync(
       'load public ipfs gateway list',

Build/build-microsoft-cdn.ts

@@ -1,70 +0,0 @@
-import { task } from './trace';
-import { SHARED_DESCRIPTION } from './constants/description';
-import { RulesetOutput } from './lib/rules/ruleset';
-import Worktank from 'worktank';
-import { RULES } from './constants/microsoft-cdn';
-
-const pool = new Worktank({
-  pool: {
-    name: 'get-microsoft-cdn',
-    size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
-  },
-  worker: {
-    autoAbort: 10000,
-    autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
-    autoInstantiate: true,
-    methods: {
-      // eslint-disable-next-line object-shorthand -- workertank
-      getMicrosoftCdnRuleset: async function (__filename: string): Promise<[domains: string[], domainSuffixes: string[]]> {
-        // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
-        const { default: module } = await import('node:module');
-        const __require = module.createRequire(__filename);
-
-        const { HostnameSmolTrie } = __require('./lib/trie');
-        const { PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } = __require('./constants/microsoft-cdn') as typeof import('./constants/microsoft-cdn');
-        const { fetchRemoteTextByLine } = __require('./lib/fetch-text-by-line') as typeof import('./lib/fetch-text-by-line');
-        const { appendArrayInPlace } = __require('foxts/append-array-in-place') as typeof import('foxts/append-array-in-place');
-        const { extractDomainsFromFelixDnsmasq } = __require('./lib/parse-dnsmasq') as typeof import('./lib/parse-dnsmasq');
-
-        const trie = new HostnameSmolTrie();
-
-        for await (const line of await fetchRemoteTextByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) {
-          const domain = extractDomainsFromFelixDnsmasq(line);
-          if (domain) {
-            trie.add(domain);
-          }
-        }
-
-        // remove blacklist domain from trie, to prevent them from being included in the later dump
-        BLACKLIST.forEach(black => trie.whitelist(black));
-
-        const domains: string[] = DOMAINS;
-        const domainSuffixes = appendArrayInPlace(PROBE_DOMAINS.flatMap(domain => trie.find(domain)), DOMAIN_SUFFIXES);
-
-        return [domains, domainSuffixes] as const;
-      }
-    }
-  }
-});
-
-const getMicrosoftCdnRulesetPromise = pool.exec(
-  'getMicrosoftCdnRuleset',
-  [__filename]
-).finally(() => pool.terminate());
-
-export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => {
-  const [domains, domainSuffixes] = await span.traceChildPromise('get microsoft cdn domains', getMicrosoftCdnRulesetPromise);
-
-  return new RulesetOutput(span, 'microsoft_cdn', 'non_ip')
-    .withTitle('Sukka\'s Ruleset - Microsoft CDN')
-    .appendDescription(SHARED_DESCRIPTION)
-    .appendDescription(
-      '',
-      'This file contains Microsoft\'s domains using their China mainland CDN servers.'
-    )
-    .addFromRuleset(RULES)
-    .appendDataSource('https://github.com/felixonmars/dnsmasq-china-list')
-    .bulkAddDomain(domains)
-    .bulkAddDomainSuffix(domainSuffixes)
-    .write();
-});

Build/build-microsoft-cdn.worker.ts

@@ -0,0 +1,42 @@
+import { task } from './trace';
+import { SHARED_DESCRIPTION } from './constants/description';
+import { RulesetOutput } from './lib/rules/ruleset';
+import { RULES, PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } from './constants/microsoft-cdn';
+import { HostnameSmolTrie } from './lib/trie';
+import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
+import { appendArrayInPlace } from 'foxts/append-array-in-place';
+import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';
+
+export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => {
+  const [domains, domainSuffixes] = await span.traceChildAsync('get microsoft cdn domains', async () => {
+    const trie = new HostnameSmolTrie();
+
+    for await (const line of await fetchRemoteTextByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) {
+      const domain = extractDomainsFromFelixDnsmasq(line);
+      if (domain) {
+        trie.add(domain);
+      }
+    }
+
+    // remove blacklist domain from trie, to prevent them from being included in the later dump
+    BLACKLIST.forEach(black => trie.whitelist(black));
+
+    const domains: string[] = DOMAINS;
+    const domainSuffixes = appendArrayInPlace(PROBE_DOMAINS.flatMap(domain => trie.find(domain)), DOMAIN_SUFFIXES);
+
+    return [domains, domainSuffixes] as [string[], string[]];
+  });
+
+  return new RulesetOutput(span, 'microsoft_cdn', 'non_ip')
+    .withTitle('Sukka\'s Ruleset - Microsoft CDN')
+    .appendDescription(SHARED_DESCRIPTION)
+    .appendDescription(
+      '',
+      'This file contains Microsoft\'s domains using their China mainland CDN servers.'
+    )
+    .addFromRuleset(RULES)
+    .appendDataSource('https://github.com/felixonmars/dnsmasq-china-list')
+    .bulkAddDomain(domains)
+    .bulkAddDomainSuffix(domainSuffixes)
+    .write();
+});

Build/build-reject-domainset.ts

@@ -18,7 +18,7 @@ import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
 import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset';
 import { foundDebugDomain } from './lib/parse-filter/shared';
-import { getPhishingDomains } from './lib/get-phishing-domains';
+import { createWorker } from './lib/worker';
 import type { MaybePromise } from './lib/misc';
 import { RulesetOutput } from './lib/rules/ruleset';
 import { fetchAssets } from './lib/fetch-assets';
@@ -39,6 +39,9 @@ const adguardFiltersExtraDownloads = ADGUARD_FILTERS_EXTRA.map(entry => processF
 const adguardFiltersWhitelistsDownloads = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry));
 
 export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
+  const phishingWorker = createWorker<typeof import('./lib/get-phishing-domains')>(
+    require.resolve('./lib/get-phishing-domains')
+  )(['getPhishingDomains']);
   const rejectDomainsetOutput = new DomainsetOutput(span, 'reject')
     .withTitle('Sukka\'s Ruleset - Reject Base')
     .appendDescription(
@@ -126,7 +129,9 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       arrayPushNonNullish(promises, domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)));
       arrayPushNonNullish(promises, domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)));
 
-      rejectPhisingDomainsetOutput.addFromDomainset(getPhishingDomains(childSpan));
+      rejectPhisingDomainsetOutput.addFromDomainset(
+        span.traceChildPromise('get phishing domains', phishingWorker.getPhishingDomains())
+      );
 
       arrayPushNonNullish(
         promises,
@@ -253,6 +258,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       rejectNonIpRulesetOutput.whitelistKeyword(keyword);
     }
 
+    // Deduplicate reject_extra and reject_phishing from the base reject domainset
     rejectDomainsetOutput.domainTrie.dump(arg => {
       rejectExtraDomainsetOutput.whitelistDomain(arg);
       rejectPhisingDomainsetOutput.whitelistDomain(arg);
@@ -295,4 +301,6 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
   await myRejectOutputAdGuardHome
     .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf')))
     .write();
+
+  await phishingWorker.end();
 });

Build/build-telegram-cidr.ts

@@ -2,10 +2,137 @@
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { RulesetOutput } from './lib/rules/ruleset';
-import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip';
+import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip';
+import picocolors from 'picocolors';
+import { $$fetch } from './lib/fetch-retry';
+import dns from 'node:dns/promises';
+import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
+import { fastIpVersion } from 'foxts/fast-ip-version';
+import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
 
 export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
-  const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise);
+  const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async () => {
+    const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
+    const lastModified = resp.headers.get('last-modified');
+    const date = lastModified ? new Date(lastModified) : new Date();
+
+    const ipcidr: string[] = [
+      // Unused secret Telegram backup CIDR, announced by AS62041
+      '95.161.64.0/20'
+    ];
+    const ipcidr6: string[] = [];
+
+    for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
+      const v = fastIpVersion(cidr);
+      if (v === 4) {
+        ipcidr.push(cidr);
+      } else if (v === 6) {
+        ipcidr6.push(cidr);
+      }
+    }
+
+    const backupIPs = new Set<string>();
+
+    // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
+
+    const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => {
+      const resolver = new dns.Resolver();
+      resolver.setServers([ip]);
+      return Object.assign(resolver, { server: ip });
+    });
+
+    // Backup IP Source 1 (DNS)
+    await Promise.all(resolvers.flatMap((resolver) => [
+      'apv3.stel.com', // prod
+      'tapv3.stel.com' // test
+    ].map(async (domain) => {
+      try {
+        // tapv3.stel.com was for testing server
+        const resp = await resolver.resolveTxt(domain);
+        const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten
+        if (strings.length !== 2) {
+          throw new TypeError(`Unexpected TXT record count: ${strings.length}`);
+        }
+
+        const str = strings[0].length > strings[1].length
+          ? strings[0] + strings[1]
+          : strings[1] + strings[0];
+
+        const ips = getTelegramBackupIPFromBase64(str);
+        ips.forEach(i => backupIPs.add(i.ip));
+
+        console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server });
+      } catch (e) {
+        console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e);
+      }
+    })));
+
+    // Backup IP Source 2: Firebase Realtime Database (test server not supported)
+    try {
+      const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
+      if (typeof text === 'string' && text.length === 344) {
+        const ips = getTelegramBackupIPFromBase64(text);
+        ips.forEach(i => backupIPs.add(i.ip));
+
+        console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
+      }
+    } catch (e) {
+      console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
+      // ignore all errors
+    }
+
+    // Backup IP Source 3: Firebase Value Store (test server not supported)
+    try {
+      const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
+        headers: {
+          Accept: '*/*',
+          Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
+        }
+      })).json();
+
+      if (
+        json && typeof json === 'object'
+        && 'fields' in json && typeof json.fields === 'object' && json.fields
+        && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
+        && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
+      ) {
+        const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
+        ips.forEach(i => backupIPs.add(i.ip));
+
+        console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
+      } else {
+        console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
+      }
+    } catch (e) {
+      console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
+    }
+
+    // Backup IP Source 4: Google App Engine
+    await Promise.all([
+      'https://dns-telegram.appspot.com',
+      'https://dns-telegram.appspot.com/test'
+    ].map(async (url) => {
+      try {
+        const text = await (await $$fetch(url)).text();
+        if (text.length === 344) {
+          const ips = getTelegramBackupIPFromBase64(text);
+          ips.forEach(i => backupIPs.add(i.ip));
+
+          console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
+        }
+      } catch (e) {
+        console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
+      }
+    }));
+
+    // tcdnb.azureedge.net no longer works
+
+    console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
+
+    ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
+
+    return { timestamp: date.getTime(), ipcidr, ipcidr6 };
+  });
 
   if (ipcidr.length + ipcidr6.length === 0) {
     throw new Error('Failed to fetch data!');

Build/build-telegram-cidr.worker.ts

@@ -0,0 +1,159 @@
+// @ts-check
+import { task } from './trace';
+import { SHARED_DESCRIPTION } from './constants/description';
+import { RulesetOutput } from './lib/rules/ruleset';
+import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip';
+import picocolors from 'picocolors';
+import { $$fetch } from './lib/fetch-retry';
+import dns from 'node:dns/promises';
+import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
+import { fastIpVersion } from 'foxts/fast-ip-version';
+import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
+
+export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
+  const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async () => {
+    const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
+    const lastModified = resp.headers.get('last-modified');
+    const date = lastModified ? new Date(lastModified) : new Date();
+
+    const ipcidr: string[] = [
+      // Unused secret Telegram backup CIDR, announced by AS62041
+      '95.161.64.0/20'
+    ];
+    const ipcidr6: string[] = [];
+
+    for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
+      const v = fastIpVersion(cidr);
+      if (v === 4) {
+        ipcidr.push(cidr);
+      } else if (v === 6) {
+        ipcidr6.push(cidr);
+      }
+    }
+
+    const backupIPs = new Set<string>();
+
+    // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
+
+    const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => {
+      const resolver = new dns.Resolver();
+      resolver.setServers([ip]);
+      return Object.assign(resolver, { server: ip });
+    });
+
+    // Backup IP Source 1 (DNS)
+    await Promise.all(resolvers.flatMap((resolver) => [
+      'apv3.stel.com', // prod
+      'tapv3.stel.com' // test
+    ].map(async (domain) => {
+      try {
+        // tapv3.stel.com was for testing server
+        const resp = await resolver.resolveTxt(domain);
+        const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten
+        if (strings.length !== 2) {
+          throw new TypeError(`Unexpected TXT record count: ${strings.length}`);
+        }
+
+        const str = strings[0].length > strings[1].length
+          ? strings[0] + strings[1]
+          : strings[1] + strings[0];
+
+        const ips = getTelegramBackupIPFromBase64(str);
+        ips.forEach(i => backupIPs.add(i.ip));
+
+        console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server });
+      } catch (e) {
+        console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e);
+      }
+    })));
+
+    // Backup IP Source 2: Firebase Realtime Database (test server not supported)
+    try {
+      const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
+      if (typeof text === 'string' && text.length === 344) {
+        const ips = getTelegramBackupIPFromBase64(text);
+        ips.forEach(i => backupIPs.add(i.ip));
+
+        console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
+      }
+    } catch (e) {
+      console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
+      // ignore all errors
+    }
+
+    // Backup IP Source 3: Firebase Value Store (test server not supported)
+    try {
+      const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
+        headers: {
+          Accept: '*/*',
+          Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
+        }
+      })).json();
+
+      if (
+        json && typeof json === 'object'
+        && 'fields' in json && typeof json.fields === 'object' && json.fields
+        && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
+        && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
+      ) {
+        const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
+        ips.forEach(i => backupIPs.add(i.ip));
+
+        console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
+      } else {
+        console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
+      }
+    } catch (e) {
+      console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
+    }
+
+    // Backup IP Source 4: Google App Engine
+    await Promise.all([
+      'https://dns-telegram.appspot.com',
+      'https://dns-telegram.appspot.com/test'
+    ].map(async (url) => {
+      try {
+        const text = await (await $$fetch(url)).text();
+        if (text.length === 344) {
+          const ips = getTelegramBackupIPFromBase64(text);
+          ips.forEach(i => backupIPs.add(i.ip));
+
+          console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
+        }
+      } catch (e) {
+        console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
+      }
+    }));
+
+    // tcdnb.azureedge.net no longer works
+
+    console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
+
+    ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
+
+    return { timestamp: date.getTime(), ipcidr, ipcidr6 };
+  });
+
+  if (ipcidr.length + ipcidr6.length === 0) {
+    throw new Error('Failed to fetch data!');
+  }
+
+  const description = [
+    ...SHARED_DESCRIPTION,
+    'Data from:',
+    ' - https://core.telegram.org/resources/cidr.txt'
+  ];
+
+  return new RulesetOutput(span, 'telegram', 'ip')
+    .withTitle('Sukka\'s Ruleset - Telegram IP CIDR')
+    .withDescription(description)
+    // .withDate(date) // With extra data source, we no longer use last-modified for file date
+    .appendDataSource(
+      'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')'
+    )
+    .bulkAddCIDR4NoResolve(ipcidr)
+    .bulkAddCIDR6NoResolve(ipcidr6)
+    .write();
+});
+
+export const ___ = '';

Build/index.ts

@@ -6,9 +6,7 @@ import { downloadPreviousBuild } from './download-previous-build';
 import { buildCommon } from './build-common';
 import { buildRejectIPList } from './build-reject-ip-list';
 import { buildAppleCdn } from './build-apple-cdn';
-import { buildCdnDownloadConf } from './build-cdn-download-conf';
 import { buildRejectDomainSet } from './build-reject-domainset';
-import { buildTelegramCIDR } from './build-telegram-cidr';
 import { buildChnCidr } from './build-chn-cidr';
 import { buildSpeedtestDomainSet } from './build-speedtest-domainset';
 import { buildDomesticRuleset } from './build-domestic-direct-lan-ruleset-dns-mapping-module';
@@ -18,11 +16,9 @@ import { buildStreamService } from './build-stream-service';
 import { buildRedirectModule } from './build-sgmodule-redirect';
 import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip';
 
-import { buildMicrosoftCdn } from './build-microsoft-cdn';
+import { createWorker } from './lib/worker';
 
 import { buildPublic } from './build-public';
-import { downloadMockAssets } from './download-mock-assets';
-
 import { buildCloudMounterRules } from './build-cloudmounter-rules';
 
 import { printStats, printTraceResult, whyIsNodeRunning } from './trace';
@@ -71,6 +67,22 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
     fs.unlinkSync(buildFinishedLock);
   }
 
+  const microsoftCdnWorker = createWorker<typeof import('./build-microsoft-cdn.worker')>(
+    require.resolve('./build-microsoft-cdn.worker')
+  )(['buildMicrosoftCdn']);
+
+  const cdnDownloadWorker = createWorker<typeof import('./build-cdn-download-conf.worker')>(
+    require.resolve('./build-cdn-download-conf.worker')
+  )(['buildCdnDownloadConf']);
+
+  const telegramCidrWorker = createWorker<typeof import('./build-telegram-cidr.worker')>(
+    require.resolve('./build-telegram-cidr.worker')
+  )(['buildTelegramCIDR']);
+
+  const mockAssetsWorker = createWorker<typeof import('./download-mock-assets.worker')>(
+    require.resolve('./download-mock-assets.worker')
+  )(['downloadMockAssets']);
+
   try {
     // only enable why-is-node-running in GitHub Actions debug mode
     if (isCI && process.env.RUNNER_DEBUG === '1') {
@@ -79,14 +91,14 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
 
     const downloadPreviousBuildPromise = downloadPreviousBuild();
 
-    await Promise.all([
+    const traces: TraceResult[] = await Promise.all([
       downloadPreviousBuildPromise,
       downloadPreviousBuildPromise.then(() => buildCommon()),
       downloadPreviousBuildPromise.then(() => buildRejectIPList()),
       downloadPreviousBuildPromise.then(() => buildAppleCdn()),
-      downloadPreviousBuildPromise.then(() => buildCdnDownloadConf()),
+      downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()),
       downloadPreviousBuildPromise.then(() => buildRejectDomainSet()),
-      downloadPreviousBuildPromise.then(() => buildTelegramCIDR()),
+      downloadPreviousBuildPromise.then(() => telegramCidrWorker.buildTelegramCIDR()),
       downloadPreviousBuildPromise.then(() => buildChnCidr()),
       downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()),
       downloadPreviousBuildPromise.then(() => buildDomesticRuleset()),
@@ -94,45 +106,29 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
       downloadPreviousBuildPromise.then(() => buildRedirectModule()),
       downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()),
       downloadPreviousBuildPromise.then(() => buildStreamService()),
-      downloadPreviousBuildPromise.then(() => buildMicrosoftCdn()),
+      downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()),
       downloadPreviousBuildPromise.then(() => buildCloudMounterRules()),
-      downloadMockAssets()
+      mockAssetsWorker.downloadMockAssets()
     ]);
 
-    await buildDeprecateFiles();
-    await buildPublic();
+    traces.push(
+      await buildDeprecateFiles(),
+      await buildPublic()
+    );
 
     // write a file to demonstrate that the build is finished
     fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n');
 
-    const traces: TraceResult[] = [];
-    [
-      downloadPreviousBuild,
-      downloadMockAssets,
-      buildCommon,
-      buildRejectIPList,
-      buildAppleCdn,
-      buildCdnDownloadConf,
-      buildRejectDomainSet,
-      buildTelegramCIDR,
-      buildChnCidr,
-      buildSpeedtestDomainSet,
-      buildDomesticRuleset,
-      buildGlobalRuleset,
-      buildRedirectModule,
-      buildAlwaysRealIPModule,
-      buildStreamService,
-      buildMicrosoftCdn,
-      buildCloudMounterRules,
-      buildPublic,
-      buildDeprecateFiles
-    ].forEach((fn) => {
-      const trace = fn.getInternalTraceResult();
-      printTraceResult(trace);
-      traces.push(trace);
+    traces.forEach((t) => {
+      printTraceResult(t);
     });
     printStats(traces);
 
+    await microsoftCdnWorker.end();
+    await cdnDownloadWorker.end();
+    await telegramCidrWorker.end();
+    await mockAssetsWorker.end();
+
     // Finish the build to avoid leaking timer/fetch ref
     await whyIsNodeRunning();
     process.exit(0);

Build/lib/get-phishing-domains.ts

@@ -1,219 +1,171 @@
-import Worktank from 'worktank';
+import picocolors from 'picocolors';
+import { parse } from 'tldts-experimental';
+import { appendArrayInPlaceCurried } from 'foxts/append-array-in-place';
 
-import { dummySpan, printTraceResult } from '../trace';
-import type { Span } from '../trace';
+import { dummySpan } from '../trace';
 import type { TldTsParsed } from './normalize-domain';
 
-const pool = new Worktank({
-  pool: {
-    name: 'process-phishing-domains',
-    size: 1
-  },
-  worker: {
-    autoAbort: 20000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects
-    autoInstantiate: true,
-    autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
-    env: {},
-    methods: {
-      // eslint-disable-next-line object-shorthand -- workertank
-      getPhishingDomains: async function (
-        importMetaUrl: string,
-        /** require.main === module */ isDebug = false
-      ): Promise<string[]> {
-      // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
-        const { default: module } = await import('node:module');
-        const __require = module.createRequire(importMetaUrl);
-
-        const picocolors = __require('picocolors') as typeof import('picocolors');
-        const tldts = __require('tldts-experimental') as typeof import('tldts-experimental');
-
-        const { appendArrayInPlaceCurried } = __require('foxts/append-array-in-place') as typeof import('foxts/append-array-in-place');
-
-        const { loosTldOptWithPrivateDomains } = __require('../constants/loose-tldts-opt') as typeof import('../constants/loose-tldts-opt');
-        const { BLACK_TLD, WHITELIST_MAIN_DOMAINS, leathalKeywords, lowKeywords, sensitiveKeywords } = __require('../constants/phishing-score-source') as typeof import('../constants/phishing-score-source');
-        const { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } = __require('../constants/reject-data-source') as typeof import('../constants/reject-data-source');
-        const { dummySpan } = __require('../trace') as typeof import('../trace');
-        const NullPrototypeObject = __require('null-prototype-object') as typeof import('null-prototype-object');
-
-        const { processHostsWithPreload } = __require('./parse-filter/hosts') as typeof import('./parse-filter/hosts');
-        const { processDomainListsWithPreload } = __require('./parse-filter/domainlists') as typeof import('./parse-filter/domainlists');
-
-        const downloads = [
-          ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)),
-          ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry))
-        ];
-
-        const domainArr: string[] = [];
-
-        const domainGroups = await Promise.all(downloads.map(task => task(dummySpan)));
-        domainGroups.forEach(appendArrayInPlaceCurried(domainArr));
-
-        // return domainArr;
-
-        const domainCountMap = new Map<string, number>();
-        const domainScoreMap: Record<string, number> = new NullPrototypeObject();
-
-        let line = '';
-        let tld: string | null = '';
-        let apexDomain: string | null = '';
-        let subdomain: string | null = '';
-        let parsed: TldTsParsed;
-
-        // const set = new Set<string>();
-        // let duplicateCount = 0;
-
-        for (let i = 0, len = domainArr.length; i < len; i++) {
-          line = domainArr[i];
-
-          // if (set.has(line)) {
-          //   duplicateCount++;
-          // } else {
-          //   set.add(line);
-          // }
-
-          parsed = tldts.parse(line, loosTldOptWithPrivateDomains);
-          if (parsed.isPrivate) {
-            continue;
-          }
+import { loosTldOptWithPrivateDomains } from '../constants/loose-tldts-opt';
+import { BLACK_TLD, WHITELIST_MAIN_DOMAINS, leathalKeywords, lowKeywords, sensitiveKeywords } from '../constants/phishing-score-source';
+import { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source';
 
-          tld = parsed.publicSuffix;
-          apexDomain = parsed.domain;
+import { processHostsWithPreload } from './parse-filter/hosts';
+import { processDomainListsWithPreload } from './parse-filter/domainlists';
 
-          if (!tld) {
-            console.log(picocolors.yellow('[phishing domains] E0001'), 'missing tld', { line, tld });
-            continue;
-          }
-          if (!apexDomain) {
-            console.log(picocolors.yellow('[phishing domains] E0002'), 'missing domain', { line, apexDomain });
-            continue;
-          }
-          if (WHITELIST_MAIN_DOMAINS.has(apexDomain)) {
-            continue;
-          }
+import process from 'node:process';
 
-          domainCountMap.set(
-            apexDomain,
-            domainCountMap.has(apexDomain)
-              ? domainCountMap.get(apexDomain)! + 1
-              : 1
-          );
-
-          let score = 0;
-
-          if (apexDomain in domainScoreMap) {
-            score = domainScoreMap[apexDomain];
-          } else {
-            if (BLACK_TLD.has(tld)) {
-              score += 3;
-            } else if (tld.length > 4) {
-              score += 2;
-            } else if (tld.length > 5) {
-              score += 4;
-            }
-            if (apexDomain.length >= 18) {
-              score += 0.5;
-            }
-          }
+export function getPhishingDomains(isDebug = false): Promise<string[]> {
+  return dummySpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync(
+    'process phishing domain set',
+    async () => {
+      const downloads = [
+        ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)),
+        ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry))
+      ];
 
-          subdomain = parsed.subdomain;
+      const domainArr: string[] = [];
 
-          if (subdomain) {
-            score += calcDomainAbuseScore(subdomain, line);
-          }
+      const domainGroups = await Promise.all(downloads.map(task => task(dummySpan)));
+      domainGroups.forEach(appendArrayInPlaceCurried(domainArr));
+
+      const domainCountMap = new Map<string, number>();
+      const domainScoreMap: Record<string, number> = Object.create(null) as Record<string, number>;
+
+      let line: string;
+      let tld: string | null;
+      let apexDomain: string | null;
+      let subdomain: string | null;
+      let parsed: TldTsParsed;
+
+      for (let i = 0, len = domainArr.length; i < len; i++) {
+        line = domainArr[i];
 
-          domainScoreMap[apexDomain] = score;
+        parsed = parse(line, loosTldOptWithPrivateDomains);
+        if (parsed.isPrivate) {
+          continue;
         }
 
-        domainCountMap.forEach((count, apexDomain) => {
-          const score = domainScoreMap[apexDomain];
-          if (
-          // !WHITELIST_MAIN_DOMAINS.has(apexDomain)
-            (score >= 24)
-            || (score >= 16 && count >= 7)
-            || (score >= 13 && count >= 11)
-            || (score >= 5 && count >= 14)
-            || (score >= 3 && count >= 21)
-            || (score >= 1 && count >= 60)
-          ) {
-            domainArr.push('.' + apexDomain);
+        tld = parsed.publicSuffix;
+        apexDomain = parsed.domain;
+
+        if (!tld) {
+          console.log(picocolors.yellow('[phishing domains] E0001'), 'missing tld', { line, tld });
+          continue;
+        }
+        if (!apexDomain) {
+          console.log(picocolors.yellow('[phishing domains] E0002'), 'missing domain', { line, apexDomain });
+          continue;
+        }
+        if (WHITELIST_MAIN_DOMAINS.has(apexDomain)) {
+          continue;
+        }
+
+        domainCountMap.set(
+          apexDomain,
+          domainCountMap.has(apexDomain)
+            ? domainCountMap.get(apexDomain)! + 1
+            : 1
+        );
+
+        let score = 0;
+
+        if (apexDomain in domainScoreMap) {
+          score = domainScoreMap[apexDomain];
+        } else {
+          if (BLACK_TLD.has(tld)) {
+            score += 3;
+          } else if (tld.length > 4) {
+            score += 2;
+          } else if (tld.length > 5) {
+            score += 4;
           }
-        });
+          if (apexDomain.length >= 18) {
+            score += 0.5;
+          }
+        }
+
+        subdomain = parsed.subdomain;
+
+        if (subdomain) {
+          score += calcDomainAbuseScore(subdomain, line);
+        }
+
+        domainScoreMap[apexDomain] = score;
+      }
 
-        if (isDebug) {
-          console.log({
-            v: 1,
-            score: domainScoreMap['com-ticketry.world'],
-            count: domainCountMap.get('com-ticketry.world'),
-            domainArrLen: domainArr.length
-          });
+      domainCountMap.forEach((count, apexDomain) => {
+        const score = domainScoreMap[apexDomain];
+        if (
+          (score >= 24)
+          || (score >= 16 && count >= 7)
+          || (score >= 13 && count >= 11)
+          || (score >= 5 && count >= 14)
+          || (score >= 3 && count >= 21)
+          || (score >= 1 && count >= 60)
+        ) {
+          domainArr.push('.' + apexDomain);
         }
+      });
+
+      if (isDebug) {
+        console.log({
+          v: 1,
+          score: domainScoreMap['com-ticketry.world'],
+          count: domainCountMap.get('com-ticketry.world'),
+          domainArrLen: domainArr.length
+        });
+      }
 
-        return domainArr;
+      return domainArr;
+    }
+  ));
+}
 
-        function calcDomainAbuseScore(subdomain: string, fullDomain: string = subdomain) {
-          if (leathalKeywords(fullDomain)) {
-            return 100;
-          }
+function calcDomainAbuseScore(subdomain: string, fullDomain: string = subdomain) {
+  if (leathalKeywords(fullDomain)) {
+    return 100;
+  }
 
-          let weight = 0;
+  let weight = 0;
 
-          const hitLowKeywords = lowKeywords(fullDomain);
-          const sensitiveKeywordsHit = sensitiveKeywords(fullDomain);
+  const hitLowKeywords = lowKeywords(fullDomain);
+  const sensitiveKeywordsHit = sensitiveKeywords(fullDomain);
 
-          if (sensitiveKeywordsHit) {
-            weight += 15;
-            if (hitLowKeywords) {
-              weight += 10;
-            }
-          } else if (hitLowKeywords) {
-            weight += 2;
-          }
+  if (sensitiveKeywordsHit) {
+    weight += 15;
+    if (hitLowKeywords) {
+      weight += 10;
+    }
+  } else if (hitLowKeywords) {
+    weight += 2;
+  }
 
-          const subdomainLength = subdomain.length;
-
-          if (subdomainLength > 6) {
-            weight += 0.015;
-
-            if (subdomainLength > 13) {
-              weight += 0.2;
-              if (subdomainLength > 20) {
-                weight += 1;
-                if (subdomainLength > 30) {
-                  weight += 5;
-                  if (subdomainLength > 40) {
-                    weight += 10;
-                  }
-                }
-              }
-
-              if (subdomain.indexOf('.', 1) > 1) {
-                weight += 1;
-              }
-            }
-          }
+  const subdomainLength = subdomain.length;
+
+  if (subdomainLength > 6) {
+    weight += 0.015;
 
-          return weight;
+    if (subdomainLength > 13) {
+      weight += 0.2;
+      if (subdomainLength > 20) {
+        weight += 1;
+        if (subdomainLength > 30) {
+          weight += 5;
+          if (subdomainLength > 40) {
+            weight += 10;
+          }
         }
       }
+
+      if (subdomain.indexOf('.', 1) > 1) {
+        weight += 1;
+      }
     }
   }
-});
 
-export function getPhishingDomains(parentSpan: Span) {
-  return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync(
-    'process phishing domain set',
-    () => pool.exec(
-      'getPhishingDomains',
-      [__filename, require.main === module]
-    ).finally(() => pool.terminate())
-  ));
+  return weight;
 }
 
-if (require.main === module) {
-  getPhishingDomains(dummySpan)
-    .catch(console.error)
-    .finally(() => {
-      dummySpan.stop();
-      printTraceResult(dummySpan.traceResult);
-    });
+if (!process.env.JEST_WORKER_ID && require.main === module) {
+  getPhishingDomains(true).catch(console.error);
 }

Build/lib/get-telegram-backup-ip.ts

@@ -7,10 +7,6 @@ import { bigint2ip } from 'fast-cidr-tools';
 
 import { base64ToUint8Array, concatUint8Arrays } from 'foxts/uint8array-utils';
 
-import Worktank from 'worktank';
-import { wait } from 'foxts/wait';
-import { once } from 'foxts/once';
-
 const mtptoto_public_rsa = `-----BEGIN RSA PUBLIC KEY-----
 MIIBCgKCAQEAyr+18Rex2ohtVy8sroGP
 BwXD3DOoKCSpjDqYoXgCqB7ioln4eDCFfOBUlfXUEvM/fnKCpF46VkAftlb4VuPD
@@ -112,160 +108,3 @@ export function getTelegramBackupIPFromBase64(base64: string) {
     }
   }));
 }
-
-const pool = new Worktank({
-  pool: {
-    name: 'get-telegram-backup-ips',
-    size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
-  },
-  worker: {
-    autoAbort: 10000,
-    autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
-    autoInstantiate: true,
-    methods: {
-      // eslint-disable-next-line object-shorthand -- workertank
-      getTelegramBackupIPs: async function (__filename: string): Promise<{ timestamp: number, ipcidr: string[], ipcidr6: string[] }> {
-        // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
-        const { default: module } = await import('node:module');
-        const __require = module.createRequire(__filename);
-
-        const picocolors = __require('picocolors') as typeof import('picocolors');
-        const { '~fetch': fetch } = __require('./fetch-retry') as typeof import('./fetch-retry');
-
-        const dns = __require('node:dns/promises') as typeof import('node:dns/promises');
-
-        const { createReadlineInterfaceFromResponse } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line');
-        const { getTelegramBackupIPFromBase64 } = __require('./get-telegram-backup-ip') as typeof import('./get-telegram-backup-ip');
-        const { fastIpVersion } = __require('foxts/fast-ip-version') as typeof import('foxts/fast-ip-version');
-        const { fastStringArrayJoin } = __require('foxts/fast-string-array-join') as typeof import('foxts/fast-string-array-join');
-
-        const resp = await fetch('https://core.telegram.org/resources/cidr.txt');
-        const lastModified = resp.headers.get('last-modified');
-        const date = lastModified ? new Date(lastModified) : new Date();
-
-        const ipcidr: string[] = [
-          // Unused secret Telegram backup CIDR, announced by AS62041
-          '95.161.64.0/20'
-        ];
-        const ipcidr6: string[] = [];
-
-        for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
-          const v = fastIpVersion(cidr);
-          if (v === 4) {
-            ipcidr.push(cidr);
-          } else if (v === 6) {
-            ipcidr6.push(cidr);
-          }
-        }
-
-        const backupIPs = new Set<string>();
-
-        // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
-
-        const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => {
-          const resolver = new dns.Resolver();
-          resolver.setServers([ip]);
-          return Object.assign(resolver, { server: ip });
-        });
-
-        // Backup IP Source 1 (DNS)
-        await Promise.all(resolvers.flatMap((resolver) => [
-          'apv3.stel.com', // prod
-          'tapv3.stel.com' // test
-        ].map(async (domain) => {
-          try {
-            // tapv3.stel.com was for testing server
-            const resp = await resolver.resolveTxt(domain);
-            const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten
-            if (strings.length !== 2) {
-              throw new TypeError(`Unexpected TXT record count: ${strings.length}`);
-            }
-
-            const str = strings[0].length > strings[1].length
-              ? strings[0] + strings[1]
-              : strings[1] + strings[0];
-
-            const ips = getTelegramBackupIPFromBase64(str);
-            ips.forEach(i => backupIPs.add(i.ip));
-
-            console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server });
-          } catch (e) {
-            console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e);
-          }
-        })));
-
-        // Backup IP Source 2: Firebase Realtime Database (test server not supported)
-        try {
-          const text = await (await fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
-          if (typeof text === 'string' && text.length === 344) {
-            const ips = getTelegramBackupIPFromBase64(text);
-            ips.forEach(i => backupIPs.add(i.ip));
-
-            console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
-          }
-        } catch (e) {
-          console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
-          // ignore all errors
-        }
-
-        // Backup IP Source 3: Firebase Value Store (test server not supported)
-        try {
-          const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
-            headers: {
-              Accept: '*/*',
-              Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
-            }
-          })).json();
-
-          // const json = await resp.json();
-          if (
-            json && typeof json === 'object'
-            && 'fields' in json && typeof json.fields === 'object' && json.fields
-            && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
-            && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
-          ) {
-            const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
-            ips.forEach(i => backupIPs.add(i.ip));
-
-            console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
-          } else {
-            console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
-          }
-        } catch (e) {
-          console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
-        }
-
-        // Backup IP Source 4: Google App Engine
-        await Promise.all([
-          'https://dns-telegram.appspot.com',
-          'https://dns-telegram.appspot.com/test'
-        ].map(async (url) => {
-          try {
-            const text = await (await fetch(url)).text();
-            if (text.length === 344) {
-              const ips = getTelegramBackupIPFromBase64(text);
-              ips.forEach(i => backupIPs.add(i.ip));
-
-              console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
-            }
-          } catch (e) {
-            console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
-          }
-        }));
-
-        // tcdnb.azureedge.net no longer works
-
-        console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
-
-        ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
-
-        return { timestamp: date.getTime(), ipcidr, ipcidr6 };
-      }
-    }
-  }
-});
-
-export const getTelegramCIDRPromise = once(() => wait(0).then(() => pool.exec(
-  'getTelegramBackupIPs',
-  [__filename]
-)).finally(() => pool.terminate()), false);

Build/lib/worker.ts

@@ -0,0 +1,28 @@
+import process from 'node:process';
+import type { JestWorkerFarm } from 'jest-worker';
+import { Worker as JestWorker } from 'jest-worker';
+
+const sharedWorkerOptions = {
+  numWorkers: 1,
+  enableWorkerThreads: true,
+  forkOptions: {
+    env: {
+      ...process.env,
+      NODE_OPTIONS: process.env.NODE_OPTIONS
+    }
+  }
+} satisfies ConstructorParameters<typeof JestWorker>[1];
+
+export function createWorker<T extends Record<string, unknown>>(workerPath: string) {
+  return <const K extends ReadonlyArray<keyof T & string>>(exposedMethods: K): JestWorkerFarm<Pick<T, K[number]>> => {
+    const worker = new JestWorker(workerPath, {
+      ...sharedWorkerOptions,
+      exposedMethods
+    }) as JestWorkerFarm<Pick<T, K[number]>>;
+
+    worker.getStdout().pipe(process.stdout);
+    worker.getStderr().pipe(process.stderr);
+
+    return worker;
+  };
+}

package.json

@@ -43,7 +43,6 @@
     "undici": "^7.24.5",
     "undici-cache-store-better-sqlite3": "^1.0.1",
     "why-is-node-running": "^3.2.2",
-    "worktank": "^3.0.2",
     "xbits": "^0.2.0",
     "yaml": "^2.8.3",
     "yauzl-promise": "^4.0.0"
@@ -62,6 +61,7 @@
     "eslint": "^10.1.0",
     "eslint-config-sukka": "^8.9.0",
     "eslint-formatter-sukka": "^8.9.0",
+    "jest-worker": "^30.3.0",
     "mitata": "^1.0.34",
     "mocha": "^11.7.5",
     "tinyexec": "^1.0.4",

pnpm-lock.yaml

@@ -86,9 +86,6 @@ importers:
       why-is-node-running:
         specifier: ^3.2.2
         version: 3.2.2
-      worktank:
-        specifier: ^3.0.2
-        version: 3.0.2
       xbits:
         specifier: ^0.2.0
         version: 0.2.0
@@ -138,6 +135,9 @@ importers:
       eslint-formatter-sukka:
         specifier: ^8.9.0
         version: 8.9.0(eslint@10.1.0)
+      jest-worker:
+        specifier: ^30.3.0
+        version: 30.3.0
       mitata:
         specifier: ^1.0.34
         version: 1.0.34
@@ -274,6 +274,18 @@ packages:
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
 
+  '@jest/pattern@30.0.1':
+    resolution: {integrity: sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA==}
+    engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0}
+
+  '@jest/schemas@30.0.5':
+    resolution: {integrity: sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==}
+    engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0}
+
+  '@jest/types@30.3.0':
+    resolution: {integrity: sha512-JHm87k7bA33hpBngtU8h6UBub/fqqA9uXfw+21j5Hmk7ooPHlboRNxHq0JcMtC+n8VJGP1mcfnD3Mk+XKe1oSw==}
+    engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0}
+
   '@mitata/counters@0.0.8':
     resolution: {integrity: sha512-f11w0Y1ETFlarDP7CePj8Z+y8Gv5Ax4gMxWsEwrqh0kH/YIY030Ezx5SUJeQg0YPTZ2OHKGcLG1oGJbIqHzaJA==}
 
@@ -529,6 +541,9 @@ packages:
   '@remusao/trie@2.1.0':
     resolution: {integrity: sha512-Er3Q8q0/2OcCJPQYJOPLmCuqO0wu7cav3SPtpjlxSbjFi1x+A1pZkkLD6c9q2rGEkGW/tkrRzfrhNMt8VQjzXg==}
 
+  '@sinclair/typebox@0.34.49':
+    resolution: {integrity: sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==}
+
   '@swc-node/core@1.14.1':
     resolution: {integrity: sha512-jrt5GUaZUU6cmMS+WTJEvGvaB6j1YNKPHPzC2PUi2BjaFbtxURHj6641Az6xN7b665hNniAIdvjxWcRml5yCnw==}
     engines: {node: '>= 10'}
@@ -639,6 +654,15 @@ packages:
   '@types/estree@1.0.8':
     resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
 
+  '@types/istanbul-lib-coverage@2.0.6':
+    resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==}
+
+  '@types/istanbul-lib-report@3.0.3':
+    resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==}
+
+  '@types/istanbul-reports@3.0.4':
+    resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==}
+
   '@types/json-schema@7.0.15':
     resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==}
 
@@ -654,6 +678,12 @@ packages:
   '@types/tar-stream@3.1.4':
     resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==}
 
+  '@types/yargs-parser@21.0.3':
+    resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==}
+
+  '@types/yargs@17.0.35':
+    resolution: {integrity: sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==}
+
   '@types/yauzl-promise@4.0.1':
     resolution: {integrity: sha512-qYEC3rJwqiJpdQ9b+bPNeuSY0c3JUM8vIuDy08qfuVN7xHm3ZDsHn2kGphUIB0ruEXrPGNXZ64nMUcu4fDjViQ==}
 
@@ -716,6 +746,9 @@ packages:
     resolution: {integrity: sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
 
+  '@ungap/structured-clone@1.3.0':
+    resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}
+
   '@unrs/resolver-binding-android-arm-eabi@1.11.1':
     resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==}
     cpu: [arm]
@@ -1404,9 +1437,6 @@ packages:
     resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==}
     engines: {node: '>= 4'}
 
-  immediato@1.1.0:
-    resolution: {integrity: sha512-6DTWQWiM3SyxAbNRDmMvFgZVwVP6wT8ciQv7GivxXejtXZFIcemC0Wlzfd/jEouJ2JroCIp4qZVloKW4BviUpQ==}
-
   imurmurhash@0.1.4:
     resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==}
     engines: {node: '>=0.8.19'}
@@ -1462,15 +1492,21 @@ packages:
   isexe@2.0.0:
     resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
 
-  isoconcurrency@1.0.0:
-    resolution: {integrity: sha512-YhuPf5V6uOtQQHt9gIkOTbq75ceXqraDvxtZZeS/XbNsre6fmM+WpJgNTSkGX5jB3+gnbwoTVqW1c3qdfyVpOA==}
-
-  isotimer@1.0.0:
-    resolution: {integrity: sha512-1p1wborMl9fFbulXx9YBpIqFnfUn/2tN8Ne9g3GLMaiQAPmN/wLlpNOKCNT822div3Sq7LKkApZJ+6JipDUusQ==}
-
   jackspeak@3.4.3:
     resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}
 
+  jest-regex-util@30.0.1:
+    resolution: {integrity: sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA==}
+    engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0}
+
+  jest-util@30.3.0:
+    resolution: {integrity: sha512-/jZDa00a3Sz7rdyu55NLrQCIrbyIkbBxareejQI315f/i8HjYN+ZWsDLLpoQSiUIEIyZF/R8fDg3BmB8AtHttg==}
+    engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0}
+
+  jest-worker@30.3.0:
+    resolution: {integrity: sha512-DrCKkaQwHexjRUFTmPzs7sHQe0TSj9nvDALKGdwmK5mW9v7j90BudWirKAJHt3QQ9Dhrg1F7DogPzhChppkJpQ==}
+    engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0}
+
   js-yaml@4.1.0:
     resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}
     hasBin: true
@@ -1513,6 +1549,9 @@ packages:
   lru-cache@10.4.3:
     resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
 
+  merge-stream@2.0.0:
+    resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
+
   mime@3.0.0:
     resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}
     engines: {node: '>=10.0.0'}
@@ -1650,9 +1689,6 @@ packages:
     resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
     engines: {node: '>= 0.8.0'}
 
-  promise-make-naked@3.0.2:
-    resolution: {integrity: sha512-B+b+kQ1YrYS7zO7P7bQcoqqMUizP06BOyNSBEnB5VJKDSWo8fsVuDkfSmwdjF0JsRtaNh83so5MMFJ95soH5jg==}
-
   pump@3.0.3:
     resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==}
 
@@ -1919,9 +1955,6 @@ packages:
     resolution: {integrity: sha512-/REy6amwPZl44DDzvRCkaI1q1bIiQB0mEFQLUrhz3z2EK91cp3n72rAjUlrTP0zV22HJIUOVHQGPxhFRjxjt+Q==}
     engines: {node: '>=4.0.0'}
 
-  webworker-shim@1.1.4:
-    resolution: {integrity: sha512-W/40L5W6ZQyGhYr3hJ7N/2SjdK5OdFtnYm94j6xlRyjckegXnIGwz0EdxdkQx6VGTglJjK8mqBhMz3fd3AY4bg==}
-
   which@2.0.2:
     resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
     engines: {node: '>= 8'}
@@ -1939,9 +1972,6 @@ packages:
   workerpool@9.3.3:
     resolution: {integrity: sha512-slxCaKbYjEdFT/o2rH9xS1hf4uRDch1w7Uo+apxhZ+sf/1d9e0ZVkn42kPNGP2dgjIx6YFvSevj0zHvbWe2jdw==}
 
-  worktank@3.0.2:
-    resolution: {integrity: sha512-ry5gPtWnakOnUBAAa2aiyWZwAFJuBtd/MwZH6o9DXnQHD4AZvidtl2uTLrb2d3Zjy9D04n84lHJNnIETQl7tuA==}
-
   wrap-ansi@7.0.0:
     resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
     engines: {node: '>=10'}
@@ -2136,6 +2166,25 @@ snapshots:
       wrap-ansi: 8.1.0
       wrap-ansi-cjs: wrap-ansi@7.0.0
 
+  '@jest/pattern@30.0.1':
+    dependencies:
+      '@types/node': 24.12.0
+      jest-regex-util: 30.0.1
+
+  '@jest/schemas@30.0.5':
+    dependencies:
+      '@sinclair/typebox': 0.34.49
+
+  '@jest/types@30.3.0':
+    dependencies:
+      '@jest/pattern': 30.0.1
+      '@jest/schemas': 30.0.5
+      '@types/istanbul-lib-coverage': 2.0.6
+      '@types/istanbul-reports': 3.0.4
+      '@types/node': 24.12.0
+      '@types/yargs': 17.0.35
+      chalk: 4.1.2
+
   '@mitata/counters@0.0.8': {}
 
   '@napi-rs/wasm-runtime@0.2.12':
@@ -2311,6 +2360,8 @@ snapshots:
 
   '@remusao/trie@2.1.0': {}
 
+  '@sinclair/typebox@0.34.49': {}
+
   '@swc-node/core@1.14.1(@swc/core@1.13.5)(@swc/types@0.1.25)':
     dependencies:
       '@swc/core': 1.13.5
@@ -2405,6 +2456,16 @@ snapshots:
 
   '@types/estree@1.0.8': {}
 
+  '@types/istanbul-lib-coverage@2.0.6': {}
+
+  '@types/istanbul-lib-report@3.0.3':
+    dependencies:
+      '@types/istanbul-lib-coverage': 2.0.6
+
+  '@types/istanbul-reports@3.0.4':
+    dependencies:
+      '@types/istanbul-lib-report': 3.0.3
+
   '@types/json-schema@7.0.15': {}
 
   '@types/mocha@10.0.10': {}
@@ -2422,6 +2483,12 @@ snapshots:
     dependencies:
       '@types/node': 24.12.0
 
+  '@types/yargs-parser@21.0.3': {}
+
+  '@types/yargs@17.0.35':
+    dependencies:
+      '@types/yargs-parser': 21.0.3
+
   '@types/yauzl-promise@4.0.1':
     dependencies:
       '@types/node': 24.12.0
@@ -2517,6 +2584,8 @@ snapshots:
       '@typescript-eslint/types': 8.57.2
       eslint-visitor-keys: 5.0.1
 
+  '@ungap/structured-clone@1.3.0': {}
+
   '@unrs/resolver-binding-android-arm-eabi@1.11.1':
     optional: true
 
@@ -3201,8 +3270,6 @@ snapshots:
 
   ignore@7.0.5: {}
 
-  immediato@1.1.0: {}
-
   imurmurhash@0.1.4: {}
 
   inherits@2.0.4: {}
@@ -3242,18 +3309,31 @@ snapshots:
 
   isexe@2.0.0: {}
 
-  isoconcurrency@1.0.0: {}
-
-  isotimer@1.0.0:
-    dependencies:
-      immediato: 1.1.0
-
   jackspeak@3.4.3:
     dependencies:
       '@isaacs/cliui': 8.0.2
     optionalDependencies:
       '@pkgjs/parseargs': 0.11.0
 
+  jest-regex-util@30.0.1: {}
+
+  jest-util@30.3.0:
+    dependencies:
+      '@jest/types': 30.3.0
+      '@types/node': 24.12.0
+      chalk: 4.1.2
+      ci-info: 4.4.0
+      graceful-fs: 4.2.11
+      picomatch: 4.0.3
+
+  jest-worker@30.3.0:
+    dependencies:
+      '@types/node': 24.12.0
+      '@ungap/structured-clone': 1.3.0
+      jest-util: 30.3.0
+      merge-stream: 2.0.0
+      supports-color: 8.1.1
+
   js-yaml@4.1.0:
     dependencies:
       argparse: 2.0.1
@@ -3294,6 +3374,8 @@ snapshots:
 
   lru-cache@10.4.3: {}
 
+  merge-stream@2.0.0: {}
+
   mime@3.0.0: {}
 
   mimic-response@3.1.0: {}
@@ -3449,8 +3531,6 @@ snapshots:
 
   prelude-ls@1.2.1: {}
 
-  promise-make-naked@3.0.2: {}
-
   pump@3.0.3:
     dependencies:
       end-of-stream: 1.4.5
@@ -3772,8 +3852,6 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  webworker-shim@1.1.4: {}
-
   which@2.0.2:
     dependencies:
       isexe: 2.0.0
@@ -3784,13 +3862,6 @@ snapshots:
 
   workerpool@9.3.3: {}
 
-  worktank@3.0.2:
-    dependencies:
-      isoconcurrency: 1.0.0
-      isotimer: 1.0.0
-      promise-make-naked: 3.0.2
-      webworker-shim: 1.1.4
-
   wrap-ansi@7.0.0:
     dependencies:
       ansi-styles: 4.3.0