Perf: offload Telegram CIDR fetching into separate thread

Build/build-cdn-download-conf.ts

@@ -79,10 +79,13 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
     downloadDomainSet,
     steamDomainSet
   ] = await Promise.all([
-    span.traceChildAsync('download public suffix list for s3', () => pool.exec(
-      'getS3OSSDomains',
-      [__filename]
-    ).finally(() => pool.terminate())),
+    span.traceChildAsync(
+      'download public suffix list for s3',
+      () => pool.exec(
+        'getS3OSSDomains',
+        [__filename]
+      ).finally(() => pool.terminate())
+    ),
     cdnDomainsListPromise,
     downloadDomainSetPromise,
     steamDomainSetPromise

Build/build-microsoft-cdn.ts

@@ -3,6 +3,7 @@ import { SHARED_DESCRIPTION } from './constants/description';
 import { RulesetOutput } from './lib/rules/ruleset';
 import Worktank from 'worktank';
 import { RULES } from './constants/microsoft-cdn';
+import { wait } from 'foxts/wait';
 
 const pool = new Worktank({
   pool: {
@@ -47,10 +48,10 @@ const pool = new Worktank({
   }
 });
 
-export const getMicrosoftCdnRulesetPromise = pool.exec(
+export const getMicrosoftCdnRulesetPromise = wait(0).then(() => pool.exec(
   'getMicrosoftCdnRuleset',
   [__filename]
-).finally(() => pool.terminate());
+)).finally(() => pool.terminate());
 
 export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => {
   const [domains, domainSuffixes] = await span.traceChildPromise('get microsoft cdn domains', getMicrosoftCdnRulesetPromise);

Build/build-sspanel-appprofile.ts

@@ -6,7 +6,7 @@ import path from 'node:path';
 
 import { ALL as AllStreamServices } from '../Source/stream';
 import { getChnCidrPromise } from './build-chn-cidr';
-import { getTelegramCIDRPromise } from './build-telegram-cidr';
+import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip';
 import { compareAndWriteFile } from './lib/create-file';
 import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn';
 import { isTruthy, nullthrow } from 'foxts/guard';
@@ -75,7 +75,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam
     // domestic - ip cidr
     getChnCidrPromise(),
     // global - ip cidr
-    getTelegramCIDRPromise(),
+    getTelegramCIDRPromise,
     // lan - ip cidr
     readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'ip/lan.conf'))
   ] as const);

Build/build-telegram-cidr.ts

@@ -1,137 +1,11 @@
 // @ts-check
-import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
-import { once } from 'foxts/once';
 import { RulesetOutput } from './lib/rules/ruleset';
-import { $$fetch } from './lib/fetch-retry';
-import { fastIpVersion } from 'foxts/fast-ip-version';
-import DNS2 from 'dns2';
-import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip';
-import picocolors from 'picocolors';
-
-export const getTelegramCIDRPromise = once(async () => {
-  const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
-  const lastModified = resp.headers.get('last-modified');
-  const date = lastModified ? new Date(lastModified) : new Date();
-
-  const ipcidr: string[] = [
-    // Unused secret Telegram backup CIDR, announced by AS62041
-    '95.161.64.0/20'
-  ];
-  const ipcidr6: string[] = [];
-
-  for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
-    const v = fastIpVersion(cidr);
-    if (v === 4) {
-      ipcidr.push(cidr);
-    } else if (v === 6) {
-      ipcidr6.push(cidr);
-    }
-  }
-
-  const backupIPs = new Set<string>();
-
-  // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
-
-  // Backup IP Source 1 (DoH)
-  await Promise.all([
-    DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }),
-    DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' })
-  ].flatMap(
-    (client) => [
-      'apv3.stel.com', // prod
-      'tapv3.stel.com' // test
-    ].map(async (domain) => {
-      try {
-        // tapv3.stel.com was for testing server
-        const resp = await client(domain, 'TXT');
-        const strings = resp.answers.map(i => i.data);
-
-        const str = strings[0]!.length > strings[1]!.length
-          ? strings[0]! + strings[1]!
-          : strings[1]! + strings[0]!;
-
-        const ips = getTelegramBackupIPFromBase64(str);
-        ips.forEach(i => backupIPs.add(i.ip));
-
-        console.log('[telegram backup ip]', picocolors.green('DoH TXT'), { domain, ips });
-      } catch (e) {
-        console.error('[telegram backup ip]', picocolors.red('DoH TXT error'), { domain }, e);
-      }
-    })
-  ));
-
-  // Backup IP Source 2: Firebase Realtime Database (test server not supported)
-  try {
-    const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
-    if (typeof text === 'string' && text.length === 344) {
-      const ips = getTelegramBackupIPFromBase64(text);
-      ips.forEach(i => backupIPs.add(i.ip));
-
-      console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
-    }
-  } catch (e) {
-    console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
-    // ignore all errors
-  }
-
-  // Backup IP Source 3: Firebase Value Store (test server not supported)
-  try {
-    const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
-      headers: {
-        Accept: '*/*',
-        Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
-      }
-    })).json();
-
-    // const json = await resp.json();
-    if (
-      json && typeof json === 'object'
-      && 'fields' in json && typeof json.fields === 'object' && json.fields
-      && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
-      && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
-    ) {
-      const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
-      ips.forEach(i => backupIPs.add(i.ip));
-
-      console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
-    } else {
-      console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
-    }
-  } catch (e) {
-    console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
-  }
-
-  // Backup IP Source 4: Google App Engine
-  await Promise.all([
-    'https://dns-telegram.appspot.com',
-    'https://dns-telegram.appspot.com/test'
-  ].map(async (url) => {
-    try {
-      const text = await (await $$fetch(url)).text();
-      if (text.length === 344) {
-        const ips = getTelegramBackupIPFromBase64(text);
-        ips.forEach(i => backupIPs.add(i.ip));
-
-        console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
-      }
-    } catch (e) {
-      console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
-    }
-  }));
-
-  // tcdnb.azureedge.net no longer works
-
-  console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
-
-  ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
-
-  return { date, ipcidr, ipcidr6 };
-});
+import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip';
 
 export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
-  const { date, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise);
+  const { timestamp, ipcidr, ipcidr6 } = await span.traceChildPromise('get telegram cidr', getTelegramCIDRPromise);
 
   if (ipcidr.length + ipcidr6.length === 0) {
     throw new Error('Failed to fetch data!');
@@ -148,7 +22,7 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async
     .withDescription(description)
     // .withDate(date) // With extra data source, we no longer use last-modified for file date
     .appendDataSource(
-      'https://core.telegram.org/resources/cidr.txt (last updated: ' + date.toISOString() + ')'
+      'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')'
     )
     .bulkAddCIDR4NoResolve(ipcidr)
     .bulkAddCIDR6NoResolve(ipcidr6)

Build/lib/fetch-retry.ts

@@ -177,6 +177,8 @@ export async function $$fetch(url: string, init: RequestInit = defaultRequestIni
   }
 }
 
+export const fetch = $$fetch;
+
 /** @deprecated -- undici.requests doesn't support gzip/br/deflate, and has difficulty w/ undidi cache */
 export async function requestWithLog(url: string, opt?: Parameters<typeof undici.request>[1]) {
   try {

Build/lib/get-phishing-domains.ts

@@ -202,17 +202,10 @@ const pool = new Worktank({
 export function getPhishingDomains(parentSpan: Span) {
   return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync(
     'process phishing domain set',
-    async () => {
-      const phishingDomains = await pool.exec(
-        'getPhishingDomains',
-        [
-          __filename,
-          require.main === module
-        ]
-      );
-      pool.terminate();
-      return phishingDomains;
-    }
+    () => pool.exec(
+      'getPhishingDomains',
+      [__filename, require.main === module]
+    ).finally(() => pool.terminate())
   ));
 }
 

Build/lib/get-telegram-backup-ip.ts

@@ -7,6 +7,9 @@ import { bigint2ip } from 'fast-cidr-tools';
 
 import { base64ToUint8Array, concatUint8Arrays } from 'foxts/uint8array-utils';
 
+import Worktank from 'worktank';
+import { wait } from 'foxts/wait';
+
 const mtptoto_public_rsa = `-----BEGIN RSA PUBLIC KEY-----
 MIIBCgKCAQEAyr+18Rex2ohtVy8sroGP
 BwXD3DOoKCSpjDqYoXgCqB7ioln4eDCFfOBUlfXUEvM/fnKCpF46VkAftlb4VuPD
@@ -108,3 +111,153 @@ export function getTelegramBackupIPFromBase64(base64: string) {
     }
   }));
 }
+
+const pool = new Worktank({
+  pool: {
+    name: 'get-telegram-backup-ips',
+    size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
+  },
+  worker: {
+    autoAbort: 10000,
+    autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
+    autoInstantiate: true,
+    methods: {
+      // eslint-disable-next-line object-shorthand -- workertank
+      getTelegramBackupIPs: async function (__filename: string): Promise<{ timestamp: number, ipcidr: string[], ipcidr6: string[] }> {
+        // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
+        const { default: module } = await import('node:module');
+        const __require = module.createRequire(__filename);
+
+        const picocolors = __require('picocolors') as typeof import('picocolors');
+        const { fetch } = __require('./fetch-retry') as typeof import('./fetch-retry');
+        const DNS2 = __require('dns2') as typeof import('dns2');
+        const { createReadlineInterfaceFromResponse } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line');
+        const { getTelegramBackupIPFromBase64 } = __require('./get-telegram-backup-ip') as typeof import('./get-telegram-backup-ip');
+        const { fastIpVersion } = __require('foxts/fast-ip-version') as typeof import('foxts/fast-ip-version');
+
+        const resp = await fetch('https://core.telegram.org/resources/cidr.txt');
+        const lastModified = resp.headers.get('last-modified');
+        const date = lastModified ? new Date(lastModified) : new Date();
+
+        const ipcidr: string[] = [
+          // Unused secret Telegram backup CIDR, announced by AS62041
+          '95.161.64.0/20'
+        ];
+        const ipcidr6: string[] = [];
+
+        for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
+          const v = fastIpVersion(cidr);
+          if (v === 4) {
+            ipcidr.push(cidr);
+          } else if (v === 6) {
+            ipcidr6.push(cidr);
+          }
+        }
+
+        const backupIPs = new Set<string>();
+
+        // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
+
+        // Backup IP Source 1 (DoH)
+        await Promise.all([
+          DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }),
+          DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' })
+        ].flatMap(
+          (client) => [
+            'apv3.stel.com', // prod
+            'tapv3.stel.com' // test
+          ].map(async (domain) => {
+            try {
+              // tapv3.stel.com was for testing server
+              const resp = await client(domain, 'TXT');
+              const strings = resp.answers.map(i => i.data);
+
+              const str = strings[0]!.length > strings[1]!.length
+                ? strings[0]! + strings[1]!
+                : strings[1]! + strings[0]!;
+
+              const ips = getTelegramBackupIPFromBase64(str);
+              ips.forEach(i => backupIPs.add(i.ip));
+
+              console.log('[telegram backup ip]', picocolors.green('DoH TXT'), { domain, ips });
+            } catch (e) {
+              console.error('[telegram backup ip]', picocolors.red('DoH TXT error'), { domain }, e);
+            }
+          })
+        ));
+
+        // Backup IP Source 2: Firebase Realtime Database (test server not supported)
+        try {
+          const text = await (await fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
+          if (typeof text === 'string' && text.length === 344) {
+            const ips = getTelegramBackupIPFromBase64(text);
+            ips.forEach(i => backupIPs.add(i.ip));
+
+            console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
+          }
+        } catch (e) {
+          console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
+          // ignore all errors
+        }
+
+        // Backup IP Source 3: Firebase Value Store (test server not supported)
+        try {
+          const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
+            headers: {
+              Accept: '*/*',
+              Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
+            }
+          })).json();
+
+          // const json = await resp.json();
+          if (
+            json && typeof json === 'object'
+            && 'fields' in json && typeof json.fields === 'object' && json.fields
+            && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
+            && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
+          ) {
+            const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
+            ips.forEach(i => backupIPs.add(i.ip));
+
+            console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
+          } else {
+            console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
+          }
+        } catch (e) {
+          console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
+        }
+
+        // Backup IP Source 4: Google App Engine
+        await Promise.all([
+          'https://dns-telegram.appspot.com',
+          'https://dns-telegram.appspot.com/test'
+        ].map(async (url) => {
+          try {
+            const text = await (await fetch(url)).text();
+            if (text.length === 344) {
+              const ips = getTelegramBackupIPFromBase64(text);
+              ips.forEach(i => backupIPs.add(i.ip));
+
+              console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
+            }
+          } catch (e) {
+            console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
+          }
+        }));
+
+        // tcdnb.azureedge.net no longer works
+
+        console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
+
+        ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
+
+        return { timestamp: date.getTime(), ipcidr, ipcidr6 };
+      }
+    }
+  }
+});
+
+export const getTelegramCIDRPromise = wait(0).then(() => pool.exec(
+  'getTelegramBackupIPs',
+  [__filename]
+)).finally(() => pool.terminate());