Chore: housekeeping

Build/build-internal-cdn-rules.ts

@@ -10,69 +10,69 @@ import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
 
 const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&');
 
-export const buildInternalCDNDomains = task(import.meta.path, async () => {
-  const set = new Set<string>();
-  const keywords = new Set<string>();
-
-  const addApexDomain = (input: string) => {
-    // We are including the private domains themselves
-    const d = tldts.getDomain(input, { allowPrivateDomains: false });
-    if (d) {
-      set.add(d);
-    }
-  };
+const addApexDomain = (input: string, set: Set<string>) => {
+  // We are including the private domains themselves
+  const d = tldts.getDomain(input, { allowPrivateDomains: false });
+  if (d) {
+    set.add(d);
+  }
+};
 
-  const processLocalDomainSet = async (domainSetPath: string) => {
-    for await (const line of readFileByLine(domainSetPath)) {
-      // console.log({ line });
+const processLocalDomainSet = async (domainSetPath: string, set: Set<string>) => {
+  for await (const line of readFileByLine(domainSetPath)) {
+    // console.log({ line });
 
-      const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false });
-      if (parsed.isIp) continue;
-      if (parsed.isIcann || parsed.isPrivate) {
-        if (parsed.domain) {
-          set.add(parsed.domain);
-        }
-        continue;
+    const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false });
+    if (parsed.isIp) continue;
+    if (parsed.isIcann || parsed.isPrivate) {
+      if (parsed.domain) {
+        set.add(parsed.domain);
       }
+      continue;
+    }
 
-      if (processLine(line)) {
-        console.warn('[drop line from domainset]', line);
-      }
+    if (processLine(line)) {
+      console.warn('[drop line from domainset]', line);
     }
-  };
+  }
+};
 
-  const processLocalRuleSet = async (ruleSetPath: string) => {
-    for await (const line of readFileByLine(ruleSetPath)) {
-      if (line.startsWith('DOMAIN-SUFFIX,')) {
-        addApexDomain(line.replace('DOMAIN-SUFFIX,', ''));
-      } else if (line.startsWith('DOMAIN,')) {
-        addApexDomain(line.replace('DOMAIN,', ''));
-      } else if (line.startsWith('DOMAIN-KEYWORD')) {
-        keywords.add(escapeRegExp(line.replace('DOMAIN-KEYWORD,', '')));
-      } else if (line.startsWith('USER-AGENT,') || line.startsWith('PROCESS-NAME,') || line.startsWith('URL-REGEX,')) {
-        // do nothing
-      } else if (processLine(line)) {
-        console.warn('[drop line from ruleset]', line);
-      }
+const processLocalRuleSet = async (ruleSetPath: string, set: Set<string>, keywords: Set<string>) => {
+  for await (const line of readFileByLine(ruleSetPath)) {
+    if (line.startsWith('DOMAIN-SUFFIX,')) {
+      addApexDomain(line.replace('DOMAIN-SUFFIX,', ''), set);
+    } else if (line.startsWith('DOMAIN,')) {
+      addApexDomain(line.replace('DOMAIN,', ''), set);
+    } else if (line.startsWith('DOMAIN-KEYWORD')) {
+      keywords.add(escapeRegExp(line.replace('DOMAIN-KEYWORD,', '')));
+    } else if (line.startsWith('USER-AGENT,') || line.startsWith('PROCESS-NAME,') || line.startsWith('URL-REGEX,')) {
+      // do nothing
+    } else if (processLine(line)) {
+      console.warn('[drop line from ruleset]', line);
     }
-  };
+  }
+};
+
+export const buildInternalCDNDomains = task(import.meta.path, async () => {
+  const proxySet = new Set<string>();
+  const proxyKeywords = new Set<string>();
 
   const gorhill = (await Promise.all([
     getGorhillPublicSuffixPromise(),
-    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf')),
-    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf')),
-    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global_plus.conf')),
-    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf')),
-    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf')),
-    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf')),
-    processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf')),
-    processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf'))
+    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), proxySet, proxyKeywords),
+    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf'), proxySet, proxyKeywords),
+    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global_plus.conf'), proxySet, proxyKeywords),
+    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf'), proxySet, proxyKeywords),
+    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'), proxySet, proxyKeywords),
+    processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf'), proxySet, proxyKeywords),
+    processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf'), proxySet),
+    processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf'), proxySet)
   ]))[0];
 
   return compareAndWriteFile(
     [
-      ...sortDomains(Array.from(set), gorhill).map(i => `SUFFIX,${i}`),
-      ...Array.from(keywords).sort().map(i => `REGEX,${i}`)
+      ...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`),
+      ...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`)
     ],
     path.resolve(import.meta.dir, '../List/internal/cdn.txt')
   );

Build/build-speedtest-domainset.ts

@@ -24,6 +24,7 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
   try {
     const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
     const key = `fetch speedtest endpoints: ${keyword}`;
+    console.log(key);
     console.time(key);
 
     const res = await fetchWithRetry(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, {

Build/build-sspanel-appprofile.ts

@@ -1,5 +1,3 @@
-import fsp from 'fs/promises';
-
 import { getAppleCdnDomainsPromise } from './build-apple-cdn';
 import { getDomesticDomainsRulesetPromise } from './build-domestic-ruleset';
 import { surgeRulesetToClashClassicalTextRuleset } from './lib/clash';

Build/build-telegram-cidr.ts

@@ -10,7 +10,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
 
 export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
-  const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit) as Response;
+  const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit);
   const lastModified = resp.headers.get('last-modified');
   const date = lastModified ? new Date(lastModified) : new Date();
 

Build/download-mock-assets.ts

@@ -18,7 +18,7 @@ export const downloadMockAssets = task(import.meta.path, () => Promise.all(Objec
   const key = picocolors.gray(`Download ${filename}`);
   console.time(key);
   const res = await fetchWithRetry(url);
-  await Bun.write(targetPath, res as Response);
+  await Bun.write(targetPath, res);
   console.timeEnd(key);
 })));
 

Build/download-previous-build.ts

@@ -48,40 +48,37 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
 
   const filesList = buildOutputList.map(f => path.join('ruleset.skk.moe-master', f));
 
-  await traceAsync(
-    'Download and extract previous build',
-    async () => {
-      const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit);
+  const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit);
 
-      if (!resp.body) {
-        throw new Error('Download previous build failed! No body found');
-      }
+  if (!resp.body) {
+    throw new Error('Download previous build failed! No body found');
+  }
 
-      const extract = tarStream.extract();
-      Readable.fromWeb(resp.body).pipe(zlib.createGunzip()).pipe(extract);
-
-      for await (const entry of extract) {
-        if (entry.header.type !== 'file') {
-          entry.resume(); // Drain the entry
-          continue;
-        }
-        // filter entry
-        if (!filesList.some(f => entry.header.name.startsWith(f))) {
-          entry.resume(); // Drain the entry
-          continue;
-        }
-
-        const relativeEntryPath = entry.header.name.replace(`ruleset.skk.moe-master${path.sep}`, '');
-        const targetPath = path.join(import.meta.dir, '..', relativeEntryPath);
-
-        await fsp.mkdir(path.dirname(targetPath), { recursive: true });
-        await pipeline(
-          entry,
-          fs.createWriteStream(targetPath)
-        );
-      }
+  const extract = tarStream.extract();
+  Readable.fromWeb(resp.body as any).pipe(zlib.createGunzip()).pipe(extract);
+
+  const pathPrefix = `ruleset.skk.moe-master${path.sep}`;
+
+  for await (const entry of extract) {
+    if (entry.header.type !== 'file') {
+      entry.resume(); // Drain the entry
+      continue;
     }
-  );
+    // filter entry
+    if (!filesList.some(f => entry.header.name.startsWith(f))) {
+      entry.resume(); // Drain the entry
+      continue;
+    }
+
+    const relativeEntryPath = entry.header.name.replace(pathPrefix, '');
+    const targetPath = path.join(import.meta.dir, '..', relativeEntryPath);
+
+    await fsp.mkdir(path.dirname(targetPath), { recursive: true });
+    await pipeline(
+      entry,
+      fs.createWriteStream(targetPath)
+    );
+  }
 });
 
 if (import.meta.main) {

Build/lib/fetch-retry.ts

@@ -60,7 +60,8 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
         retries: MAX_RETRIES,
         factor: FACTOR,
         maxRetryAfter: MAX_RETRY_AFTER,
-        retryOnAborted: false
+        retryOnAborted: false,
+        retryOnNon2xx: true
       },
       opts.retry
     );
@@ -85,6 +86,9 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
             }
             throw new ResponseError(res);
           } else {
+            if (!res.ok && retryOpts.retryOnNon2xx) {
+              throw new ResponseError(res);
+            }
             return res;
           }
         } catch (err: unknown) {

Build/lib/fetch-text-by-line.ts

@@ -79,5 +79,5 @@ export async function *createReadlineInterfaceFromResponse(resp: Response): Asyn
 }
 
 export function fetchRemoteTextByLine(url: string | URL) {
-  return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res as Response));
+  return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res));
 }

Build/lib/get-phishing-domains.ts

@@ -1,5 +1,3 @@
-import fsp from 'fs/promises';
-import path from 'path';
 import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
 import { processDomainLists, processHosts } from './parse-filter';
 import { traceAsync, traceSync } from './trace-runner';