Chore: use more make-fetch-happen

Build/build-apple-cdn.ts

@@ -2,19 +2,10 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
-import { deserializeArray, fsFetchCache, serializeArray, getFileContentHash } from './lib/cache-filesystem';
 import { DomainsetOutput } from './lib/create-file';
+import { $fetch } from './lib/make-fetch-happen';
 
-const url = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf';
-export const getAppleCdnDomainsPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304(
-  url,
-  getFileContentHash(__filename),
-  parseFelixDnsmasqFromResp,
-  {
-    serializer: serializeArray,
-    deserializer: deserializeArray
-  }
-));
+export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp));
 
 export const buildAppleCdn = task(require.main === module, __filename)(async (span) => {
   const res: string[] = await span.traceChildPromise('get apple cdn domains', getAppleCdnDomainsPromise());

Build/build-cdn-download-conf.ts

@@ -1,9 +1,8 @@
 import path from 'node:path';
-import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
-import { createTrie } from './lib/trie';
+import { fetchRemoteTextByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
+import { HostnameTrie } from './lib/trie';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
-import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 import { SOURCE_DIR } from './constants/dir';
 import { processLine } from './lib/process-line';
@@ -11,16 +10,14 @@ import { DomainsetOutput } from './lib/create-file';
 import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
 
 const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
-  const trie = createTrie((await getPublicSuffixListTextPromise()).reduce<string[]>(
-    (acc, cur) => {
-      const tmp = processLine(cur);
-      if (tmp) {
-        acc.push(tmp);
-      }
-      return acc;
-    },
-    []
-  ));
+  const trie = new HostnameTrie();
+
+  for await (const line of await fetchRemoteTextByLine('https://publicsuffix.org/list/public_suffix_list.dat')) {
+    const tmp = processLine(line);
+    if (tmp) {
+      trie.add(tmp);
+    }
+  }
 
   /**
    * Extract OSS domain from publicsuffix list
@@ -68,10 +65,11 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
     readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf'))
   ]);
 
+  // Move S3 domains to download domain set, since S3 files may be large
   appendArrayInPlace(downloadDomainSet, S3OSSDomains.map(domain => `.${domain}`));
   appendArrayInPlace(downloadDomainSet, steamDomainSet);
 
-  // we have whitelisted the crashlytics domain, but it doesn't mean we can't put it in CDN policy
+  // we have whitelisted the crashlytics domain, and we also want to put it in CDN policy
   appendArrayInPlace(cdnDomainsList, CRASHLYTICS_WHITELIST);
 
   return Promise.all([

Build/build-reject-ip-list.ts

@@ -8,33 +8,25 @@ import { fsFetchCache, getFileContentHash } from './lib/cache-filesystem';
 import { processLine } from './lib/process-line';
 import { RulesetOutput } from './lib/create-file';
 import { SOURCE_DIR } from './constants/dir';
+import { $fetch } from './lib/make-fetch-happen';
 
 const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf';
+const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => {
+  const ipv4: string[] = [];
+  const ipv6: string[] = [];
 
-const getBogusNxDomainIPsPromise = fsFetchCache.applyWithHttp304(
-  BOGUS_NXDOMAIN_URL,
-  getFileContentHash(__filename),
-  async (resp) => {
-    const ipv4: string[] = [];
-    const ipv6: string[] = [];
-
-    for await (const line of createReadlineInterfaceFromResponse(resp)) {
-      if (line.startsWith('bogus-nxdomain=')) {
-        const ip = line.slice(15).trim();
-        if (isProbablyIpv4(ip)) {
-          ipv4.push(ip);
-        } else if (isProbablyIpv6(ip)) {
-          ipv6.push(ip);
-        }
+  for await (const line of createReadlineInterfaceFromResponse(resp)) {
+    if (line.startsWith('bogus-nxdomain=')) {
+      const ip = line.slice(15).trim();
+      if (isProbablyIpv4(ip)) {
+        ipv4.push(ip);
+      } else if (isProbablyIpv6(ip)) {
+        ipv6.push(ip);
       }
     }
-    return [ipv4, ipv6] as const;
-  },
-  {
-    serializer: JSON.stringify,
-    deserializer: JSON.parse
   }
-);
+  return [ipv4, ipv6] as const;
+});
 
 const BOTNET_FILTER_URL = 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt';
 const BOTNET_FILTER_MIRROR_URL = [

Build/lib/cache-filesystem.ts

@@ -278,10 +278,6 @@ export class Cache<S = string> {
       return fn(await fetchAssetsWithout304(primaryUrl, mirrorUrls));
     }
 
-    if (mirrorUrls.length === 0) {
-      return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.body.text()), opt);
-    }
-
     const baseKey = primaryUrl + '$' + extraCacheKey;
     const getETagKey = (url: string) => baseKey + '$' + url + '$etag';
     const cachedKey = baseKey + '$cached';
@@ -346,10 +342,12 @@ export class Cache<S = string> {
     };
 
     try {
-      const text = await Promise.any([
-        createFetchFallbackPromise(primaryUrl, -1),
-        ...mirrorUrls.map(createFetchFallbackPromise)
-      ]);
+      const text = mirrorUrls.length === 0
+        ? await createFetchFallbackPromise(primaryUrl, -1)
+        : await Promise.any([
+          createFetchFallbackPromise(primaryUrl, -1),
+          ...mirrorUrls.map(createFetchFallbackPromise)
+        ]);
 
       console.log(picocolors.yellow('[cache] miss'), primaryUrl);
       const serializer = 'serializer' in opt ? opt.serializer : identity as any;

Build/lib/download-publicsuffixlist.ts

@@ -1,15 +0,0 @@
-import { deserializeArray, fsFetchCache, getFileContentHash, serializeArray } from './cache-filesystem';
-import { createMemoizedPromise } from './memo-promise';
-
-export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304<string[]>(
-  'https://publicsuffix.org/list/public_suffix_list.dat',
-  getFileContentHash(__filename),
-  (r) => r.body.text().then(text => text.split('\n')),
-  {
-    // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml
-    // Though the action runs every 24 hours, the IANA list is updated every 7 days.
-    // So a 3 day TTL should be enough.
-    serializer: serializeArray,
-    deserializer: deserializeArray
-  }
-));

Build/lib/fetch-retry.ts

@@ -7,7 +7,6 @@ import undici, {
 
 import type {
   Dispatcher,
-  RequestInit,
   Response
 } from 'undici';
 
@@ -143,33 +142,33 @@ export const defaultRequestInit = {
   }
 };
 
-export async function fetchWithLog(url: string, init?: RequestInit) {
-  try {
-    const res = await undici.fetch(url, init);
-    if (res.status >= 400) {
-      throw new ResponseError(res, url);
-    }
-
-    if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) {
-      throw new ResponseError(res, url);
-    }
-
-    return res;
-  } catch (err: unknown) {
-    if (typeof err === 'object' && err !== null && 'name' in err) {
-      if ((
-        err.name === 'AbortError'
-        || ('digest' in err && err.digest === 'AbortError')
-      )) {
-        console.log(picocolors.gray('[fetch abort]'), url);
-      }
-    } else {
-      console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err);
-    }
-
-    throw err;
-  }
-}
+// export async function fetchWithLog(url: string, init?: RequestInit) {
+//   try {
+//     const res = await undici.fetch(url, init);
+//     if (res.status >= 400) {
+//       throw new ResponseError(res, url);
+//     }
+
+//     if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) {
+//       throw new ResponseError(res, url);
+//     }
+
+//     return res;
+//   } catch (err: unknown) {
+//     if (typeof err === 'object' && err !== null && 'name' in err) {
+//       if ((
+//         err.name === 'AbortError'
+//         || ('digest' in err && err.digest === 'AbortError')
+//       )) {
+//         console.log(picocolors.gray('[fetch abort]'), url);
+//       }
+//     } else {
+//       console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err);
+//     }
+
+//     throw err;
+//   }
+// }
 
 export async function requestWithLog(url: string, opt?: Parameters<typeof undici.request>[1]) {
   try {