Grab public IPFS gateway list to CDN Hosts

Build/build-cdn-download-conf.ts

@@ -7,8 +7,10 @@ import { SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput } from './lib/rules/domainset';
 import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
 import Worktank from 'worktank';
+import { $$fetch } from './lib/fetch-retry';
+import { fastUri } from 'fast-uri';
 
-const cdnDomainsListPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf'));
+const cdnDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf'));
 const downloadDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf'));
 const steamDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf'));
 
@@ -75,6 +77,7 @@ const pool = new Worktank({
 export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => {
   const [
     S3OSSDomains,
+    IPFSDomains,
     cdnDomainsList,
     downloadDomainSet,
     steamDomainSet
@@ -86,7 +89,27 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
         [__filename]
       ).finally(() => pool.terminate())
     ),
-    cdnDomainsListPromise,
+    span.traceChildAsync(
+      'load public ipfs gateway list',
+      async () => {
+        const data = await (await $$fetch('https://cdn.jsdelivr.net/gh/ipfs/public-gateway-checker@refs/heads/main/gateways.json')).json();
+        if (!Array.isArray(data)) {
+          console.error('Invalid IPFS gateway list format');
+          return [];
+        }
+        return data.reduce<string[]>((acc, gateway) => {
+          if (typeof gateway !== 'string') {
+            return acc;
+          }
+          const hn = fastUri.parse(gateway).host;
+          if (hn) {
+            acc.push(hn.trim());
+          }
+          return acc;
+        }, []);
+      }
+    ),
+    cdnDomainSetPromise,
     downloadDomainSetPromise,
     steamDomainSetPromise
   ]);
@@ -107,6 +130,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
         'This file contains object storage and static assets CDN domains.'
       )
       .addFromDomainset(cdnDomainsList)
+      .bulkAddDomainSuffix(IPFSDomains)
       .write(),
 
     new DomainsetOutput(span, 'download')