Update build tools

Build/build-apple-cdn.js

@@ -1,26 +1,14 @@
 const path = require('path');
 
-const { isDomainLoose } = require('./lib/is-domain-loose');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
 const { withBannerArray } = require('./lib/with-banner');
 
-const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
+const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
 
 (async () => {
   console.time('Total Time - build-apple-cdn-conf');
 
-  const rl = await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
-
-  /** @type {string[]} */
-  const res = [];
-  for await (const line of rl) {
-    if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
-      const domain = line.replace('server=/', '').replace('/114.114.114.114', '');
-      if (isDomainLoose(domain)) {
-        res.push(domain);
-      }
-    }
-  }
+  const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
 
   await Promise.all([
     compareAndWriteFile(

Build/build-cdn-conf.js

@@ -6,32 +6,29 @@ const { minifyRules } = require('./lib/minify-rules');
 const { domainDeduper } = require('./lib/domain-deduper');
 const { processLine } = require('./lib/process-line');
 const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
+const Trie = require('./lib/trie');
 
 (async () => {
   console.time('Total Time - build-cdn-conf');
 
+  const trie = new Trie();
+  for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://publicsuffix.org/list/public_suffix_list.dat')) {
+    trie.add(line);
+  }
+
   /**
    * Extract OSS domain from publicsuffix list
    * @type {Set<string>}
    */
   const S3OSSDomains = new Set();
 
-  for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://publicsuffix.org/list/public_suffix_list.dat')) {
-    if (
-      line
-      && (
-        line.startsWith('s3-')
-        || line.startsWith('s3.')
-      )
-      && (
-        line.endsWith('.amazonaws.com')
-        || line.endsWith('.scw.cloud')
-      )
-      && !line.includes('cn-')
-    ) {
-      S3OSSDomains.add(line);
-    }
-  }
+  trie.find('.amazonaws.com')
+    .filter(line => (line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-'))
+    .forEach(line => S3OSSDomains.add(line));
+
+  trie.find('.scw.cloud')
+    .filter(line => (line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-'))
+    .forEach(line => S3OSSDomains.add(line));
 
   /** @type {string[]} */
   const cdnDomainsList = [];

Build/build-internal-chn-domains.js

@@ -1,25 +1,15 @@
 // @ts-check
-const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
-const { processLine } = require('./lib/process-line');
 const path = require('path');
 const fse = require('fs-extra');
 const fs = require('fs');
+const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
 
 (async () => {
-  /** @type {Set<string>} */
-  const result = new Set();
-  for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) {
-    const l = processLine(line);
-    if (l) {
-      result.add(
-        l.replace('server=/', '').replace('/114.114.114.114', '')
-      );
-    }
-  }
+  const result = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf');
 
   await fse.ensureDir(path.resolve(__dirname, '../List/internal'));
   await fs.promises.writeFile(
     path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt'),
-    `${Array.from(result).map(line => `SUFFIX,${line}`).join('\n')}\n`
+    `${result.map(line => `SUFFIX,${line}`).join('\n')}\n`
   );
 })();

Build/build-speedtest-domainset.js

@@ -57,7 +57,8 @@ const querySpeedtestApi = async (keyword) => {
     '.speedtest.gslnetworks.com',
     '.speedtest.jsinfo.net',
     '.speedtest.i3d.net',
-    '.speedtestkorea.com'
+    '.speedtestkorea.com',
+    '.speedtest.telus.com'
   ]);
 
   const hostnameGroups = await Promise.all([

Build/lib/parse-dnsmasq.js

@@ -0,0 +1,22 @@
+const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line');
+const { isDomainLoose } = require('./is-domain-loose');
+
+/**
+ * @param {string | URL} url
+ */
+const parseFelixDnsmasq = async (url) => {
+  /** @type {string[]} */
+  const res = [];
+  for await (const line of await fetchRemoteTextAndCreateReadlineInterface(url)) {
+    if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
+      const domain = line.replace('server=/', '').replace('/114.114.114.114', '');
+      if (isDomainLoose(domain)) {
+        res.push(domain);
+      }
+    }
+  }
+
+  return res;
+};
+
+module.exports.parseFelixDnsmasq = parseFelixDnsmasq;