Refactor/Chore: prepare build infra for further optimization

Build/build-anti-bogus-domain.js

@@ -6,7 +6,7 @@ const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('.
 const { processLine } = require('./lib/process-line');
 const { runner } = require('./lib/trace-runner');
 
-runner(__filename, async () => {
+const buildAntiBogusDomain = async () => {
   /** @type {string[]} */
   const res = [];
   for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) {
@@ -56,4 +56,10 @@ runner(__filename, async () => {
     path.resolve(__dirname, '../List/ip/reject.conf'),
     path.resolve(__dirname, '../Clash/ip/reject.txt')
   ));
-});
+};
+
+module.exports.buildAntiBogusDomain = buildAntiBogusDomain;
+
+if (require.main === module) {
+  runner(__filename, buildAntiBogusDomain);
+}

Build/build-apple-cdn.js

@@ -3,7 +3,7 @@ const { createRuleset } = require('./lib/create-file');
 const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
 const { runner } = require('./lib/trace-runner');
 
-runner(__filename, async () => {
+const buildAppleCdn = async () => {
   const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
 
   const description = [
@@ -40,4 +40,10 @@ runner(__filename, async () => {
       path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt')
     )
   ]);
-});
+};
+
+module.exports.buildAppleCdn = buildAppleCdn;
+
+if (require.main === module) {
+  runner(__filename, buildAppleCdn);
+}

Build/build-cdn-conf.js

@@ -1,15 +1,15 @@
 // @ts-check
 const path = require('path');
 const { createRuleset } = require('./lib/create-file');
-const { minifyRules } = require('./lib/minify-rules');
 const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const Trie = require('./lib/trie');
 const { runner } = require('./lib/trace-runner');
 const fs = require('fs');
+const { processLine } = require('./lib/process-line');
 
 const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix-list_dat.txt');
 
-runner(__filename, async () => {
+const buildCdnConf = async () => {
   const trie = new Trie();
 
   if (fs.existsSync(publicSuffixPath)) {
@@ -42,10 +42,11 @@ runner(__filename, async () => {
 
   /** @type {string[]} */
   const cdnDomainsList = [];
-  for await (const line of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) {
+  for await (const l of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) {
+    const line = processLine(l);
     if (line === '# --- [AWS S3 Replace Me] ---') {
       S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`));
-    } else {
+    } else if (line) {
       cdnDomainsList.push(line);
     }
   }
@@ -57,15 +58,20 @@ runner(__filename, async () => {
     '',
     'This file contains object storage and static assets CDN domains.'
   ];
-  const ruleset = minifyRules(cdnDomainsList);
 
   return Promise.all(createRuleset(
     'Sukka\'s Ruleset - CDN Domains',
     description,
     new Date(),
-    ruleset,
+    cdnDomainsList,
     'ruleset',
     path.resolve(__dirname, '../List/non_ip/cdn.conf'),
     path.resolve(__dirname, '../Clash/non_ip/cdn.txt')
   ));
-});
+};
+
+module.exports.buildCdnConf = buildCdnConf;
+
+if (require.main === module) {
+  runner(__filename, buildCdnConf);
+}

Build/build-chn-cidr.js

@@ -12,7 +12,7 @@ const EXCLUDE_CIDRS = [
   '223.120.0.0/15'
 ];
 
-runner(__filename, async () => {
+const buildChnCidr = async () => {
   const { exclude: excludeCidrs } = await import('cidr-tools-wasm');
 
   /** @type {string[]} */
@@ -56,4 +56,10 @@ runner(__filename, async () => {
       pathResolve(__dirname, '../Clash/ip/china_ip.txt')
     )
   ]);
-});
+};
+
+module.exports.buildChnCidr = buildChnCidr;
+
+if (require.main === module) {
+  runner(__filename, buildChnCidr);
+}

Build/build-common.js

@@ -0,0 +1,152 @@
+// @ts-check
+
+const path = require('path');
+const { PathScurry } = require('path-scurry');
+const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
+const { processLine } = require('./lib/process-line');
+const { createRuleset } = require('./lib/create-file');
+const { domainDeduper } = require('./lib/domain-deduper');
+const { runner } = require('./lib/trace-runner');
+
+const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
+const MAGIC_COMMAND_TITLE = '# $ meta_title ';
+const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
+
+const sourceDir = path.resolve(__dirname, '../Source');
+const outputSurgeDir = path.resolve(__dirname, '../List');
+const outputClashDir = path.resolve(__dirname, '../Clash');
+
+const buildCommon = async () => {
+  /** @type {Promise<void>[]} */
+  const promises = [];
+
+  const pw = new PathScurry(sourceDir);
+  for await (const entry of pw) {
+    if (entry.isFile()) {
+      if (path.extname(entry.name) === '.js') {
+        continue;
+      }
+
+      const relativePath = entry.relative();
+      if (relativePath.startsWith('domainset/')) {
+        promises.push(transformDomainset(entry.fullpath(), relativePath));
+        continue;
+      }
+      if (
+        relativePath.startsWith('ip/')
+        || relativePath.startsWith('non_ip/')
+      ) {
+        promises.push(transformRuleset(entry.fullpath(), relativePath));
+        continue;
+      }
+    }
+  }
+
+  return Promise.all(promises);
+};
+
+module.exports.buildCommon = buildCommon;
+
+if (require.main === module) {
+  runner(__filename, buildCommon);
+}
+
+/**
+ * @param {string} sourcePath
+ */
+const processFile = async (sourcePath) => {
+  /** @type {string[]} */
+  const lines = [];
+
+  let title = '';
+  /** @type {string[]} */
+  const descriptions = [];
+
+  for await (const line of readFileByLine(sourcePath)) {
+    if (line === MAGIC_COMMAND_SKIP) {
+      return;
+    }
+
+    if (line.startsWith(MAGIC_COMMAND_TITLE)) {
+      title = line.slice(MAGIC_COMMAND_TITLE.length).trim();
+      continue;
+    }
+
+    if (line.startsWith(MAGIC_COMMAND_DESCRIPTION)) {
+      descriptions.push(line.slice(MAGIC_COMMAND_DESCRIPTION.length).trim());
+      continue;
+    }
+
+    const l = processLine(line);
+    if (l) {
+      lines.push(l);
+    }
+  }
+
+  return /** @type {const} */ ([title, descriptions, lines]);
+};
+
+/**
+ * @param {string} sourcePath
+ * @param {string} relativePath
+ */
+async function transformDomainset(sourcePath, relativePath) {
+  const res = await processFile(sourcePath);
+  if (!res) return;
+  const [title, descriptions, lines] = res;
+
+  const deduped = domainDeduper(lines);
+  const description = [
+    'License: AGPL 3.0',
+    'Homepage: https://ruleset.skk.moe',
+    'GitHub: https://github.com/SukkaW/Surge',
+    ...(
+      descriptions.length
+        ? ['', ...descriptions]
+        : []
+    )
+  ];
+
+  await Promise.all(createRuleset(
+    title,
+    description,
+    new Date(),
+    deduped,
+    'domainset',
+    path.resolve(outputSurgeDir, relativePath),
+    path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
+  ));
+}
+
+/**
+ * Output Surge RULE-SET and Clash classical text format
+ *
+ * @param {string} sourcePath
+ * @param {string} relativePath
+ */
+async function transformRuleset(sourcePath, relativePath) {
+  const res = await processFile(sourcePath);
+  if (!res) return;
+  const [title, descriptions, lines] = res;
+
+  const description = [
+    'License: AGPL 3.0',
+    'Homepage: https://ruleset.skk.moe',
+    'GitHub: https://github.com/SukkaW/Surge',
+    ...(
+      descriptions.length
+        ? ['', ...descriptions]
+        : []
+    )
+  ];
+
+  await Promise.all(createRuleset(
+    title,
+    description,
+    new Date(),
+    lines,
+    'ruleset',
+    path.resolve(outputSurgeDir, relativePath),
+    path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
+  ));
+}

Build/build-domestic-ruleset.js

@@ -7,7 +7,7 @@ const { compareAndWriteFile, createRuleset } = require('./lib/create-file');
 const domainSorter = require('./lib/stable-sort-domain');
 const { runner } = require('./lib/trace-runner');
 
-runner(__filename, async () => {
+const buildDomesticRuleset = async () => {
   const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
   const results = [];
   for await (const l of rl) {
@@ -67,4 +67,10 @@ runner(__filename, async () => {
       path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule')
     )
   ]);
-});
+};
+
+module.exports.buildDomesticRuleset = buildDomesticRuleset;
+
+if (require.main === module) {
+  runner(__filename, buildDomesticRuleset);
+}

Build/build-internal-cdn-rules.js

@@ -15,7 +15,7 @@ const escapeRegExp = (string) => {
   return string.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
 };
 
-runner(__filename, async () => {
+const buildInternalCDNDomains = async () => {
   const set = new Set();
   const keywords = new Set();
 
@@ -87,4 +87,10 @@ runner(__filename, async () => {
     ],
     path.resolve(__dirname, '../List/internal/cdn.txt')
   );
-});
+};
+
+module.exports.buildInternalCDNDomains = buildInternalCDNDomains;
+
+if (require.main === module) {
+  runner(__filename, buildInternalCDNDomains);
+}

Build/build-internal-chn-domains.js

@@ -5,7 +5,7 @@ const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
 const { runner } = require('./lib/trace-runner');
 const { compareAndWriteFile } = require('./lib/create-file');
 
-runner(__filename, async () => {
+const buildInternalChnDomains = async () => {
   const [result] = await Promise.all([
     parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
     fse.ensureDir(path.resolve(__dirname, '../List/internal'))
@@ -15,4 +15,10 @@ runner(__filename, async () => {
     result.map(line => `SUFFIX,${line}`),
     path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt')
   );
-});
+};
+
+module.exports.buildInternalChnDomains = buildInternalChnDomains;
+
+if (require.main === module) {
+  runner(__filename, buildInternalChnDomains);
+}

Build/build-internal-reverse-chn-cidr.js

@@ -24,7 +24,7 @@ const RESERVED_IPV4_CIDR = [
   '240.0.0.0/4'
 ];
 
-runner(__filename, async () => {
+const buildInternalReverseChnCIDR = async () => {
   const { exclude } = await import('cidr-tools-wasm');
 
   /** @type {string[]} */
@@ -47,4 +47,10 @@ runner(__filename, async () => {
     path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'),
     `${reversedCidr.join('\n')}\n`
   );
-});
+};
+
+module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR;
+
+if (require.main === module) {
+  runner(__filename, buildInternalReverseChnCIDR);
+}

Build/build-phishing-domainset.js

@@ -60,7 +60,7 @@ const BLACK_TLD = new Set([
   'com.cn'
 ]);
 
-runner(__filename, async () => {
+const buildPhishingDomainSet = async () => {
   const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
   const domainCountMap = {};
 
@@ -151,4 +151,10 @@ runner(__filename, async () => {
     path.resolve(__dirname, '../List/domainset/reject_phishing.conf'),
     path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt')
   ));
-});
+};
+
+module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
+
+if (require.main === module) {
+  runner(__filename, buildPhishingDomainSet);
+}

Build/build-public.js

@@ -2,6 +2,7 @@ const listDir = require('@sukka/listdir');
 const path = require('path');
 const fs = require('fs');
 const fse = require('fs-extra');
+const { runner } = require('./lib/trace-runner');
 
 const rootPath = path.resolve(__dirname, '../');
 const publicPath = path.resolve(__dirname, '../public');
@@ -16,7 +17,7 @@ const folderAndFilesToBeDeployed = [
   'README.md'
 ];
 
-(async () => {
+const buildPublicHtml = async () => {
   await fse.ensureDir(publicPath);
   await Promise.all(folderAndFilesToBeDeployed.map(dir => fse.copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir))));
 
@@ -28,7 +29,13 @@ const folderAndFilesToBeDeployed = [
   const html = template(list);
 
   await fs.promises.writeFile(path.join(publicPath, 'index.html'), html, 'utf-8');
-})();
+};
+
+module.exports.buildPublicHtml = buildPublicHtml;
+
+if (require.main === module) {
+  runner(__filename, buildPublicHtml);
+}
 
 /**
  * @param {string[]} urlList

Build/build-reject-domainset.js

@@ -23,7 +23,7 @@ const domainKeywordsSet = new Set();
 /** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
 const domainSuffixSet = new Set();
 
-runner(__filename, async () => {
+const buildRejectDomainSet = async () => {
   /** @type Set<string> */
   const domainSets = new Set();
 
@@ -228,4 +228,10 @@ runner(__filename, async () => {
     // Copy reject_sukka.conf for backward compatibility
     fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
   ]);
-});
+};
+
+module.exports.buildRejectDomainSet = buildRejectDomainSet;
+
+if (require.main === module) {
+  runner(__filename, buildRejectDomainSet);
+}

Build/build-speedtest-domainset.js

@@ -46,7 +46,7 @@ const querySpeedtestApi = async (keyword) => {
   }
 };
 
-runner(__filename, async () => {
+const buildSpeedtestDomainSet = async () => {
   /** @type {Set<string>} */
   const domains = new Set([
     '.speedtest.net',
@@ -123,4 +123,10 @@ runner(__filename, async () => {
     path.resolve(__dirname, '../List/domainset/speedtest.conf'),
     path.resolve(__dirname, '../Clash/domainset/speedtest.txt')
   ));
-});
+};
+
+module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet;
+
+if (require.main === module) {
+  runner(__filename, buildSpeedtestDomainSet);
+}

Build/build-telegram-cidr.js

@@ -6,7 +6,7 @@ const { processLine } = require('./lib/process-line');
 const { createRuleset } = require('./lib/create-file');
 const { runner } = require('./lib/trace-runner');
 
-runner(__filename, async () => {
+const buildTelegramCIDR = async () => {
   /** @type {Response} */
   const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
   const lastModified = resp.headers.get('last-modified');
@@ -49,4 +49,10 @@ runner(__filename, async () => {
     path.resolve(__dirname, '../List/ip/telegram.conf'),
     path.resolve(__dirname, '../Clash/ip/telegram.txt')
   ));
-});
+};
+
+module.exports.buildTelegramCIDR = buildTelegramCIDR;
+
+if (require.main === module) {
+  runner(__filename, buildTelegramCIDR);
+}

Build/build.js

@@ -1,146 +0,0 @@
-// @ts-check
-
-const path = require('path');
-const { PathScurry } = require('path-scurry');
-const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
-const { processLine } = require('./lib/process-line');
-const { createRuleset } = require('./lib/create-file');
-const { domainDeduper } = require('./lib/domain-deduper');
-const { runner } = require('./lib/trace-runner');
-
-const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
-const MAGIC_COMMAND_TITLE = '# $ meta_title ';
-const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
-
-const sourceDir = path.resolve(__dirname, '../Source');
-const outputSurgeDir = path.resolve(__dirname, '../List');
-const outputClashDir = path.resolve(__dirname, '../Clash');
-
-runner(__filename, async () => {
-  /** @type {Promise<void>[]} */
-  const promises = [];
-
-  const pw = new PathScurry(sourceDir);
-  for await (const entry of pw) {
-    if (entry.isFile()) {
-      if (path.extname(entry.name) === '.js') {
-        continue;
-      }
-
-      const relativePath = entry.relative();
-      if (relativePath.startsWith('domainset/')) {
-        promises.push(transformDomainset(entry.fullpath(), relativePath));
-        continue;
-      }
-      if (
-        relativePath.startsWith('ip/')
-        || relativePath.startsWith('non_ip/')
-      ) {
-        promises.push(transformRuleset(entry.fullpath(), relativePath));
-        continue;
-      }
-    }
-  }
-
-  return Promise.all(promises);
-});
-
-/**
- * @param {string} sourcePath
- */
-const processFile = async (sourcePath) => {
-  /** @type {string[]} */
-  const lines = [];
-
-  let title = '';
-  /** @type {string[]} */
-  const descriptions = [];
-
-  for await (const line of readFileByLine(sourcePath)) {
-    if (line === MAGIC_COMMAND_SKIP) {
-      return;
-    }
-
-    if (line.startsWith(MAGIC_COMMAND_TITLE)) {
-      title = line.slice(MAGIC_COMMAND_TITLE.length).trim();
-      continue;
-    }
-
-    if (line.startsWith(MAGIC_COMMAND_DESCRIPTION)) {
-      descriptions.push(line.slice(MAGIC_COMMAND_DESCRIPTION.length).trim());
-      continue;
-    }
-
-    const l = processLine(line);
-    if (l) {
-      lines.push(l);
-    }
-  }
-
-  return /** @type {const} */ ([title, descriptions, lines]);
-};
-
-/**
- * @param {string} sourcePath
- * @param {string} relativePath
- */
-async function transformDomainset(sourcePath, relativePath) {
-  const res = await processFile(sourcePath);
-  if (!res) return;
-  const [title, descriptions, lines] = res;
-
-  const deduped = domainDeduper(lines);
-  const description = [
-    'License: AGPL 3.0',
-    'Homepage: https://ruleset.skk.moe',
-    'GitHub: https://github.com/SukkaW/Surge',
-    ...(
-      descriptions.length
-        ? ['', ...descriptions]
-        : []
-    )
-  ];
-
-  await Promise.all(createRuleset(
-    title,
-    description,
-    new Date(),
-    deduped,
-    'domainset',
-    path.resolve(outputSurgeDir, relativePath),
-    path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
-  ));
-}
-
-/**
- * Output Surge RULE-SET and Clash classical text format
- *
- * @param {string} sourcePath
- * @param {string} relativePath
- */
-async function transformRuleset(sourcePath, relativePath) {
-  const res = await processFile(sourcePath);
-  if (!res) return;
-  const [title, descriptions, lines] = res;
-
-  const description = [
-    'License: AGPL 3.0',
-    'Homepage: https://ruleset.skk.moe',
-    'GitHub: https://github.com/SukkaW/Surge',
-    ...(
-      descriptions.length
-        ? ['', ...descriptions]
-        : []
-    )
-  ];
-
-  await Promise.all(createRuleset(
-    title,
-    description,
-    new Date(),
-    lines,
-    'ruleset',
-    path.resolve(outputSurgeDir, relativePath),
-    path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
-  ));
-}

Build/download-previous-build.js

@@ -98,9 +98,14 @@ const downloadPublicSuffixList = async () => {
   );
 };
 
-runner(__filename, () => {
-  return Promise.all([
-    downloadPreviousBuild(),
-    downloadPublicSuffixList()
-  ]);
-});
+module.exports.downloadPreviousBuild = downloadPreviousBuild;
+module.exports.downloadPublicSuffixList = downloadPublicSuffixList;
+
+if (require.main === module) {
+  runner(__filename, () => {
+    return Promise.all([
+      downloadPreviousBuild(),
+      downloadPublicSuffixList()
+    ]);
+  });
+}

Build/lib/minify-rules.js

@@ -1,13 +0,0 @@
-// @ts-check
-/**
- * @param {string[]} rules
- */
-exports.minifyRules = (rules) => rules.filter(line => {
-  if (line[0] === '#') {
-    return false;
-  }
-  if (line.trim().length === 0) {
-    return false;
-  }
-  return true;
-});

Build/validate-domainset.js

@@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => {
   }
 };
 
-runner(__filename, async () => {
+const validate = async () => {
   const [domainsetFiles, _rulesetFiles] = await Promise.all([
     listDir(path.resolve(__dirname, '../List/domainset')),
     listDir(path.resolve(__dirname, '../List/non_ip'))
@@ -67,4 +67,9 @@ runner(__filename, async () => {
     domainsetFiles.map(file => validateDomainSet(file))
     // rulesetFiles.map(file => validateRuleset(file))
   );
-});
+};
+module.exports.validate = validate;
+
+if (require.main === module) {
+  runner(__filename, validate);
+}

Modules/sukka_enhance_adblock.sgmodule

@@ -1,6 +1,5 @@
 #!name=[Sukka] Enhance Better ADBlock for Surge
 #!desc=增强 ADBlock 效果、恢复网站正常功能
-# Use Build/build.js to generate the list
 
 [MITM]
 hostname = %APPEND% *.google-analytics.com, *.googletagmanager.com, *.googlesyndication.com, *.googletagservices.com, *.doubleclick.net, cdn.ampproject.org, *.addthis.com

Modules/sukka_mitm_hostnames.sgmodule

@@ -1,6 +1,6 @@
 #!name=[Sukka] Surge Reject MITM
 #!desc=为 URL Regex 规则组启用 MITM
-# Use Build/build.js to generate the list
+# Use Build/build-mitm-hostname.js to generate the list
 
 [MITM]
 

package.json

@@ -27,7 +27,7 @@
       "command": "node ./Build/download-previous-build.js"
     },
     "build:common": {
-      "command": "node ./Build/build.js",
+      "command": "node ./Build/build-common.js",
       "dependencies": [
         "download-previous-build"
       ]