Clean up

.gitignore

@@ -13,3 +13,4 @@ List/non_ip/apple_cdn.conf
 List/ip/telegram.conf
 List/ip/reject.conf
 List/ip/china_ip.conf
+List/internal/

Build/build-internal-rules.js

@@ -0,0 +1,57 @@
+// @ts-check
+const fs = require('fs');
+const fse = require('fs-extra');
+const path = require('path');
+const readline = require('readline');
+const { isDomainLoose } = require('./lib/is-domain-loose');
+
+(async () => {
+  const results = [];
+
+  for await (
+    const line of readline.createInterface({
+      input: fs.createReadStream(path.resolve(__dirname, '../List/non_ip/cdn.conf')),
+      crlfDelay: Infinity
+    })
+  ) {
+    if (line.startsWith('DOMAIN-SUFFIX,')) {
+      results.push(line.replace('DOMAIN-SUFFIX,', 'SUFFIX,'));
+    } else if (line.startsWith('DOMAIN,')) {
+      results.push(line.replace('DOMAIN,', 'SUFFIX,'));
+    }
+  }
+
+  for await (
+    const line of readline.createInterface({
+      input: fs.createReadStream(path.resolve(__dirname, '../List/domainset/cdn.conf')),
+      crlfDelay: Infinity
+    })
+  ) {
+    if (line[0] === '.') {
+      results.push(`SUFFIX,${line.slice(1)}`);
+    } else if (isDomainLoose(line)) {
+      results.push(`SUFFIX,${line}`);
+    }
+  }
+
+  for await (
+    const line of readline.createInterface({
+      input: fs.createReadStream(path.resolve(__dirname, '../List/domainset/download.conf')),
+      crlfDelay: Infinity
+    })
+  ) {
+    if (line[0] === '.') {
+      results.push(`SUFFIX,${line.slice(1)}`);
+    } else if (isDomainLoose(line)) {
+      results.push(`SUFFIX,${line}`);
+    }
+  }
+
+  results.push('');
+
+  await fse.ensureDir(path.resolve(__dirname, '../List/internal'));
+  await fs.promises.writeFile(
+    path.resolve(__dirname, '../List/internal/cdn.csv'),
+    results.join('\n')
+  );
+})();

Build/download-previous-build.js

@@ -14,46 +14,45 @@ const fileExists = (path) => {
 };
 
 (async () => {
-  const filesList = (
-    await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' })
-  )
+  const filesList = (await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' }))
     .split('\n')
-    .filter(p => p.startsWith('List/'));
+    .filter(p => p.startsWith('List/') && !p.endsWith('/'));
 
   if (
-    (await Promise.all(
+    !((await Promise.all(
       filesList.map(p => fileExists(join(__dirname, '..', p)))
-    )).some(exist => !exist)
+    )).some(exist => !exist))
   ) {
-    const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`);
-    const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages');
-    const readableNodeStream = Readable.fromWeb(resp.body);
-    await pipeline(
-      readableNodeStream,
-      fs.createWriteStream(tempFile)
-    );
+    console.log('All files exists, skip download.');
+    return;
+  }
 
-    const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`);
-    await fse.ensureDir(extractedPath);
-    await tar.x({
-      file: tempFile,
-      cwd: extractedPath,
-      filter: (p) => {
-        return p.split('/')[1] === 'List'
-      }
-    });
+  const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`);
+  const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages');
+  const readableNodeStream = Readable.fromWeb(resp.body);
+  await pipeline(
+    readableNodeStream,
+    fs.createWriteStream(tempFile)
+  );
 
-    await Promise.all(filesList.map(p => fse.copy(
-      join(extractedPath, 'Surge-gh-pages', p),
-      join(__dirname, '..', p),
-      {
-        overwrite: true
-      }
-    )))
+  const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`);
+  await fse.ensureDir(extractedPath);
+  await tar.x({
+    file: tempFile,
+    cwd: extractedPath,
+    filter: (p) => {
+      return p.split('/')[1] === 'List'
+    }
+  });
 
-    await fs.promises.unlink(tempFile).catch(() => { });
-    await fs.promises.unlink(extractedPath).catch(() => { });
-  } else {
-    console.log('All files exists, skip download.');
-  }
+  await Promise.all(filesList.map(p => fse.copy(
+    join(extractedPath, 'Surge-gh-pages', p),
+    join(__dirname, '..', p),
+    {
+      overwrite: true
+    }
+  )))
+
+  await fs.promises.unlink(tempFile).catch(() => { });
+  await fs.promises.unlink(extractedPath).catch(() => { });
 })();

Build/validate-domainset.js

@@ -81,9 +81,7 @@ const validateRuleset = async (filePath) => {
   ]);
   await Promise.all(
     domainsetFiles.map(file => validateDomainSet(file))
+    // rulesetFiles.map(file => validateRuleset(file))
   );
-  // await Promise.all(
-  //   rulesetFiles.map(file => validateRuleset(file))
-  // );
 })();
 

List/domainset/my_proxy.conf

@@ -1,49 +1 @@
-.mikuclub.xyz
-.mikuclub.cn
-.saucenao.com
-.fork.dev
-.nextdns.io
-.ibb.co
-.sku.moe
-.sukeycz.edu.kg
-.services.googleapis.cn
-.hostloc.com
-.t66y.com
-.zendesk.com
-.influxdata.com
-
-.services.mozilla.com
-vscode-sync.trafficmanager.net
-.cloudflare.com
-.wakatime.com
-
-.grammarly.com
-.1password.com
-.1password.eu
-.1password.ca
-
-.surge-activation.com
-.yandex.com
-
-ip.istatmenus.app
-.exp-tas.com
-.firefox.com
-.visualstudio.com
-
-.azurecontainer.io
-.azurewebsites.net
-.azure-mobile.net
-.cloudapp.net
-.azurestaticapps.net
-
-.elasticbeanstalk.com
-.compute.amazonaws.com
-.compute-1.amazonaws.com
-
-dash.cloudflare.com
-.he.net
-
-.webpkgcache.com
-vscodeexperiments.azureedge.net
-
-.gitlab.com
+# deprecated

List/non_ip/my_proxy.conf

@@ -1 +1,51 @@
-# Deprecated
+DOMAIN_SUFFIX,mikuclub.xyz
+DOMAIN_SUFFIX,mikuclub.cn
+DOMAIN_SUFFIX,saucenao.com
+DOMAIN_SUFFIX,fork.dev
+DOMAIN_SUFFIX,nextdns.io
+DOMAIN_SUFFIX,ibb.co
+DOMAIN_SUFFIX,sku.moe
+DOMAIN_SUFFIX,sukeycz.edu.kg
+DOMAIN_SUFFIX,services.googleapis.cn
+DOMAIN_SUFFIX,hostloc.com
+DOMAIN_SUFFIX,t66y.com
+DOMAIN_SUFFIX,zendesk.com
+DOMAIN_SUFFIX,influxdata.com
+
+DOMAIN_SUFFIX,services.mozilla.com
+DOMAIN_SUFFIX,cloudflare.com
+DOMAIN_SUFFIX,wakatime.com
+
+DOMAIN_SUFFIX,grammarly.com
+DOMAIN_SUFFIX,1password.com
+DOMAIN_SUFFIX,1password.eu
+DOMAIN_SUFFIX,1password.ca
+
+DOMAIN_SUFFIX,surge-activation.com
+DOMAIN_SUFFIX,yandex.com
+
+DOMAIN,ip.istatmenus.app
+DOMAIN_SUFFIX,exp-tas.com
+DOMAIN_SUFFIX,firefox.com
+DOMAIN_SUFFIX,visualstudio.com
+
+DOMAIN_SUFFIX,azurecontainer.io
+DOMAIN_SUFFIX,azurewebsites.net
+DOMAIN_SUFFIX,azure-mobile.net
+DOMAIN_SUFFIX,cloudapp.net
+DOMAIN_SUFFIX,azurestaticapps.net
+
+DOMAIN_SUFFIX,elasticbeanstalk.com
+DOMAIN_SUFFIX,compute.amazonaws.com
+DOMAIN_SUFFIX,compute-1.amazonaws.com
+
+DOMAIN_SUFFIX,cloudflare.com
+DOMAIN_SUFFIX,he.net
+
+DOMAIN_SUFFIX,webpkgcache.com
+DOMAIN,vscodeexperiments.azureedge.net
+
+DOMAIN_SUFFIX,gitlab.com
+
+DOMAIN_SUFFIX,bgp.tools
+DOMAIN_SUFFIX,setapp.com

List/non_ip/my_proxy_plus.conf

@@ -1,5 +0,0 @@
-DOMAIN-SUFFIX,tailwindcss.com
-DOMAIN-SUFFIX,ytoo.li
-DOMAIN-SUFFIX,flowercloud.net
-DOMAIN-KEYWORD,blogspot
-DOMAIN-SUFFIX,codeberg.org

package.json

@@ -14,6 +14,7 @@
     "build:phishing-domainset": "wireit",
     "build:telegram-cidr": "wireit",
     "build:chn-cidr": "wireit",
+    "build:internal-rules": "wireit",
     "validate:domainset": "wireit"
   },
   "wireit": {
@@ -82,6 +83,12 @@
         "build:cdn-conf"
       ]
     },
+    "build:internal-rules": {
+      "command": "node ./Build/build-internal-rules.js",
+      "dependencies": [
+        "build:cdn-conf"
+      ]
+    },
     "build": {
       "dependencies": [
         "build:anti-bogus-domain",