Collect speedtest.net endpoint

.gitignore

@@ -9,6 +9,7 @@ List/domainset/cdn.conf
 List/domainset/reject_phishing.conf
 List/domainset/reject_sukka.conf
 List/domainset/apple_cdn.conf
+List/domainset/speedtest.conf
 List/non_ip/cdn.conf
 List/non_ip/apple_cdn.conf
 List/ip/telegram.conf

Build/build-speedtest-domainset.js

@@ -0,0 +1,117 @@
+const { fetch } = require('undici');
+const { domainDeduper } = require('./lib/domain-deduper');
+const path = require('path');
+const { withBannerArray } = require('./lib/with-banner.js');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
+
+const { Sema } = require('async-sema');
+const s = new Sema(2);
+
+/**
+ * @param {string} keyword
+ * @returns {string[]}
+ */
+const querySpeedtestApi = async (keyword) => {
+  await s.acquire();
+
+  try {
+    const res = await fetch(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, {
+      headers: {
+        dnt: '1',
+        Referer: 'https://www.speedtest.net/',
+        accept: 'application/json, text/plain, */*',
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Sec-Ch-Ua': '"Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Fetch-Dest': 'empty',
+        'Sec-Fetch-Mode': 'cors',
+        'Sec-Fetch-Site': 'same-origin',
+        'Sec-Gpc': '1'
+      }
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(text);
+    }
+
+    /** @type {{ url: string }[]} */
+    const json = await res.json();
+    s.release();
+    return json.map(({ url }) => new URL(url).hostname);
+  } catch (e) {
+    s.release();
+    console.log(e);
+  }
+};
+
+(async () => {
+  /** @type {Set<string>} */
+  const domains = new Set([
+    '.speedtest.net',
+    '.ooklaserver.net',
+    '.speed.misaka.one',
+    'speed.cloudflare.com',
+    '.speedtest.rt.ru',
+    '.speedtest.aptg.com.tw',
+    '.speedtest.gslnetworks.com',
+    '.speedtest.jsinfo.net',
+    '.speedtest.i3d.net',
+    '.speedtestkorea.com'
+  ]);
+
+  const hostnameGroups = await Promise.all([
+    'Hong Kong',
+    'Taiwan',
+    'China Telecom',
+    'China Mobile',
+    'China Unicom',
+    'Japan',
+    'Tokyo',
+    'Singapore',
+    'Korea',
+    'Canada',
+    'Toronto',
+    'Montreal',
+    'Los Ang',
+    'San Jos',
+    'Seattle',
+    'New York',
+    'Dallas',
+    'Miami',
+    'Berlin',
+    'Frankfurt',
+    'London',
+    'Paris',
+    'Amsterdam',
+    'Moscow',
+    'Australia',
+    'Sydney',
+    'Brazil',
+    'Turkey'
+  ].map(querySpeedtestApi));
+
+  for (const hostnames of hostnameGroups) {
+    if (Array.isArray(hostnames)) {
+      for (const hostname of hostnames) {
+        domains.add(hostname);
+      }
+    }
+  }
+
+  const reduped = domainDeduper(Array.from(domains)).sort();
+
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - Speedtest Domains',
+      [
+        'License: AGPL 3.0',
+        'Homepage: https://ruleset.skk.moe',
+        'GitHub: https://github.com/SukkaW/Surge'
+      ],
+      new Date(),
+      reduped
+    ),
+    path.resolve(__dirname, '../List/domainset/speedtest.conf')
+  );
+})();

Build/download-previous-build.js

@@ -46,11 +46,16 @@ const fileExists = (path) => {
     })
   );
 
-  await Promise.all(filesList.map(p => fse.copy(
-    join(extractedPath, 'Surge-gh-pages', p),
-    join(__dirname, '..', p),
-    { overwrite: true }
-  )));
+  await Promise.all(filesList.map(async p => {
+    const src = join(extractedPath, 'Surge-gh-pages', p);
+    if (await fileExists(src)) {
+      return fse.copy(
+        src,
+        join(__dirname, '..', p),
+        { overwrite: true }
+      );
+    }
+  }));
 
   await fs.promises.unlink(extractedPath).catch(() => { });
 })();

Build/lib/string-array-compare.js

@@ -1,11 +1,13 @@
 // @ts-check
 const { promises: fsPromises } = require('fs');
+const fse = require('fs-extra');
 
 /**
  * @param {string[]} linesA
  * @param {string} filePath
  */
 async function compareAndWriteFile(linesA, filePath) {
+  await fse.ensureFile(filePath);
   const linesB = (await fsPromises.readFile(filePath, { encoding: 'utf-8' })).split('\n');
 
   if (!stringArrayCompare(linesA, linesB)) {

List/non_ip/speedtest.conf

@@ -1,8 +1 @@
-DOMAIN-SUFFIX,speedtest.net
-DOMAIN-SUFFIX,speed.misaka.one
-DOMAIN-SUFFIX,speedtest.i3d.net
-DOMAIN-SUFFIX,lg-hkg.fdcservers.net
-DOMAIN-SUFFIX,speed.cloudflare.com
-DOMAIN-SUFFIX,ooklaserver.net
-DOMAIN-SUFFIX,speedtestbb.hk.chinamobile.com
-DOMAIN-SUFFIX,ookla-speedtest-central.hgconair.hgc.com.hk
+# Deprecated

package.json

@@ -14,6 +14,7 @@
     "build:phishing-domainset": "wireit",
     "build:telegram-cidr": "wireit",
     "build:chn-cidr": "wireit",
+    "build:speedtest-domainset": "wireit",
     "build:internal-cdn-rules": "wireit",
     "build:internal-reverse-chn-cidr": "wireit",
     "build:internal-chn-domains": "wireit",
@@ -66,6 +67,12 @@
         "download-previous-build"
       ]
     },
+    "build:speedtest-domainset": {
+      "command": "node ./Build/build-speedtest-domainset.js",
+      "dependencies": [
+        "download-previous-build"
+      ]
+    },
     "build:internal-cdn-rules": {
       "command": "node ./Build/build-internal-cdn-rules.js",
       "dependencies": [
@@ -88,7 +95,8 @@
         "build:internal-reverse-chn-cidr",
         "build:internal-chn-domains",
         "build:reject-domainset",
-        "build:telegram-cidr"
+        "build:telegram-cidr",
+        "build:speedtest-domainset"
       ]
     },
     "validate:domainset": {
@@ -111,6 +119,7 @@
         "build:internal-cdn-rules",
         "build:internal-reverse-chn-cidr",
         "build:internal-chn-domains",
+        "build:speedtest-domainset",
         "build:public",
         "validate:domainset"
       ]
@@ -126,6 +135,7 @@
     "@cliqz/adblocker": "^1.26.6",
     "@sukka/listdir": "^0.2.0",
     "@vercel/fetch-retry": "^5.1.3",
+    "async-sema": "^3.1.1",
     "ci-info": "^3.8.0",
     "cidr-tools-wasm": "^0.0.11",
     "fs-extra": "^11.1.1",

pnpm-lock.yaml

@@ -19,6 +19,9 @@ dependencies:
   '@vercel/fetch-retry':
     specifier: ^5.1.3
     version: 5.1.3(patch_hash=jczsp2gadg65vd4ekschpibzda)(node-fetch@2.6.11)
+  async-sema:
+    specifier: ^3.1.1
+    version: 3.1.1
   ci-info:
     specifier: ^3.8.0
     version: 3.8.0
@@ -333,6 +336,10 @@ packages:
       retry: 0.13.1
     dev: false
 
+  /async-sema@3.1.1:
+    resolution: {integrity: sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==}
+    dev: false
+
   /balanced-match@1.0.2:
     resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
     dev: true