Fetch ookla servers from `speedtest-net-servers.cdn.skk.moe`

Build/build-speedtest-domainset.ts

@@ -7,50 +7,35 @@ import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 
 import { DomainsetOutput } from './lib/rules/domainset';
 import { OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir';
-import { newQueue } from '@henrygd/queue';
 import { $$fetch } from './lib/fetch-retry';
 
-const KEYWORDS = [
-  'Hong Kong',
-  'Taiwan',
-  'China Telecom',
-  'China Mobile',
-  'China Unicom',
-  'Japan',
-  'Tokyo',
-  'Singapore',
-  'Korea',
-  'Seoul',
-  'Canada',
-  'Toronto',
-  'Montreal',
-  'Los Ang',
-  'San Jos',
-  'Seattle',
-  'New York',
-  'Dallas',
-  'Miami',
-  'Berlin',
-  'Frankfurt',
-  'London',
-  'Paris',
-  'Amsterdam',
-  'Moscow',
-  'Australia',
-  'Sydney',
-  'Brazil',
-  'Turkey'
-];
-
-const s = newQueue(2);
-
-const latestTopUserAgentsPromise = $$fetch('https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src/desktop.json')
-  .then(res => res.json() as Promise<string[]>)
-  .then((userAgents: string[]) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 ')));
-const getSpeedtestHostsGroupsPromise = Promise.all(KEYWORDS.flatMap(querySpeedtestApi));
+interface SpeedTestServer {
+  url: string,
+  lat: string,
+  lon: string,
+  distance: number,
+  name: string,
+  country: string,
+  cc: string,
+  sponsor: string,
+  id: string,
+  preferred: number,
+  https_functional: number,
+  host: string
+}
 
-export const buildSpeedtestDomainSet = task(require.main === module, __filename)(async (span) => {
-  const output = new DomainsetOutput(span, 'speedtest')
+const getSpeedtestHostsGroupsPromise = $$fetch('https://speedtest-net-servers.cdn.skk.moe/servers.json')
+  .then(res => res.json() as Promise<SpeedTestServer[]>)
+  .then((data) => data.reduce<string[]>((prev, cur) => {
+    const hn = tldts.getHostname(cur.host || cur.url, { detectIp: false, validateHostname: true });
+    if (hn) {
+      prev.push(hn);
+    }
+    return prev;
+  }, []));
+
+export const buildSpeedtestDomainSet = task(require.main === module, __filename)(
+  async (span) => new DomainsetOutput(span, 'speedtest')
     .withTitle('Sukka\'s Ruleset - Speedtest Domains')
     .withDescription([
       ...SHARED_DESCRIPTION,
@@ -58,55 +43,7 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename)
       'This file contains common speedtest endpoints.'
     ])
     .addFromDomainset(readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'domainset/speedtest.conf')))
-    .addFromDomainset(readFileIntoProcessedArray(path.resolve(OUTPUT_SURGE_DIR, 'domainset/speedtest.conf')));
-
-  const hostnameGroup = await span.traceChildPromise('get speedtest hosts groups', getSpeedtestHostsGroupsPromise);
-
-  hostnameGroup.forEach(output.bulkAddDomain.bind(output));
-
-  return output.write();
-});
-
-async function querySpeedtestApi(keyword: string) {
-  const topUserAgents = await latestTopUserAgentsPromise;
-
-  const url = `https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`;
-
-  try {
-    const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
-
-    const data = await s.add<Array<{ url: string, host: string }>>(() => $$fetch(url, {
-      headers: {
-        dnt: '1',
-        Referer: 'https://www.speedtest.net/',
-        accept: 'application/json, text/plain, */*',
-        'User-Agent': randomUserAgent,
-        'Accept-Language': 'en-US,en;q=0.9',
-        ...(randomUserAgent.includes('Chrome')
-          ? {
-            'Sec-Ch-Ua-Mobile': '?0',
-            'Sec-Fetch-Dest': 'empty',
-            'Sec-Fetch-Mode': 'cors',
-            'Sec-Fetch-Site': 'same-origin',
-            'Sec-Gpc': '1'
-          }
-          : {})
-      },
-      signal: AbortSignal.timeout(1000 * 60)
-    }).then(res => res.json() as any));
-
-    return data.reduce<string[]>(
-      (prev, cur) => {
-        const hn = tldts.getHostname(cur.host || cur.url, { detectIp: false, validateHostname: true });
-        if (hn) {
-          prev.push(hn);
-        }
-        return prev;
-      },
-      []
-    );
-  } catch (e) {
-    console.error(e);
-    return [];
-  }
-}
+    .addFromDomainset(readFileIntoProcessedArray(path.resolve(OUTPUT_SURGE_DIR, 'domainset/speedtest.conf')))
+    .bulkAddDomain(await span.traceChildPromise('get speedtest hosts groups', getSpeedtestHostsGroupsPromise))
+    .write()
+);