Chore: create rule set / Add China IP sing-box

Build/build-apple-cdn.ts

@@ -1,11 +1,11 @@
 // @ts-check
-import path from 'path';
 import { createRuleset } from './lib/create-file';
 import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
 import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
+import { output } from './lib/misc';
 
 const cacheKey = createCacheKey(__filename);
 
@@ -42,9 +42,7 @@ export const buildAppleCdn = task(require.main === module, __filename)(async (sp
       new Date(),
       ruleset,
       'ruleset',
-      path.resolve(__dirname, '../List/non_ip/apple_cdn.conf'),
-      path.resolve(__dirname, '../Clash/non_ip/apple_cdn.txt'),
-      path.resolve(__dirname, '../sing-box/non_ip/apple_cdn.json')
+      ...output('apple_cdn', 'non_ip')
     ),
     createRuleset(
       span,
@@ -53,10 +51,7 @@ export const buildAppleCdn = task(require.main === module, __filename)(async (sp
       new Date(),
       domainset,
       'domainset',
-      path.resolve(__dirname, '../List/domainset/apple_cdn.conf'),
-      path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt'),
-      path.resolve(__dirname, '../sing-box/domainset/apple_cdn.json'),
-      path.resolve(__dirname, '../Clash/clash_mrs_domain/apple_cdn.mrs')
+      ...output('apple_cdn', 'domainset')
     )
   ]);
 });

Build/build-cdn-download-conf.ts

@@ -8,6 +8,7 @@ import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist'
 import { domainDeduper } from './lib/domain-deduper';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 import { sortDomains } from './lib/stable-sort-domain';
+import { output } from './lib/misc';
 
 const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
   const trie = createTrie(
@@ -77,10 +78,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
       new Date(),
       sortDomains(domainDeduper(cdnDomainsList)),
       'domainset',
-      path.resolve(__dirname, '../List/domainset/cdn.conf'),
-      path.resolve(__dirname, '../Clash/domainset/cdn.txt'),
-      path.resolve(__dirname, '../sing-box/domainset/cdn.json'),
-      path.resolve(__dirname, '../Clash/clash_mrs_domain/cdn.mrs')
+      ...output('cdn', 'domainset')
     ),
     createRuleset(
       span,
@@ -93,10 +91,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
       new Date(),
       sortDomains(domainDeduper(downloadDomainSet)),
       'domainset',
-      path.resolve(__dirname, '../List/domainset/download.conf'),
-      path.resolve(__dirname, '../Clash/domainset/download.txt'),
-      path.resolve(__dirname, '../sing-box/domainset/download.json'),
-      path.resolve(__dirname, '../Clash/clash_mrs_domain/download.mrs')
+      ...output('download', 'domainset')
     )
   ]);
 });

Build/build-chn-cidr.ts

@@ -1,6 +1,5 @@
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
-import { resolve as pathResolve } from 'path';
-import { compareAndWriteFile, withBannerArray } from './lib/create-file';
+import { createRuleset } from './lib/create-file';
 import { processLineFromReadline } from './lib/process-line';
 import { task } from './trace';
 
@@ -8,6 +7,7 @@ import { exclude } from 'fast-cidr-tools';
 import { createMemoizedPromise } from './lib/memo-promise';
 import { CN_CIDR_NOT_INCLUDED_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
 import { appendArrayInPlace } from './lib/append-array-in-place';
+import { output } from './lib/misc';
 
 export const getChnCidrPromise = createMemoizedPromise(async () => {
   const cidr4 = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'));
@@ -30,57 +30,29 @@ export const buildChnCidr = task(require.main === module, __filename)(async (spa
 
   // Can not use createRuleset here, as Clash support advanced ipset syntax
   return Promise.all([
-    compareAndWriteFile(
+    createRuleset(
       span,
-      withBannerArray(
-        'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
-        [
-          ...description,
-          'Data from https://misaka.io (misakaio @ GitHub)'
-        ],
-        new Date(),
-        filteredCidr4.map(i => `IP-CIDR,${i}`)
-      ),
-      pathResolve(__dirname, '../List/ip/china_ip.conf')
+      'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
+      [
+        ...description,
+        'Data from https://misaka.io (misakaio @ GitHub)'
+      ],
+      new Date(),
+      filteredCidr4,
+      'ipcidr',
+      ...output('china_ip', 'ip')
     ),
-    compareAndWriteFile(
+    createRuleset(
       span,
-      withBannerArray(
-        'Sukka\'s Ruleset - Mainland China IPv6 CIDR',
-        [
-          ...description,
-          'Data from https://github.com/gaoyifan/china-operator-ip'
-        ],
-        new Date(),
-        cidr6.map(i => `IP-CIDR6,${i}`)
-      ),
-      pathResolve(__dirname, '../List/ip/china_ip_ipv6.conf')
-    ),
-    compareAndWriteFile(
-      span,
-      withBannerArray(
-        'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
-        [
-          ...description,
-          'Data from https://misaka.io (misakaio @ GitHub)'
-        ],
-        new Date(),
-        filteredCidr4
-      ),
-      pathResolve(__dirname, '../Clash/ip/china_ip.txt')
-    ),
-    compareAndWriteFile(
-      span,
-      withBannerArray(
-        'Sukka\'s Ruleset - Mainland China IPv6 CIDR',
-        [
-          ...description,
-          'Data from https://github.com/gaoyifan/china-operator-ip'
-        ],
-        new Date(),
-        cidr6
-      ),
-      pathResolve(__dirname, '../Clash/ip/china_ip_ipv6.txt')
+      'Sukka\'s Ruleset - Mainland China IPv6 CIDR',
+      [
+        ...description,
+        'Data from https://github.com/gaoyifan/china-operator-ip'
+      ],
+      new Date(),
+      cidr6,
+      'ipcidr6',
+      ...output('china_ip_ipv6', 'ip')
     )
   ]);
 });

Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts

@@ -9,7 +9,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
 import * as yaml from 'yaml';
 import { appendArrayInPlace } from './lib/append-array-in-place';
-import { writeFile } from './lib/misc';
+import { output, writeFile } from './lib/misc';
 
 export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(async () => {
   const domestics = await readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
@@ -48,9 +48,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as
       new Date(),
       res[0],
       'ruleset',
-      path.resolve(__dirname, '../List/non_ip/domestic.conf'),
-      path.resolve(__dirname, '../Clash/non_ip/domestic.txt'),
-      path.resolve(__dirname, '../sing-box/non_ip/domestic.json')
+      ...output('domestic', 'non_ip')
     ),
     createRuleset(
       span,
@@ -63,9 +61,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as
       new Date(),
       res[1],
       'ruleset',
-      path.resolve(__dirname, '../List/non_ip/direct.conf'),
-      path.resolve(__dirname, '../Clash/non_ip/direct.txt'),
-      path.resolve(__dirname, '../sing-box/non_ip/direct.json')
+      ...output('direct', 'non_ip')
     ),
     createRuleset(
       span,
@@ -78,9 +74,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as
       new Date(),
       res[2],
       'ruleset',
-      path.resolve(__dirname, '../List/non_ip/lan.conf'),
-      path.resolve(__dirname, '../Clash/non_ip/lan.txt'),
-      path.resolve(__dirname, '../sing-box/non_ip/lan.json')
+      ...output('lan', 'non_ip')
     ),
     compareAndWriteFile(
       span,

Build/build-microsoft-cdn.ts

@@ -1,4 +1,3 @@
-import path from 'path';
 import { task } from './trace';
 import { createRuleset } from './lib/create-file';
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
@@ -7,6 +6,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
 import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';
 import { sortDomains } from './lib/stable-sort-domain';
+import { output } from './lib/misc';
 
 const PROBE_DOMAINS = ['.microsoft.com', '.windows.net', '.windows.com', '.windowsupdate.com', '.windowssearch.com', '.office.net'];
 
@@ -63,8 +63,6 @@ export const buildMicrosoftCdn = task(require.main === module, __filename)(async
     new Date(),
     res,
     'ruleset',
-    path.resolve(__dirname, '../List/non_ip/microsoft_cdn.conf'),
-    path.resolve(__dirname, '../Clash/non_ip/microsoft_cdn.txt'),
-    path.resolve(__dirname, '../sing-box/non_ip/microsoft_cdn.json')
+    ...output('microsoft_cdn', 'non_ip')
   );
 });

Build/build-reject-domainset.ts

@@ -18,6 +18,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
 import { getPhishingDomains } from './lib/get-phishing-domains';
 
 import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array';
+import { output } from './lib/misc';
 
 const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/reject_sukka.conf'));
 
@@ -190,10 +191,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       new Date(),
       span.traceChildSync('sort reject domainset (base)', () => sortDomains(dudupedDominArray, domainArrayMainDomainMap, domainArraySubdomainMap)),
       'domainset',
-      path.resolve(__dirname, '../List/domainset/reject.conf'),
-      path.resolve(__dirname, '../Clash/domainset/reject.txt'),
-      path.resolve(__dirname, '../sing-box/domainset/reject.json'),
-      path.resolve(__dirname, '../Clash/clash_mrs_domain/reject.mrs')
+      ...output('reject', 'domainset')
     ),
     createRuleset(
       span,
@@ -212,10 +210,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       new Date(),
       span.traceChildSync('sort reject domainset (extra)', () => sortDomains(dudupedDominArrayExtra, domainArrayMainDomainMap, domainArraySubdomainMap)),
       'domainset',
-      path.resolve(__dirname, '../List/domainset/reject_extra.conf'),
-      path.resolve(__dirname, '../Clash/domainset/reject_extra.txt'),
-      path.resolve(__dirname, '../sing-box/domainset/reject_extra.json'),
-      path.resolve(__dirname, '../Clash/clash_mrs_domain/reject_extra.mrs')
+      ...output('reject_extra', 'domainset')
     ),
     compareAndWriteFile(
       span,

Build/build-reject-ip-list.ts

@@ -9,6 +9,7 @@ import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } f
 import { fetchAssets } from './lib/fetch-assets';
 import { processLine } from './lib/process-line';
 import { appendArrayInPlace } from './lib/append-array-in-place';
+import { output } from './lib/misc';
 
 const cacheKey = createCacheKey(__filename);
 
@@ -100,8 +101,6 @@ export const buildRejectIPList = task(require.main === module, __filename)(async
     new Date(),
     result,
     'ruleset',
-    path.resolve(__dirname, '../List/ip/reject.conf'),
-    path.resolve(__dirname, '../Clash/ip/reject.txt'),
-    path.resolve(__dirname, '../sing-box/ip/reject.json')
+    ...output('reject', 'ip')
   );
 });

Build/build-speedtest-domainset.ts

@@ -12,6 +12,7 @@ import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
 
 import { createTrie } from './lib/trie';
+import { output } from './lib/misc';
 
 const s = new Sema(2);
 const cacheKey = createCacheKey(__filename);
@@ -250,9 +251,6 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename)
     new Date(),
     deduped,
     'domainset',
-    path.resolve(__dirname, '../List/domainset/speedtest.conf'),
-    path.resolve(__dirname, '../Clash/domainset/speedtest.txt'),
-    path.resolve(__dirname, '../sing-box/domainset/speedtest.json'),
-    path.resolve(__dirname, '../Clash/clash_mrs_domain/speedtest.mrs')
+    ...output('speedtest', 'domainset')
   );
 });

Build/build-stream-service.ts

@@ -2,11 +2,11 @@
 import type { Span } from './trace';
 import { task } from './trace';
 
-import path from 'path';
 import { createRuleset } from './lib/create-file';
 
 import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
 import { SHARED_DESCRIPTION } from './lib/constants';
+import { output } from './lib/misc';
 
 export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
   return span.traceChildAsync(fileId, async (childSpan) => Promise.all([
@@ -22,9 +22,7 @@ export const createRulesetForStreamService = (span: Span, fileId: string, title:
       new Date(),
       streamServices.flatMap((i) => i.rules),
       'ruleset',
-      path.resolve(__dirname, `../List/non_ip/${fileId}.conf`),
-      path.resolve(__dirname, `../Clash/non_ip/${fileId}.txt`),
-      path.resolve(__dirname, `../sing-box/non_ip/${fileId}.json`)
+      ...output(fileId, 'non_ip')
     ),
     // IP
     createRuleset(
@@ -45,9 +43,7 @@ export const createRulesetForStreamService = (span: Span, fileId: string, title:
           : []
       )),
       'ruleset',
-      path.resolve(__dirname, `../List/ip/${fileId}.conf`),
-      path.resolve(__dirname, `../Clash/ip/${fileId}.txt`),
-      path.resolve(__dirname, `../sing-box/ip/${fileId}.json`)
+      ...output(fileId, 'ip')
     )
   ]));
 };

Build/build-telegram-cidr.ts

@@ -1,13 +1,13 @@
 // @ts-check
 import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry';
 import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
-import path from 'path';
 import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip';
 import { processLine } from './lib/process-line';
 import { createRuleset } from './lib/create-file';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
+import { output } from './lib/misc';
 
 export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
   const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit);
@@ -52,8 +52,6 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async
     date,
     results,
     'ruleset',
-    path.resolve(__dirname, '../List/ip/telegram.conf'),
-    path.resolve(__dirname, '../Clash/ip/telegram.txt'),
-    path.resolve(__dirname, '../sing-box/ip/telegram.json')
+    ...output('telegram', 'ip')
   );
 });

Build/lib/create-file.ts

@@ -7,7 +7,7 @@ import fs from 'fs';
 import { fastStringArrayJoin, writeFile } from './misc';
 import { readFileByLine } from './fetch-text-by-line';
 import stringify from 'json-stringify-pretty-compact';
-import { surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
+import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
 
 export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
   let isEqual = true;
@@ -79,7 +79,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
   });
 }
 
-export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
+const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
   return [
     '#########################################',
     `# ${title}`,
@@ -154,15 +154,32 @@ const MARK = 'this_ruleset_is_made_by_sukkaw.ruleset.skk.moe';
 export const createRuleset = (
   parentSpan: Span,
   title: string, description: string[] | readonly string[], date: Date, content: string[],
-  type: ('ruleset' | 'domainset' | string & {}),
+  type: 'ruleset' | 'domainset' | 'ipcidr' | 'ipcidr6',
   surgePath: string, clashPath: string, singBoxPath: string, _clashMrsPath?: string
 ) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn(async (childSpan) => {
-  const surgeContent = withBannerArray(
-    title, description, date,
-    type === 'domainset'
-      ? [MARK, ...content]
-      : sortRuleSet([`DOMAIN,${MARK}`, ...content])
-  );
+  content = sortRuleSet(content);
+  const surgeContent = childSpan.traceChildSync('process surge ruleset', () => {
+    let _surgeContent;
+    switch (type) {
+      case 'domainset':
+        _surgeContent = [MARK, ...content];
+        break;
+      case 'ruleset':
+        _surgeContent = [`DOMAIN,${MARK}`, ...content];
+        break;
+      case 'ipcidr':
+        _surgeContent = [`DOMAIN,${MARK}`, ...content.map(i => `IP-CIDR,${i}`)];
+        break;
+      case 'ipcidr6':
+        _surgeContent = [`DOMAIN,${MARK}`, ...content.map(i => `IP-CIDR6,${i}`)];
+        break;
+      default:
+        throw new TypeError(`Unknown type: ${type}`);
+    }
+
+    return withBannerArray(title, description, date, _surgeContent);
+  });
+
   const clashContent = childSpan.traceChildSync('convert incoming ruleset to clash', () => {
     let _clashContent;
     switch (type) {
@@ -172,6 +189,10 @@ export const createRuleset = (
       case 'ruleset':
         _clashContent = [`DOMAIN,${MARK}`, ...surgeRulesetToClashClassicalTextRuleset(content)];
         break;
+      case 'ipcidr':
+      case 'ipcidr6':
+        _clashContent = content;
+        break;
       default:
         throw new TypeError(`Unknown type: ${type}`);
     }
@@ -186,6 +207,10 @@ export const createRuleset = (
       case 'ruleset':
         _singBoxContent = surgeRulesetToSingbox([`DOMAIN,${MARK}`, ...content]);
         break;
+      case 'ipcidr':
+      case 'ipcidr6':
+        _singBoxContent = ipCidrListToSingbox(content);
+        break;
       default:
         throw new TypeError(`Unknown type: ${type}`);
     }

Build/lib/misc.ts

@@ -1,4 +1,4 @@
-import { dirname } from 'path';
+import path, { dirname } from 'path';
 import fs from 'fs';
 import fsp from 'fs/promises';
 import { makeRe } from 'picomatch';
@@ -33,3 +33,15 @@ export const writeFile: Write = async (destination: string, input, dir = dirname
 export const domainWildCardToRegex = (domain: string) => {
   return makeRe(domain, { contains: false, strictSlashes: true }).source;
 };
+
+const OUTPUT_SURGE_DIR = path.resolve(__dirname, '../../List');
+const OUTPUT_CLASH_DIR = path.resolve(__dirname, '../../Clash');
+const OUTPUT_SINGBOX_DIR = path.resolve(__dirname, '../../sing-box');
+
+export const output = (id: string, type: 'non_ip' | 'ip' | 'domainset') => {
+  return [
+    path.join(OUTPUT_SURGE_DIR, type, id + '.conf'),
+    path.join(OUTPUT_CLASH_DIR, type, id + '.txt'),
+    path.join(OUTPUT_SINGBOX_DIR, type, id + '.json')
+  ] as const;
+};

Build/lib/singbox.ts

@@ -110,3 +110,12 @@ export const surgeDomainsetToSingbox = (domainset: string[]) => {
     rules: [rule]
   };
 };
+
+export const ipCidrListToSingbox = (ipCidrList: string[]): SingboxSourceFormat => {
+  return {
+    version: 2,
+    rules: [{
+      ip_cidr: ipCidrList
+    }]
+  };
+};

Build/lib/text-decoder-stream.ts

@@ -1,54 +0,0 @@
-// Copyright 2016 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Polyfill for TextEncoderStream and TextDecoderStream
-// Modified by Sukka (https://skk.moe) to increase compatibility and performance with Bun.
-
-export class PolyfillTextDecoderStream extends TransformStream<Uint8Array, string> {
-  readonly fatal: boolean;
-  readonly ignoreBOM: boolean;
-
-  constructor(
-    public readonly encoding: BufferEncoding = 'utf-8',
-    {
-      fatal = false,
-      ignoreBOM = false
-    }: ConstructorParameters<typeof TextDecoder>[1] = {}
-  ) {
-    const decoder = new TextDecoder(encoding, { fatal, ignoreBOM });
-
-    const nonLastChunkDecoderOpt = { stream: true };
-
-    super({
-      transform(chunk: Uint8Array, controller: TransformStreamDefaultController<string>) {
-        const decoded = decoder.decode(chunk, nonLastChunkDecoderOpt);
-        controller.enqueue(decoded);
-      },
-      flush(controller: TransformStreamDefaultController<string>) {
-        // If {fatal: false} is in options (the default), then the final call to
-        // decode() can produce extra output (usually the unicode replacement
-        // character 0xFFFD). When fatal is true, this call is just used for its
-        // side-effect of throwing a TypeError exception if the input is
-        // incomplete.
-        const output = decoder.decode();
-        if (output.length > 0) {
-          controller.enqueue(output);
-        }
-      }
-    });
-
-    this.fatal = fatal;
-    this.ignoreBOM = ignoreBOM;
-  }
-}