Chore: maintainance

Build/build-apple-cdn.ts

@@ -2,7 +2,7 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { createMemoizedPromise } from './lib/memo-promise';
-import { DomainsetOutput } from './lib/create-file';
+import { DomainsetOutput } from './lib/rules/domainset';
 import { $$fetch } from './lib/fetch-retry';
 
 export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp));

Build/build-cdn-download-conf.ts

@@ -5,7 +5,7 @@ import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 import { SOURCE_DIR } from './constants/dir';
-import { DomainsetOutput } from './lib/create-file';
+import { DomainsetOutput } from './lib/rules/domainset';
 import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
 import { appendSetElementsToArray } from 'foxts/append-set-elements-to-array';
 

Build/build-chn-cidr.ts

@@ -5,7 +5,7 @@ import { contains as containsCidr, exclude as excludeCidr } from 'fast-cidr-tool
 import { createMemoizedPromise } from './lib/memo-promise';
 import { CN_CIDR_MISSING_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
 import { appendArrayInPlace } from './lib/append-array-in-place';
-import { IPListOutput } from './lib/create-file';
+import { IPListOutput } from './lib/rules/ip';
 import { cachedOnlyFail } from './lib/fs-memo';
 import { createFileDescription } from './constants/description';
 

Build/build-cloudmounter-rules.ts

@@ -1,7 +1,7 @@
 import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { task } from './trace';
-import { RulesetOutput } from './lib/create-file';
+import { RulesetOutput } from './lib/rules/ruleset';
 
 export const buildCloudMounterRules = task(require.main === module, __filename)(async (span) => {
   // AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com))

Build/build-common.ts

@@ -9,7 +9,8 @@ import { SHARED_DESCRIPTION } from './constants/description';
 import { fdir as Fdir } from 'fdir';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 import { SOURCE_DIR } from './constants/dir';
-import { DomainsetOutput, RulesetOutput } from './lib/create-file';
+import { DomainsetOutput } from './lib/rules/domainset';
+import { RulesetOutput } from './lib/rules/ruleset';
 
 const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
 const MAGIC_COMMAND_TITLE = '# $ meta_title ';

Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts

@@ -11,7 +11,7 @@ import { createMemoizedPromise } from './lib/memo-promise';
 import * as yaml from 'yaml';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, OUTPUT_MODULES_RULES_DIR, SOURCE_DIR } from './constants/dir';
-import { RulesetOutput } from './lib/create-file';
+import { RulesetOutput } from './lib/rules/ruleset';
 import { SurgeOnlyRulesetOutput } from './lib/rules/ruleset';
 
 export function createGetDnsMappingRule(allowWildcard: boolean) {

Build/build-microsoft-cdn.ts

@@ -4,7 +4,7 @@ import { HostnameSmolTrie } from './lib/trie';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { createMemoizedPromise } from './lib/memo-promise';
 import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';
-import { RulesetOutput } from './lib/create-file';
+import { RulesetOutput } from './lib/rules/ruleset';
 import { appendArrayInPlace } from './lib/append-array-in-place';
 
 const PROBE_DOMAINS = ['.microsoft.com', '.windows.net', '.windows.com', '.windowsupdate.com', '.windowssearch.com', '.office.net'];

Build/build-reject-domainset.ts

@@ -17,7 +17,7 @@ import { getPhishingDomains } from './lib/get-phishing-domains';
 
 import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
 import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
-import { DomainsetOutput } from './lib/create-file';
+import { DomainsetOutput } from './lib/rules/domainset';
 import { foundDebugDomain } from './lib/parse-filter/shared';
 import { AdGuardHomeOutput } from './lib/rules/domainset';
 

Build/build-reject-ip-list.ts

@@ -3,12 +3,13 @@ import path from 'node:path';
 import { createReadlineInterfaceFromResponse, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
-import { compareAndWriteFile, RulesetOutput } from './lib/create-file';
+import { compareAndWriteFile } from './lib/create-file';
 import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
 import { $$fetch } from './lib/fetch-retry';
 import { fetchAssets } from './lib/fetch-assets';
 import { fastIpVersion } from './lib/misc';
 import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
+import { RulesetOutput } from './lib/rules/ruleset';
 
 const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf';
 const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => {

Build/build-speedtest-domainset.ts

@@ -5,7 +5,7 @@ import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 
-import { DomainsetOutput } from './lib/create-file';
+import { DomainsetOutput } from './lib/rules/domainset';
 import { OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir';
 import { newQueue } from '@henrygd/queue';
 import { $$fetch } from './lib/fetch-retry';

Build/build-stream-service.ts

@@ -4,7 +4,7 @@ import { task } from './trace';
 
 import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
 import { SHARED_DESCRIPTION } from './constants/description';
-import { RulesetOutput } from './lib/create-file';
+import { RulesetOutput } from './lib/rules/ruleset';
 
 function createRulesetForStreamService(
   span: Span,

Build/build-telegram-cidr.ts

@@ -3,7 +3,7 @@ import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { createMemoizedPromise } from './lib/memo-promise';
-import { RulesetOutput } from './lib/create-file';
+import { RulesetOutput } from './lib/rules/ruleset';
 import { $$fetch } from './lib/fetch-retry';
 import { fastIpVersion } from './lib/misc';
 

Build/lib/create-file.ts

@@ -1,4 +1,109 @@
-export { DomainsetOutput } from './rules/domainset';
-export { IPListOutput } from './rules/ip';
-export { RulesetOutput } from './rules/ruleset';
-export { fileEqual, compareAndWriteFile } from './rules/base';
+import { asyncWriteToStream } from 'foxts/async-write-to-stream';
+import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
+import fs from 'node:fs';
+import picocolors from 'picocolors';
+import type { Span } from '../trace';
+import { readFileByLine } from './fetch-text-by-line';
+import { writeFile } from './misc';
+
+export async function fileEqual(linesA: string[], source: AsyncIterable<string> | Iterable<string>): Promise<boolean> {
+  if (linesA.length === 0) {
+    return false;
+  }
+
+  const linesABound = linesA.length - 1;
+
+  let index = -1;
+  for await (const lineB of source) {
+    index++;
+
+    if (index > linesABound) {
+      return (index === linesA.length && lineB.length === 0);
+    }
+
+    const lineA = linesA[index];
+
+    if (lineA.length === 0) {
+      if (lineB.length === 0) {
+        // both lines are empty, check next line
+        continue;
+      }
+      // lineA is empty but lineB is not
+      return false;
+    }
+    // now lineA can not be empty
+    if (lineB.length === 0) {
+      // lineB is empty but lineA is not
+      return false;
+    }
+
+    // now both lines can not be empty
+
+    const firstCharA = lineA.charCodeAt(0);
+    const firstCharB = lineB.charCodeAt(0);
+
+    if (firstCharA !== firstCharB) {
+      return false;
+    }
+
+    // now firstCharA is equal to firstCharB, we only need to check the first char
+    if (firstCharA === 35 /* # */) {
+      continue;
+    }
+    // adguard conf
+    if (firstCharA === 33 /* ! */) {
+      continue;
+    }
+
+    if (
+      firstCharA === 47 /* / */
+      && lineA[1] === '/' && lineB[1] === '/'
+      && lineA[3] === '#' && lineB[3] === '#'
+    ) {
+      continue;
+    }
+
+    if (lineA !== lineB) {
+      return false;
+    }
+  }
+
+  // The file becomes larger
+  return !(index < linesABound);
+}
+
+export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
+  const linesALen = linesA.length;
+
+  const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => {
+    if (fs.existsSync(filePath)) {
+      return fileEqual(linesA, readFileByLine(filePath));
+    }
+
+    console.log(`${filePath} does not exists, writing...`);
+    return false;
+  });
+
+  if (isEqual) {
+    console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`)));
+    return;
+  }
+
+  await span.traceChildAsync(`writing ${filePath}`, async () => {
+    // The default highwater mark is normally 16384,
+    // So we make sure direct write to file if the content is
+    // most likely less than 500 lines
+    if (linesALen < 500) {
+      return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n');
+    }
+
+    const writeStream = fs.createWriteStream(filePath);
+    for (let i = 0; i < linesALen; i++) {
+      const p = asyncWriteToStream(writeStream, linesA[i] + '\n');
+      // eslint-disable-next-line no-await-in-loop -- stream high water mark
+      if (p) await p;
+    }
+
+    writeStream.end();
+  });
+}

Build/lib/rules/base.ts

@@ -1,13 +1,7 @@
 import type { Span } from '../../trace';
 import { HostnameSmolTrie } from '../trie';
 import { invariant, not } from 'foxts/guard';
-import picocolors from 'picocolors';
-import fs from 'node:fs';
-import { writeFile } from '../misc';
 import type { MaybePromise } from '../misc';
-import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
-import { readFileByLine } from '../fetch-text-by-line';
-import { asyncWriteToStream } from 'foxts/async-write-to-stream';
 import type { BaseWriteStrategy } from '../writing-strategy/base';
 import { merge as mergeCidr } from 'fast-cidr-tools';
 import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
@@ -496,105 +490,3 @@ export class FileOutput {
     return this;
   }
 }
-
-export async function fileEqual(linesA: string[], source: AsyncIterable<string> | Iterable<string>): Promise<boolean> {
-  if (linesA.length === 0) {
-    return false;
-  }
-
-  const linesABound = linesA.length - 1;
-
-  let index = -1;
-  for await (const lineB of source) {
-    index++;
-
-    if (index > linesABound) {
-      return (index === linesA.length && lineB.length === 0);
-    }
-
-    const lineA = linesA[index];
-
-    if (lineA.length === 0) {
-      if (lineB.length === 0) {
-        // both lines are empty, check next line
-        continue;
-      }
-      // lineA is empty but lineB is not
-      return false;
-    }
-    // now lineA can not be empty
-    if (lineB.length === 0) {
-      // lineB is empty but lineA is not
-      return false;
-    }
-
-    // now both lines can not be empty
-
-    const firstCharA = lineA.charCodeAt(0);
-    const firstCharB = lineB.charCodeAt(0);
-
-    if (firstCharA !== firstCharB) {
-      return false;
-    }
-
-    // now firstCharA is equal to firstCharB, we only need to check the first char
-    if (firstCharA === 35 /* # */) {
-      continue;
-    }
-    // adguard conf
-    if (firstCharA === 33 /* ! */) {
-      continue;
-    }
-
-    if (
-      firstCharA === 47 /* / */
-      && lineA[1] === '/' && lineB[1] === '/'
-      && lineA[3] === '#' && lineB[3] === '#'
-    ) {
-      continue;
-    }
-
-    if (lineA !== lineB) {
-      return false;
-    }
-  }
-
-  // The file becomes larger
-  return !(index < linesABound);
-}
-
-export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
-  const linesALen = linesA.length;
-
-  const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => {
-    if (fs.existsSync(filePath)) {
-      return fileEqual(linesA, readFileByLine(filePath));
-    }
-
-    console.log(`${filePath} does not exists, writing...`);
-    return false;
-  });
-
-  if (isEqual) {
-    console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`)));
-    return;
-  }
-
-  await span.traceChildAsync(`writing ${filePath}`, async () => {
-    // The default highwater mark is normally 16384,
-    // So we make sure direct write to file if the content is
-    // most likely less than 500 lines
-    if (linesALen < 500) {
-      return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n');
-    }
-
-    const writeStream = fs.createWriteStream(filePath);
-    for (let i = 0; i < linesALen; i++) {
-      const p = asyncWriteToStream(writeStream, linesA[i] + '\n');
-      // eslint-disable-next-line no-await-in-loop -- stream high water mark
-      if (p) await p;
-    }
-
-    writeStream.end();
-  });
-}