Refactor: migrate to `hntrie`

Build/build-cdn-download-conf.worker.ts

@@ -6,7 +6,7 @@ import { appendArrayInPlace } from 'foxts/append-array-in-place';
 import { SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput } from './lib/rules/domainset';
 import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
-import { HostnameTrie } from './lib/trie';
+import { HostnameTrie } from 'hntrie';
 import { $$fetch } from './lib/fetch-retry';
 import { fastUri } from 'fast-uri';
 

Build/build-microsoft-cdn.worker.ts

@@ -2,7 +2,7 @@ import { task } from './trace';
 import { SHARED_DESCRIPTION } from './constants/description';
 import { RulesetOutput } from './lib/rules/ruleset';
 import { RULES, PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } from './constants/microsoft-cdn';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { appendArrayInPlace } from 'foxts/append-array-in-place';
 import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';

Build/build-reject-domainset.ts

@@ -263,7 +263,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
     }
 
     // Deduplicate reject_extra and reject_phishing from the base reject domainset
-    rejectDomainsetOutput.domainTrie.dump(arg => {
+    rejectDomainsetOutput.domainTrie.dump((domain, includeSubdomain) => {
+      const arg = includeSubdomain ? '.' + domain : domain;
       rejectExtraDomainsetOutput.whitelistDomain(arg);
       rejectPhisingDomainsetOutput.whitelistDomain(arg);
 

Build/lib/fetch-text-by-line.ts

@@ -30,6 +30,7 @@ export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | U
   }
 
   const resultStream = webStream
+    // @ts-expect-error -- mismatched Node.js and web types
     .pipeThrough(new TextDecoderStream())
     .pipeThrough(new TextLineStream({ skipEmptyLines: processLine }));
 

Build/lib/rules/base.ts

@@ -1,5 +1,6 @@
 import type { Span } from '../../trace';
-import { HostnameSmolTrie } from '../trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
+import { domainToASCII } from 'node:url';
 import { not, nullthrow } from 'foxts/guard';
 import { fastIpVersion } from 'foxts/fast-ip-version';
 import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
@@ -20,7 +21,7 @@ export class FileOutput {
 
   protected dataSource = new Set<string>();
 
-  public domainTrie = new HostnameSmolTrie(null);
+  public domainTrie = new HostnameSmolTrie();
   public wildcardSet = new Set<string>();
 
   protected domainKeywords = new Set<string>();
@@ -116,14 +117,18 @@ export class FileOutput {
     for (let i = 0, len = domains.length; i < len; i++) {
       const d = domains[i];
       if (d !== null) {
-        this.domainTrie.add(d, false, null, 0);
+        this.domainTrie.add(d);
       }
     }
     return this;
   }
 
-  addDomainSuffix(domain: string, lineFromDot = domain[0] === '.') {
-    this.domainTrie.add(domain, true, null, lineFromDot ? 1 : 0);
+  addDomainSuffix(domain: string) {
+    if (domain[0] === '.') {
+      this.domainTrie.add(domain);
+    } else {
+      this.domainTrie.addSubdomain(domain);
+    }
     return this;
   }
 
@@ -179,9 +184,9 @@ export class FileOutput {
       }
 
       if (line[0] === '.') {
-        this.addDomainSuffix(line, true);
+        this.addDomainSuffix(line);
       } else {
-        this.domainTrie.add(line, false, null, 0);
+        this.domainTrie.add(line);
       }
     }
   }
@@ -210,10 +215,10 @@ export class FileOutput {
 
       switch (type) {
         case 'DOMAIN':
-          this.domainTrie.add(value, false, null, 0);
+          this.domainTrie.add(value);
           break;
         case 'DOMAIN-SUFFIX':
-          this.addDomainSuffix(value, false);
+          this.domainTrie.addSubdomain(value);
           break;
         case 'DOMAIN-KEYWORD':
           this.addDomainKeyword(value);
@@ -416,9 +421,16 @@ export class FileOutput {
 
     const strategiesLen = this.strategies.length;
 
-    this.domainTrie.dumpWithoutDot((domain, includeAllSubdomain) => {
+    const domainEntries: Array<[domain: string, subdomain: boolean]> = [];
+    this.domainTrie.dump((domain, includeSubdomain) => {
+      const d = domainToASCII(domain);
+      if (d) domainEntries.push([d, includeSubdomain]);
+    });
+    domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+    for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) {
+      const [domain, includeAllSubdomain] = domainEntries[j];
       if (kwfilter(domain)) {
-        return;
+        continue;
       }
 
       for (let i = 0; i < strategiesLen; i++) {
@@ -429,7 +441,7 @@ export class FileOutput {
           strategy.writeDomain(domain);
         }
       }
-    }, true);
+    }
 
     // Now, we whitelisted out DOMAIN-KEYWORD
     const whiteKwfilter = createKeywordFilter(Array.from(this.whitelistKeywords));

Build/lib/trie.test.ts

@@ -1,411 +0,0 @@
-import { describe, it } from 'mocha';
-import { expect } from 'earl';
-import { HostnameSmolTrie, HostnameTrie } from './trie';
-
-function createTrie<Meta = any>(from: string[] | Set<string> | null, smolTree: true): HostnameSmolTrie<Meta>;
-function createTrie<Meta = any>(from?: string[] | Set<string> | null, smolTree?: false): HostnameTrie<Meta>;
-function createTrie<_Meta = any>(from?: string[] | Set<string> | null, smolTree = true) {
-  if (smolTree) {
-    return new HostnameSmolTrie(from);
-  }
-  return new HostnameTrie(from);
-};
-
-// describe('hostname to tokens', () => {
-//   it('should split hostname into tokens.', () => {
-//     expect(hostnameToTokens('.blog.skk.moe')).toEqual([
-//       '.',
-//       'blog',
-//       '.',
-//       'skk',
-//       '.',
-//       'moe'
-//     ]);
-
-//     expect(hostnameToTokens('blog.skk.moe')).toEqual([
-//       'blog',
-//       '.',
-//       'skk',
-//       '.',
-//       'moe'
-//     ]);
-
-//     expect(hostnameToTokens('skk.moe')).toEqual([
-//       'skk',
-//       '.',
-//       'moe'
-//     ]);
-
-//     expect(hostnameToTokens('moe')).toEqual([
-//       'moe'
-//     ]);
-//   });
-// });
-
-describe('Trie', () => {
-  it('should be possible to add domains to a Trie.', () => {
-    const trie = createTrie(null, false);
-
-    trie.add('a.skk.moe');
-    trie.add('skk.moe');
-    trie.add('anotherskk.moe');
-
-    expect(trie.size).toEqual(3);
-
-    expect(trie.has('a.skk.moe')).toEqual(true);
-    expect(trie.has('skk.moe')).toEqual(true);
-    expect(trie.has('anotherskk.moe')).toEqual(true);
-    expect(trie.has('example.com')).toEqual(false);
-    expect(trie.has('skk.mo')).toEqual(false);
-    expect(trie.has('another.skk.moe')).toEqual(false);
-  });
-
-  it('adding the same item several times should not increase size.', () => {
-    const trie = createTrie(null, false);
-
-    trie.add('skk.moe');
-    trie.add('blog.skk.moe');
-    // eslint-disable-next-line sukka/no-element-overwrite -- deliberately do testing
-    trie.add('skk.moe');
-
-    expect(trie.size).toEqual(2);
-    expect(trie.has('skk.moe')).toEqual(true);
-  });
-
-  it('should be possible to set the null sequence.', () => {
-    const trie = createTrie(null, false);
-
-    trie.add('');
-    expect(trie.has('')).toEqual(true);
-
-    const trie2 = createTrie(null, true);
-    trie2.add('');
-    expect(trie2.has('')).toEqual(true);
-  });
-
-  it('should be possible to delete items.', () => {
-    const trie = createTrie(null, false);
-
-    trie.add('skk.moe');
-    trie.add('blog.skk.moe');
-    trie.add('example.com');
-    trie.add('moe.sb');
-
-    expect(trie.delete('no-match.com')).toEqual(false);
-    expect(trie.delete('example.org')).toEqual(false);
-
-    expect(trie.delete('skk.moe')).toEqual(true);
-    expect(trie.has('skk.moe')).toEqual(false);
-    expect(trie.has('moe.sb')).toEqual(true);
-
-    expect(trie.size).toEqual(3);
-
-    expect(trie.delete('example.com')).toEqual(true);
-    expect(trie.size).toEqual(2);
-    expect(trie.delete('moe.sb')).toEqual(true);
-    expect(trie.size).toEqual(1);
-  });
-
-  it('should be possible to check the existence of a sequence in the Trie.', () => {
-    const trie = createTrie(null, true);
-
-    trie.add('example.org.skk.moe');
-
-    expect(trie.has('example.org.skk.moe')).toEqual(true);
-    expect(trie.has('skk.moe')).toEqual(false);
-    expect(trie.has('example.org')).toEqual(false);
-    expect(trie.has('')).toEqual(false);
-  });
-
-  it('should be possible to retrieve items matching the given prefix.', () => {
-    const trie = createTrie(null, false);
-
-    trie.add('example.com');
-    trie.add('blog.example.com');
-    trie.add('cdn.example.com');
-    trie.add('example.org');
-
-    expect(trie.find('example.com')).toEqual(['example.com', 'cdn.example.com', 'blog.example.com']);
-    expect(trie.find('com')).toEqual(['example.com', 'cdn.example.com', 'blog.example.com']);
-    expect(trie.find('.example.com')).toEqual(['cdn.example.com', 'blog.example.com']);
-    expect(trie.find('org')).toEqual(['example.org']);
-    expect(trie.find('example.net')).toEqual([]);
-    expect(trie.dump()).toEqual(['example.org', 'example.com', 'cdn.example.com', 'blog.example.com']);
-  });
-
-  it('should be possible to retrieve items matching the given prefix even with a smol trie', () => {
-    const trie = createTrie(null, true);
-
-    trie.add('.example.com');
-    trie.add('example.com');
-    trie.add('blog.example.com');
-    trie.add('cdn.example.com');
-    trie.add('example.org');
-
-    expect(trie.find('example.com')).toEqual(['.example.com']);
-    expect(trie.find('com')).toEqual(['.example.com']);
-    expect(trie.find('.example.com')).toEqual(['.example.com']);
-    expect(trie.find('org')).toEqual(['example.org']);
-    expect(trie.find('example.net')).toEqual([]);
-    expect(trie.dump()).toEqual(['example.org', '.example.com']);
-  });
-
-  it('should be possible to create a trie from an arbitrary iterable.', () => {
-    let trie = createTrie(['skk.moe', 'blog.skk.moe'], false);
-
-    expect(trie.size).toEqual(2);
-    expect(trie.has('skk.moe')).toEqual(true);
-
-    trie = createTrie(new Set(['skk.moe', 'example.com']), false);
-    expect(trie.size).toEqual(2);
-    expect(trie.has('skk.moe')).toEqual(true);
-  });
-});
-
-describe('surge domainset dedupe', () => {
-  it('should not remove same entry', () => {
-    const trie = createTrie(['.skk.moe', 'noc.one'], false);
-
-    expect(trie.find('.skk.moe')).toEqual(['.skk.moe']);
-    expect(trie.find('noc.one')).toEqual(['noc.one']);
-  });
-
-  it('should match subdomain - 1', () => {
-    const trie = createTrie(['www.noc.one', 'www.sukkaw.com', 'blog.skk.moe', 'image.cdn.skk.moe', 'cdn.sukkaw.net'], false);
-
-    expect(trie.find('.skk.moe')).toEqual(['image.cdn.skk.moe', 'blog.skk.moe']);
-    expect(trie.find('.sukkaw.com')).toEqual(['www.sukkaw.com']);
-  });
-
-  it('should match subdomain - 2', () => {
-    const trie = createTrie(['www.noc.one', 'www.sukkaw.com', '.skk.moe', 'blog.skk.moe', 'image.cdn.skk.moe', 'cdn.sukkaw.net'], false);
-
-    expect(trie.find('.skk.moe')).toEqual(['.skk.moe', 'image.cdn.skk.moe', 'blog.skk.moe']);
-    expect(trie.find('.sukkaw.com')).toEqual(['www.sukkaw.com']);
-  });
-
-  it('should not remove non-subdomain', () => {
-    const trie = createTrie(['skk.moe', 'sukkaskk.moe'], false);
-    expect(trie.find('.skk.moe')).toEqual([]);
-  });
-});
-
-describe('smol tree', () => {
-  it('should init tree', () => {
-    const trie = createTrie([
-      'skk.moe',
-      'anotherskk.moe',
-      'blog.anotherskk.moe',
-      'blog.skk.moe',
-      '.cdn.local',
-      'blog.img.skk.local',
-      'img.skk.local'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      'img.skk.local',
-      'blog.img.skk.local',
-      '.cdn.local',
-      'anotherskk.moe',
-      'blog.anotherskk.moe',
-      'skk.moe',
-      'blog.skk.moe'
-    ]);
-  });
-
-  it('should create simple tree - 1', () => {
-    const trie = createTrie([
-      '.skk.moe', 'blog.skk.moe', '.cdn.skk.moe', 'skk.moe',
-      'www.noc.one', 'cdn.noc.one',
-      '.blog.sub.example.com', 'sub.example.com', 'cdn.sub.example.com', '.sub.example.com'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      '.sub.example.com',
-      'cdn.noc.one',
-      'www.noc.one',
-      '.skk.moe'
-    ]);
-  });
-
-  it('should create simple tree - 2', () => {
-    const trie = createTrie([
-      '.skk.moe', 'blog.skk.moe', '.cdn.skk.moe', 'skk.moe'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      '.skk.moe'
-    ]);
-  });
-
-  it('should create simple tree - 3', () => {
-    const trie = createTrie([
-      '.blog.sub.example.com', 'cdn.sub.example.com', '.sub.example.com'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      '.sub.example.com'
-    ]);
-
-    trie.add('.sub.example.com');
-    expect(trie.dump()).toEqual([
-      '.sub.example.com'
-    ]);
-  });
-
-  it('should create simple tree - 3', () => {
-    const trie = createTrie([
-      'commercial.shouji.360.cn',
-      'act.commercial.shouji.360.cn',
-      'cdn.creative.medialytics.com',
-      'px.cdn.creative.medialytics.com'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      'cdn.creative.medialytics.com',
-      'px.cdn.creative.medialytics.com',
-      'commercial.shouji.360.cn',
-      'act.commercial.shouji.360.cn'
-    ]);
-  });
-
-  it('should dedupe subdomain properly', () => {
-    const trie = createTrie([
-      'skk.moe',
-      'anotherskk.moe',
-      'blog.anotherskk.moe',
-      'blog.skk.moe'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      'anotherskk.moe',
-      'blog.anotherskk.moe',
-      'skk.moe',
-      'blog.skk.moe'
-    ]);
-  });
-
-  it('should effctly whitelist domains', () => {
-    const trie = createTrie([
-      'skk.moe',
-      'anotherskk.moe',
-      'blog.anotherskk.moe',
-      'blog.skk.moe',
-      '.cdn.local',
-      'blog.img.skk.local',
-      'img.skk.local'
-    ], true);
-
-    trie.whitelist('.skk.moe');
-
-    expect(trie.dump()).toEqual([
-      'img.skk.local',
-      'blog.img.skk.local',
-      '.cdn.local',
-      'anotherskk.moe',
-      'blog.anotherskk.moe'
-    ]);
-
-    trie.whitelist('anotherskk.moe');
-    expect(trie.dump()).toEqual([
-      'img.skk.local',
-      'blog.img.skk.local',
-      '.cdn.local',
-      'blog.anotherskk.moe'
-    ]);
-
-    trie.add('anotherskk.moe');
-    trie.whitelist('.anotherskk.moe');
-
-    expect(trie.dump()).toEqual([
-      'img.skk.local',
-      'blog.img.skk.local',
-      '.cdn.local'
-    ]);
-
-    trie.whitelist('img.skk.local');
-    expect(trie.dump()).toEqual([
-      'blog.img.skk.local',
-      '.cdn.local'
-    ]);
-
-    trie.whitelist('cdn.local');
-    expect(trie.dump()).toEqual([
-      'blog.img.skk.local'
-    ]);
-
-    trie.whitelist('.skk.local');
-    expect(trie.dump()).toEqual([]);
-  });
-
-  it('should whitelist trie correctly', () => {
-    const trie = createTrie([
-      '.t.co',
-      't.co',
-      'example.t.co',
-      '.skk.moe',
-      'blog.cdn.example.com',
-      'cdn.example.com'
-    ], true);
-
-    expect(trie.dump()).toEqual([
-      'cdn.example.com', 'blog.cdn.example.com',
-      '.skk.moe',
-      '.t.co'
-    ]);
-
-    trie.whitelist('.t.co');
-    expect(trie.dump()).toEqual([
-      'cdn.example.com', 'blog.cdn.example.com', '.skk.moe'
-    ]);
-
-    trie.whitelist('skk.moe');
-    expect(trie.dump()).toEqual(['cdn.example.com', 'blog.cdn.example.com']);
-
-    trie.whitelist('cdn.example.com');
-    expect(trie.dump()).toEqual(['blog.cdn.example.com']);
-  });
-
-  it('contains - normal', () => {
-    const trie = createTrie([
-      'skk.moe',
-      'anotherskk.moe',
-      'blog.anotherskk.moe',
-      'blog.skk.moe'
-    ], true);
-
-    expect(trie.contains('skk.moe')).toEqual(true);
-    expect(trie.contains('blog.skk.moe')).toEqual(true);
-    expect(trie.contains('anotherskk.moe')).toEqual(true);
-    expect(trie.contains('blog.anotherskk.moe')).toEqual(true);
-
-    expect(trie.contains('example.com')).toEqual(false);
-    expect(trie.contains('blog.example.com')).toEqual(false);
-    expect(trie.contains('skk.mo')).toEqual(false);
-    expect(trie.contains('cdn.skk.moe')).toEqual(false);
-  });
-
-  it('contains - subdomain', () => {
-    const trie = createTrie([
-      'index.rubygems.org'
-    ], true);
-
-    expect(trie.contains('rubygems.org')).toEqual(false);
-    expect(trie.contains('index.rubygems.org')).toEqual(true);
-    expect(trie.contains('sub.index.rubygems.org')).toEqual(false);
-  });
-
-  it('contains - include subdomains', () => {
-    const trie = createTrie([
-      '.skk.moe'
-    ], true);
-
-    expect(trie.contains('skk.moe')).toEqual(true);
-    expect(trie.contains('blog.skk.moe')).toEqual(true);
-    expect(trie.contains('image.cdn.skk.moe')).toEqual(true);
-
-    expect(trie.contains('example.com')).toEqual(false);
-    expect(trie.contains('blog.example.com')).toEqual(false);
-    expect(trie.contains('skk.mo')).toEqual(false);
-  });
-});

Build/lib/trie.ts

@@ -1,665 +0,0 @@
-/**
- * Hostbane-Optimized Trie based on Mnemonist Trie
- */
-
-import { fastStringCompare } from 'foxts/fast-string-compare';
-import util from 'node:util';
-import { noop } from 'foxts/noop';
-import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
-
-import { deleteBit, getBit, missingBit, setBit } from 'foxts/bitwise';
-import { domainToASCII } from 'node:url';
-
-const START = 1 << 1;
-const INCLUDE_ALL_SUBDOMAIN = 1 << 2;
-
-type TrieNode<Meta = any> = [
-  /** end, includeAllSubdomain (.example.org, ||example.com) */ flag: number,
-  /** parent */ TrieNode | null,
-  /** children */ Map<string, TrieNode>,
-  /** token */ token: string,
-  /** meta */ Meta
-];
-
-function deepTrieNodeToJSON<Meta = unknown>(node: TrieNode,
-  unpackMeta: ((meta?: Meta) => string) | undefined) {
-  const obj: Record<string, unknown> = {
-    ['[start]']: getBit(node[0], START),
-    ['[subdomain]']: getBit(node[0], INCLUDE_ALL_SUBDOMAIN)
-  };
-
-  if (node[4] != null) {
-    if (unpackMeta) {
-      obj['[meta]'] = unpackMeta(node[4]);
-    } else {
-      obj['[meta]'] = node[4];
-    }
-  }
-  node[2].forEach((value, key) => {
-    obj[key] = deepTrieNodeToJSON(value, unpackMeta);
-  });
-  return obj;
-}
-
-const createNode = <Meta = unknown>(token: string, parent: TrieNode | null = null): TrieNode => [1, parent, new Map<string, TrieNode>(), token, null] as TrieNode<Meta>;
-
-function hostnameToTokens(hostname: string, hostnameFromIndex: number): string[] {
-  const tokens = hostname.split('.');
-  const results: string[] = [];
-  let token = '';
-
-  for (let i = hostnameFromIndex, l = tokens.length; i < l; i++) {
-    token = tokens[i];
-    if (token.length > 0) {
-      results.push(token);
-    } else {
-      throw new TypeError(JSON.stringify({ hostname, hostnameFromIndex }, null, 2));
-    }
-  }
-
-  return results;
-}
-
-function walkHostnameTokens(hostname: string, onToken: (token: string) => boolean | null, hostnameFromIndex: number): boolean | null {
-  const tokens = hostname.split('.');
-
-  const l = tokens.length - 1;
-
-  // we are at the first of hostname, no splitor there
-  let token = '';
-
-  for (let i = l; i >= hostnameFromIndex; i--) {
-    token = tokens[i];
-    if (token.length > 0) {
-      const t = onToken(token);
-      if (t === null) {
-        return null;
-      }
-      // if the callback returns true, we should skip the rest
-      if (t) {
-        return true;
-      }
-    }
-  }
-
-  return false;
-}
-
-interface FindSingleChildLeafResult<Meta> {
-  node: TrieNode<Meta>,
-  toPrune: TrieNode<Meta> | null,
-  tokenToPrune: string | null,
-  parent: TrieNode<Meta>
-}
-
-abstract class Triebase<Meta = unknown> {
-  protected readonly $root: TrieNode<Meta> = createNode('$root');
-  protected $size = 0;
-
-  get root() {
-    return this.$root;
-  }
-
-  constructor(from?: string[] | Set<string> | null) {
-    // Actually build trie
-    if (Array.isArray(from)) {
-      for (let i = 0, l = from.length; i < l; i++) {
-        this.add(from[i]);
-      }
-    } else if (from) {
-      from.forEach((value) => this.add(value));
-    }
-  }
-
-  public abstract add(suffix: string, includeAllSubdomain?: boolean, meta?: Meta, hostnameFromIndex?: number): void;
-
-  protected walkIntoLeafWithTokens(
-    tokens: string[],
-    onLoop: (node: TrieNode, parent: TrieNode, token: string) => void = noop
-  ) {
-    let node: TrieNode = this.$root;
-    let parent: TrieNode = node;
-
-    let token: string;
-    let child: Map<string, TrieNode<Meta>> = node[2];
-
-    // reverse lookup from end to start
-    for (let i = tokens.length - 1; i >= 0; i--) {
-      token = tokens[i];
-
-      // if (token === '') {
-      //   break;
-      // }
-
-      parent = node;
-
-      child = node[2];
-      // cache node index access is 20% faster than direct access when doing twice
-      if (child.has(token)) {
-        node = child.get(token)!;
-      } else {
-        return null;
-      }
-
-      onLoop(node, parent, token);
-    }
-
-    return { node, parent };
-  };
-
-  protected walkIntoLeafWithSuffix(
-    suffix: string,
-    hostnameFromIndex: number,
-    onLoop: (node: TrieNode, parent: TrieNode, token: string) => void = noop
-  ) {
-    let node: TrieNode = this.$root;
-    let parent: TrieNode = node;
-
-    let child: Map<string, TrieNode<Meta>> = node[2];
-
-    const onToken = (token: string) => {
-      // if (token === '') {
-      //   return true;
-      // }
-
-      parent = node;
-
-      child = node[2];
-
-      if (child.has(token)) {
-        node = child.get(token)!;
-      } else {
-        return null;
-      }
-
-      onLoop(node, parent, token);
-
-      return false;
-    };
-
-    if (walkHostnameTokens(suffix, onToken, hostnameFromIndex) === null) {
-      return null;
-    }
-
-    return { node, parent };
-  };
-
-  public contains(suffix: string, includeAllSubdomain = suffix[0] === '.'): boolean {
-    const hostnameFromIndex = suffix[0] === '.' ? 1 : 0;
-
-    let node: TrieNode = this.$root;
-    // let parent: TrieNode = node;
-
-    let child: Map<string, TrieNode<Meta>> = node[2];
-
-    let result = false;
-
-    const onToken = (token: string) => {
-      // if (token === '') {
-      //   return true;
-      // }
-
-      // parent = node;
-
-      child = node[2];
-
-      if (child.has(token)) {
-        node = child.get(token)!;
-      } else {
-        if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) {
-          result = true;
-        }
-        return null;
-      }
-
-      return false;
-    };
-
-    if (walkHostnameTokens(suffix, onToken, hostnameFromIndex) === null) {
-      return result;
-    }
-
-    if (includeAllSubdomain) return getBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-    return getBit(node[0], START);
-
-    // if (res === null) return false;
-    // if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN);
-    // return true;
-  };
-
-  private static bfsResults: [node: TrieNode | null, suffix: string[]] = [null, []];
-
-  private static dfs<Meta>(this: void, nodeStack: Array<TrieNode<Meta>>, suffixStack: string[][]) {
-    const node = nodeStack.pop()!;
-    const suffix = suffixStack.pop()!;
-
-    node[2].forEach((childNode, k) => {
-      // Pushing the child node to the stack for next iteration of DFS
-      nodeStack.push(childNode);
-
-      suffixStack.push([k, ...suffix]);
-    });
-
-    Triebase.bfsResults[0] = node;
-    Triebase.bfsResults[1] = suffix;
-
-    return Triebase.bfsResults;
-  }
-
-  private static dfsWithSort<Meta>(this: void, nodeStack: Array<TrieNode<Meta>>, suffixStack: string[][]) {
-    const node = nodeStack.pop()!;
-    const suffix = suffixStack.pop()!;
-
-    const child = node[2];
-
-    if (child.size) {
-      const keys = Array.from(child.keys()).sort(Triebase.compare);
-
-      for (let i = 0, l = keys.length; i < l; i++) {
-        const key = keys[i];
-        const childNode = child.get(key)!;
-
-        // Pushing the child node to the stack for next iteration of DFS
-        nodeStack.push(childNode);
-        suffixStack.push([key, ...suffix]);
-      }
-    }
-
-    Triebase.bfsResults[0] = node;
-    Triebase.bfsResults[1] = suffix;
-
-    return Triebase.bfsResults;
-  }
-
-  private walk(
-    onMatches: (suffix: string[], subdomain: boolean, meta: Meta) => void,
-    withSort = false,
-    initialNode = this.$root,
-    initialSuffix: string[] = []
-  ) {
-    const dfsImpl = withSort ? Triebase.dfsWithSort : Triebase.dfs;
-
-    const nodeStack: Array<TrieNode<Meta>> = [initialNode];
-
-    // Resolving initial string (begin the start of the stack)
-    const suffixStack: string[][] = [initialSuffix];
-
-    let node: TrieNode<Meta> = initialNode;
-
-    do {
-      const r = dfsImpl(nodeStack, suffixStack);
-      node = r[0]!;
-      const suffix = r[1];
-
-      // If the node is a sentinel, we push the suffix to the results
-      if (getBit(node[0], START)) {
-        onMatches(suffix, getBit(node[0], INCLUDE_ALL_SUBDOMAIN), node[4]);
-      }
-    } while (nodeStack.length);
-  };
-
-  static compare(this: void, a: string, b: string) {
-    if (a === b) return 0;
-    return (a.length - b.length) || fastStringCompare(a, b);
-  }
-
-  protected getSingleChildLeaf(tokens: string[]): FindSingleChildLeafResult<Meta> | null {
-    let toPrune: TrieNode | null = null;
-    let tokenToPrune: string | null = null;
-
-    const onLoop = (node: TrieNode, parent: TrieNode, token: string) => {
-      // Keeping track of a potential branch to prune
-
-      const child = node[2];
-
-      const childSize = child.size + (getBit(node[0], INCLUDE_ALL_SUBDOMAIN) ? 1 : 0);
-
-      if (toPrune !== null) { // the most near branch that could potentially being pruned
-        if (childSize >= 1) {
-          // The branch has some children, the branch need retain.
-          // And we need to abort prune that parent branch, so we set it to null
-          toPrune = null;
-          tokenToPrune = null;
-        }
-      } else if (childSize < 1) {
-        // There is only one token child, or no child at all, we can prune it safely
-        // It is now the top-est branch that could potentially being pruned
-        toPrune = parent;
-        tokenToPrune = token;
-      }
-    };
-
-    const res = this.walkIntoLeafWithTokens(tokens, onLoop);
-
-    if (res === null) return null;
-    return { node: res.node, toPrune, tokenToPrune, parent: res.parent };
-  };
-
-  /**
-   * Method used to retrieve every item in the trie with the given prefix.
-   */
-  public find(
-    inputSuffix: string,
-    subdomainOnly = inputSuffix[0] === '.',
-    hostnameFromIndex = inputSuffix[0] === '.' ? 1 : 0
-    // /** @default true */ includeEqualWithSuffix = true
-  ): string[] {
-    const inputTokens = hostnameToTokens(inputSuffix, hostnameFromIndex);
-    const res = this.walkIntoLeafWithTokens(inputTokens);
-    if (res === null) return [];
-
-    const results: string[] = [];
-
-    const onMatches = subdomainOnly
-      ? (suffix: string[], subdomain: boolean) => { // fast path (default option)
-        const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
-        if (!subdomain && subStringEqual(inputSuffix, d, 1)) return;
-
-        results.push(subdomain ? '.' + d : d);
-      }
-      : (suffix: string[], subdomain: boolean) => { // fast path (default option)
-        const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
-        results.push(subdomain ? '.' + d : d);
-      };
-
-    this.walk(
-      onMatches,
-      false,
-      res.node, // Performing DFS from prefix
-      inputTokens
-    );
-
-    return results;
-  };
-
-  /**
-   * Method used to delete a prefix from the trie.
-   */
-  public remove(suffix: string): boolean {
-    const res = this.getSingleChildLeaf(hostnameToTokens(suffix, 0));
-    if (res === null) return false;
-
-    if (missingBit(res.node[0], START)) return false;
-
-    this.$size--;
-    const { node, toPrune, tokenToPrune } = res;
-
-    if (tokenToPrune && toPrune) {
-      toPrune[2].delete(tokenToPrune);
-    } else {
-      node[0] = deleteBit(node[0], START);
-    }
-
-    return true;
-  };
-
-  // eslint-disable-next-line @typescript-eslint/unbound-method -- safe
-  public delete = this.remove;
-
-  /**
-   * Method used to assert whether the given prefix exists in the Trie.
-   */
-  public has(suffix: string, includeAllSubdomain = suffix[0] === '.'): boolean {
-    const hostnameFromIndex = suffix[0] === '.' ? 1 : 0;
-
-    const res = this.walkIntoLeafWithSuffix(suffix, hostnameFromIndex);
-
-    if (res === null) return false;
-    if (missingBit(res.node[0], START)) return false;
-    if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN);
-    return true;
-  };
-
-  public dumpWithoutDot(onSuffix: (suffix: string, subdomain: boolean) => void, withSort = false) {
-    const handleSuffix = (suffix: string[], subdomain: boolean) => {
-      onSuffix(domainToASCII(fastStringArrayJoin(suffix, '.')), subdomain);
-    };
-
-    this.walk(handleSuffix, withSort);
-  }
-
-  public dump(onSuffix: (suffix: string) => void, withSort?: boolean): void;
-  public dump(onSuffix?: null, withSort?: boolean): string[];
-  public dump(onSuffix?: ((suffix: string) => void) | null, withSort = false): string[] | void {
-    const results: string[] = [];
-
-    const handleSuffix = onSuffix
-      ? (suffix: string[], subdomain: boolean) => {
-        const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
-        onSuffix(subdomain ? '.' + d : d);
-      }
-      : (suffix: string[], subdomain: boolean) => {
-        const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
-        results.push(subdomain ? '.' + d : d);
-      };
-
-    this.walk(handleSuffix, withSort);
-
-    return results;
-  };
-
-  public dumpMeta(onMeta: (meta: Meta) => void, withSort?: boolean): void;
-  public dumpMeta(onMeta?: null, withSort?: boolean): Meta[];
-  public dumpMeta(onMeta?: ((meta: Meta) => void) | null, withSort = false): Meta[] | void {
-    const results: Meta[] = [];
-
-    const handleMeta = onMeta
-      ? (_suffix: string[], _subdomain: boolean, meta: Meta) => onMeta(meta)
-      : (_suffix: string[], _subdomain: boolean, meta: Meta) => results.push(meta);
-
-    this.walk(handleMeta, withSort);
-
-    return results;
-  };
-
-  public dumpWithMeta(onSuffix: (suffix: string, meta: Meta | undefined) => void, withSort?: boolean): void;
-  public dumpWithMeta(onMeta?: null, withSort?: boolean): Array<[string, Meta | undefined]>;
-  public dumpWithMeta(onSuffix?: ((suffix: string, meta: Meta | undefined) => void) | null, withSort = false): Array<[string, Meta | undefined]> | void {
-    const results: Array<[string, Meta | undefined]> = [];
-
-    const handleSuffix = onSuffix
-      ? (suffix: string[], subdomain: boolean, meta: Meta | undefined) => {
-        const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
-        return onSuffix(subdomain ? '.' + d : d, meta);
-      }
-      : (suffix: string[], subdomain: boolean, meta: Meta | undefined) => {
-        const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
-        results.push([subdomain ? '.' + d : d, meta]);
-      };
-
-    this.walk(handleSuffix, withSort);
-
-    return results;
-  };
-
-  public inspect(depth: number, unpackMeta?: (meta?: Meta) => any) {
-    return fastStringArrayJoin(
-      JSON.stringify(deepTrieNodeToJSON(this.$root, unpackMeta), null, 2).split('\n').map((line) => ' '.repeat(depth) + line),
-      '\n'
-    );
-  }
-
-  public [util.inspect.custom](depth: number) {
-    return this.inspect(depth);
-  };
-
-  public merge(trie: Triebase<Meta>) {
-    const handleSuffix = (suffix: string[], subdomain: boolean, meta: Meta) => {
-      this.add(fastStringArrayJoin(suffix, '.'), subdomain, meta);
-    };
-
-    trie.walk(handleSuffix);
-
-    return this;
-  }
-}
-
-export class HostnameSmolTrie<Meta = unknown> extends Triebase<Meta> {
-  public smolTree = true;
-
-  add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void {
-    let node: TrieNode<Meta> = this.$root;
-    let curNodeChildren: Map<string, TrieNode<Meta>> = node[2];
-
-    const onToken = (token: string) => {
-      curNodeChildren = node[2];
-      if (curNodeChildren.has(token)) {
-        node = curNodeChildren.get(token)!;
-
-        // During the adding of `[start]blog|.skk.moe` and find out that there is a `[start].skk.moe` in the trie, skip adding the rest of the node
-        if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) {
-          return true;
-        }
-      } else {
-        const newNode = createNode(token, node);
-        curNodeChildren.set(token, newNode);
-        node = newNode;
-      }
-
-      return false;
-    };
-
-    // When walkHostnameTokens returns true, we should skip the rest
-    if (walkHostnameTokens(suffix, onToken, hostnameFromIndex)) {
-      return;
-    }
-
-    // If we are in smolTree mode, we need to do something at the end of the loop
-    if (includeAllSubdomain) {
-      // Trying to add `[.]sub.example.com` where there is already a `blog.sub.example.com` in the trie
-
-      // Make sure parent `[start]sub.example.com` (without dot) is removed (SETINEL to false)
-      // (/** parent */ node[2]!)[0] = false;
-
-      // Removing the rest of the parent's child nodes
-      node[2].clear();
-      // The SENTINEL of this node will be set to true at the end of the function, so we don't need to set it here
-
-      // we can use else-if here, because the children is now empty, we don't need to check the leading "."
-    } else if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) {
-      // Trying to add `example.com` when there is already a `.example.com` in the trie
-      // No need to increment size and set SENTINEL to true (skip this "new" item)
-      return;
-    }
-
-    node[0] = setBit(node[0], START);
-    if (includeAllSubdomain) {
-      node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-    } else {
-      node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-    }
-    node[4] = meta!;
-  }
-
-  public whitelist(suffix: string, includeAllSubdomain = suffix[0] === '.', hostnameFromIndex = suffix[0] === '.' ? 1 : 0) {
-    const tokens = hostnameToTokens(suffix, hostnameFromIndex);
-    const res = this.getSingleChildLeaf(tokens);
-    if (res === null) return;
-
-    const { node, toPrune, tokenToPrune } = res;
-
-    // Trying to whitelist `[start].sub.example.com` where there might already be a `[start]blog.sub.example.com` in the trie
-    if (includeAllSubdomain) {
-      // If there is a `[start]sub.example.com` here, remove it
-      node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-      // Removing all the child nodes by empty the children
-      node[2].clear();
-      // we do not remove sub.example.com for now, we will do that later
-    } else {
-      // Trying to whitelist `example.com` when there is already a `.example.com` in the trie
-      node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-    }
-
-    if (includeAllSubdomain) {
-      node[1]?.[2].delete(node[3]);
-    } else if (missingBit(node[0], START) && node[1]) {
-      return;
-    }
-
-    if (toPrune && tokenToPrune) {
-      toPrune[2].delete(tokenToPrune);
-    } else {
-      node[0] = deleteBit(node[0], START);
-    }
-
-    cleanUpEmptyTrailNode(node);
-  };
-}
-
-function cleanUpEmptyTrailNode(node: TrieNode<unknown>) {
-  let current = node;
-  while (
-    // the current node is not an "end node", a.k.a. not the start of a domain
-    missingBit(current[0], START)
-    // also no leading "." (no subdomain)
-    && missingBit(current[0], INCLUDE_ALL_SUBDOMAIN)
-    // child is empty
-    && current[2].size === 0
-    // has parent: we need to delete the current node from the parent
-    // we also need to clean up the parent node
-    && current[1]
-  ) {
-    current[1][2].delete(current[3]);
-    current = current[1];
-  }
-}
-
-export class HostnameTrie<Meta = unknown> extends Triebase<Meta> {
-  get size() {
-    return this.$size;
-  }
-
-  add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void {
-    let node: TrieNode<Meta> = this.$root;
-    let child: Map<string, TrieNode<Meta>> = node[2];
-
-    const onToken = (token: string) => {
-      child = node[2];
-      if (child.has(token)) {
-        node = child.get(token)!;
-      } else {
-        const newNode = createNode(token, node);
-        child.set(token, newNode);
-        node = newNode;
-      }
-
-      return false;
-    };
-
-    // When walkHostnameTokens returns true, we should skip the rest
-    if (walkHostnameTokens(suffix, onToken, hostnameFromIndex)) {
-      return;
-    }
-
-    // if same entry has been added before, skip
-    if (getBit(node[0], START)) {
-      return;
-    }
-
-    this.$size++;
-
-    node[0] = setBit(node[0], START);
-    if (includeAllSubdomain) {
-      node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-    } else {
-      node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
-    }
-    node[4] = meta!;
-  }
-}
-
-// function deepEqualArray(a: string[], b: string[]) {
-//   let len = a.length;
-//   if (len !== b.length) return false;
-//   while (len--) {
-//     if (a[len] !== b[len]) return false;
-//   }
-//   return true;
-// };
-
-function subStringEqual(needle: string, haystack: string, needleIndex = 0) {
-  for (let i = 0, l = haystack.length; i < l; i++) {
-    if (needle[i + needleIndex] !== haystack[i]) return false;
-  }
-  return true;
-}

Build/tools-dedupe-src.ts

@@ -4,7 +4,7 @@ import fsp from 'node:fs/promises';
 import { SOURCE_DIR } from './constants/dir';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { processLine } from './lib/process-line';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
 import { task } from './trace';
 
 const ENFORCED_WHITELIST = [
@@ -48,6 +48,15 @@ task(require.main === module, __filename)(async (span) => {
   await Promise.all(files.map(file => span.traceChildAsync('dedupe ' + file, () => dedupeFile(file, whiteTrie))));
 });
 
+function trieHasEntry(trie: HostnameSmolTrie, line: string): boolean {
+  if (line[0] === '.') return trie.hasSubdomain(line.slice(1));
+  return trie.has(line) || trie.hasSubdomain(line);
+}
+
+function trieContains(trie: HostnameSmolTrie, line: string): boolean {
+  return trie.match(line);
+}
+
 async function dedupeFile(file: string, whitelist: HostnameSmolTrie) {
   const result: string[] = [];
 
@@ -55,8 +64,7 @@ async function dedupeFile(file: string, whitelist: HostnameSmolTrie) {
 
   let line: string | null = '';
 
-  // eslint-disable-next-line @typescript-eslint/unbound-method -- .call
-  let trieHasOrContains = HostnameSmolTrie.prototype.has;
+  let trieCheck = trieHasEntry;
 
   for await (const l of readFileByLine(file)) {
     line = processLine(l);
@@ -66,19 +74,18 @@ async function dedupeFile(file: string, whitelist: HostnameSmolTrie) {
         return;
       }
       if (l.startsWith('# $ dedupe_use_trie_contains')) {
-        // eslint-disable-next-line @typescript-eslint/unbound-method -- .call
-        trieHasOrContains = HostnameSmolTrie.prototype.contains;
+        trieCheck = trieContains;
       }
 
       result.push(l); // keep all comments and blank lines
       continue;
     }
 
-    if (trieHasOrContains.call(trie, line)) {
+    if (trieCheck(trie, line)) {
       continue; // drop duplicate
     }
 
-    if (whitelist.has(line)) {
+    if (trieHasEntry(whitelist, line)) {
       continue; // drop whitelisted items
     }
 

Build/tools-lum-apex-domains.ts

@@ -1,6 +1,7 @@
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import tldts from 'tldts-experimental';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
+import { domainToASCII } from 'node:url';
 import path from 'node:path';
 import { SOURCE_DIR } from './constants/dir';
 import runAgainstSourceFile from './lib/run-against-source-file';
@@ -32,11 +33,16 @@ import runAgainstSourceFile from './lib/run-against-source-file';
   }
 
   await runAgainstSourceFile(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), (domain, includeAllSubDomain) => {
-    trie.whitelist(domain, includeAllSubDomain);
+    trie.whitelist(includeAllSubDomain ? '.' + domain : domain);
   }, 'domainset');
   await runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip', 'reject.conf'), (domain, includeAllSubDomain) => {
-    trie.whitelist(domain, includeAllSubDomain);
+    trie.whitelist(includeAllSubDomain ? '.' + domain : domain);
   }, 'ruleset');
 
-  console.log(trie.dump().map(i => '.' + i).join('\n'));
+  const dump: string[] = [];
+  trie.dump((rawDomain) => {
+    const domain = domainToASCII(rawDomain);
+    if (domain) dump.push(domain);
+  });
+  console.log(dump.map(i => '.' + i).join('\n'));
 })();

Build/tools-migrate-domains.ts

@@ -2,7 +2,8 @@
 import path from 'node:path';
 import { processFilterRulesWithPreload } from './lib/parse-filter/filters';
 import { processHosts } from './lib/parse-filter/hosts';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
+import { domainToASCII } from 'node:url';
 import { dummySpan } from './trace';
 import { SOURCE_DIR } from './constants/dir';
 import { PREDEFINED_WHITELIST } from './constants/reject-data-source';
@@ -15,7 +16,7 @@ import runAgainstSourceFile from './lib/run-against-source-file';
   // const { whiteDomainSuffixes, whiteDomains } = await writeFiltersToTrie(trie, 'https://cdn.jsdelivr.net/gh/Perflyst/PiHoleBlocklist@master/SmartTV-AGH.txt', true);
 
   const callback = (domain: string, includeAllSubDomain: boolean) => {
-    trie.whitelist(domain, includeAllSubDomain);
+    trie.whitelist(includeAllSubDomain ? '.' + domain : domain);
   };
 
   await runAgainstSourceFile(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), callback, 'domainset');
@@ -26,7 +27,12 @@ import runAgainstSourceFile from './lib/run-against-source-file';
   }
 
   console.log('---------------------------');
-  console.log(trie.dump().join('\n'));
+  const dump: string[] = [];
+  trie.dump((rawDomain, includeSubdomain) => {
+    const domain = domainToASCII(rawDomain);
+    if (domain) dump.push(includeSubdomain ? '.' + domain : domain);
+  });
+  console.log(dump.join('\n'));
   console.log('---------------------------');
   // console.log('whitelist domain suffixes:');
   // console.log(whiteDomainSuffixes.join('\n'));
@@ -46,16 +52,16 @@ async function writeHostsToTrie(trie: HostnameSmolTrie, hostsUrl: string, includ
 async function writeFiltersToTrie(trie: HostnameSmolTrie, filterUrl: string, includeThirdParty = false) {
   const { whiteDomainSuffixes, whiteDomains, blackDomainSuffixes, blackDomains } = await processFilterRulesWithPreload(filterUrl, [], includeThirdParty)(dummySpan);
   for (let i = 0, len = blackDomainSuffixes.length; i < len; i++) {
-    trie.add(blackDomainSuffixes[i], true);
+    trie.addSubdomain(blackDomainSuffixes[i]);
   }
   for (let i = 0, len = blackDomains.length; i < len; i++) {
-    trie.add(blackDomains[i], false);
+    trie.add(blackDomains[i]);
   }
   for (let i = 0, len = whiteDomainSuffixes.length; i < len; i++) {
-    trie.whitelist(whiteDomainSuffixes[i], true);
+    trie.whitelist('.' + whiteDomainSuffixes[i]);
   }
   for (let i = 0, len = whiteDomains.length; i < len; i++) {
-    trie.whitelist(whiteDomains[i], false);
+    trie.whitelist(whiteDomains[i]);
   }
 
   return { whiteDomainSuffixes, whiteDomains };

Build/validate-domestic.ts

@@ -4,7 +4,8 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
 import { $$fetch } from './lib/fetch-retry';
 import runAgainstSourceFile from './lib/run-against-source-file';
 import { getTopOneMillionDomains } from './validate-gfwlist';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
+import { domainToASCII } from 'node:url';
 import tldts from 'tldts-experimental';
 import { DOMESTICS } from '../Source/non_ip/domestic';
 
@@ -15,15 +16,15 @@ export async function parseDomesticList() {
 
   const resultTrie = new HostnameSmolTrie();
 
-  topDomainTrie.dumpWithoutDot((domain) => {
+  topDomainTrie.dump((domain) => {
     const apexDomain = tldts.getDomain(domain);
 
     if (apexDomain && allChinaDomains.has(apexDomain)) {
-      resultTrie.add(apexDomain, false);
+      resultTrie.add(apexDomain);
     }
   });
 
-  const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(domain, includeAllSubdomain);
+  const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(includeAllSubdomain ? '.' + domain : domain);
 
   // await Promise.all([
   await runAgainstSourceFile(
@@ -40,11 +41,11 @@ export async function parseDomesticList() {
       switch (domain[0]) {
         case '+':
         case '$': {
-          resultTrie.whitelist(domain.slice(1), true);
+          resultTrie.whitelist('.' + domain.slice(1));
           break;
         }
         default: {
-          resultTrie.whitelist(domain, true);
+          resultTrie.whitelist('.' + domain);
           break;
         }
       }
@@ -58,7 +59,12 @@ export async function parseDomesticList() {
   //   }
   // }
   // ]);
-  const dump = resultTrie.dump(null, true);
+  const dump: string[] = [];
+  resultTrie.dump((rawDomain, includeSubdomain) => {
+    const domain = domainToASCII(rawDomain);
+    if (domain) dump.push(includeSubdomain ? '.' + domain : domain);
+  });
+  dump.sort((a, b) => (a.length - b.length) || (a < b ? -1 : a > b ? 1 : 0));
 
   console.log(dump.join('\n') + '\n');
 

Build/validate-gfwlist.ts

@@ -1,6 +1,7 @@
 import { processLine } from './lib/process-line';
 import { fastNormalizeDomain } from './lib/normalize-domain';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
+import { domainToASCII } from 'node:url';
 import yauzl from 'yauzl-promise';
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import path from 'node:path';
@@ -120,8 +121,9 @@ export async function parseGfwList() {
   const keywordSet = new Set<string>();
 
   const callback = (domain: string, includeAllSubdomain: boolean) => {
-    gfwListTrie.whitelist(domain, includeAllSubdomain);
-    topDomainTrie.whitelist(domain, includeAllSubdomain);
+    const d = includeAllSubdomain ? '.' + domain : domain;
+    gfwListTrie.whitelist(d);
+    topDomainTrie.whitelist(d);
   };
   await Promise.all([
     runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/global.conf'), callback, 'ruleset', keywordSet),
@@ -149,17 +151,18 @@ export async function parseGfwList() {
     });
   });
 
-  whiteSet.forEach(domain => gfwListTrie.whitelist(domain, true));
+  whiteSet.forEach(domain => gfwListTrie.whitelist(domain[0] === '.' ? domain : '.' + domain));
 
   let dedupedGfwListSize = 0;
-  gfwListTrie.dump(() => dedupedGfwListSize++);
+  gfwListTrie.dump(() => { dedupedGfwListSize++; });
 
   const kwfilter = createKeywordFilter([...keywordSet]);
 
   const missingTop10000Gfwed = new Set<string>();
 
-  topDomainTrie.dump((domain) => {
-    if (gfwListTrie.has(domain) && !kwfilter(domain)) {
+  topDomainTrie.dump((rawDomain) => {
+    const domain = domainToASCII(rawDomain);
+    if (domain && gfwListTrie.match(domain) && !kwfilter(domain)) {
       missingTop10000Gfwed.add(domain);
     }
   });

Build/validate-global-tld.ts

@@ -1,5 +1,6 @@
 import path from 'node:path';
-import { HostnameSmolTrie } from './lib/trie';
+import { HostnameSmolTrie } from 'hntrie/smol';
+import { domainToASCII } from 'node:url';
 import { OUTPUT_SURGE_DIR } from './constants/dir';
 import { ICP_TLD } from './constants/domains';
 import tldts from 'tldts-experimental';
@@ -22,11 +23,20 @@ import { MARKER_DOMAIN } from './constants/description';
   }, 'ruleset');
 
   await runAgainstSourceFile(path.join(OUTPUT_SURGE_DIR, 'non_ip', 'global.conf'), (domain, includeAllSubdomain) => {
-    trie.add(domain, includeAllSubdomain);
+    if (includeAllSubdomain) {
+      trie.addSubdomain(domain);
+    } else {
+      trie.add(domain);
+    }
   }, 'ruleset');
 
-  ICP_TLD.forEach(tld => trie.whitelist(tld, true));
-  extraWhiteTLDs.forEach(tld => trie.whitelist(tld, true));
+  ICP_TLD.forEach(tld => trie.whitelist('.' + tld));
+  extraWhiteTLDs.forEach(tld => trie.whitelist('.' + tld));
 
-  console.log(trie.dump().join('\n'));
+  const dump: string[] = [];
+  trie.dump((rawDomain, includeSubdomain) => {
+    const domain = domainToASCII(rawDomain);
+    if (domain) dump.push(includeSubdomain ? '.' + domain : domain);
+  });
+  console.log(dump.join('\n'));
 })();

package.json

@@ -31,6 +31,7 @@
     "fast-uri": "^4.1.0",
     "fdir": "^6.5.0",
     "hash-wasm": "^4.12.0",
+    "hntrie": "^1.0.1",
     "json-stringify-pretty-compact": "4.0.0",
     "null-prototype-object": "^1.2.7",
     "picocolors": "^1.1.1",

pnpm-lock.yaml

@@ -50,6 +50,9 @@ importers:
       hash-wasm:
         specifier: ^4.12.0
         version: 4.12.0
+      hntrie:
+        specifier: ^1.0.1
+        version: 1.0.1
       json-stringify-pretty-compact:
         specifier: 4.0.0
         version: 4.0.0
@@ -1409,6 +1412,9 @@ packages:
     resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}
     hasBin: true
 
+  hntrie@1.0.1:
+    resolution: {integrity: sha512-k3OIFupPUGL1WrAiZ0Xa5on/vE/MKlub7ZQtEH0AfzfskMqq4/H6xgSqMeCXzVgNDWruP/Ff+9O5mHRzL4XZyg==}
+
   htmlparser2@6.1.0:
     resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==}
 
@@ -3210,6 +3216,10 @@ snapshots:
 
   he@1.2.0: {}
 
+  hntrie@1.0.1:
+    dependencies:
+      foxts: 5.5.1
+
   htmlparser2@6.1.0:
     dependencies:
       domelementtype: 2.3.0