Feat: support reject ip from adguard filter

Build/build-reject-domainset.ts

@@ -6,7 +6,7 @@ import { processHostsWithPreload } from './lib/parse-filter/hosts';
 import { processDomainListsWithPreload } from './lib/parse-filter/domainlists';
 import { processFilterRulesWithPreload } from './lib/parse-filter/filters';
 
-import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA } from './constants/reject-data-source';
+import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, BOTNET_FILTER, BOGUS_NXDOMAIN_DNSMASQ } from './constants/reject-data-source';
 import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { task } from './trace';
 // tldts-experimental is way faster than tldts, but very little bit inaccurate
@@ -21,10 +21,14 @@ import { foundDebugDomain } from './lib/parse-filter/shared';
 import { AdGuardHomeOutput } from './lib/rules/domainset';
 import { getPhishingDomains } from './lib/get-phishing-domains';
 import type { MaybePromise } from './lib/misc';
+import { RulesetOutput } from './lib/rules/ruleset';
+import { fetchAssets } from './lib/fetch-assets';
+import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
 
 const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject.conf'));
 const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_extra.conf'));
 const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
+const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
 
 const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry));
 const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
@@ -75,6 +79,36 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       ...PHISHING_DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`)
     ]);
 
+  const rejectIPOutput = new RulesetOutput(span, 'reject', 'ip')
+    .withTitle('Sukka\'s Ruleset - Anti Bogus Domain')
+    .withDescription([
+      ...SHARED_DESCRIPTION,
+      '',
+      'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers, and botnet controller IPs.',
+      '',
+      'Data from:',
+      ' - https://github.com/felixonmars/dnsmasq-china-list',
+      ' - https://github.com/curbengh/botnet-filter',
+      ' - And other sources mentioned in /domainset/reject file'
+    ])
+    .bulkAddIPASN(AUGUST_ASN)
+    .bulkAddIPASN(HUIZE_ASN);
+
+  // Dedupe domainSets (no need to await this)
+  // Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
+  // DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
+  // It is faster to add base than add others first then whitelist
+  rejectOutput.addFromRuleset(readLocalRejectRulesetPromise);
+  rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise);
+
+  rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise);
+  rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise);
+  rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise);
+
+  rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise);
+
+  rejectIPOutput.addFromRuleset(readLocalRejectIpListPromise);
+
   const appendArrayToRejectOutput = (source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) => rejectOutput.addFromDomainset(source);
   const appendArrayToRejectExtraOutput = (source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) => rejectExtraOutput.addFromDomainset(source);
 
@@ -85,18 +119,6 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
   await span
     .traceChild('download and process hosts / adblock filter rules')
     .traceAsyncFn((childSpan) => Promise.all([
-      // Dedupe domainSets
-      // Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
-      // DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
-      // It is faster to add base than add others first then whitelist
-      rejectOutput.addFromRuleset(readLocalRejectRulesetPromise),
-      rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise),
-
-      rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise),
-      rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise),
-      rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise),
-
-      rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise),
 
       // Parse from remote hosts & domain lists
       hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
@@ -108,21 +130,23 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
       rejectPhisingOutput.addFromDomainset(getPhishingDomains(childSpan)),
 
       adguardFiltersDownloads.map(
-        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
+        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, blackIPs }) => {
           addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
           addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
 
           rejectOutput.bulkAddDomain(blackDomains);
           rejectOutput.bulkAddDomainSuffix(blackDomainSuffixes);
+          rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
         })
       ),
       adguardFiltersExtraDownloads.map(
-        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
+        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, blackIPs }) => {
           addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
           addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
 
           rejectExtraOutput.bulkAddDomain(blackDomains);
           rejectExtraOutput.bulkAddDomainSuffix(blackDomainSuffixes);
+          rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
         })
       ),
       adguardFiltersWhitelistsDownloads.map(
@@ -132,7 +156,27 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
           addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains);
           addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix);
         })
-      )
+      ),
+
+      span.traceChildAsync(
+        'get botnet ips',
+        () => fetchAssets(...BOTNET_FILTER, true, true)
+      ).then(arr => rejectIPOutput.bulkAddAnyCIDR(arr, false)),
+      span.traceChildAsync(
+        'get bogus nxdomain ips',
+        () => fetchAssets(...BOGUS_NXDOMAIN_DNSMASQ, true, false)
+          .then(arr => {
+            for (let i = 0, len = arr.length; i < len; i++) {
+              const line = arr[i];
+              if (line.startsWith('bogus-nxdomain=')) {
+                arr[i] = line.slice(15).trim();
+              }
+            }
+
+            return arr;
+          })
+        // bogus nxdomain needs to be blocked even after resolved
+      ).then(arr => rejectIPOutput.bulkAddAnyCIDR(arr, false))
     ].flat()));
 
   if (foundDebugDomain.value) {
@@ -143,7 +187,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
   await Promise.all([
     rejectOutput.done(),
     rejectExtraOutput.done(),
-    rejectPhisingOutput.done()
+    rejectPhisingOutput.done(),
+    rejectIPOutput.done()
   ]);
 
   // whitelist
@@ -161,7 +206,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
   await Promise.all([
     rejectOutput.write(),
     rejectExtraOutput.write(),
-    rejectPhisingOutput.write()
+    rejectPhisingOutput.write(),
+    rejectIPOutput.write()
   ]);
 
   // we are going to re-use rejectOutput's domainTrie and mutate it

Build/build-reject-ip-list.ts

@@ -1,87 +1,12 @@
 // @ts-check
 import path from 'node:path';
-import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { task } from './trace';
-import { SHARED_DESCRIPTION } from './constants/description';
 import { compareAndWriteFile } from './lib/create-file';
-import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
-import { fetchAssets } from './lib/fetch-assets';
-import { fastIpVersion } from './lib/misc';
+import { OUTPUT_INTERNAL_DIR } from './constants/dir';
 import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
-import { RulesetOutput } from './lib/rules/ruleset';
 
-const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(
-  'https://cdn.jsdelivr.net/gh/felixonmars/dnsmasq-china-list@master/bogus-nxdomain.china.conf',
-  ['https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'],
-  true
-).then((arr) => {
-  const ipv4: string[] = [];
-  const ipv6: string[] = [];
-
-  for (let i = 0, len = arr.length; i < len; i++) {
-    const line = arr[i];
-    if (line.startsWith('bogus-nxdomain=')) {
-      const ip = line.slice(15).trim();
-      const v = fastIpVersion(ip);
-      if (v === 4) {
-        ipv4.push(ip);
-      } else if (v === 6) {
-        ipv6.push(ip);
-      }
-    }
-  }
-
-  return [ipv4, ipv6] as const;
-});
-
-const BOTNET_FILTER_URL = 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt';
-const BOTNET_FILTER_MIRROR_URL = [
-  'https://botnet-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt',
-  'https://malware-filter.gitlab.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt',
-  'https://malware-filter.gitlab.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt'
-  // 'https://curbengh.github.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt',
-  // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt
-];
-
-const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL, true, true).then(arr => arr.reduce<[ipv4: string[], ipv6: string[]]>((acc, ip) => {
-  const v = fastIpVersion(ip);
-  if (v === 4) {
-    acc[0].push(ip);
-  } else if (v === 6) {
-    acc[1].push(ip);
-  }
-  return acc;
-}, [[], []]));
-
-const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
-
-export const buildRejectIPList = task(require.main === module, __filename)(async (span) => {
-  const [bogusNxDomainIPs, botNetIPs] = await Promise.all([
-    span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise),
-    span.traceChildPromise('get botnet ips', getBotNetFilterIPsPromise)
-  ]);
-
-  return Promise.all([
-    new RulesetOutput(span, 'reject', 'ip')
-      .withTitle('Sukka\'s Ruleset - Anti Bogus Domain')
-      .withDescription([
-        ...SHARED_DESCRIPTION,
-        '',
-        'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers, and botnet controller IPs.',
-        '',
-        'Data from:',
-        ' - https://github.com/felixonmars/dnsmasq-china-list',
-        ' - https://github.com/curbengh/botnet-filter'
-      ])
-      .addFromRuleset(readLocalRejectIpListPromise)
-      .bulkAddCIDR4NoResolve(bogusNxDomainIPs[0])
-      .bulkAddCIDR6NoResolve(bogusNxDomainIPs[1])
-      .bulkAddCIDR4NoResolve(botNetIPs[0])
-      .bulkAddCIDR6NoResolve(botNetIPs[1])
-      .bulkAddIPASN(AUGUST_ASN)
-      .bulkAddIPASN(HUIZE_ASN)
-      .write(),
-    compareAndWriteFile(span, [AUGUST_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'august_asn.txt')),
-    compareAndWriteFile(span, [HUIZE_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'huize_asn.txt'))
-  ]);
-});
+// Notice: botnet and bogus_nxdomain has been moved to build-reject-domainset
+export const buildRejectIPList = task(require.main === module, __filename)(async (span) => Promise.all([
+  compareAndWriteFile(span, [AUGUST_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'august_asn.txt')),
+  compareAndWriteFile(span, [HUIZE_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'huize_asn.txt'))
+]));

Build/constants/reject-data-source.ts

@@ -562,3 +562,19 @@ export const PREDEFINED_WHITELIST = [
   'hbbtv.redbutton.de',
   'hbbtv.kika.de'
 ];
+
+export const BOTNET_FILTER = [
+  'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt',
+  [
+    'https://botnet-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt',
+    'https://malware-filter.gitlab.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt',
+    'https://malware-filter.gitlab.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt'
+    // 'https://curbengh.github.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt',
+    // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt
+  ]
+] as const;
+
+export const BOGUS_NXDOMAIN_DNSMASQ = [
+  'https://cdn.jsdelivr.net/gh/felixonmars/dnsmasq-china-list@master/bogus-nxdomain.china.conf',
+  ['https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf']
+] as const;

Build/lib/parse-filter/filters.ts

@@ -28,7 +28,7 @@ export function processFilterRulesWithPreload(
 ) {
   const downloadPromise = fetchAssets(filterRulesUrl, fallbackUrls);
 
-  return (span: Span) => span.traceChildAsync<Record<'whiteDomains' | 'whiteDomainSuffixes' | 'blackDomains' | 'blackDomainSuffixes', string[]>>(`process filter rules: ${filterRulesUrl}`, async (span) => {
+  return (span: Span) => span.traceChildAsync<Record<'whiteDomains' | 'whiteDomainSuffixes' | 'blackDomains' | 'blackDomainSuffixes' | 'blackIPs', string[]>>(`process filter rules: ${filterRulesUrl}`, async (span) => {
     const filterRules = await span.traceChildPromise('download', downloadPromise);
 
     const whiteDomains = new Set<string>();
@@ -39,6 +39,8 @@ export function processFilterRulesWithPreload(
 
     const warningMessages: string[] = [];
 
+    const blackIPs: string[] = [];
+
     const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', ParseType.NotParsed];
     /**
        * @param {string} line
@@ -78,6 +80,9 @@ export function processFilterRulesWithPreload(
         case ParseType.ErrorMessage:
           warningMessages.push(hostname);
           break;
+        case ParseType.BlackIP:
+          blackIPs.push(hostname);
+          break;
         default:
           break;
       }
@@ -107,7 +112,8 @@ export function processFilterRulesWithPreload(
       whiteDomains: Array.from(whiteDomains),
       whiteDomainSuffixes: Array.from(whiteDomainSuffixes),
       blackDomains: Array.from(blackDomains),
-      blackDomainSuffixes: Array.from(blackDomainSuffixes)
+      blackDomainSuffixes: Array.from(blackDomainSuffixes),
+      blackIPs
     };
   });
 }

Build/lib/rules/base.ts

@@ -1,6 +1,7 @@
 import type { Span } from '../../trace';
 import { HostnameSmolTrie } from '../trie';
 import { not, nullthrow } from 'foxts/guard';
+import { fastIpVersion } from '../misc';
 import type { MaybePromise } from '../misc';
 import type { BaseWriteStrategy } from '../writing-strategy/base';
 import { merge as mergeCidr } from 'fast-cidr-tools';
@@ -225,6 +226,27 @@ export class FileOutput {
     return ip + '/128';
   };
 
+  bulkAddAnyCIDR(cidrs: string[], noResolve = false) {
+    const list4 = noResolve ? this.ipcidrNoResolve : this.ipcidr;
+    const list6 = noResolve ? this.ipcidr6NoResolve : this.ipcidr6;
+
+    for (let i = 0, len = cidrs.length; i < len; i++) {
+      let cidr = cidrs[i];
+      const version = fastIpVersion(cidr);
+      if (version === 0) {
+        continue; // skip invalid IPs
+      }
+      cidr = FileOutput.ipToCidr(cidr, version);
+
+      if (version === 4) {
+        list4.add(cidr);
+      } else /* if (version === 6) */ {
+        list6.add(cidr);
+      }
+    }
+    return this;
+  }
+
   bulkAddCIDR4(cidrs: string[]) {
     for (let i = 0, len = cidrs.length; i < len; i++) {
       this.ipcidr.add(FileOutput.ipToCidr(cidrs[i], 4));