Update Download / Global Hosts

Build/lib/normalize-domain.ts

@@ -0,0 +1,15 @@
+import * as tldts from './cached-tld-parse';
+import { isProbablyIpv4 } from './is-fast-ip';
+export const normalizeDomain = (domain: string) => {
+  if (!domain) return null;
+  if (isProbablyIpv4(domain)) return null;
+
+  const parsed = tldts.parse2(domain);
+  if (parsed.isIp) return null;
+  if (!parsed.isIcann && !parsed.isPrivate) return null;
+
+  const h = parsed.hostname;
+  if (!h) return null;
+
+  return h[0] === '.' ? h.slice(1) : h;
+};

Build/lib/parse-filter.ts

@@ -1,14 +1,15 @@
 // @ts-check
 import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
-import * as tldts from './cached-tld-parse';
+
 import { fetchRemoteTextAndReadByLine } from './fetch-text-by-line';
 import { NetworkFilter } from '@cliqz/adblocker';
 import { processLine } from './process-line';
 import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
 import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
-import { isProbablyIpv4 } from './is-fast-ip';
+
 import { traceAsync } from './trace-runner';
 import picocolors from 'picocolors';
+import { normalizeDomain } from './normalize-domain';
 
 const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
 let foundDebugDomain = false;
@@ -23,20 +24,6 @@ const warnOnce = (url: string, isWhite: boolean, ...message: any[]) => {
   console.warn(url, isWhite ? '(white)' : '(black)', ...message);
 };
 
-const normalizeDomain = (domain: string) => {
-  if (!domain) return null;
-  if (isProbablyIpv4(domain)) return null;
-
-  const parsed = tldts.parse2(domain);
-  if (parsed.isIp) return null;
-  if (!parsed.isIcann && !parsed.isPrivate) return null;
-
-  const h = parsed.hostname;
-  if (!h) return null;
-
-  return h[0] === '.' ? h.slice(1) : h;
-};
-
 export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false) {
   return traceAsync(`- processDomainLists: ${domainListsUrl}`, async () => {
     const domainSets = new Set<string>();

Build/validate-gfwlist.ts

@@ -0,0 +1,117 @@
+import { processLine } from './lib/process-line';
+import { normalizeDomain } from './lib/normalize-domain';
+import { createTrie } from './lib/trie';
+import { Readable } from 'stream';
+import { parse } from 'csv-parse';
+import { readFileByLine } from './lib/fetch-text-by-line';
+import path from 'path';
+
+export const parseGfwList = async () => {
+  const whiteSet = new Set<string>();
+  const blackSet = new Set<string>();
+
+  const text = await (await fetch('https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt')).text();
+  for (const l of atob(text).split('\n')) {
+    const line = processLine(l);
+    if (!line) continue;
+    if (line[0] === '[') {
+      continue;
+    }
+    if (line.includes('.*')) {
+      continue;
+    }
+    if (line.includes('*')) {
+      continue;
+    }
+    if (line.startsWith('@@||')) {
+      whiteSet.add(line.slice(4));
+      continue;
+    }
+    if (line.startsWith('@@|http://')) {
+      whiteSet.add(line.slice(8));
+      continue;
+    }
+    if (line.startsWith('@@|https://')) {
+      whiteSet.add(line.slice(9));
+      continue;
+    }
+    if (line.startsWith('||')) {
+      blackSet.add(line.slice(2));
+      continue;
+    }
+    if (line.startsWith('|')) {
+      blackSet.add(line.slice(1));
+      continue;
+    }
+    if (line.startsWith('.')) {
+      blackSet.add(line.slice(1));
+      continue;
+    }
+    const d = normalizeDomain(line);
+    if (d) {
+      blackSet.add(d);
+      continue;
+    }
+  }
+
+  const top500Gfwed = new Set<string>();
+
+  const res = await fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=843&top=1000');
+  const stream = Readable.fromWeb(res.body!).pipe(parse());
+
+  const trie = createTrie(blackSet);
+
+  for await (const [domain] of stream) {
+    if (trie.has(domain)) {
+      top500Gfwed.add(domain);
+    }
+  }
+
+  const notIncludedTop500Gfwed = new Set<string>(top500Gfwed);
+
+  const runAgainstRuleset = async (ruleset: string) => {
+    for await (const l of readFileByLine(ruleset)) {
+      const line = processLine(l);
+      if (!line) continue;
+      const [type, domain] = line.split(',');
+      if (type === 'DOMAIN-SUFFIX') {
+        if (top500Gfwed.has(domain)) {
+          notIncludedTop500Gfwed.delete(domain);
+        }
+      } else if (type === 'DOMAIN-KEYWORD') {
+        for (const d of top500Gfwed) {
+          if (d.includes(domain)) {
+            notIncludedTop500Gfwed.delete(d);
+          }
+        }
+      }
+    }
+  };
+
+  await Promise.all([
+    runAgainstRuleset(path.resolve(import.meta.dir, '../Source/non_ip/global_plus.conf')),
+    runAgainstRuleset(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'))
+  ]);
+
+  // for await (const l of readFileByLine(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'))) {
+  //   const line = processLine(l);
+  //   if (!line) continue;
+  //   const domain = line[0] === '.' ? line.slice(1) : line;
+  //   if (top500Gfwed.has(domain)) {
+  //     notIncludedTop500Gfwed.delete(domain);
+  //   }
+  // }
+
+  console.log(notIncludedTop500Gfwed);
+
+  return [
+    whiteSet,
+    blackSet,
+    trie,
+    top500Gfwed
+  ] as const;
+};
+
+if (import.meta.main) {
+  parseGfwList();
+}

Source/domainset/download.conf

@@ -169,6 +169,7 @@ fedorapeople.org
 mirrors.edge.kernel.org
 .mirrors.dotsrc.org
 .mirror.clarkson.edu
+.mirror.constant.com
 # WhatPulse
 releases.whatpulse.org
 # GIMP

Source/non_ip/global.conf

@@ -16,8 +16,6 @@ DOMAIN,news-edge.apple.com
 
 # >> Google
 DOMAIN-SUFFIX,abc.xyz
-DOMAIN-SUFFIX,ampproject.org
-DOMAIN-SUFFIX,android.com
 DOMAIN-SUFFIX,androidify.com
 DOMAIN-SUFFIX,autodraw.com
 DOMAIN-SUFFIX,capitalg.com
@@ -35,7 +33,6 @@ DOMAIN-SUFFIX,getmdl.io
 DOMAIN-SUFFIX,gmodules.com
 DOMAIN-SUFFIX,godoc.org
 DOMAIN-SUFFIX,golang.org
-DOMAIN-SUFFIX,gstatic.com
 DOMAIN-SUFFIX,gv.com
 DOMAIN-SUFFIX,gwtproject.org
 DOMAIN-SUFFIX,itasoftware.com
@@ -68,13 +65,6 @@ DOMAIN-SUFFIX,bing.com
 # >> Oracle
 DOMAIN-SUFFIX,oracle.com
 
-# >> Snapchat
-DOMAIN-SUFFIX,sc-cdn.net
-DOMAIN-SUFFIX,snap-dev.net
-DOMAIN-SUFFIX,snap.com
-DOMAIN-SUFFIX,snapchat.com
-DOMAIN-SUFFIX,snapkit.co
-
 # >> 451
 DOMAIN-SUFFIX,aicoin.com
 DOMAIN-SUFFIX,aimoon.com

Source/non_ip/global_plus.conf

@@ -5,9 +5,13 @@
 DOMAIN-SUFFIX,cloudflareresolve.com
 
 # >> Google
+DOMAIN-SUFFIX,ampproject.org
+DOMAIN-SUFFIX,android.com
 DOMAIN-SUFFIX,appspot.com
 DOMAIN-SUFFIX,blogger.com
 DOMAIN-SUFFIX,blogblog.com
+DOMAIN-SUFFIX,cloudfunctions.net
+DOMAIN-SUFFIX,firebaseio.com
 DOMAIN-SUFFIX,getoutline.org
 DOMAIN-SUFFIX,gvt0.com
 DOMAIN-SUFFIX,gvt1.com
@@ -15,6 +19,7 @@ DOMAIN-SUFFIX,gvt2.com
 DOMAIN-SUFFIX,gvt3.com
 DOMAIN-SUFFIX,googleapis.cn
 DOMAIN-KEYWORD,google
+DOMAIN-SUFFIX,gstatic.com
 DOMAIN-SUFFIX,gmail.com
 DOMAIN-SUFFIX,ggpht.com
 DOMAIN-KEYWORD,blogspot
@@ -22,6 +27,10 @@ DOMAIN-SUFFIX,youtu.be
 DOMAIN-SUFFIX,yt.be
 DOMAIN-SUFFIX,ytimg.com
 DOMAIN-SUFFIX,g.co
+DOMAIN-SUFFIX,goo.gl
+DOMAIN-SUFFIX,dns.google
+# ocsp.pki.goog is available in Mainland China
+DOMAIN,pki.goog
 
 # >> Facebook
 DOMAIN-SUFFIX,cdninstagram.com
@@ -111,6 +120,19 @@ DOMAIN-SUFFIX,archive.ph
 DOMAIN-SUFFIX,archive.is
 DOMAIN-SUFFIX,archive.today
 
+# >> Snapchat
+DOMAIN-SUFFIX,sc-cdn.net
+DOMAIN-SUFFIX,snap-dev.net
+DOMAIN-SUFFIX,snap.com
+DOMAIN-SUFFIX,snapchat.com
+DOMAIN-SUFFIX,snapkit.co
+
+# Signal
+DOMAIN-SUFFIX,signal.art
+DOMAIN-SUFFIX,signal.org
+DOMAIN-SUFFIX,signalusers.org
+DOMAIN-SUFFIX,whispersystems.org
+
 # >> Other
 DOMAIN-SUFFIX,abc.net.au
 DOMAIN-SUFFIX,amazon.co.jp
@@ -126,6 +148,7 @@ DOMAIN-SUFFIX,bibox.com
 DOMAIN-SUFFIX,binance.com
 DOMAIN-SUFFIX,bitfinex.com
 DOMAIN-SUFFIX,booklive.jp
+DOMAIN-SUFFIX,brave.com
 DOMAIN-SUFFIX,bwh1.net
 DOMAIN-SUFFIX,cbc.ca
 DOMAIN-SUFFIX,character.ai
@@ -142,7 +165,9 @@ DOMAIN-SUFFIX,exhentai.org
 DOMAIN-SUFFIX,feedly.com
 DOMAIN-SUFFIX,flickr.com
 DOMAIN-SUFFIX,gate.io
+DOMAIN-SUFFIX,godaddy.com
 DOMAIN-SUFFIX,goodreads.com
+DOMAIN-SUFFIX,homedepot.com
 DOMAIN-SUFFIX,huggingface.co
 DOMAIN-SUFFIX,initiummall.com
 DOMAIN-SUFFIX,issuu.com
@@ -171,6 +196,7 @@ DOMAIN-SUFFIX,pixiv.net
 DOMAIN-SUFFIX,pixiv.org
 DOMAIN-SUFFIX,pximg.net
 DOMAIN-SUFFIX,pornhub.com
+DOMAIN-SUFFIX,phncdn.com
 DOMAIN-SUFFIX,quora.com
 DOMAIN-SUFFIX,quoracdn.net
 DOMAIN-SUFFIX,redd.it
@@ -185,10 +211,12 @@ DOMAIN-SUFFIX,shadowsocks.org
 DOMAIN-SUFFIX,slideshare.net
 DOMAIN-SUFFIX,soundcloud.com
 DOMAIN-SUFFIX,steamcommunity.com
+DOMAIN-SUFFIX,surfshark.com
 DOMAIN-SUFFIX,theinitium.com
 DOMAIN-SUFFIX,theguardian.com
 DOMAIN-SUFFIX,thisav.com
 DOMAIN-SUFFIX,tineye.com
+DOMAIN-SUFFIX,tradingview.com
 DOMAIN-SUFFIX,tumblr.com
 DOMAIN-SUFFIX,turbobit.net
 DOMAIN-SUFFIX,twitch.tv
@@ -198,11 +226,13 @@ DOMAIN-SUFFIX,v2fly.org
 DOMAIN-SUFFIX,v2ray.com
 DOMAIN-SUFFIX,vimeo.com
 DOMAIN-SUFFIX,vine.co
+DOMAIN-SUFFIX,viber.com
 DOMAIN-SUFFIX,voachinese.com
 DOMAIN-SUFFIX,washingtonpost.com
 DOMAIN-SUFFIX,whoer.net
 DOMAIN-SUFFIX,wikibooks.org
 DOMAIN-SUFFIX,wikidata.org
+DOMAIN-SUFFIX,wikimedia.org
 DOMAIN-SUFFIX,wikinews.org
 DOMAIN-SUFFIX,wikipedia.org
 DOMAIN-SUFFIX,wikiquote.org
@@ -210,9 +240,14 @@ DOMAIN-SUFFIX,wikisource.org
 DOMAIN-SUFFIX,wikiversity.org
 DOMAIN-SUFFIX,wikivoyage.org
 DOMAIN-SUFFIX,wiktionary.org
+DOMAIN-SUFFIX,wordpress.com
 DOMAIN-SUFFIX,wsj.com
 DOMAIN-SUFFIX,wsj.net
+DOMAIN-SUFFIX,xfinity.com
+DOMAIN-SUFFIX,xhamster.com
+DOMAIN-SUFFIX,xnxx.com
 DOMAIN-SUFFIX,xvideos.com
+DOMAIN-SUFFIX,xvideos-cdn.com
 DOMAIN-SUFFIX,yahoo.com
 DOMAIN,search.yahoo.co.jp
 

package.json

@@ -19,6 +19,7 @@
     "async-retry": "^1.3.3",
     "async-sema": "^3.1.1",
     "ci-info": "^4.0.0",
+    "csv-parse": "^5.5.3",
     "fast-cidr-tools": "^0.2.2",
     "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js",
     "mnemonist": "^0.39.6",