Update Rules

Build/build-reject-domainset.js

@@ -2,19 +2,42 @@ const { simpleGet } = require('./util-http-get');
 const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 
-let cliProgress;
 let Piscina;
 try {
   Piscina = require('piscina');
-  cliProgress = require('cli-progress');
 } catch (e) {
   console.log('Dependencies not found');
-  console.log('"npm i cli-progress piscina" then try again!');
+  console.log('"npm i piscina" then try again!');
 
   console.error(e);
   process.exit(1);
 }
 
+/**
+ * @param {string | URL} domainListsUrl
+ */
+async function processDomainLists(domainListsUrl) {
+  if (typeof domainListsUrl === 'string') {
+    domainListsUrl = new URL(domainListsUrl);
+  }
+
+  /** @type Set<string> */
+  const domainSets = new Set();
+  /** @type string[] */
+  const domains = (await simpleGet.https(domainListsUrl)).split('\n');
+  domains.forEach(line => {
+    if (line.startsWith('#')) {
+      return;
+    }
+    if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
+      return;
+    }
+    domainSets.add(line.trim());
+  });
+
+  return [...domainSets];
+}
+
 /**
  * @param {string | URL} hostsUrl
  */
@@ -29,7 +52,7 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
   /** @type string[] */
   const hosts = (await simpleGet.https(hostsUrl)).split('\n');
   hosts.forEach(line => {
-    if (line.startsWith('#')) {
+    if (line.includes('#')) {
       return;
     }
     if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
@@ -58,8 +81,20 @@ async function processFilterRules(filterRulesUrl) {
   /** @type Set<string> */
   const whitelistDomainSets = new Set([
     'localhost',
+    'broadcasthost',
+    'ip6-loopback',
+    'ip6-localnet',
+    'ip6-mcastprefix',
+    'ip6-allnodes',
+    'ip6-allrouters',
+    'ip6-allhosts',
+    'mcastprefix',
     'analytics.google.com',
-    'msa.cdn.mediaset.net' // Added manually using DOMAIN-KEYWORDS
+    'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
+    'cloud.answerhub.com',
+    'ae01.alicdn.com',
+    'whoami.akamai.net',
+    'whoami.ds.akahelp.net'
   ]);
   /** @type Set<string> */
   const blacklistDomainSets = new Set();
@@ -116,7 +151,7 @@ async function processFilterRules(filterRulesUrl) {
   /** @type Set<string> */
   const domainSets = new Set();
 
-  // Parse from remote hosts
+  // Parse from remote hosts & domain lists
   (await Promise.all([
     processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext', true),
     processHosts('https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'),
@@ -129,7 +164,8 @@ async function processFilterRules(filterRulesUrl) {
     });
   });
 
-  console.log(`Import ${domainSets.size} rules from hosts files!`);
+  const hostsSize = domainSets.size;
+  console.log(`Import ${hostsSize} rules from hosts files!`);
 
   await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
     data.split('\n').forEach(line => {
@@ -150,7 +186,8 @@ async function processFilterRules(filterRulesUrl) {
     });
   });
 
-  console.log(`Import rules from reject_sukka.conf!`);
+  const sukkaSize = domainSets.size - hostsSize;
+  console.log(`Import ${sukkaSize} rules from reject_sukka.conf!`);
 
   // Parse from AdGuard Filters
   /** @type Set<string> */
@@ -167,38 +204,54 @@ async function processFilterRules(filterRulesUrl) {
     black.forEach(i => domainSets.add(i));
   });
 
-  console.log(`Import rules from adguard filters!`);
+  const adguardSize = domainSets.size - hostsSize - sukkaSize;
+  console.log(`Import ${adguardSize} rules from adguard filters!`);
 
   // Read DOMAIN Keyword
   const domainKeywordsSet = new Set();
+  const domainSuffixSet = new Set();
   await fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
     data.split('\n').forEach(line => {
       if (line.startsWith('DOMAIN-KEYWORD')) {
         const [, ...keywords] = line.split(',');
         domainKeywordsSet.add(keywords.join(',').trim());
+      } else if (line.startsWith('DOMAIN-SUFFIX')) {
+        const [, ...keywords] = line.split(',');
+        domainSuffixSet.add(keywords.join(',').trim());
       }
     });
   });
 
   console.log(`Import ${domainKeywordsSet.size} black keywords!`);
 
+  const beforeDeduping = domainSets.size;
   // Dedupe domainSets
-  console.log(`Start deduping!`);
+  console.log(`Start deduping! (${beforeDeduping})`);
 
   const piscina = new Piscina({
     filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js')
   });
 
-  const res = await Promise.all([
-    piscina.run({ keywords: domainKeywordsSet, input: domainSets }, { name: 'dedupeKeywords' }),
+  const res2 = await Promise.all([
+    piscina.run({ keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets }, { name: 'dedupeKeywords' }),
     piscina.run({ whiteList: filterRuleWhitelistDomainSets, input: domainSets }, { name: 'whitelisted' }),
-    ...sliceIntoChunks(Array.from(domainSets), 5000).map(chunk => piscina.run({ input: chunk, fullSet: domainSets }, { name: 'dedupe' }))
-  ]);
 
-  res.forEach(set => {
+    Array.from(domainSets).reduce((result, element, index) => {
+      const chunk = index % 12;
+      result[chunk] = result[chunk] ?? [];
+
+      result[chunk].push(element);
+      return result;
+    }, []).map(chunk => piscina.run({ input: chunk, fullSet: domainSets }, { name: 'dedupe' }))
+  ]);
+  res2.forEach(set => {
     set.forEach(i => domainSets.delete(i));
   });
 
+  const diffDeduping = beforeDeduping - domainSets.size;
+
+  console.log(`Deduped ${diffDeduping} rules!`);
+
   return fsPromises.writeFile(pathResolve(__dirname, '../List/domainset/reject.conf'), `${[...domainSets].join('\n')}\n`);
 })();
 

Build/worker/build-reject-domainset-worker.js

@@ -1,17 +1,17 @@
 exports.dedupe = ({ fullSet, input }) => {
   const output = new Set();
 
-  for (const domain of input) {
-    for (const domain2 of fullSet) {
+  for (const domainFromInput of input) {
+    for (const domainFromFullSet of fullSet) {
       if (
-        domain2.startsWith('.')
-        && domain2 !== domain
+        domainFromFullSet.startsWith('.')
+        && domainFromFullSet !== domainFromInput
         && (
-          domain.endsWith(domain2)
-          || `.${domain}` === domain2
+          domainFromInput.endsWith(domainFromFullSet)
+          || `.${domainFromInput}` === domainFromFullSet
         )
       ) {
-        output.add(domain);
+        output.add(domainFromInput);
         break;
       }
     }
@@ -35,7 +35,7 @@ exports.whitelisted = ({ whiteList, input }) => {
   return output;
 };
 
-exports.dedupeKeywords = ({ keywords, input }) => {
+exports.dedupeKeywords = ({ keywords, suffixes, input }) => {
   const output = new Set();
 
   for (const domain of input) {
@@ -45,6 +45,12 @@ exports.dedupeKeywords = ({ keywords, input }) => {
         break;
       }
     }
+    for (const suffix of suffixes) {
+      if (domain.endsWith(suffix)) {
+        output.add(domain);
+        break;
+      }
+    }
   }
 
   return output;

List/domainset/cdn.conf

@@ -56,6 +56,7 @@ platform.twitter.com
 .ampproject.org
 
 # >> Google CDN
+dl.google.com
 fonts.googleapis.com
 fonts.gstatic.com
 ajax.googleapis.com
@@ -129,7 +130,7 @@ img.vim-cn.com
 
 # >> General CDN
 # AWS S3
-s3.amazonaws.com
+.s3.amazonaws.com
 # WordPress
 .files.wordpress.com
 # Instagram
@@ -142,6 +143,8 @@ cdn.sstatic.net
 .fonts.net
 # Naver
 .pstatic.net
+# Steam
+.steamstatic.com
 # Others
 .v2ex.co
 cdn.v2ex.com

List/domainset/reject_sukka.conf

@@ -365,7 +365,6 @@ xy-log.tagtic.cn
 .vrtcal.com
 .webspectator.com
 .xadchina.cn
-.xdrig.com
 .yldbt.com
 .ymtracking.com
 .adwo.com
@@ -388,6 +387,7 @@ ad.huajiao.com
 ad.hzyoka.com
 ad.jiemian.com
 ad.qingting.fm
+.qtad.qingting.fm
 adunion.wifi8.com
 adfile.wifi8.com
 ad.yun.wifi8.com
@@ -396,7 +396,6 @@ adsapi.manhuaren.com
 adsdk.dmzj.com
 adshows.21cn.com
 adui.tg.meitu.com
-adx.hupu.com
 .ad.cmvideo.cn
 app-ad.variflight.com
 applog.mobike.com
@@ -412,7 +411,6 @@ mopnativeadv.037201.com
 .jiguang.cn
 .jpush.io
 static.vuevideo.net
-stats.jpush.cn
 stats.umsns.com
 trackapp.guahao.cn
 traffic.mogujie.com
@@ -424,6 +422,10 @@ ssp.api.tappx.com
 .zynga.com
 .zyngaplayersupport.com
 .adx.io
+advertpay-vip-ssl.xunlei.com
+.advertising.speed.cdn.vip.xunlei.com
+.advertpay.vip.xunlei.com
+.youmi.net
 
 .adcolony.com
 .adpro.cn
@@ -514,12 +516,18 @@ adcloud.jp
 .districtm.io
 .buysellads.net
 
+.ad.wx.com
+.ad.weixin.qq.com
+.guomob.com
+.ad1.udn.com
+.ad2.udn.com
+.ad3.udn.com
+.ad4.udn.com
+.ad5.udn.com
+
 # >> Tracking
 tracking.yorg.app
 stats.wp.com
-data.alexa.com
-device-metrics-us.amazon.com
-device-metrics-us-2.amazon.com
 js.monitor.azure.com
 .cloudflareinsights.com
 .histats.com
@@ -559,6 +567,7 @@ analytics.slashdotmedia.com
 .bkrtx.com
 .ads.pro-market.net
 .stat.xtom.com
+.analytics.archive.org
 
 .adjust.io
 .airbrake.io
@@ -575,7 +584,6 @@ analytics.slashdotmedia.com
 .do-not-tracker.org
 .eviltracker.net
 .getexceptional.com
-.inmobi.com
 .jumptap.com
 .localytics.com
 .mixpanel.com
@@ -588,6 +596,15 @@ analytics.slashdotmedia.com
 .usergrid.com
 .vungle.com
 
+.telemetry.mozilla.org
+.union.6.cn
+api-analytics-cn.huami.com
+
+# >> Alexa
+data.alexa.com
+device-metrics-us.amazon.com
+device-metrics-us-2.amazon.com
+
 # >> Apple
 metrics.apple.com
 securemetrics.apple.com
@@ -633,7 +650,6 @@ corp.sts.microsoft.com
 diagnostics.support.microsoft.com
 i1.services.social.microsoft.com
 cache.datamart.windows.com
-diagnostics.support.microsoft.com
 spynet2.microsoft.com
 spynetalt.microsoft.com
 .msads.net
@@ -669,6 +685,7 @@ logservice.hicloud.com
 logservice1.hicloud.com
 
 # >> JD
+jrad.jd.com
 .union.jd.com
 c-nfa.jd.com
 cps.360buy.com
@@ -912,9 +929,18 @@ news.qhstatic.com
 show.look.360.cn
 .dftoutiao.com
 new.taobc.com
-s.360.cn
-
-# >> Youku
+.s.360.cn
+mclean.lato.cloud.360safe.com
+mvconf.lato.cloud.360safe.com
+mclean.cloud.360safe.com
+mvconf.cloud.360safe.com
+mclean.uk.cloud.360safe.com
+mvconf.uk.cloud.360safe.com
+aicleaner.shouji.360.cn
+ssl.360antivirus.org
+ad.360in.com
+
+# >> Youku + Tudou
 .atm.youku.com
 actives.youku.com
 ad.api.3g.youku.com
@@ -961,6 +987,12 @@ wan.youku.com
 ykatr.youku.com
 ykrec.youku.com
 
+iwstat.tudou.com
+nstat.tudou.com
+adplay.tudou.com
+stats.tudou.com
+.ad.cmvideo.cn
+
 # >> Netease
 ad.bn.netease.com
 ad.yixin.im
@@ -1008,8 +1040,11 @@ log.tbs.qq.com
 .mpush.qq.com
 .push.qq.com
 pgdt.gtimg.cn
+pgdt.ugdtimg.com
+union.gdtimg.com
 .bugly.qq.com
 .gdt.qq.com
+.adnet.qq.com
 .rqd.qq.com
 oth.eve.mdt.qq.com
 oth.str.mdt.qq.com
@@ -1030,6 +1065,11 @@ splashqqlive.gtimg.com
 access-sngapm.qcloud.com
 beacon.gtimg.com
 .l.qq.com
+ad.tencentmusic.com
+adstats.tencentmusic.com
+adcdn.tencentmusic.com
+adexpo.tencentmusic.com
+adclick.tencentmusic.com
 
 # >> Sina
 ad4.sina.com.cn
@@ -1130,6 +1170,7 @@ cdn.media.innity.net
 ssl-cdn.media.innity.net
 
 # >> iQiyi
+.cupid.ptqy.gitv.tv
 a.ckm.iqiyi.com
 ad.m.iqiyi.com
 afp.iqiyi.com
@@ -1143,13 +1184,14 @@ msg.video.qiyi.com
 msg2.video.qiyi.com
 policy.video.iqiyi.com
 yuedu.iqiyi.com
-cupid.iqiyi.com
+.cupid.iqiyi.com
 ifacelog.iqiyi.com
 mbdlog.iqiyi.com
 msg.71.am
 msg.qy.net
 
 # >> Kugou and Kuwo
+ads.bssdl.kugou.com
 adserviceretry.kugou.com
 ads.service.kugou.com
 adsfile.bssdlbig.kugou.com
@@ -1234,10 +1276,9 @@ pro.letv.com
 stat.letv.com
 static.app.m.letv.com
 
-# >> Xiaomi
+# >> Xiaomi (MIUI)
 ad.intl.xiaomi.com
 .ad.mi.com
-.ad.xiaomi.com
 .admob.xiaomi.com
 ad1.xiaomi.com
 mistat.xiaomi.com
@@ -1249,11 +1290,23 @@ adeng.hpplay.cn
 .kuyun.com
 .ad.xiaomi.com
 .ad.intl.xiaomi.com
-.data.mistat.xiaomi.com
-adv.sec.miui.com
 o2o.api.xiaomi.com
 api.miui.security.xiaomi.com
 .auth.be.sec.miui.com
+.adfuture.cn
+sa.api.intl.miui.com
+.data.mistat.xiaomi.com
+.data.mistat.intl.xiaomi.com
+.data.mistat.india.xiaomi.com
+.data.mistat.rus.xiaomi.com
+.tracking.miui.com
+.tracking.intl.miui.com
+.tracking.india.miui.com
+.tracking.rus.miui.com
+.xlmc.sec.miui.com
+wtradv.market.xiaomi.com
+video.market.xiaomi.com
+.pandora.xiaomi.com
 
 # >> PCOnline
 ad-analysis.pconline.com.cn
@@ -1271,12 +1324,18 @@ cdn.adapi.fotoable.com
 ads.aplus.pptv.com
 asimgs.pplive.cn
 de.as.pptv.com
+ads.data.pplive.com
+gas.data.pplive.com
+plt.data.pplive.com
+web.data.pplive.com
+.game.pps.tv
 
 # >> ele
 log.star.ele.me
 
 # >> HunanTV
 .da.mgtv.com
+.log.mgtv.com
 .da.hunantv.com
 .log.hunantv.com
 log.v2.hunantv.com
@@ -1366,12 +1425,26 @@ galaxy.bjcathay.com
 mdrecv.app.cntvwb.cn
 sdapprecv.app.cntvwb.cn
 vdapprecv.app.cntvwb.cn
+.ad.cctv.com
 
 # >> Kingsoft
 ad-stat.ksosoft.com
 img.auction-ads.wpscdn.cn
 minfo.wps.cn
 
+# >> FunShion
+adm.funshion.com
+conf.funshion.com
+vs.funshion.com
+rt.funshion.net
+stat.funshion.net
+
+# >> Hupu
+adx.hupu.com
+shence.hupu.com
+ccdace.hupu.com
+hd.hupu.com
+
 # >> Tonghuahshun
 adm.10jqka.com.cn
 stat.10jqka.com.cn
@@ -1412,6 +1485,16 @@ disqusads.com
 # >> XiMaLaYa
 ad.ximalaya.com
 adse.ximalaya.com
+adse.wsa.ximalaya.com
+adsebs.ximalaya.com
+adsehera.ximalaya.com
+adbehavior.ximalaya.com
+adwbs.ximalaya.com
+adbs.ximalaya.com
+dns.ximalaya.com
+mobwsa.ximalaya.com
+location.ximalaya.com
+xdcs-collector.ximalaya.com
 
 # Roku
 logs.roku.com
@@ -1455,28 +1538,28 @@ ntp.msn.cn
 assets.msn.cn
 api.msn.com
 browser.events.data.msn.com
-#edge.activity.windows.com
 img-s-msn-com.akamaized.net
 
-# >> MIUI
-.adfuture.cn
-sa.api.intl.miui.com
-.data.mistat.xiaomi.com
-.data.mistat.intl.xiaomi.com
-.data.mistat.india.xiaomi.com
-.data.mistat.rus.xiaomi.com
-.tracking.miui.com
-.tracking.intl.miui.com
-.tracking.india.miui.com
-.tracking.rus.miui.com
-.xlmc.sec.miui.com
-wtradv.market.xiaomi.com
-.pandora.xiaomi.com
-
 # >> OPPO
-
 adsfs.oppomobile.com
 .ads.oppomobile.com
+.adsfs-sdkconfig.heytapimage.com
+httpdns.push.oppomobile.com
+
+# >> VIVO
+adlog.vivo.com.cn
+adreq.vivo.com.cn
+adsdk.vivo.com.cn
+adsstatic.vivo.com.cn
+adxlog.vivo.com.cn
+stnetsdk.appstore.vivo.com.cn
+monitor-stsdk.vivo.com.cn
+onrt-stsdk.vivo.com.cn
+ort-stsdk.vivo.com.cn
+pnrt-stsdk.vivo.com.cn
+prt-stsdk.vivo.com.cn
+stnetsdk.vivo.com.cn
+.stsdk.vivo.com.cn
 
 # >> 勾正数据
 .gz-data.com

List/non_ip/reject.conf

@@ -30,7 +30,18 @@ DOMAIN-KEYWORD,adserver
 DOMAIN-KEYWORD,adspace
 DOMAIN-KEYWORD,advertmarket
 DOMAIN-KEYWORD,adsyndication
-DOMAIN-KEYWORD,doubleclick
+DOMAIN-KEYWORD,doubleclick.
+DOMAIN-KEYWORD,adjust.
+DOMAIN-KEYWORD,appsflyer
+
+# Important: Force add the following domains without whitelisting
+DOMAIN-SUFFIX,openx.net
+DOMAIN-SUFFIX,mmstat.com
+DOMAIN-SUFFIX,amplitude.com
+DOMAIN-KEYWORD,advertising.com
+
+# Tencent AD KEYWORD
+DOMAIN-KEYWORD,wxsnsdy
 
 DOMAIN-KEYWORD,bahoom,REJECT
 DOMAIN,daisydiskapp.com,REJECT