Refactor: track download stats and enable HTTP/2

Build/build-reject-ip-list.ts

@@ -49,6 +49,8 @@ const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fet
   return acc;
 }, [[], []]));
 
+const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
+
 export const buildRejectIPList = task(require.main === module, __filename)(async (span) => {
   const [bogusNxDomainIPs, botNetIPs] = await Promise.all([
     span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise),
@@ -66,7 +68,7 @@ export const buildRejectIPList = task(require.main === module, __filename)(async
       ' - https://github.com/felixonmars/dnsmasq-china-list',
       ' - https://github.com/curbengh/botnet-filter'
     ])
-    .addFromRuleset(await readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf')))
+    .addFromRuleset(readLocalRejectIpListPromise)
     .bulkAddCIDR4NoResolve(bogusNxDomainIPs[0])
     .bulkAddCIDR6NoResolve(bogusNxDomainIPs[1])
     .bulkAddCIDR4NoResolve(botNetIPs[0])

Build/lib/fetch-retry.ts

@@ -23,7 +23,7 @@ if (!fs.existsSync(CACHE_DIR)) {
   fs.mkdirSync(CACHE_DIR, { recursive: true });
 }
 
-const agent = new Agent({});
+const agent = new Agent({ allowH2: true });
 
 setGlobalDispatcher(agent.compose(
   interceptors.retry({

Build/lib/parse-filter/domainlists.ts

@@ -6,9 +6,8 @@ import { fetchAssetsWithout304 } from '../fetch-assets';
 import type { Span } from '../../trace';
 
 function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string) {
-  let line = processLine(l);
+  const line = processLine(l);
   if (!line) return;
-  line = line.toLowerCase();
 
   const domain = normalizeDomain(line);
   if (!domain) return;
@@ -33,7 +32,7 @@ export function processDomainLists(
   domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false
 ) {
   return span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => {
-    const text = await span.traceChildAsync(`process domainlist: ${domainListsUrl}`, () => fetchAssetsWithout304(
+    const text = await span.traceChildAsync('download', () => fetchAssetsWithout304(
       domainListsUrl,
       mirrors
     ));

Build/lib/parse-filter/filters.ts

@@ -27,7 +27,7 @@ export async function processFilterRules(
   allowThirdParty = false
 ): Promise<{ white: string[], black: string[] }> {
   const [white, black, warningMessages] = await parentSpan.traceChild(`process filter rules: ${filterRulesUrl}`).traceAsyncFn(async (span) => {
-    const text = await fetchAssetsWithout304(filterRulesUrl, fallbackUrls);
+    const text = await span.traceChildAsync('download', () => fetchAssetsWithout304(filterRulesUrl, fallbackUrls));
 
     const whitelistDomainSets = new Set<string>();
     const blacklistDomainSets = new Set<string>();

Build/lib/rules/base.ts

@@ -404,15 +404,16 @@ export async function fileEqual(linesA: string[], source: AsyncIterable<string>)
 }
 
 export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
-  let isEqual = true;
   const linesALen = linesA.length;
 
-  if (fs.existsSync(filePath)) {
-    isEqual = await fileEqual(linesA, readFileByLine(filePath));
-  } else {
+  const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => {
+    if (fs.existsSync(filePath)) {
+      return fileEqual(linesA, readFileByLine(filePath));
+    }
+
     console.log(`${filePath} does not exists, writing...`);
-    isEqual = false;
-  }
+    return false;
+  });
 
   if (isEqual) {
     console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`)));