Refactor domain alive check

Build/lib/is-domain-alive.ts

@@ -90,7 +90,7 @@ export async function isDomainAlive(
   }
   const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
   if (!apexDomain) {
-    console.log(picocolors.gray('[domain invalid]'), picocolors.gray('no apex domain'), { domain });
+    // console.log(picocolors.gray('[domain invalid]'), picocolors.gray('no apex domain'), { domain });
     domainAliveMap.set('.' + domain, true);
     return true;
   }

Build/validate-domain-alive.ts

@@ -4,6 +4,11 @@ import { isDomainAlive } from './lib/is-domain-alive';
 import { fdir as Fdir } from 'fdir';
 import runAgainstSourceFile from './lib/run-against-source-file';
 
+import cliProgress from 'cli-progress';
+import { newQueue } from '@henrygd/queue';
+
+const queue = newQueue(32);
+
 const deadDomains: string[] = [];
 
 (async () => {
@@ -26,7 +31,8 @@ const deadDomains: string[] = [];
     .crawl(SOURCE_DIR + path.sep + 'non_ip')
     .withPromise();
 
-  const promises: Array<Promise<void>> = [];
+  const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+  bar.start(0, 0);
 
   await Promise.all([
     ...domainRules,
@@ -34,18 +40,26 @@ const deadDomains: string[] = [];
   ].map(
     filepath => runAgainstSourceFile(
       filepath,
-      (domain: string, includeAllSubdomain: boolean) => promises.push(
-        isDomainAlive(domain, includeAllSubdomain).then((alive) => {
-          if (alive) {
-            return;
-          }
-          deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
-        })
-      )
+      (domain: string, includeAllSubdomain: boolean) => {
+        bar.setTotal(bar.getTotal() + 1);
+
+        return queue.add(
+          () => isDomainAlive(domain, includeAllSubdomain).then((alive) => {
+            bar.increment();
+
+            if (alive) {
+              return;
+            }
+            deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
+          })
+        );
+      }
     ).then(() => console.log('[crawl]', filepath))
   ));
 
-  await Promise.all(promises);
+  await queue.done();
+
+  bar.stop();
 
   console.log();
   console.log();

package.json

@@ -26,6 +26,7 @@
     "async-retry": "^1.3.3",
     "better-sqlite3": "^11.10.0",
     "ci-info": "^4.2.0",
+    "cli-progress": "^3.12.0",
     "csv-parse": "^5.6.0",
     "dns2": "^2.1.0",
     "fast-cidr-tools": "^0.3.2",
@@ -55,6 +56,7 @@
     "@swc/core": "^1.11.31",
     "@types/async-retry": "^1.4.9",
     "@types/better-sqlite3": "^7.6.13",
+    "@types/cli-progress": "^3.11.6",
     "@types/dns2": "^2.0.9",
     "@types/fast-fifo": "^1.3.0",
     "@types/mocha": "^10.0.10",

pnpm-lock.yaml

@@ -34,6 +34,9 @@ importers:
       ci-info:
         specifier: ^4.2.0
         version: 4.2.0
+      cli-progress:
+        specifier: ^3.12.0
+        version: 3.12.0
       csv-parse:
         specifier: ^5.6.0
         version: 5.6.0
@@ -116,6 +119,9 @@ importers:
       '@types/better-sqlite3':
         specifier: ^7.6.13
         version: 7.6.13
+      '@types/cli-progress':
+        specifier: ^3.11.6
+        version: 3.11.6
       '@types/dns2':
         specifier: ^2.0.9
         version: 2.0.9
@@ -574,6 +580,9 @@ packages:
   '@types/better-sqlite3@7.6.13':
     resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==}
 
+  '@types/cli-progress@3.11.6':
+    resolution: {integrity: sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA==}
+
   '@types/dns2@2.0.9':
     resolution: {integrity: sha512-+eiPSuo/KfvaaW7DiMf/vrt9mhxbZCCgxoCa/c2qMVSuZQus4BzBuKRh8XBwcUXWAtvI8QuZOp13UODJNq9DXg==}
 
@@ -900,6 +909,10 @@ packages:
     resolution: {integrity: sha512-cYY9mypksY8NRqgDB1XD1RiJL338v/551niynFTGkZOO2LHuB2OmOYxDIe/ttN9AHwrqdum1360G3ald0W9kCg==}
     engines: {node: '>=8'}
 
+  cli-progress@3.12.0:
+    resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==}
+    engines: {node: '>=4'}
+
   cliui@8.0.1:
     resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==}
     engines: {node: '>=12'}
@@ -2358,6 +2371,10 @@ snapshots:
     dependencies:
       '@types/node': 22.15.30
 
+  '@types/cli-progress@3.11.6':
+    dependencies:
+      '@types/node': 22.15.30
+
   '@types/dns2@2.0.9':
     dependencies:
       '@types/node': 22.15.30
@@ -2676,6 +2693,10 @@ snapshots:
 
   ci-info@4.2.0: {}
 
+  cli-progress@3.12.0:
+    dependencies:
+      string-width: 4.2.3
+
   cliui@8.0.1:
     dependencies:
       string-width: 4.2.3