Chore: improve domain alive validation

.github/workflows/check-source-domain.yml

@@ -51,7 +51,7 @@ jobs:
       - run: pnpm install
       - run: pnpm run node Build/validate-domain-alive.ts
         env:
-          DEBUG: domain-alive:*
+          DEBUG: domain-alive:dead-domain:*
       - name: Cache cache.db
         if: always()
         uses: actions/cache/save@v4

Build/lib/is-domain-alive.ts

@@ -1,4 +1,4 @@
-import { createDomainAliveChecker } from 'domain-alive';
+import { createDomainAliveChecker, createRegisterableDomainAliveChecker } from 'domain-alive';
 
 const dnsServers = [
   '8.8.8.8',
@@ -42,10 +42,20 @@ const dnsServers = [
   // 'dns.rabbitdns.org'
 ].map(dns => 'https://' + dns + '/dns-query');
 
-console.log({ dnsServers });
+const resultCache = new Map();
+const registerableDomainResultCache = new Map();
+
+export const isRegisterableDomainAlive = createRegisterableDomainAliveChecker({
+  dns: {
+    dnsServers
+  },
+  registerableDomainResultCache
+});
 
 export const isDomainAlive = createDomainAliveChecker({
   dns: {
     dnsServers
-  }
+  },
+  registerableDomainResultCache,
+  resultCache
 });

Build/validate-domain-alive.ts

@@ -1,6 +1,6 @@
 import { SOURCE_DIR } from './constants/dir';
 import path from 'node:path';
-import { isDomainAlive } from './lib/is-domain-alive';
+import { isDomainAlive, isRegisterableDomainAlive } from './lib/is-domain-alive';
 import { fdir as Fdir } from 'fdir';
 import runAgainstSourceFile from './lib/run-against-source-file';
 
@@ -43,17 +43,24 @@ const deadDomains: string[] = [];
       (domain: string, includeAllSubdomain: boolean) => {
         bar.setTotal(bar.getTotal() + 1);
 
-        return queue.add(
-          () => isDomainAlive(domain).then(({ alive, registerableDomainAlive, registerableDomain }) => {
-            bar.increment();
+        return queue.add(async () => {
+          let registerableDomainAlive, registerableDomain, alive: boolean | undefined;
 
-            if (!registerableDomainAlive) {
-              deadDomains.push('.' + registerableDomain);
-            } else if (!alive) {
-              deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
-            }
-          })
-        );
+          if (includeAllSubdomain) {
+            // we only need to check apex domain, because we don't know if there is any stripped subdomain
+            ({ alive: registerableDomainAlive, registerableDomain } = await isRegisterableDomainAlive(domain));
+          } else {
+            ({ alive, registerableDomainAlive, registerableDomain } = await isDomainAlive(domain));
+          }
+
+          bar.increment();
+
+          if (!registerableDomainAlive) {
+            deadDomains.push('.' + registerableDomain);
+          } else if (!includeAllSubdomain && alive != null && !alive) {
+            deadDomains.push(domain);
+          }
+        });
       }
     ).then(() => console.log('[crawl]', filepath))
   ));