Browse Source

Perf: speed up tldts

SukkaW 2 years ago
parent
commit
5701fc9b70

+ 1 - 1
Build/build-internal-cdn-rules.js

@@ -35,7 +35,7 @@ const buildInternalCDNDomains = task(__filename, async () => {
    */
    */
   const processLocalDomainSet = async (domainSetPath) => {
   const processLocalDomainSet = async (domainSetPath) => {
     for await (const line of readFileByLine(domainSetPath)) {
     for await (const line of readFileByLine(domainSetPath)) {
-      const parsed = tldts.parse(line, { allowPrivateDomains: true });
+      const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false });
       if (parsed.isIp) continue;
       if (parsed.isIp) continue;
       if (parsed.isIcann || parsed.isPrivate) {
       if (parsed.isIcann || parsed.isPrivate) {
         if (parsed.domain) {
         if (parsed.domain) {

+ 1 - 1
Build/build-phishing-domainset.js

@@ -142,7 +142,7 @@ const buildPhishingDomainSet = task(__filename, async () => {
         }
         }
 
 
         if (domainCountMap[apexDomain] < 5) {
         if (domainCountMap[apexDomain] < 5) {
-          const subdomain = tldts.getSubdomain(line);
+          const subdomain = tldts.getSubdomain(line, { detectIp: false });
           if (subdomain?.includes('.')) {
           if (subdomain?.includes('.')) {
             domainCountMap[apexDomain] += 1.5;
             domainCountMap[apexDomain] += 1.5;
           }
           }

+ 1 - 1
Build/build-redirect-module.js

@@ -74,7 +74,7 @@ const REDIRECT = /** @type {const} */ ([
 ]);
 ]);
 
 
 const buildRedirectModule = task(__filename, async () => {
 const buildRedirectModule = task(__filename, async () => {
-  const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from)))).filter(Boolean);
+  const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean);
 
 
   return compareAndWriteFile(
   return compareAndWriteFile(
     [
     [

+ 1 - 1
Build/build-reject-domainset.js

@@ -179,7 +179,7 @@ const buildRejectDomainSet = task(__filename, async () => {
     '* Collect reject domain stats',
     '* Collect reject domain stats',
     () => Object.entries(
     () => Object.entries(
       dudupedDominArray.reduce((acc, cur) => {
       dudupedDominArray.reduce((acc, cur) => {
-        const suffix = tldts.getDomain(cur, { allowPrivateDomains: false });
+        const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false });
         if (suffix) {
         if (suffix) {
           acc[suffix] = (acc[suffix] ?? 0) + 1;
           acc[suffix] = (acc[suffix] ?? 0) + 1;
         }
         }

+ 3 - 2
Build/lib/get-gorhill-publicsuffix.js

@@ -1,10 +1,11 @@
 const { toASCII } = require('punycode/');
 const { toASCII } = require('punycode/');
 const fs = require('fs');
 const fs = require('fs');
 const path = require('path');
 const path = require('path');
+const { traceAsync } = require('./trace-runner');
 
 
 const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
 const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
 
 
-const getGorhillPublicSuffix = async () => {
+const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix instance', async () => {
   const customFetch = async (url) => {
   const customFetch = async (url) => {
     const buf = await fs.promises.readFile(url);
     const buf = await fs.promises.readFile(url);
     return {
     return {
@@ -26,7 +27,7 @@ const getGorhillPublicSuffix = async () => {
   await gorhill.enableWASM({ customFetch });
   await gorhill.enableWASM({ customFetch });
 
 
   return gorhill;
   return gorhill;
-};
+});
 
 
 /** @type {Promise<import('gorhill-publicsuffixlist').default> | null} */
 /** @type {Promise<import('gorhill-publicsuffixlist').default> | null} */
 let gorhillPublicSuffixPromise = null;
 let gorhillPublicSuffixPromise = null;