SukkaSurge/Build/build-phishing-domainset.js

const tldts = require('tldts');
const { processFilterRules } = require('./lib/parse-filter.js');
const fs = require('fs');
const path = require('path');
const { withBannerArray } = require('./lib/with-banner.js');
const { stringArrayCompare, compareAndWriteFile } = require('./lib/string-array-compare');

const WHITELIST_DOMAIN = new Set([
  'w3s.link',
  'dweb.link',
  'nftstorage.link',
  'square.site'
]);
const BLACK_TLD = Array.from(new Set([
  '.xyz',
  '.top',
  '.win',
  '.vip',
  '.site',
  '.space',
  '.online',
  '.icu',
  '.fun',
  '.shop',
  '.cool',
  '.cyou',
  '.id',
  '.pro',
  '.za.com',
  '.sa.com',
  '.ltd',
  '.group',
  '.rest',
  '.tech',
  '.link',
  '.ink',
  '.bar',
  '.tokyo'
]));

(async () => {
  const domainSet = Array.from(
    (
      await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')
    ).black
  );
  const domainCountMap = {};

  for (let i = 0, len = domainSet.length; i < len; i++) {
    const line = domainSet[i];
    // starts with #
    if (line.charCodeAt(0) === 35) {
      continue;
    }
    if (line.trim().length === 0) {
      continue;
    }

    const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;

    if (domain.length > 19) {
      const apexDomain = tldts.getDomain(domain, { allowPrivateDomains: true });

      if (apexDomain) {
        if (WHITELIST_DOMAIN.has(apexDomain)) {
          continue;
        }

        domainCountMap[apexDomain] ||= 0;
        domainCountMap[apexDomain] += 1;

        // Add more weight if the domain is long enough
        if (domain.length > 45) {
          domainCountMap[apexDomain] += 1.5;
        } else if (domain.length > 35) {
          domainCountMap[apexDomain] += 1;
        } else if (domain.length > 30) {
          domainCountMap[apexDomain] += 0.5;
        } else if (domain.length > 25) {
          domainCountMap[apexDomain] += 0.25;
        }

        const subdomain = tldts.getSubdomain(domain, { allowPrivateDomains: true });
        if (subdomain && subdomain.includes('.')) {
          domainCountMap[apexDomain] += 0.5;
        }
      }
    }
  }

  const results = [];
  Object.entries(domainCountMap).forEach(([domain, count]) => {
    if (
      count >= 5
      && BLACK_TLD.some(tld => domain.endsWith(tld))
    ) {
      results.push('.' + domain);
    }
  });

  results.sort();

  await compareAndWriteFile(
    withBannerArray(
      'Sukka\'s Surge Rules - Reject Phishing',
      [
        'License: AGPL 3.0',
        'Homepage: https://ruleset.skk.moe',
        'GitHub: https://github.com/SukkaW/Surge',
        '',
        'The domainset supports enhanced phishing protection',
        'Build from:',
        ' - https://gitlab.com/malware-filter/phishing-filter'
      ],
      new Date(),
      results
    ),
    path.resolve(__dirname, '../List/domainset/reject_phishing.conf')
  )
})();