|
|
@@ -4,7 +4,7 @@ const path = require('path');
|
|
|
const { createRuleset } = require('./lib/create-file');
|
|
|
const { processLine } = require('./lib/process-line.js');
|
|
|
const domainSorter = require('./lib/stable-sort-domain');
|
|
|
-const { runner, traceSync, task } = require('./lib/trace-runner.js');
|
|
|
+const { traceSync, task } = require('./lib/trace-runner.js');
|
|
|
|
|
|
const WHITELIST_DOMAIN = new Set([
|
|
|
'w3s.link',
|
|
|
@@ -61,67 +61,76 @@ const BLACK_TLD = new Set([
|
|
|
]);
|
|
|
|
|
|
const buildPhishingDomainSet = task(__filename, async () => {
|
|
|
- const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
|
|
|
+ const domainSet = Array.from((await processFilterRules(
|
|
|
+ 'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
|
|
|
+ // [
|
|
|
+ // 'https://malware-filter.gitlab.io/phishing-filter/phishing-filter-agh.txt',
|
|
|
+ // 'https://malware-filter.pages.dev/phishing-filter-agh.txt',
|
|
|
+ // 'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
|
|
|
+ // ]
|
|
|
+ )).black);
|
|
|
const domainCountMap = {};
|
|
|
|
|
|
- for (let i = 0, len = domainSet.length; i < len; i++) {
|
|
|
- const line = processLine(domainSet[i]);
|
|
|
- if (!line) continue;
|
|
|
+ traceSync('* process domain set', () => {
|
|
|
+ for (let i = 0, len = domainSet.length; i < len; i++) {
|
|
|
+ const line = processLine(domainSet[i]);
|
|
|
+ if (!line) continue;
|
|
|
|
|
|
- const parsed = tldts.parse(line, { allowPrivateDomains: true });
|
|
|
- const apexDomain = parsed.domain;
|
|
|
+ const parsed = tldts.parse(line, { allowPrivateDomains: true });
|
|
|
+ const apexDomain = parsed.domain;
|
|
|
|
|
|
- if (apexDomain) {
|
|
|
- if (WHITELIST_DOMAIN.has(apexDomain)) {
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- domainCountMap[apexDomain] ||= 0;
|
|
|
+ if (apexDomain) {
|
|
|
+ if (WHITELIST_DOMAIN.has(apexDomain)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
- let isPhishingDomainMockingAmazon = false;
|
|
|
- if (line.startsWith('.amaz')) {
|
|
|
- domainCountMap[apexDomain] += 0.5;
|
|
|
+ domainCountMap[apexDomain] ||= 0;
|
|
|
|
|
|
- isPhishingDomainMockingAmazon = true;
|
|
|
+ let isPhishingDomainMockingAmazon = false;
|
|
|
+ if (line.startsWith('.amaz')) {
|
|
|
+ domainCountMap[apexDomain] += 0.5;
|
|
|
|
|
|
- if (line.startsWith('.amazon-')) {
|
|
|
- domainCountMap[apexDomain] += 4.5;
|
|
|
- }
|
|
|
- } else if (line.startsWith('.customer')) {
|
|
|
- domainCountMap[apexDomain] += 0.25;
|
|
|
- }
|
|
|
- if (line.includes('-co-jp')) {
|
|
|
- domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5);
|
|
|
- }
|
|
|
+ isPhishingDomainMockingAmazon = true;
|
|
|
|
|
|
- const tld = parsed.publicSuffix;
|
|
|
- if (!tld || !BLACK_TLD.has(tld)) continue;
|
|
|
-
|
|
|
- domainCountMap[apexDomain] += 1;
|
|
|
-
|
|
|
- if (line.length > 19) {
|
|
|
- // Add more weight if the domain is long enough
|
|
|
- if (line.length > 44) {
|
|
|
- domainCountMap[apexDomain] += 3.5;
|
|
|
- } else if (line.length > 34) {
|
|
|
- domainCountMap[apexDomain] += 2.5;
|
|
|
- } else if (line.length > 29) {
|
|
|
- domainCountMap[apexDomain] += 1.5;
|
|
|
- } else if (line.length > 24) {
|
|
|
- domainCountMap[apexDomain] += 0.75;
|
|
|
- } else if (line.length > 19) {
|
|
|
+ if (line.startsWith('.amazon-')) {
|
|
|
+ domainCountMap[apexDomain] += 4.5;
|
|
|
+ }
|
|
|
+ } else if (line.startsWith('.customer')) {
|
|
|
domainCountMap[apexDomain] += 0.25;
|
|
|
}
|
|
|
+ if (line.includes('-co-jp')) {
|
|
|
+ domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5);
|
|
|
+ }
|
|
|
+
|
|
|
+ const tld = parsed.publicSuffix;
|
|
|
+ if (!tld || !BLACK_TLD.has(tld)) continue;
|
|
|
|
|
|
- if (domainCountMap[apexDomain] < 5) {
|
|
|
- const subdomain = parsed.subdomain;
|
|
|
- if (subdomain && subdomain.includes('.')) {
|
|
|
+ domainCountMap[apexDomain] += 1;
|
|
|
+
|
|
|
+ if (line.length > 19) {
|
|
|
+ // Add more weight if the domain is long enough
|
|
|
+ if (line.length > 44) {
|
|
|
+ domainCountMap[apexDomain] += 3.5;
|
|
|
+ } else if (line.length > 34) {
|
|
|
+ domainCountMap[apexDomain] += 2.5;
|
|
|
+ } else if (line.length > 29) {
|
|
|
domainCountMap[apexDomain] += 1.5;
|
|
|
+ } else if (line.length > 24) {
|
|
|
+ domainCountMap[apexDomain] += 0.75;
|
|
|
+ } else if (line.length > 19) {
|
|
|
+ domainCountMap[apexDomain] += 0.25;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (domainCountMap[apexDomain] < 5) {
|
|
|
+ const subdomain = parsed.subdomain;
|
|
|
+ if (subdomain?.includes('.')) {
|
|
|
+ domainCountMap[apexDomain] += 1.5;
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- }
|
|
|
+ });
|
|
|
|
|
|
const results = traceSync('* get final results', () => Object.entries(domainCountMap)
|
|
|
.reduce((acc, [apexDomain, count]) => {
|
|
|
@@ -156,5 +165,5 @@ const buildPhishingDomainSet = task(__filename, async () => {
|
|
|
module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
|
|
|
|
|
|
if (require.main === module) {
|
|
|
- runner(__filename, buildPhishingDomainSet);
|
|
|
+ buildPhishingDomainSet();
|
|
|
}
|