|
@@ -1,6 +1,8 @@
|
|
|
// @ts-check
|
|
// @ts-check
|
|
|
-const { promises: fsPromises } = require('fs');
|
|
|
|
|
|
|
+const fs = require('fs');
|
|
|
const fse = require('fs-extra');
|
|
const fse = require('fs-extra');
|
|
|
|
|
+const readline = require('readline');
|
|
|
|
|
+
|
|
|
const { resolve: pathResolve } = require('path');
|
|
const { resolve: pathResolve } = require('path');
|
|
|
const { processHosts, processFilterRules, preprocessFullDomainSetBeforeUsedAsWorkerData } = require('./lib/parse-filter');
|
|
const { processHosts, processFilterRules, preprocessFullDomainSetBeforeUsedAsWorkerData } = require('./lib/parse-filter');
|
|
|
const { getDomain } = require('tldts');
|
|
const { getDomain } = require('tldts');
|
|
@@ -23,145 +25,148 @@ const domainSuffixSet = new Set();
|
|
|
/** @type Set<string> */
|
|
/** @type Set<string> */
|
|
|
const domainSets = new Set();
|
|
const domainSets = new Set();
|
|
|
|
|
|
|
|
- console.log('Downloading hosts file...');
|
|
|
|
|
- console.time('* Download and process Hosts');
|
|
|
|
|
-
|
|
|
|
|
- // Parse from remote hosts & domain lists
|
|
|
|
|
- (await Promise.all(HOSTS.map(entry => processHosts(entry[0], entry[1]))))
|
|
|
|
|
- .forEach(hosts => {
|
|
|
|
|
- hosts.forEach(host => {
|
|
|
|
|
- if (host) {
|
|
|
|
|
- domainSets.add(host);
|
|
|
|
|
- }
|
|
|
|
|
- });
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
- console.timeEnd('* Download and process Hosts');
|
|
|
|
|
-
|
|
|
|
|
- let previousSize = domainSets.size;
|
|
|
|
|
- console.log(`Import ${previousSize} rules from hosts files!`);
|
|
|
|
|
-
|
|
|
|
|
// Parse from AdGuard Filters
|
|
// Parse from AdGuard Filters
|
|
|
- console.time('* Download and process AdBlock Filter Rules');
|
|
|
|
|
|
|
+ console.time('* Download and process Hosts / AdBlock Filter Rules');
|
|
|
|
|
|
|
|
let shouldStop = false;
|
|
let shouldStop = false;
|
|
|
- await Promise.all(ADGUARD_FILTERS.map(input => {
|
|
|
|
|
- const promise = typeof input === 'string'
|
|
|
|
|
- ? processFilterRules(input, undefined, false)
|
|
|
|
|
- : processFilterRules(input[0], input[1] || undefined, input[2] ?? false)
|
|
|
|
|
-
|
|
|
|
|
- return promise.then((i) => {
|
|
|
|
|
- if (i) {
|
|
|
|
|
- const { white, black, foundDebugDomain } = i;
|
|
|
|
|
- if (foundDebugDomain) {
|
|
|
|
|
- shouldStop = true;
|
|
|
|
|
- }
|
|
|
|
|
- white.forEach(i => {
|
|
|
|
|
- if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
|
|
|
|
- return;
|
|
|
|
|
- }
|
|
|
|
|
- filterRuleWhitelistDomainSets.add(i);
|
|
|
|
|
- });
|
|
|
|
|
- black.forEach(i => domainSets.add(i));
|
|
|
|
|
- } else {
|
|
|
|
|
- process.exit(1);
|
|
|
|
|
- }
|
|
|
|
|
- });
|
|
|
|
|
- }));
|
|
|
|
|
|
|
|
|
|
await Promise.all([
|
|
await Promise.all([
|
|
|
- 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
|
|
|
|
|
- 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
|
|
|
|
|
- ].map(
|
|
|
|
|
- input => processFilterRules(input).then((i) => {
|
|
|
|
|
- if (i) {
|
|
|
|
|
- const { white, black } = i;
|
|
|
|
|
- white.forEach(i => {
|
|
|
|
|
- if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
|
|
|
|
- return;
|
|
|
|
|
|
|
+ // Parse from remote hosts & domain lists
|
|
|
|
|
+ Promise.all(HOSTS.map(entry => processHosts(entry[0], entry[1])))
|
|
|
|
|
+ .then(r => r.forEach(hosts => {
|
|
|
|
|
+ hosts.forEach(host => {
|
|
|
|
|
+ if (host) {
|
|
|
|
|
+ domainSets.add(host);
|
|
|
}
|
|
}
|
|
|
- filterRuleWhitelistDomainSets.add(i)
|
|
|
|
|
});
|
|
});
|
|
|
- black.forEach(i => {
|
|
|
|
|
- if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
|
|
|
|
- return;
|
|
|
|
|
|
|
+ })),
|
|
|
|
|
+ Promise.all(ADGUARD_FILTERS.map(input => {
|
|
|
|
|
+ const promise = typeof input === 'string'
|
|
|
|
|
+ ? processFilterRules(input, undefined, false)
|
|
|
|
|
+ : processFilterRules(input[0], input[1] || undefined, input[2] ?? false);
|
|
|
|
|
+
|
|
|
|
|
+ return promise.then((i) => {
|
|
|
|
|
+ if (i) {
|
|
|
|
|
+ const { white, black, foundDebugDomain } = i;
|
|
|
|
|
+ if (foundDebugDomain) {
|
|
|
|
|
+ shouldStop = true;
|
|
|
}
|
|
}
|
|
|
- filterRuleWhitelistDomainSets.add(i)
|
|
|
|
|
- });
|
|
|
|
|
- } else {
|
|
|
|
|
- process.exit(1);
|
|
|
|
|
- }
|
|
|
|
|
- })
|
|
|
|
|
- ));
|
|
|
|
|
|
|
+ white.forEach(i => {
|
|
|
|
|
+ // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
|
|
|
|
+ // return;
|
|
|
|
|
+ // }
|
|
|
|
|
+ filterRuleWhitelistDomainSets.add(i);
|
|
|
|
|
+ });
|
|
|
|
|
+ black.forEach(i => domainSets.add(i));
|
|
|
|
|
+ } else {
|
|
|
|
|
+ process.exit(1);
|
|
|
|
|
+ }
|
|
|
|
|
+ });
|
|
|
|
|
+ })),
|
|
|
|
|
+ Promise.all([
|
|
|
|
|
+ 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
|
|
|
|
|
+ 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
|
|
|
|
|
+ ].map(
|
|
|
|
|
+ input => processFilterRules(input).then((i) => {
|
|
|
|
|
+ if (i) {
|
|
|
|
|
+ const { white, black } = i;
|
|
|
|
|
+ white.forEach(i => {
|
|
|
|
|
+ // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
|
|
|
|
+ // return;
|
|
|
|
|
+ // }
|
|
|
|
|
+ filterRuleWhitelistDomainSets.add(i);
|
|
|
|
|
+ });
|
|
|
|
|
+ black.forEach(i => {
|
|
|
|
|
+ // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
|
|
|
|
+ // return;
|
|
|
|
|
+ // }
|
|
|
|
|
+ filterRuleWhitelistDomainSets.add(i);
|
|
|
|
|
+ });
|
|
|
|
|
+ } else {
|
|
|
|
|
+ process.exit(1);
|
|
|
|
|
+ }
|
|
|
|
|
+ })
|
|
|
|
|
+ ))
|
|
|
|
|
+ ]);
|
|
|
|
|
+
|
|
|
|
|
+ const trie0 = Trie.from(Array.from(filterRuleWhitelistDomainSets));
|
|
|
|
|
+ PREDEFINED_ENFORCED_BACKLIST.forEach(enforcedBlack => {
|
|
|
|
|
+ trie0.find(enforcedBlack).forEach(found => filterRuleWhitelistDomainSets.delete(found));
|
|
|
|
|
+ });
|
|
|
|
|
|
|
|
- console.timeEnd('* Download and process AdBlock Filter Rules');
|
|
|
|
|
|
|
+ console.timeEnd('* Download and process Hosts / AdBlock Filter Rules');
|
|
|
|
|
|
|
|
if (shouldStop) {
|
|
if (shouldStop) {
|
|
|
process.exit(1);
|
|
process.exit(1);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- previousSize = domainSets.size - previousSize;
|
|
|
|
|
- console.log(`Import ${previousSize} rules from adguard filters!`);
|
|
|
|
|
-
|
|
|
|
|
- await fsPromises.readFile(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
|
|
|
|
|
- data.split('\n').forEach(line => {
|
|
|
|
|
- const trimmed = line.trim();
|
|
|
|
|
- if (
|
|
|
|
|
- line.startsWith('#')
|
|
|
|
|
- || line.startsWith(' ')
|
|
|
|
|
- || line.startsWith('\r')
|
|
|
|
|
- || line.startsWith('\n')
|
|
|
|
|
- || trimmed === ''
|
|
|
|
|
- ) {
|
|
|
|
|
- return;
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ let previousSize = domainSets.size;
|
|
|
|
|
+ console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules!`);
|
|
|
|
|
|
|
|
- domainSets.add(trimmed);
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ const rl1 = readline.createInterface({
|
|
|
|
|
+ input: fs.createReadStream(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }),
|
|
|
|
|
+ crlfDelay: Infinity
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
|
|
+ for await (const line of rl1) {
|
|
|
|
|
+ if (
|
|
|
|
|
+ line.startsWith('#')
|
|
|
|
|
+ || line.startsWith(' ')
|
|
|
|
|
+ || line.startsWith('\r')
|
|
|
|
|
+ || line.startsWith('\n')
|
|
|
|
|
+ ) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const trimmed = line.trim();
|
|
|
|
|
+ if (trimmed === '') continue;
|
|
|
|
|
+
|
|
|
|
|
+ domainSets.add(trimmed);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
previousSize = domainSets.size - previousSize;
|
|
previousSize = domainSets.size - previousSize;
|
|
|
console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
|
|
console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
|
|
|
|
|
|
|
|
- await Promise.all([
|
|
|
|
|
- // Copy reject_sukka.conf for backward compatibility
|
|
|
|
|
- fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf')),
|
|
|
|
|
- fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
|
|
|
|
|
- data.split('\n').forEach(line => {
|
|
|
|
|
- if (line.startsWith('DOMAIN-KEYWORD')) {
|
|
|
|
|
- const [, ...keywords] = line.split(',');
|
|
|
|
|
- domainKeywordsSet.add(keywords.join(',').trim());
|
|
|
|
|
- } else if (line.startsWith('DOMAIN-SUFFIX')) {
|
|
|
|
|
- const [, ...keywords] = line.split(',');
|
|
|
|
|
- domainSuffixSet.add(keywords.join(',').trim());
|
|
|
|
|
- }
|
|
|
|
|
- });
|
|
|
|
|
- }),
|
|
|
|
|
- // Read Special Phishing Suffix list
|
|
|
|
|
- fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
|
|
|
|
|
- data.split('\n').forEach(line => {
|
|
|
|
|
- const trimmed = line.trim();
|
|
|
|
|
- if (
|
|
|
|
|
- line.startsWith('#')
|
|
|
|
|
- || line.startsWith(' ')
|
|
|
|
|
- || line.startsWith('\r')
|
|
|
|
|
- || line.startsWith('\n')
|
|
|
|
|
- || trimmed === ''
|
|
|
|
|
- ) {
|
|
|
|
|
- return;
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ const rl2 = readline.createInterface({
|
|
|
|
|
+ input: fs.createReadStream(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }),
|
|
|
|
|
+ crlfDelay: Infinity
|
|
|
|
|
+ });
|
|
|
|
|
+ for await (const line of rl2) {
|
|
|
|
|
+ if (line.startsWith('DOMAIN-KEYWORD')) {
|
|
|
|
|
+ const [, ...keywords] = line.split(',');
|
|
|
|
|
+ domainKeywordsSet.add(keywords.join(',').trim());
|
|
|
|
|
+ } else if (line.startsWith('DOMAIN-SUFFIX')) {
|
|
|
|
|
+ const [, ...keywords] = line.split(',');
|
|
|
|
|
+ domainSuffixSet.add(keywords.join(',').trim());
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- domainSuffixSet.add(trimmed);
|
|
|
|
|
- });
|
|
|
|
|
- })
|
|
|
|
|
- ]);
|
|
|
|
|
|
|
+ const rl3 = readline.createInterface({
|
|
|
|
|
+ input: fs.createReadStream(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }),
|
|
|
|
|
+ crlfDelay: Infinity
|
|
|
|
|
+ });
|
|
|
|
|
+ for await (const line of rl3) {
|
|
|
|
|
+ if (
|
|
|
|
|
+ line.startsWith('#')
|
|
|
|
|
+ || line.startsWith(' ')
|
|
|
|
|
+ || line.startsWith('\r')
|
|
|
|
|
+ || line.startsWith('\n')
|
|
|
|
|
+ ) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const trimmed = line.trim();
|
|
|
|
|
+ if (trimmed === '') continue;
|
|
|
|
|
+
|
|
|
|
|
+ domainSuffixSet.add(trimmed);
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
|
|
console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
|
|
|
|
|
|
|
|
previousSize = domainSets.size;
|
|
previousSize = domainSets.size;
|
|
|
// Dedupe domainSets
|
|
// Dedupe domainSets
|
|
|
console.log(`Start deduping from black keywords/suffixes! (${previousSize})`);
|
|
console.log(`Start deduping from black keywords/suffixes! (${previousSize})`);
|
|
|
- console.time(`* Dedupe from black keywords/suffixes`);
|
|
|
|
|
|
|
+ console.time('* Dedupe from black keywords/suffixes');
|
|
|
|
|
|
|
|
const trie1 = Trie.from(Array.from(domainSets));
|
|
const trie1 = Trie.from(Array.from(domainSets));
|
|
|
domainSuffixSet.forEach(suffix => {
|
|
domainSuffixSet.forEach(suffix => {
|
|
@@ -174,16 +179,14 @@ const domainSuffixSet = new Set();
|
|
|
// Build whitelist trie, to handle case like removing `g.msn.com` due to white `.g.msn.com` (`@@||g.msn.com`)
|
|
// Build whitelist trie, to handle case like removing `g.msn.com` due to white `.g.msn.com` (`@@||g.msn.com`)
|
|
|
const trieWhite = Trie.from(Array.from(filterRuleWhitelistDomainSets));
|
|
const trieWhite = Trie.from(Array.from(filterRuleWhitelistDomainSets));
|
|
|
for (const domain of domainSets) {
|
|
for (const domain of domainSets) {
|
|
|
- if (domain[0] !== '.' && trieWhite.has(`.${domain}`)) {
|
|
|
|
|
- domainSets.delete(domain);
|
|
|
|
|
- continue;
|
|
|
|
|
- }
|
|
|
|
|
if (domain[0] === '.') {
|
|
if (domain[0] === '.') {
|
|
|
- const found = trieWhite.find(domain);
|
|
|
|
|
- if (found.length > 0) {
|
|
|
|
|
|
|
+ if (trieWhite.contains(domain)) {
|
|
|
domainSets.delete(domain);
|
|
domainSets.delete(domain);
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
|
|
+ } else if (trieWhite.has(`.${domain}`)) {
|
|
|
|
|
+ domainSets.delete(domain);
|
|
|
|
|
+ continue;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Remove keyword
|
|
// Remove keyword
|
|
@@ -192,7 +195,7 @@ const domainSuffixSet = new Set();
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- console.timeEnd(`* Dedupe from black keywords/suffixes`);
|
|
|
|
|
|
|
+ console.timeEnd('* Dedupe from black keywords/suffixes');
|
|
|
console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`);
|
|
console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`);
|
|
|
|
|
|
|
|
previousSize = domainSets.size;
|
|
previousSize = domainSets.size;
|
|
@@ -212,7 +215,7 @@ const domainSuffixSet = new Set();
|
|
|
if (found.length) {
|
|
if (found.length) {
|
|
|
found.forEach(f => {
|
|
found.forEach(f => {
|
|
|
domainSets.delete(f);
|
|
domainSets.delete(f);
|
|
|
- })
|
|
|
|
|
|
|
+ });
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const a = domainStartsWithADotAndFromFullSet.slice(1);
|
|
const a = domainStartsWithADotAndFromFullSet.slice(1);
|
|
@@ -237,12 +240,10 @@ const domainSuffixSet = new Set();
|
|
|
};
|
|
};
|
|
|
const sortedDomainSets = Array.from(domainSets)
|
|
const sortedDomainSets = Array.from(domainSets)
|
|
|
.map((v) => {
|
|
.map((v) => {
|
|
|
- return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v)?.toLowerCase() || v };
|
|
|
|
|
|
|
+ return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v) || v };
|
|
|
})
|
|
})
|
|
|
.sort(sorter)
|
|
.sort(sorter)
|
|
|
- .map((i) => {
|
|
|
|
|
- return i.v;
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ .map((i) => i.v);
|
|
|
|
|
|
|
|
await compareAndWriteFile(
|
|
await compareAndWriteFile(
|
|
|
withBannerArray(
|
|
withBannerArray(
|
|
@@ -256,7 +257,7 @@ const domainSuffixSet = new Set();
|
|
|
'',
|
|
'',
|
|
|
'Build from:',
|
|
'Build from:',
|
|
|
...HOSTS.map(host => ` - ${host[0]}`),
|
|
...HOSTS.map(host => ` - ${host[0]}`),
|
|
|
- ...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`),
|
|
|
|
|
|
|
+ ...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`)
|
|
|
],
|
|
],
|
|
|
new Date(),
|
|
new Date(),
|
|
|
sortedDomainSets
|
|
sortedDomainSets
|
|
@@ -264,6 +265,9 @@ const domainSuffixSet = new Set();
|
|
|
pathResolve(__dirname, '../List/domainset/reject.conf')
|
|
pathResolve(__dirname, '../List/domainset/reject.conf')
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
|
|
+ // Copy reject_sukka.conf for backward compatibility
|
|
|
|
|
+ await fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'));
|
|
|
|
|
+
|
|
|
console.timeEnd('* Write reject.conf');
|
|
console.timeEnd('* Write reject.conf');
|
|
|
|
|
|
|
|
console.timeEnd('Total Time - build-reject-domain-set');
|
|
console.timeEnd('Total Time - build-reject-domain-set');
|