| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051 |
- const Piscina = require('piscina');
- const fullsetDomainStartsWithADot = Piscina.workerData
- const totalLen = fullsetDomainStartsWithADot.length;
- module.exports.dedupe = ({ chunk }) => {
- const chunkLength = chunk.length;
- const outputToBeRemoved = new Int8Array(chunkLength);
- for (let i = 0; i < chunkLength; i++) {
- const domainFromInput = chunk[i];
- for (let j = 0; j < totalLen; j++) {
- const domainFromFullSet = fullsetDomainStartsWithADot[j];
- // domainFromFullSet is always startsWith "."
- if (domainFromFullSet === domainFromInput) continue;
- const domainFromInputLen = domainFromInput.length;
- const domainFromFullSetLen = domainFromFullSet.length;
- // !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet
- if (domainFromInput.charCodeAt(0) !== 46) {
- if (domainFromInputLen + 1 === domainFromFullSetLen) {
- let shouldBeRemoved = true;
- for (let k = 0; k < domainFromInputLen; k++) {
- if (domainFromFullSet.charCodeAt(k + 1) !== domainFromInput.charCodeAt(k)) {
- shouldBeRemoved = false;
- break;
- }
- }
- if (shouldBeRemoved) {
- outputToBeRemoved[i] = 1;
- break;
- }
- }
- } else if (domainFromInputLen > domainFromFullSetLen) {
- // domainFromInput is now startsWith a "."
- if (domainFromInput.endsWith(domainFromFullSet)) {
- outputToBeRemoved[i] = 1;
- break;
- }
- }
- }
- }
- return Piscina.move(outputToBeRemoved);
- };
|