build-reject-domainset-worker.js 1.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051
  1. const Piscina = require('piscina');
  2. const fullsetDomainStartsWithADot = Piscina.workerData
  3. const totalLen = fullsetDomainStartsWithADot.length;
  4. module.exports.dedupe = ({ chunk }) => {
  5. const chunkLength = chunk.length;
  6. const outputToBeRemoved = new Int8Array(chunkLength);
  7. for (let i = 0; i < chunkLength; i++) {
  8. const domainFromInput = chunk[i];
  9. for (let j = 0; j < totalLen; j++) {
  10. const domainFromFullSet = fullsetDomainStartsWithADot[j];
  11. // domainFromFullSet is always startsWith "."
  12. if (domainFromFullSet === domainFromInput) continue;
  13. const domainFromInputLen = domainFromInput.length;
  14. const domainFromFullSetLen = domainFromFullSet.length;
  15. // !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet
  16. if (domainFromInput.charCodeAt(0) !== 46) {
  17. if (domainFromInputLen + 1 === domainFromFullSetLen) {
  18. let shouldBeRemoved = true;
  19. for (let k = 0; k < domainFromInputLen; k++) {
  20. if (domainFromFullSet.charCodeAt(k + 1) !== domainFromInput.charCodeAt(k)) {
  21. shouldBeRemoved = false;
  22. break;
  23. }
  24. }
  25. if (shouldBeRemoved) {
  26. outputToBeRemoved[i] = 1;
  27. break;
  28. }
  29. }
  30. } else if (domainFromInputLen > domainFromFullSetLen) {
  31. // domainFromInput is now startsWith a "."
  32. if (domainFromInput.endsWith(domainFromFullSet)) {
  33. outputToBeRemoved[i] = 1;
  34. break;
  35. }
  36. }
  37. }
  38. }
  39. return Piscina.move(outputToBeRemoved);
  40. };