build-reject-domainset-worker.js 1.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657
  1. const Piscina = require('piscina');
  2. const { isCI } = require('ci-info');
  3. const fullsetDomainStartsWithADot = Piscina.workerData
  4. const totalLen = fullsetDomainStartsWithADot.length;
  5. const log = isCI ? () => { } : console.log.bind(console);
  6. module.exports.dedupe = ({ chunk }) => {
  7. const chunkLength = chunk.length;
  8. const outputToBeRemoved = new Int8Array(chunkLength);
  9. for (let i = 0; i < chunkLength; i++) {
  10. const domainFromInput = chunk[i];
  11. for (let j = 0; j < totalLen; j++) {
  12. const domainFromFullSet = fullsetDomainStartsWithADot[j];
  13. // domainFromFullSet is always startsWith "."
  14. if (domainFromFullSet === domainFromInput) continue;
  15. const domainFromInputLen = domainFromInput.length;
  16. const domainFromFullSetLen = domainFromFullSet.length;
  17. // !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet
  18. if (domainFromInput.charCodeAt(0) !== 46) {
  19. if (domainFromInputLen + 1 === domainFromFullSetLen) {
  20. let shouldBeRemoved = true;
  21. for (let k = 0; k < domainFromInputLen; k++) {
  22. if (domainFromFullSet[k + 1] !== domainFromInput[k]) {
  23. shouldBeRemoved = false;
  24. break;
  25. }
  26. }
  27. if (shouldBeRemoved) {
  28. outputToBeRemoved[i] = 1;
  29. log(domainFromInput, domainFromFullSet)
  30. break;
  31. }
  32. }
  33. }
  34. if (domainFromInputLen > domainFromFullSetLen) {
  35. // domainFromInput is now startsWith a "."
  36. if (domainFromInput.endsWith(domainFromFullSet)) {
  37. outputToBeRemoved[i] = 1;
  38. log(domainFromInput, domainFromFullSet)
  39. break;
  40. }
  41. }
  42. }
  43. }
  44. return Piscina.move(outputToBeRemoved);
  45. };