build-reject-domainset-worker.js 1.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354
  1. const Piscina = require('piscina');
  2. const { isCI } = require('ci-info');
  3. const fullsetDomainStartsWithADot = Piscina.workerData
  4. const totalLen = fullsetDomainStartsWithADot.length;
  5. const log = isCI ? () => { } : console.log.bind(console);
  6. module.exports.dedupe = ({ chunk }) => {
  7. const chunkLength = chunk.length;
  8. const outputToBeRemoved = new Int8Array(chunkLength);
  9. for (let i = 0; i < chunkLength; i++) {
  10. const domainFromInput = chunk[i];
  11. for (let j = 0; j < totalLen; j++) {
  12. const domainFromFullSet = fullsetDomainStartsWithADot[j];
  13. // domainFromFullSet is always startsWith "."
  14. if (domainFromFullSet === domainFromInput) continue;
  15. const domainFromInputLen = domainFromInput.length;
  16. const domainFromFullSetLen = domainFromFullSet.length;
  17. // !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet
  18. if (domainFromInput[0] !== '.' && domainFromInputLen + 1 === domainFromFullSetLen) {
  19. let shouldBeRemoved = true;
  20. for (let k = 0; k < domainFromInputLen; k++) {
  21. if (domainFromFullSet[k + 1] !== domainFromInput[k]) {
  22. shouldBeRemoved = false;
  23. break;
  24. }
  25. }
  26. if (shouldBeRemoved) {
  27. outputToBeRemoved[i] = 1;
  28. log(domainFromInput, domainFromFullSet)
  29. break;
  30. }
  31. }
  32. if (domainFromInputLen > domainFromFullSetLen) {
  33. // domainFromInput is now startsWith a "."
  34. if (domainFromInput.endsWith(domainFromFullSet)) {
  35. outputToBeRemoved[i] = 1;
  36. log(domainFromInput, domainFromFullSet)
  37. break;
  38. }
  39. }
  40. }
  41. }
  42. return Piscina.move(outputToBeRemoved);
  43. };