build-reject-domainset-worker.js 1.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445
  1. const { workerData } = require('piscina');
  2. const len = workerData.length;
  3. exports.dedupe = ({ chunk }) => {
  4. const outputToBeRemoved = new Set();
  5. for (let i = 0, l = chunk.length; i < l; i++) {
  6. const domainFromInput = chunk[i];
  7. for (let j = 0; j < len; j++) {
  8. const domainFromFullSet = workerData[j];
  9. if (domainFromFullSet === domainFromInput) continue;
  10. if (domainFromFullSet.charCodeAt(0) !== 46) continue;
  11. // domainFromFullSet is now startsWith a "."
  12. if (domainFromInput.charCodeAt(0) !== 46) {
  13. let shouldBeRemoved = true;
  14. for (let k = 0, l2 = domainFromInput.length; k < l2; k++) {
  15. if (domainFromFullSet.charCodeAt(k + 1) !== domainFromInput.charCodeAt(k)) {
  16. shouldBeRemoved = false;
  17. break;
  18. }
  19. }
  20. if (shouldBeRemoved) {
  21. outputToBeRemoved.add(domainFromInput);
  22. break;
  23. }
  24. }
  25. // domainFromInput is now startsWith a "."
  26. if (domainFromInput.length >= domainFromFullSet.length) {
  27. if (domainFromInput.endsWith(domainFromFullSet)) {
  28. outputToBeRemoved.add(domainFromInput);
  29. break;
  30. }
  31. }
  32. }
  33. }
  34. return outputToBeRemoved;
  35. };