Chore: add fetch retry support

Build/build-anti-bogus-domain.js

@@ -1,10 +1,10 @@
-const { fetch } = require('undici');
+const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
 const { isIP } = require('net');
 
 (async () => {
-  const res = (await (await fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text())
+  const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text())
     .split('\n')
     .map(line => {
       if (line.startsWith('bogus-nxdomain=')) {

Build/build-apple-cdn.js

@@ -1,11 +1,11 @@
-const { fetch } = require('undici');
+const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
 
 const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m;
 
 (async () => {
-  const res = (await (await fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text())
+  const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text())
     .split('\n')
     .map(line => {
       if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {

Build/build-cdn-conf.js

@@ -1,9 +1,9 @@
-const { fetch } = require('undici');
+const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
 
 (async () => {
-  const domains = (await (await fetch('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n');
+  const domains = (await (await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n');
 
   const S3OSSDomains = domains.filter(line => {
     if (line) {

Build/build-cidr.js

@@ -1,9 +1,9 @@
-const { fetch } = require('undici');
+const { fetchWithRetry } = require('./lib/fetch-retry');
 const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 
 (async () => {
-  const cidr = (await (await fetch('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n');
+  const cidr = (await (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n');
 
   const filteredCidr = cidr.filter(line => {
     if (line) {

Build/build-telegram-cidr.js

@@ -1,10 +1,10 @@
-const { fetch } = require('undici');
+const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 
 (async () => {
-  const resp = await fetch('https://core.telegram.org/resources/cidr.txt');
+  const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
   const lastModified = new Date(resp.headers.get('last-modified'));
 
   const res = (await resp.text())

Build/lib/fetch-retry.js

@@ -0,0 +1,3 @@
+const { fetch } = require('undici');
+const fetchWithRetry = require('@vercel/fetch-retry')(fetch);
+module.exports.fetchWithRetry = fetchWithRetry;

Build/lib/parse-filter.js

@@ -1,5 +1,5 @@
 const { isIP } = require('net');
-const { fetch } = require('undici');
+const { fetchWithRetry } = require('./fetch-retry');
 
 const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m
 
@@ -26,7 +26,7 @@ async function processDomainLists (domainListsUrl) {
   /** @type Set<string> */
   const domainSets = new Set();
   /** @type string[] */
-  const domains = (await (await fetch(domainListsUrl)).text()).split('\n');
+  const domains = (await (await fetchWithRetry(domainListsUrl)).text()).split('\n');
   domains.forEach(line => {
     if (
       line.startsWith('#')
@@ -63,7 +63,7 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
   const domainSets = new Set();
 
   /** @type string[] */
-  const hosts = (await (await fetch(hostsUrl)).text()).split('\n');
+  const hosts = (await (await fetchWithRetry(hostsUrl)).text()).split('\n');
   hosts.forEach(line => {
     if (line.includes('#')) {
       return;
@@ -105,7 +105,7 @@ async function processFilterRules (filterRulesUrl) {
   const blacklistDomainSets = new Set();
 
   /** @type string[] */
-  const filterRules = (await (await fetch(filterRulesUrl)).text()).split('\n').map(line => line.trim());
+  const filterRules = (await (await fetchWithRetry(filterRulesUrl)).text()).split('\n').map(line => line.trim());
 
   filterRules.forEach(line => {
     const lineStartsWithDoubleVerticalBar = line.startsWith('||');

List/domainset/cdn.conf

@@ -769,3 +769,4 @@ img.perfops.net
 assets.ipstack.com
 vice-web-statics-cdn.vice.com
 video-images.vice.com
+.operacdn.com

package.json

@@ -57,6 +57,7 @@
   "license": "ISC",
   "dependencies": {
     "@sukka/listdir": "^0.2.0",
+    "@vercel/fetch-retry": "^5.1.3",
     "ci-info": "^3.3.2",
     "picocolors": "^1.0.0",
     "piscina": "^3.2.0",

pnpm-lock.yaml

@@ -2,6 +2,7 @@ lockfileVersion: 5.4
 
 specifiers:
   '@sukka/listdir': ^0.2.0
+  '@vercel/fetch-retry': ^5.1.3
   ci-info: ^3.3.2
   picocolors: ^1.0.0
   piscina: ^3.2.0
@@ -11,6 +12,7 @@ specifiers:
 
 dependencies:
   '@sukka/listdir': 0.2.0
+  '@vercel/fetch-retry': 5.1.3
   ci-info: 3.3.2
   picocolors: 1.0.0
   piscina: 3.2.0
@@ -51,6 +53,17 @@ packages:
     resolution: {integrity: sha512-UyVirNhAOXKwjiDehjUaGtpfk0QwNHyiXrlLb/FmWMtI+BGhaEvB9MypSfEAtiiMI3g6QTfG38ayNAorEuz5ow==}
     dev: false
 
+  /@vercel/fetch-retry/5.1.3:
+    resolution: {integrity: sha512-UIbFc4VsEZHOr6dWuE+kxY4NxnOLXFMCWm0fSKRRHUEtrIzaJLzHpWk2QskCXTSzFgFvhkLAvSrBK2XZg7NSzg==}
+    peerDependencies:
+      node-fetch: ^2.6.7
+    dependencies:
+      async-retry: 1.3.3
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /ajv/8.8.2:
     resolution: {integrity: sha512-x9VuX+R/jcFj1DHo/fCp99esgGDWiHENrKxaCENuCxpoMCmAt/COCGVDwA7kleEpEzJjDnvh3yGoOuLu0Dtllw==}
     dependencies:
@@ -85,6 +98,12 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /async-retry/1.3.3:
+    resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
+    dependencies:
+      retry: 0.13.1
+    dev: false
+
   /base64-js/1.5.1:
     resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
     dev: false
@@ -131,6 +150,18 @@ packages:
     resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
     dev: false
 
+  /debug/4.3.4:
+    resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+    dependencies:
+      ms: 2.1.2
+    dev: false
+
   /emoji-regex/8.0.0:
     resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
     dev: false
@@ -252,6 +283,10 @@ packages:
       picomatch: 2.3.1
     dev: true
 
+  /ms/2.1.2:
+    resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
+    dev: false
+
   /nice-napi/1.0.2:
     resolution: {integrity: sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA==}
     os: ['!win32']
@@ -335,6 +370,11 @@ packages:
     engines: {node: '>= 4'}
     dev: true
 
+  /retry/0.13.1:
+    resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
+    engines: {node: '>= 4'}
+    dev: false
+
   /reusify/1.0.4:
     resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
     engines: {iojs: '>=1.0.0', node: '>=0.10.0'}