Update CHN IP CIDR generation

Build/build-chn-cidr.js

@@ -1,19 +1,26 @@
 const { fetchWithRetry } = require('./lib/fetch-retry');
+const { withBanner } = require('./lib/with-banner');
 const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 
 (async () => {
   console.time('Total Time - build-chnroutes-cidr');
 
-  const cidr = (await (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n');
+  const [rawCidr, { merge: mergeCidrs }] = await Promise.all([
+    (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text(),
+    import('cidr-tools')
+  ]);
+  const cidr = rawCidr.split('\n');
 
-  const filteredCidr = cidr.filter(line => {
+  console.log('Before Merge:', cidr.length);
+  const filteredCidr = mergeCidrs(cidr.filter(line => {
     if (line) {
       return !line.startsWith('#')
     }
 
     return false;
-  })
+  }));
+  console.log('After Merge:', filteredCidr.length);
 
   await fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
 
@@ -23,10 +30,10 @@ const { resolve: pathResolve } = require('path');
 function makeCidrList(cidr) {
   const date = new Date();
 
-  return `############################
-# Mainland China IPv4 CIDR
-# Data from misaka.io (misakaio @ GitHub)
-# Last Updated: ${date.toISOString()}
-# Routes: ${cidr.length}
-############################\n` + cidr.map(i => `IP-CIDR,${i}`).join('\n') + '\n########### END ############\n';
+  return withBanner(
+    'Mainland China IPv4 CIDR',
+    ['Data from misaka.io (misakaio @ GitHub)', 'License: CC BY-SA 2.0'],
+    date,
+    cidr.map(i => `IP-CIDR,${i}`)
+  );
 };

Build/lib/with-banner.js

@@ -0,0 +1,17 @@
+/**
+ * @param {string} title
+ * @param {string[]} description
+ * @param {Date} date
+ * @param {string[]} content
+ * @returns {string}
+ */
+const withBanner = (title, description, date, content) => {
+    return `########################################
+# ${title}
+${description.map(line => `# ${line}`).join('\n')}
+# Last Updated: ${date.toISOString()}
+# Size: ${content.length}
+########################################\n` + content.join('\n') + '\n################# END ###################\n';
+};
+
+module.exports.withBanner = withBanner;

package.json

@@ -79,6 +79,7 @@
     "@sukka/listdir": "^0.2.0",
     "@vercel/fetch-retry": "^5.1.3",
     "ci-info": "^3.6.1",
+    "cidr-tools": "^5.1.2",
     "fs-extra": "^10.1.0",
     "picocolors": "^1.0.0",
     "piscina": "^3.2.0",

pnpm-lock.yaml

@@ -4,6 +4,7 @@ specifiers:
   '@sukka/listdir': ^0.2.0
   '@vercel/fetch-retry': ^5.1.3
   ci-info: ^3.6.1
+  cidr-tools: ^5.1.2
   fs-extra: ^10.1.0
   picocolors: ^1.0.0
   piscina: ^3.2.0
@@ -16,6 +17,7 @@ dependencies:
   '@sukka/listdir': 0.2.0
   '@vercel/fetch-retry': 5.1.3
   ci-info: 3.6.1
+  cidr-tools: 5.1.2
   fs-extra: 10.1.0
   picocolors: 1.0.0
   piscina: 3.2.0
@@ -151,6 +153,23 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /cidr-regex/4.0.2:
+    resolution: {integrity: sha512-xvqVyyE8emYjipWVxsA+Vcsbt8TbJJsTHbOeMMA5qsmXUu5VZJzNBGwI4rr1IZ0FQI2z7bfry4NNACP5Cf67CA==}
+    engines: {node: '>=14'}
+    dependencies:
+      ip-regex: 5.0.0
+    dev: false
+
+  /cidr-tools/5.1.2:
+    resolution: {integrity: sha512-Ed9OBqB1E3cUWDQ0A81eilNVSAfd9b0vlAOa+oIcU4eGNbwFQuPqvWDq+A3csv+l4NE/IbBCDfTKUpCnvmIBAg==}
+    engines: {node: '>=14'}
+    dependencies:
+      cidr-regex: 4.0.2
+      ip-bigint: 5.0.3
+      ip-regex: 5.0.0
+      string-natural-compare: 3.0.1
+    dev: false
+
   /color-convert/2.0.1:
     resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
     engines: {node: '>=7.0.0'}
@@ -249,6 +268,18 @@ packages:
     resolution: {integrity: sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw==}
     dev: false
 
+  /ip-bigint/5.0.3:
+    resolution: {integrity: sha512-Tvl0inX9ACOQ0eLKg89nz2gS0l/bxvkqSDs43q4KWUaA2ZlDdeCXNtMx/EEoXPSuLEgWn9Zyr3Ze5N0EpYRnFQ==}
+    engines: {node: '>=14'}
+    dependencies:
+      ip-regex: 5.0.0
+    dev: false
+
+  /ip-regex/5.0.0:
+    resolution: {integrity: sha512-fOCG6lhoKKakwv+C6KdsOnGvgXnmgfmp0myi3bcNwj3qfwPAxRKWEuFhvEFF7ceYIz6+1jRZ+yguLFAmUNPEfw==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dev: false
+
   /is-binary-path/2.1.0:
     resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
     engines: {node: '>=8'}
@@ -432,6 +463,10 @@ packages:
     engines: {node: '>=10.0.0'}
     dev: false
 
+  /string-natural-compare/3.0.1:
+    resolution: {integrity: sha512-n3sPwynL1nwKi3WJ6AIsClwBMa0zTi54fn2oLU6ndfTSIO05xaznjSf15PcBZU6FNWbmN5Q6cxT4V5hGvB4taw==}
+    dev: false
+
   /string-width/4.2.3:
     resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
     engines: {node: '>=8'}