SukkaSurge/Build/build-mitm-hostname.js

const fs = require('fs');
const { promises: fsPromises } = fs;
const pathFn = require('path');
let table;

const PRESET_MITM_HOSTNAMES = [
  '*baidu.com',
  '*ydstatic.com',
  '*snssdk.com',
  '*musical.com',
  '*musical.ly',
  '*snssdk.ly',
  'api.chelaile.net.cn',
  'atrace.chelaile.net.cn',
  '*.meituan.net',
  'ctrl.playcvn.com',
  'ctrl.playcvn.net',
  'ctrl.zmzapi.com',
  'ctrl.zmzapi.net',
  'api.zhuishushenqi.com',
  'b.zhuishushenqi.com',
  '*.music.126.net',
  '*.prod.hosts.ooklaserver.net'
];

try {
  table = require('table');
} catch (e) {
  console.log('Dependency "table" not found');
  console.log('"npm i table" then try again!');
  process.exit(1);
}

(async () => {
  const folderListPath = pathFn.resolve(__dirname, '../List/');
  const rulesets = await listDir(folderListPath);
  let urlRegexPaths = [];

  urlRegexPaths.push(
    ...(await fsPromises.readFile(pathFn.join(__dirname, '../Modules/sukka_url_rewrite.sgmodule'), { encoding: 'utf-8' }))
      .split('\n')
      .filter(
        i => !i.startsWith('#')
          && !i.startsWith('[')
      )
      .map(i => i.split(' ')[0])
      .map(i => ({
        origin: i,
        processed: i
          .replaceAll('(www.)?', '{www or not}')
          .replaceAll('^https?://', '')
          .replaceAll('^https://', '')
          .replaceAll('^http://', '')
          .split('/')[0]
          .replaceAll('\\.', '.')
          .replaceAll('.+', '*')
          .replaceAll('(.*)', '*')
      }))
  );

  const bothWwwApexDomains = [];
  urlRegexPaths = urlRegexPaths.map(i => {
    if (!i.processed.includes('{www or not}')) return i;

    const d = i.processed.replace('{www or not}', '');
    bothWwwApexDomains.push({
      origin: i.origin,
      processed: `www.${d}`
    });

    return {
      origin: i.origin,
      processed: d
    };
  });

  urlRegexPaths.push(...bothWwwApexDomains);

  await Promise.all(rulesets.map(async file => {
    const content = (await fsPromises.readFile(pathFn.join(folderListPath, file), { encoding: 'utf-8' })).split('\n');
    urlRegexPaths.push(
      ...content
        .filter(i => i.startsWith('URL-REGEX'))
        .map(i => i.split(',')[1])
        .map(i => ({
          origin: i,
          processed: i
            .replaceAll('^https?://', '')
            .replaceAll('^https://', '')
            .replaceAll('^http://', '')
            .replaceAll('\\.', '.')
            .replaceAll('.+', '*')
            .replaceAll('\\d', '*')
            .replaceAll('([a-z])', '*')
            .replaceAll('[a-z]', '*')
            .replaceAll('([0-9])', '*')
            .replaceAll('[0-9]', '*')
            .replaceAll(/{.+?}/g, '')
            .replaceAll(/\*+/g, '*')
        }))
    );
  }));

  let mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed
  const parsedFailures = new Set();

  const dedupedUrlRegexPaths = [...new Set(urlRegexPaths)];

  dedupedUrlRegexPaths.forEach(i => {
    const result = parseDomain(i.processed);

    if (result.success) {
      mitmDomains.add(result.hostname.trim());
    } else {
      parsedFailures.add(i.origin);
    }
  });

  mitmDomains = [...mitmDomains].filter(i => {
    return i.length > 3
      && !i.includes('.mp4') // Special Case
      && i !== '(www.)' // Special Case
      && !(i !== '*baidu.com' && i.endsWith('baidu.com')) // Special Case
      && !(i !== '*.meituan.net' && i.endsWith('.meituan.net'))
      && !i.startsWith('.')
      && !i.endsWith('.')
      && !i.endsWith('*')
  });

  const mitmDomainsRegExpArray = mitmDomains.map(i => {
    return new RegExp(
      escapeRegExp(i)
        .replaceAll('{www or not}', '(www.)?')
        .replaceAll('\\*', '(.*)')
    )
  });

  const parsedDomainsData = [];
  dedupedUrlRegexPaths.forEach(i => {
    const result = parseDomain(i.processed);

    if (result.success) {
      if (matchWithRegExpArray(result.hostname.trim(), mitmDomainsRegExpArray)) {
        parsedDomainsData.push([green(result.hostname), i.origin]);
      } else {
        parsedDomainsData.push([yellow(result.hostname), i.origin]);
      }
    }
  });

  console.log('Mitm Hostnames:');
  console.log('hostname = %APPEND% ' + mitmDomains.join(', '));
  console.log('--------------------');
  console.log('Parsed Sucessed:');
  console.log(table.table(parsedDomainsData, {
    border: table.getBorderCharacters('void'),
    columnDefault: {
      paddingLeft: 0,
      paddingRight: 3
    },
    drawHorizontalLine: () => false
  }));
  console.log('--------------------');
  console.log('Parsed Failed');
  console.log([...parsedFailures].join('\n'));
})();

/** Util function */
function green(...args) {
  return `\u001b[32m${args.join(' ')}\u001b[0m`;
}
function yellow(...args) {
  return `\u001b[33m${args.join(' ')}\u001b[0m`;
}

function parseDomain(input) {
  try {
    const url = new URL(`https://${input}`);
    return {
      success: true,
      hostname: url.hostname
    }
  } catch {
    return {
      success: false
    }
  }
}

function matchWithRegExpArray(input, regexps = []) {
  for (const r of regexps) {
    if (r.test(input)) return true;
  }

  return false;
}

function escapeRegExp(string = '') {
  const reRegExpChar = /[\\^$.*+?()[\]{}|]/g;
  const reHasRegExpChar = RegExp(reRegExpChar.source);

  return string && reHasRegExpChar.test(string)
    ? string.replace(reRegExpChar, '\\$&')
    : string;
}

function listDir(path, options) {
  const results = [];
  options = Object.assign({ ignoreHidden: true, ignorePattern: null }, options);
  return listDirWalker(path, results, '', options).then(() => results);
}
function listDirWalker(path, results, parent, options) {
  const promises = [];
  return readAndFilterDir(path, options).then(items => {
    items.forEach(item => {
      const currentPath = pathFn.join(parent, item.name);
      if (item.isDirectory()) {
        promises.push(listDirWalker(pathFn.join(path, item.name), results, currentPath, options));
      }
      else {
        results.push(currentPath);
      }
    });
  }).then(() => Promise.all(promises));
}
function readAndFilterDir(path, options) {
  const { ignoreHidden = true, ignorePattern } = options;
  return fs.promises.readdir(path, Object.assign(Object.assign({}, options), { withFileTypes: true }))
    .then(results => {
      if (ignoreHidden) {
        results = results.filter(({ name }) => !name.startsWith('.'));
      }
      if (ignorePattern) {
        results = results.filter(({ name }) => !ignorePattern.test(name));
      }
      return results;
    });
}