Fix: an fetch option to allow remote upstream to be empty

https://feodotracker.abuse.ch/faq/

Build/build-reject-ip-list.ts

@@ -43,7 +43,7 @@ const BOTNET_FILTER_MIRROR_URL = [
   // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt
 ];
 
-const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL, true).then(arr => arr.reduce<[ipv4: string[], ipv6: string[]]>((acc, ip) => {
+const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL, true, true).then(arr => arr.reduce<[ipv4: string[], ipv6: string[]]>((acc, ip) => {
   const v = fastIpVersion(ip);
   if (v === 4) {
     acc[0].push(ip);

Build/constants/reject-data-source.ts

@@ -1,6 +1,6 @@
 export const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
 
-type HostsSource = [main: string, mirrors: string[] | null, includeAllSubDomain: boolean];
+type HostsSource = [main: string, mirrors: string[] | null, includeAllSubDomain: boolean, allowEmptyRemote?: boolean];
 
 export const HOSTS: HostsSource[] = [
   // WindowsSpyBlocker hasn't been updated since 2022-06-16, its content has been merged into domainset/reject.conf

Build/lib/fetch-assets.ts

@@ -13,7 +13,7 @@ export class CustomAbortError extends Error {
 
 const reusedCustomAbortError = new CustomAbortError();
 
-export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false) {
+export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false, allowEmpty = false) {
   const controller = new AbortController();
 
   const createFetchFallbackPromise = async (url: string, index: number) => {
@@ -38,7 +38,7 @@ export async function fetchAssets(url: string, fallbackUrls: null | undefined |
     }
     const arr = await Array.fromAsync(stream);
 
-    if (arr.length < 1) {
+    if (arr.length < 1 && !allowEmpty) {
       throw new ResponseError(res, url, 'empty response w/o 304');
     }
 

Build/lib/parse-filter/domainlists.ts

@@ -22,9 +22,10 @@ function domainListLineCbIncludeAllSubdomain(line: string, set: string[], meta:
 }
 export function processDomainListsWithPreload(
   domainListsUrl: string, mirrors: string[] | null,
-  includeAllSubDomain = false
+  includeAllSubDomain = false,
+  allowEmptyRemote = false
 ) {
-  const downloadPromise = fetchAssets(domainListsUrl, mirrors, true);
+  const downloadPromise = fetchAssets(domainListsUrl, mirrors, true, allowEmptyRemote);
   const lineCb = includeAllSubDomain ? domainListLineCbIncludeAllSubdomain : domainListLineCb;
 
   return (span: Span) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => {

Build/lib/parse-filter/hosts.ts

@@ -56,8 +56,8 @@ export function processHosts(
   });
 }
 
-export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false) {
-  const downloadPromise = fetchAssets(hostsUrl, mirrors, true);
+export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, allowEmptyRemote = false) {
+  const downloadPromise = fetchAssets(hostsUrl, mirrors, true, allowEmptyRemote);
   const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb;
 
   return (span: Span) => span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => {