|
|
@@ -7,6 +7,8 @@ import { ProcessLineStream } from './process-line';
|
|
|
import { AdGuardFilterIgnoreUnsupportedLinesStream } from './parse-filter/filters';
|
|
|
import { appendArrayInPlace } from 'foxts/append-array-in-place';
|
|
|
|
|
|
+import { newQueue } from '@henrygd/queue';
|
|
|
+
|
|
|
// eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better
|
|
|
class CustomAbortError extends Error {
|
|
|
public readonly name = 'AbortError';
|
|
|
@@ -15,6 +17,8 @@ class CustomAbortError extends Error {
|
|
|
|
|
|
const reusedCustomAbortError = new CustomAbortError();
|
|
|
|
|
|
+const queue = newQueue(16);
|
|
|
+
|
|
|
export async function fetchAssets(
|
|
|
url: string, fallbackUrls: null | undefined | string[] | readonly string[],
|
|
|
processLine = false, allowEmpty = false, filterAdGuardUnsupportedLines = false
|
|
|
@@ -25,7 +29,7 @@ export async function fetchAssets(
|
|
|
if (index >= 0) {
|
|
|
// To avoid wasting bandwidth, we will wait for a few time before downloading from the fallback URL.
|
|
|
try {
|
|
|
- await waitWithAbort(200 + (index + 1) * 400, controller.signal);
|
|
|
+ await waitWithAbort(1800 + (index + 1) * 1200, controller.signal);
|
|
|
} catch {
|
|
|
console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url));
|
|
|
throw reusedCustomAbortError;
|
|
|
@@ -38,6 +42,8 @@ export async function fetchAssets(
|
|
|
if (index >= 0) {
|
|
|
console.log(picocolors.yellowBright('[fetch fallback begin]'), picocolors.gray(url));
|
|
|
}
|
|
|
+
|
|
|
+ // we don't queue add here
|
|
|
const res = await $$fetch(url, { signal: controller.signal, ...defaultRequestInit });
|
|
|
|
|
|
let stream = nullthrow(res.body, url + ' has an empty body')
|
|
|
@@ -49,7 +55,9 @@ export async function fetchAssets(
|
|
|
if (filterAdGuardUnsupportedLines) {
|
|
|
stream = stream.pipeThrough(new AdGuardFilterIgnoreUnsupportedLinesStream());
|
|
|
}
|
|
|
- const arr = await Array.fromAsync(stream);
|
|
|
+
|
|
|
+ // we does queue during downloading
|
|
|
+ const arr = await queue.add(() => Array.fromAsync(stream));
|
|
|
|
|
|
if (arr.length < 1 && !allowEmpty) {
|
|
|
throw new ResponseError(res, url, 'empty response w/o 304');
|