19e42c9c52
- Move external URL checks to its own module under `tests/`. This separates them from integration test, addressing long runs and frequent failures that led to ignoring test results. - Move `check-desktop-runtime-errors` to `tests/checks` to keep all test-related checks into one directory. - Replace `ts-node` with `vite` for running `check-desktop-runtime-errors` to maintain a consistent execution environment across checks. - Implement a timeout for each fetch call. - Be nice to external sources, wait 5 seconds before sending another request to an URL under same domain. This solves rate-limiting issues. - Instead of running test on every push/pull request, run them only weekly. - Do not run tests on each commit/PR but only scheduled (weekly) to minimize noise. - Fix URLs are not captured correctly inside backticks or parenthesis.
76 lines
2.1 KiB
TypeScript
76 lines
2.1 KiB
TypeScript
import { sleep } from '@/infrastructure/Threading/AsyncSleep';
|
|
import { IUrlStatus } from './IUrlStatus';
|
|
import { getUrlStatus, IRequestOptions } from './Requestor';
|
|
import { groupUrlsByDomain } from './UrlPerDomainGrouper';
|
|
|
|
export async function getUrlStatusesInParallel(
|
|
urls: string[],
|
|
options?: IBatchRequestOptions,
|
|
): Promise<IUrlStatus[]> {
|
|
// urls = [ 'https://privacy.sexy' ]; // Here to comment out when testing
|
|
const uniqueUrls = Array.from(new Set(urls));
|
|
options = { ...DefaultOptions, ...options };
|
|
console.log('Options: ', options);
|
|
const results = await request(uniqueUrls, options);
|
|
return results;
|
|
}
|
|
|
|
export interface IBatchRequestOptions {
|
|
domainOptions?: IDomainOptions;
|
|
requestOptions?: IRequestOptions;
|
|
}
|
|
|
|
interface IDomainOptions {
|
|
sameDomainParallelize?: boolean;
|
|
sameDomainDelayInMs?: number;
|
|
}
|
|
|
|
const DefaultOptions: IBatchRequestOptions = {
|
|
domainOptions: {
|
|
sameDomainParallelize: false,
|
|
sameDomainDelayInMs: 3 /* sec */ * 1000,
|
|
},
|
|
requestOptions: {
|
|
retryExponentialBaseInMs: 5 /* sec */ * 1000,
|
|
requestTimeoutInMs: 60 /* sec */ * 1000,
|
|
additionalHeaders: {},
|
|
},
|
|
};
|
|
|
|
function request(
|
|
urls: string[],
|
|
options: IBatchRequestOptions,
|
|
): Promise<IUrlStatus[]> {
|
|
if (!options.domainOptions.sameDomainParallelize) {
|
|
return runOnEachDomainWithDelay(
|
|
urls,
|
|
(url) => getUrlStatus(url, options.requestOptions),
|
|
options.domainOptions.sameDomainDelayInMs,
|
|
);
|
|
}
|
|
return Promise.all(urls.map((url) => getUrlStatus(url, options.requestOptions)));
|
|
}
|
|
|
|
async function runOnEachDomainWithDelay(
|
|
urls: string[],
|
|
action: (url: string) => Promise<IUrlStatus>,
|
|
delayInMs: number,
|
|
): Promise<IUrlStatus[]> {
|
|
const grouped = groupUrlsByDomain(urls);
|
|
const tasks = grouped.map(async (group) => {
|
|
const results = new Array<IUrlStatus>();
|
|
/* eslint-disable no-await-in-loop */
|
|
for (const url of group) {
|
|
const status = await action(url);
|
|
results.push(status);
|
|
if (results.length !== group.length) {
|
|
await sleep(delayInMs);
|
|
}
|
|
}
|
|
/* eslint-enable no-await-in-loop */
|
|
return results;
|
|
});
|
|
const r = await Promise.all(tasks);
|
|
return r.flat();
|
|
}
|