be7a886225
This commit improves the URL health checking mechanism to reduce false negatives. - Treat all 2XX status codes as successful, addressing issues with codes like `204`. - Exclude URLs within Markdown inline code blocks. - Send the Host header for improved handling of webpages behind proxies. - Improve formatting and context for output messages. - Fix the defaulting options for redirects and cookie handling. - Add URL exclusion support for non-responsive URLs. - Update the user agent pool to modern browsers and platforms. - Improve CI/CD workflow to respond to modifications in the `test/checks/external-urls` directory, offering immediate feedback on potential impacts to the external URL test. - Add support for randomizing TLS fingerprint to mimic various clients better, improving the effectiveness of checks. However, this is not fully supported by Node.js's HTTP client; see nodejs/undici#1983 for more details. - Use `AbortSignal` instead of `AbortController` as more modern and simpler way to handle timeouts.
57 lines
2.0 KiB
TypeScript
57 lines
2.0 KiB
TypeScript
import { sleep } from '@/infrastructure/Threading/AsyncSleep';
|
|
import { indentText } from '@tests/shared/Text';
|
|
import { type UrlStatus, formatUrlStatus } from './UrlStatus';
|
|
|
|
const DefaultBaseRetryIntervalInMs = 5 /* sec */ * 1000;
|
|
|
|
export async function retryWithExponentialBackOff(
|
|
action: () => Promise<UrlStatus>,
|
|
baseRetryIntervalInMs: number = DefaultBaseRetryIntervalInMs,
|
|
currentRetry = 1,
|
|
): Promise<UrlStatus> {
|
|
const maxTries = 3;
|
|
const status = await action();
|
|
if (shouldRetry(status)) {
|
|
if (currentRetry <= maxTries) {
|
|
const exponentialBackOffInMs = getRetryTimeoutInMs(currentRetry, baseRetryIntervalInMs);
|
|
console.log([
|
|
`Attempt ${currentRetry}: Retrying in ${exponentialBackOffInMs / 1000} seconds.`,
|
|
'Details:',
|
|
indentText(formatUrlStatus(status)),
|
|
].join('\n'));
|
|
await sleep(exponentialBackOffInMs);
|
|
return retryWithExponentialBackOff(action, baseRetryIntervalInMs, currentRetry + 1);
|
|
}
|
|
console.warn('💀 All retry attempts failed. Final failure to retrieve URL:', indentText(formatUrlStatus(status)));
|
|
}
|
|
return status;
|
|
}
|
|
|
|
function shouldRetry(status: UrlStatus): boolean {
|
|
if (status.error) {
|
|
return true;
|
|
}
|
|
if (status.code === undefined) {
|
|
return true;
|
|
}
|
|
return isTransientError(status.code)
|
|
|| status.code === 429; // Too Many Requests
|
|
}
|
|
|
|
function isTransientError(statusCode: number): boolean {
|
|
return statusCode >= 500 && statusCode <= 599;
|
|
}
|
|
|
|
function getRetryTimeoutInMs(
|
|
currentRetry: number,
|
|
baseRetryIntervalInMs: number = DefaultBaseRetryIntervalInMs,
|
|
): number {
|
|
const retryRandomFactor = 0.5; // Retry intervals are between 50% and 150%
|
|
// of the exponentially increasing base amount
|
|
const minRandom = 1 - retryRandomFactor;
|
|
const maxRandom = 1 + retryRandomFactor;
|
|
const randomization = (Math.random() * (maxRandom - minRandom)) + maxRandom;
|
|
const exponential = 2 ** (currentRetry - 1);
|
|
return Math.ceil(exponential * baseRetryIntervalInMs * randomization);
|
|
}
|