db62ed7f3a
This commit: - Fixes broken URLs using archive.org or other references. - Replaces tenforums.com URLs with better documentation as they tend to return HTTP status code 403 to tests and also are low quality source. - Changes all insecure http sources to https alternatives - Adds integration tests to check for broken URLs - There's logic implemented for having a delay inbetween when sending requests to same domains, however it's not used as the sources can respond to totally parallelized requests. - Run test pipeline weekly to get notified about broken URls without commits
45 lines
1.8 KiB
TypeScript
45 lines
1.8 KiB
TypeScript
import { sleepAsync } from '@/infrastructure/Threading/AsyncSleep';
|
||
import { IUrlStatus } from './IUrlStatus';
|
||
|
||
const DefaultBaseRetryIntervalInMs = 5 /* sec */ * 1000;
|
||
|
||
export async function retryWithExponentialBackOffAsync(
|
||
action: () => Promise<IUrlStatus>,
|
||
baseRetryIntervalInMs: number = DefaultBaseRetryIntervalInMs,
|
||
currentRetry = 1): Promise<IUrlStatus> {
|
||
const maxTries: number = 3;
|
||
const status = await action();
|
||
if (shouldRetry(status)) {
|
||
if (currentRetry <= maxTries) {
|
||
const exponentialBackOffInMs = getRetryTimeoutInMs(currentRetry, baseRetryIntervalInMs);
|
||
// tslint:disable-next-line: no-console
|
||
console.log(`Retrying (${currentRetry}) in ${exponentialBackOffInMs / 1000} seconds`, status);
|
||
await sleepAsync(exponentialBackOffInMs);
|
||
return retryWithExponentialBackOffAsync(action, baseRetryIntervalInMs, currentRetry + 1);
|
||
}
|
||
}
|
||
return status;
|
||
}
|
||
|
||
function shouldRetry(status: IUrlStatus) {
|
||
if (status.error) {
|
||
return true;
|
||
}
|
||
return isTransientError(status.statusCode)
|
||
|| status.statusCode === 429; // Too Many Requests
|
||
}
|
||
|
||
function isTransientError(statusCode: number) {
|
||
return statusCode >= 500 && statusCode <= 599;
|
||
}
|
||
|
||
function getRetryTimeoutInMs(currentRetry: number, baseRetryIntervalInMs: number = DefaultBaseRetryIntervalInMs) {
|
||
const retryRandomFactor = 0.5; // Retry intervals are between 50% and 150%
|
||
// of the exponentially increasing base amount
|
||
const minRandom = 1 - retryRandomFactor;
|
||
const maxRandom = 1 + retryRandomFactor;
|
||
const randomization = (Math.random() * (maxRandom - minRandom)) + maxRandom;
|
||
const exponential = Math.pow(2, currentRetry - 1);
|
||
return Math.ceil(exponential * baseRetryIntervalInMs * randomization);
|
||
}
|