db62ed7f3a
This commit: - Fixes broken URLs using archive.org or other references. - Replaces tenforums.com URLs with better documentation as they tend to return HTTP status code 403 to tests and also are low quality source. - Changes all insecure http sources to https alternatives - Adds integration tests to check for broken URLs - There's logic implemented for having a delay inbetween when sending requests to same domains, however it's not used as the sources can respond to totally parallelized requests. - Run test pipeline weekly to get notified about broken URls without commits
48 lines
1.6 KiB
TypeScript
48 lines
1.6 KiB
TypeScript
import { retryWithExponentialBackOffAsync } from './ExponentialBackOffRetryHandler';
|
||
import { IUrlStatus } from './IUrlStatus';
|
||
import fetch from 'cross-fetch';
|
||
|
||
export interface IRequestOptions {
|
||
retryExponentialBaseInMs?: number;
|
||
additionalHeaders?: Record<string, string>;
|
||
}
|
||
|
||
export async function getUrlStatusAsync(
|
||
url: string,
|
||
options: IRequestOptions = DefaultOptions): Promise<IUrlStatus> {
|
||
options = { ...DefaultOptions, ...options };
|
||
const fetchOptions = getFetchOptions(options);
|
||
return retryWithExponentialBackOffAsync(async () => {
|
||
console.log('Requesting', url); // tslint:disable-line: no-console
|
||
try {
|
||
const response = await fetch(url, fetchOptions);
|
||
return { url, statusCode: response.status};
|
||
} catch (err) {
|
||
return { url, error: err};
|
||
}
|
||
}, options.retryExponentialBaseInMs);
|
||
}
|
||
|
||
const DefaultOptions: IRequestOptions = {
|
||
retryExponentialBaseInMs: 5000,
|
||
additionalHeaders: {},
|
||
};
|
||
|
||
function getFetchOptions(options: IRequestOptions) {
|
||
return {
|
||
method: 'GET',
|
||
headers: { ...DefaultHeaders, ...options.additionalHeaders },
|
||
};
|
||
}
|
||
|
||
const DefaultHeaders: Record<string, string> = {
|
||
/* Chrome on macOS */
|
||
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
|
||
'upgrade-insecure-requests': '1',
|
||
'connection': 'keep-alive',
|
||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||
'accept-encoding': 'gzip, deflate, br',
|
||
'cache-control': 'max-age=0',
|
||
'accept-language': 'en-US,en;q=0.9',
|
||
};
|