949fac1a7c
This commit applies `strictNullChecks` to the entire codebase to improve maintainability and type safety. Key changes include: - Remove some explicit null-checks where unnecessary. - Add necessary null-checks. - Refactor static factory functions for a more functional approach. - Improve some test names and contexts for better debugging. - Add unit tests for any additional logic introduced. - Refactor `createPositionFromRegexFullMatch` to its own function as the logic is reused. - Prefer `find` prefix on functions that may return `undefined` and `get` prefix for those that always return a value.
71 lines
2.2 KiB
TypeScript
71 lines
2.2 KiB
TypeScript
import { retryWithExponentialBackOff } from './ExponentialBackOffRetryHandler';
|
|
import { IUrlStatus } from './IUrlStatus';
|
|
import { fetchFollow, IFollowOptions, DefaultFollowOptions } from './FetchFollow';
|
|
import { getRandomUserAgent } from './UserAgents';
|
|
|
|
export function getUrlStatus(
|
|
url: string,
|
|
options: IRequestOptions = DefaultOptions,
|
|
): Promise<IUrlStatus> {
|
|
const defaultedOptions = { ...DefaultOptions, ...options };
|
|
const fetchOptions = getFetchOptions(url, defaultedOptions);
|
|
return retryWithExponentialBackOff(async () => {
|
|
console.log('Requesting', url);
|
|
let result: IUrlStatus;
|
|
try {
|
|
const response = await fetchFollow(
|
|
url,
|
|
defaultedOptions.requestTimeoutInMs,
|
|
fetchOptions,
|
|
defaultedOptions.followOptions,
|
|
);
|
|
result = { url, code: response.status };
|
|
} catch (err) {
|
|
result = { url, error: JSON.stringify(err, null, '\t') };
|
|
}
|
|
return result;
|
|
}, defaultedOptions.retryExponentialBaseInMs);
|
|
}
|
|
|
|
export interface IRequestOptions {
|
|
readonly retryExponentialBaseInMs?: number;
|
|
readonly additionalHeaders?: Record<string, string>;
|
|
readonly additionalHeadersUrlIgnore?: string[];
|
|
readonly followOptions?: IFollowOptions;
|
|
readonly requestTimeoutInMs: number;
|
|
}
|
|
|
|
const DefaultOptions: Required<IRequestOptions> = {
|
|
retryExponentialBaseInMs: 5000,
|
|
additionalHeaders: {},
|
|
additionalHeadersUrlIgnore: [],
|
|
requestTimeoutInMs: 60 /* seconds */ * 1000,
|
|
followOptions: DefaultFollowOptions,
|
|
};
|
|
|
|
function getFetchOptions(url: string, options: Required<IRequestOptions>): RequestInit {
|
|
const additionalHeaders = options.additionalHeadersUrlIgnore
|
|
.some((ignorePattern) => url.startsWith(ignorePattern))
|
|
? {}
|
|
: options.additionalHeaders;
|
|
return {
|
|
method: 'HEAD',
|
|
headers: {
|
|
...getDefaultHeaders(),
|
|
...additionalHeaders,
|
|
},
|
|
};
|
|
}
|
|
|
|
function getDefaultHeaders(): Record<string, string> {
|
|
return {
|
|
'user-agent': getRandomUserAgent(),
|
|
'upgrade-insecure-requests': '1',
|
|
connection: 'keep-alive',
|
|
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
|
'accept-encoding': 'gzip, deflate, br',
|
|
'cache-control': 'max-age=0',
|
|
'accept-language': 'en-US,en;q=0.9',
|
|
};
|
|
}
|