799fb091b8
Implement following redirects over `fetch` supporting cookies.
`node-fetch` does not support sending cookies during redirect. However,
this is needed to not end-up in a redirect loop for a sign-in callback.
Fix integration tests failing due to redirects and 403 errors:
- Many redirects from `answers.microsoft.com` was throwing: throwing
`FetchError: maximum redirect reached` error. It was caused by not
having cookies when following redirects therefore having an infinite
sign-in callback for the webpage.
- Fixes integration tests failing due to additional referer header being
sent by the application. It adds support for making exceptions to
additional header sending through a list of regexes.
Add in-depth documentation for URL status checking.
55 lines
2.0 KiB
TypeScript
55 lines
2.0 KiB
TypeScript
import { retryWithExponentialBackOffAsync } from './ExponentialBackOffRetryHandler';
|
||
import { IUrlStatus } from './IUrlStatus';
|
||
import { fetchFollow, IFollowOptions } from './FetchFollow';
|
||
|
||
export async function getUrlStatusAsync(
|
||
url: string,
|
||
options: IRequestOptions = DefaultOptions): Promise<IUrlStatus> {
|
||
options = { ...DefaultOptions, ...options };
|
||
const fetchOptions = getFetchOptions(url, options);
|
||
return retryWithExponentialBackOffAsync(async () => {
|
||
console.log('Requesting', url); // tslint:disable-line: no-console
|
||
let result: IUrlStatus;
|
||
try {
|
||
const response = await fetchFollow(url, fetchOptions, options.followOptions);
|
||
result = { url, code: response.status };
|
||
} catch (err) {
|
||
result = { url, error: err };
|
||
}
|
||
return result;
|
||
}, options.retryExponentialBaseInMs);
|
||
}
|
||
|
||
export interface IRequestOptions {
|
||
retryExponentialBaseInMs?: number;
|
||
additionalHeaders?: Record<string, string>;
|
||
additionalHeadersUrlIgnore?: string[];
|
||
followOptions?: IFollowOptions;
|
||
}
|
||
|
||
const DefaultOptions: IRequestOptions = {
|
||
retryExponentialBaseInMs: 5000,
|
||
additionalHeaders: {},
|
||
additionalHeadersUrlIgnore: [],
|
||
};
|
||
|
||
function getFetchOptions(url: string, options: IRequestOptions): RequestInit {
|
||
const additionalHeaders = options.additionalHeadersUrlIgnore.some(
|
||
(ignorePattern) => url.match(ignorePattern)) ? {} : options.additionalHeaders;
|
||
return {
|
||
method: 'GET',
|
||
headers: { ...DefaultHeaders, ...additionalHeaders },
|
||
};
|
||
}
|
||
|
||
const DefaultHeaders: Record<string, string> = {
|
||
/* Chrome on macOS */
|
||
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
|
||
'upgrade-insecure-requests': '1',
|
||
'connection': 'keep-alive',
|
||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||
'accept-encoding': 'gzip, deflate, br',
|
||
'cache-control': 'max-age=0',
|
||
'accept-language': 'en-US,en;q=0.9',
|
||
};
|