Fix failing URL status checking integration tests

Implement following redirects over `fetch` supporting cookies.
`node-fetch` does not support sending cookies during redirect. However,
this is needed to not end-up in a redirect loop for a sign-in callback.

Fix integration tests failing due to redirects and 403 errors:
  - Many redirects from `answers.microsoft.com` was throwing: throwing
    `FetchError: maximum redirect reached` error. It was caused by not
    having cookies when following redirects therefore having an infinite
    sign-in callback for the webpage.
  - Fixes integration tests failing due to additional referer header being
    sent by the application. It adds support for making exceptions to
    additional header sending through a list of regexes.

Add in-depth documentation for URL status checking.

tests/integration/application/collections/StatusChecker/FetchFollow.ts

@@ -0,0 +1,66 @@
+import fetch from 'cross-fetch';
+
+export function fetchFollow(
+    url: string, fetchOptions: RequestInit, followOptions: IFollowOptions): Promise<Response> {
+    followOptions = { ...DefaultOptions, ...followOptions };
+    if (!followOptions.followRedirects
+        || followOptions.maximumRedirectFollowDepth === 0) {
+        return fetch(url, fetchOptions);
+    }
+    fetchOptions = { ...fetchOptions, redirect: 'manual' /* handled manually */ };
+    const cookies = new CookieStorage(followOptions.enableCookies);
+    return followRecursivelyWithCookies(
+        url, fetchOptions, followOptions.maximumRedirectFollowDepth, cookies);
+}
+
+export interface IFollowOptions {
+    followRedirects?: boolean;
+    maximumRedirectFollowDepth?: number;
+    enableCookies?: boolean;
+}
+
+const DefaultOptions: IFollowOptions = {
+    followRedirects: true,
+    maximumRedirectFollowDepth: 20,
+    enableCookies: true,
+};
+
+async function followRecursivelyWithCookies(
+    url: string, options: RequestInit, followDepth: number, cookies: CookieStorage): Promise<Response> {
+    if (cookies.hasAny()) {
+        options = { ...options, headers: { ...options.headers, cookie: cookies.getHeader() } };
+    }
+    const response = await fetch(url, options);
+    if (!isRedirect(response.status)) {
+        return response;
+    }
+    if (--followDepth < 0) {
+        throw new Error(`[max-redirect] maximum redirect reached at: ${url}`);
+    }
+    const cookieHeader = response.headers.get('set-cookie');
+    cookies.addHeader(cookieHeader);
+    const nextUrl = response.headers.get('location');
+    return followRecursivelyWithCookies(nextUrl, options, followDepth, cookies);
+}
+
+function isRedirect(code: number): boolean {
+    return code === 301 || code === 302 || code === 303 || code === 307 || code === 308;
+}
+
+class CookieStorage {
+    public cookies = new Array<string>();
+    constructor(private readonly enabled: boolean) {
+    }
+    public hasAny() {
+        return this.enabled && this.cookies.length > 0;
+    }
+    public addHeader(header: string) {
+        if (!this.enabled || !header) {
+            return;
+        }
+        this.cookies.push(header);
+    }
+    public getHeader() {
+        return this.cookies.join(' ; ');
+    }
+}