Fix failing URL status checking integration tests

Implement following redirects over `fetch` supporting cookies.
`node-fetch` does not support sending cookies during redirect. However,
this is needed to not end-up in a redirect loop for a sign-in callback.

Fix integration tests failing due to redirects and 403 errors:
  - Many redirects from `answers.microsoft.com` was throwing: throwing
    `FetchError: maximum redirect reached` error. It was caused by not
    having cookies when following redirects therefore having an infinite
    sign-in callback for the webpage.
  - Fixes integration tests failing due to additional referer header being
    sent by the application. It adds support for making exceptions to
    additional header sending through a list of regexes.

Add in-depth documentation for URL status checking.

tests/integration/application/collections/StatusChecker/Requestor.ts

@@ -1,37 +1,44 @@
 import { retryWithExponentialBackOffAsync } from './ExponentialBackOffRetryHandler';
 import { IUrlStatus } from './IUrlStatus';
-import fetch from 'cross-fetch';
-
-export interface IRequestOptions {
-    retryExponentialBaseInMs?: number;
-    additionalHeaders?: Record<string, string>;
-}
+import { fetchFollow, IFollowOptions } from './FetchFollow';
 
 export async function getUrlStatusAsync(
     url: string,
     options: IRequestOptions = DefaultOptions): Promise<IUrlStatus> {
     options = { ...DefaultOptions, ...options };
-    const fetchOptions = getFetchOptions(options);
+    const fetchOptions = getFetchOptions(url, options);
     return retryWithExponentialBackOffAsync(async () => {
         console.log('Requesting', url); // tslint:disable-line: no-console
+        let result: IUrlStatus;
         try {
-            const response = await fetch(url, fetchOptions);
-            return { url, statusCode: response.status};
+            const response = await fetchFollow(url, fetchOptions, options.followOptions);
+            result = { url, code: response.status };
         } catch (err) {
-            return { url, error: err};
+            result = { url, error: err };
         }
+        return result;
     }, options.retryExponentialBaseInMs);
 }
 
+export interface IRequestOptions {
+    retryExponentialBaseInMs?: number;
+    additionalHeaders?: Record<string, string>;
+    additionalHeadersUrlIgnore?: string[];
+    followOptions?: IFollowOptions;
+}
+
 const DefaultOptions: IRequestOptions = {
     retryExponentialBaseInMs: 5000,
     additionalHeaders: {},
+    additionalHeadersUrlIgnore: [],
 };
 
-function getFetchOptions(options: IRequestOptions) {
+function getFetchOptions(url: string, options: IRequestOptions): RequestInit {
+    const additionalHeaders = options.additionalHeadersUrlIgnore.some(
+        (ignorePattern) => url.match(ignorePattern)) ? {} : options.additionalHeaders;
     return {
         method: 'GET',
-        headers: { ...DefaultHeaders, ...options.additionalHeaders },
+        headers: { ...DefaultHeaders, ...additionalHeaders },
     };
 }