fix broken URLs and automate broken URL checks #70

This commit:
- Fixes broken URLs using archive.org or other references.
- Replaces tenforums.com URLs with better documentation as they tend to return HTTP status code 403 to tests and also are low quality source.
- Changes all insecure http sources to https alternatives
- Adds integration tests to check for broken URLs
  - There's logic implemented for having a delay inbetween when sending requests to same domains, however it's not used as the sources can respond to totally parallelized requests.
- Run test pipeline weekly to get notified about broken URls without commits

tests/integration/application/collections/StatusChecker/Requestor.ts

@@ -0,0 +1,47 @@
+import { retryWithExponentialBackOffAsync } from './ExponentialBackOffRetryHandler';
+import { IUrlStatus } from './IUrlStatus';
+import fetch from 'cross-fetch';
+
+export interface IRequestOptions {
+    retryExponentialBaseInMs?: number;
+    additionalHeaders?: Record<string, string>;
+}
+
+export async function getUrlStatusAsync(
+    url: string,
+    options: IRequestOptions = DefaultOptions): Promise<IUrlStatus> {
+    options = { ...DefaultOptions, ...options };
+    const fetchOptions = getFetchOptions(options);
+    return retryWithExponentialBackOffAsync(async () => {
+        console.log('Requesting', url); // tslint:disable-line: no-console
+        try {
+            const response = await fetch(url, fetchOptions);
+            return { url, statusCode: response.status};
+        } catch (err) {
+            return { url, error: err};
+        }
+    }, options.retryExponentialBaseInMs);
+}
+
+const DefaultOptions: IRequestOptions = {
+    retryExponentialBaseInMs: 5000,
+    additionalHeaders: {},
+};
+
+function getFetchOptions(options: IRequestOptions) {
+    return {
+        method: 'GET',
+        headers: { ...DefaultHeaders, ...options.additionalHeaders },
+    };
+}
+
+const DefaultHeaders: Record<string, string> = {
+    /* Chrome on macOS */
+    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
+    'upgrade-insecure-requests': '1',
+    'connection': 'keep-alive',
+    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+    'accept-encoding': 'gzip, deflate, br',
+    'cache-control': 'max-age=0',
+    'accept-language': 'en-US,en;q=0.9',
+};