fix broken URLs and automate broken URL checks #70

This commit:
- Fixes broken URLs using archive.org or other references.
- Replaces tenforums.com URLs with better documentation as they tend to return HTTP status code 403 to tests and also are low quality source.
- Changes all insecure http sources to https alternatives
- Adds integration tests to check for broken URLs
  - There's logic implemented for having a delay inbetween when sending requests to same domains, however it's not used as the sources can respond to totally parallelized requests.
- Run test pipeline weekly to get notified about broken URls without commits

tests/integration/application/collections/NoDeadDocumentationUrls.spec.ts

@@ -0,0 +1,48 @@
+import 'mocha';
+import { expect } from 'chai';
+import { parseApplication } from '@/application/Parser/ApplicationParser';
+import { IApplication } from '@/domain/IApplication';
+import { IUrlStatus } from './StatusChecker/IUrlStatus';
+import { getUrlStatusesInParallelAsync, IBatchRequestOptions } from './StatusChecker/BatchStatusChecker';
+
+describe('collections', () => {
+    // arrange
+    const app = parseApplication();
+    const urls = collectUniqueUrls(app);
+    const options: IBatchRequestOptions = {
+        domainOptions: {
+            sameDomainParallelize: true, // no need to be so nice until sources start failing
+            // sameDomainDelayInMs: 2 /* sec */ * 1000,
+        },
+        requestOptions: {
+            retryExponentialBaseInMs: 3 /* sec */ * 1000,
+            additionalHeaders: { referer: app.info.homepage },
+        },
+    };
+    const testTimeoutInMs = urls.length * 60000 /* 1 minute */;
+    it('have no dead urls', async () => {
+        // act
+        const results = await getUrlStatusesInParallelAsync(urls, options);
+        // assert
+        const deadUrls = results.filter((r) => r.statusCode !== 200);
+        expect(deadUrls).to.have.lengthOf(0, printUrls(deadUrls));
+    }).timeout(testTimeoutInMs);
+});
+
+function collectUniqueUrls(app: IApplication): string[] {
+    return app
+        .collections
+        .flatMap((a) => a.getAllScripts())
+        .flatMap((script) => script.documentationUrls)
+        .filter((url, index, array) => array.indexOf(url) === index);
+}
+
+function printUrls(statuses: IUrlStatus[]): string {
+    return '\n' +
+        statuses.map((status) =>
+            `- ${status.url}\n` +
+            (status.statusCode ? `\tResponse code: ${status.statusCode}` : '') +
+            (status.error ? `\tException: ${JSON.stringify(status.error, null, '\t')}` : ''))
+        .join(`\n`)
+        + '\n';
+}