db62ed7f3a
This commit: - Fixes broken URLs using archive.org or other references. - Replaces tenforums.com URLs with better documentation as they tend to return HTTP status code 403 to tests and also are low quality source. - Changes all insecure http sources to https alternatives - Adds integration tests to check for broken URLs - There's logic implemented for having a delay inbetween when sending requests to same domains, however it's not used as the sources can respond to totally parallelized requests. - Run test pipeline weekly to get notified about broken URls without commits
49 lines
1.8 KiB
TypeScript
49 lines
1.8 KiB
TypeScript
import 'mocha';
|
|
import { expect } from 'chai';
|
|
import { parseApplication } from '@/application/Parser/ApplicationParser';
|
|
import { IApplication } from '@/domain/IApplication';
|
|
import { IUrlStatus } from './StatusChecker/IUrlStatus';
|
|
import { getUrlStatusesInParallelAsync, IBatchRequestOptions } from './StatusChecker/BatchStatusChecker';
|
|
|
|
describe('collections', () => {
|
|
// arrange
|
|
const app = parseApplication();
|
|
const urls = collectUniqueUrls(app);
|
|
const options: IBatchRequestOptions = {
|
|
domainOptions: {
|
|
sameDomainParallelize: true, // no need to be so nice until sources start failing
|
|
// sameDomainDelayInMs: 2 /* sec */ * 1000,
|
|
},
|
|
requestOptions: {
|
|
retryExponentialBaseInMs: 3 /* sec */ * 1000,
|
|
additionalHeaders: { referer: app.info.homepage },
|
|
},
|
|
};
|
|
const testTimeoutInMs = urls.length * 60000 /* 1 minute */;
|
|
it('have no dead urls', async () => {
|
|
// act
|
|
const results = await getUrlStatusesInParallelAsync(urls, options);
|
|
// assert
|
|
const deadUrls = results.filter((r) => r.statusCode !== 200);
|
|
expect(deadUrls).to.have.lengthOf(0, printUrls(deadUrls));
|
|
}).timeout(testTimeoutInMs);
|
|
});
|
|
|
|
function collectUniqueUrls(app: IApplication): string[] {
|
|
return app
|
|
.collections
|
|
.flatMap((a) => a.getAllScripts())
|
|
.flatMap((script) => script.documentationUrls)
|
|
.filter((url, index, array) => array.indexOf(url) === index);
|
|
}
|
|
|
|
function printUrls(statuses: IUrlStatus[]): string {
|
|
return '\n' +
|
|
statuses.map((status) =>
|
|
`- ${status.url}\n` +
|
|
(status.statusCode ? `\tResponse code: ${status.statusCode}` : '') +
|
|
(status.error ? `\tException: ${JSON.stringify(status.error, null, '\t')}` : ''))
|
|
.join(`\n`)
|
|
+ '\n';
|
|
}
|