61b475fa8d
TSLint deprecated and is being replaced by ESLint. Add Vue CLI plugin (@vue/cli-plugin-eslint) using: `vue add @vue/cli-plugin-eslint`. It also adds `.eslintrc.js` manually for Cypress since Vue CLI for ESLint misses it (vuejs/vue-cli#6892). Also rename `npm run lint:vue` to `npm run lint:eslint` for better clarification. This commit disables all rules that the current code is not compliant with. This allows for enabling them gradually and separating commits instead of mixing ESLint introduction with other code changes. AirBnb is chosen as base configuration. "Standard" is not chosen due to its poor defaults. It makes code cleaner but harder to maintain: - It converts interfaces to types which is harder to read. - Removes semicolons that helps to eliminate some ambigious code. "Airbnb" on the other hand helps for easier future changes and maintinability: - Includes more useful rules. - Keeps the semicolons and interfaces. - Enforces trailing commas that makes it easier to delete lines later on. - Delete branches: standard, prettier.
69 lines
2.2 KiB
TypeScript
69 lines
2.2 KiB
TypeScript
import { sleep } from '@/infrastructure/Threading/AsyncSleep';
|
|
import { IUrlStatus } from './IUrlStatus';
|
|
import { getUrlStatus, IRequestOptions } from './Requestor';
|
|
import { groupUrlsByDomain } from './UrlPerDomainGrouper';
|
|
|
|
export async function getUrlStatusesInParallel(
|
|
urls: string[],
|
|
options?: IBatchRequestOptions): Promise<IUrlStatus[]> {
|
|
// urls = [ 'https://privacy.sexy' ]; // Here to comment out when testing
|
|
const uniqueUrls = Array.from(new Set(urls));
|
|
options = { ...DefaultOptions, ...options };
|
|
console.log('Options: ', options);
|
|
const results = await request(uniqueUrls, options);
|
|
return results;
|
|
}
|
|
|
|
export interface IBatchRequestOptions {
|
|
domainOptions?: IDomainOptions;
|
|
requestOptions?: IRequestOptions;
|
|
}
|
|
|
|
interface IDomainOptions {
|
|
sameDomainParallelize?: boolean;
|
|
sameDomainDelayInMs?: number;
|
|
}
|
|
|
|
const DefaultOptions: IBatchRequestOptions = {
|
|
domainOptions: {
|
|
sameDomainParallelize: false,
|
|
sameDomainDelayInMs: 3 /* sec */ * 1000,
|
|
},
|
|
requestOptions: {
|
|
retryExponentialBaseInMs: 5 /* sec */ * 1000,
|
|
additionalHeaders: {},
|
|
},
|
|
};
|
|
|
|
function request(urls: string[], options: IBatchRequestOptions): Promise<IUrlStatus[]> {
|
|
if (!options.domainOptions.sameDomainParallelize) {
|
|
return runOnEachDomainWithDelay(
|
|
urls,
|
|
(url) => getUrlStatus(url, options.requestOptions),
|
|
options.domainOptions.sameDomainDelayInMs);
|
|
} else {
|
|
return Promise.all(
|
|
urls.map((url) => getUrlStatus(url, options.requestOptions)));
|
|
}
|
|
}
|
|
|
|
async function runOnEachDomainWithDelay(
|
|
urls: string[],
|
|
action: (url: string) => Promise<IUrlStatus>,
|
|
delayInMs: number): Promise<IUrlStatus[]> {
|
|
const grouped = groupUrlsByDomain(urls);
|
|
const tasks = grouped.map(async (group) => {
|
|
const results = new Array<IUrlStatus>();
|
|
for (const url of group) {
|
|
const status = await action(url);
|
|
results.push(status);
|
|
if (results.length !== group.length) {
|
|
await sleep(delayInMs);
|
|
}
|
|
}
|
|
return results;
|
|
});
|
|
const r = await Promise.all(tasks);
|
|
return r.flat();
|
|
}
|