diff --git a/.github/workflows/checks.desktop-runtime-errors.yaml b/.github/workflows/checks.desktop-runtime-errors.yaml index d2f9d430..f1c82040 100644 --- a/.github/workflows/checks.desktop-runtime-errors.yaml +++ b/.github/workflows/checks.desktop-runtime-errors.yaml @@ -6,7 +6,7 @@ on: pull_request: jobs: - build-desktop: + run-check: strategy: matrix: os: [ macos, ubuntu, windows ] @@ -60,7 +60,9 @@ jobs: - name: Test shell: bash - run: npm run check:desktop -- --screenshot + run: |- + export SCREENSHOT=true + npm run check:desktop - name: Upload screenshot if: always() # Run even if previous step fails diff --git a/.github/workflows/checks.external-urls.yaml b/.github/workflows/checks.external-urls.yaml new file mode 100644 index 00000000..baca35a0 --- /dev/null +++ b/.github/workflows/checks.external-urls.yaml @@ -0,0 +1,18 @@ +name: checks.external-urls + +on: + schedule: + - cron: '0 0 * * 0' # at 00:00 on every Sunday + +jobs: + run-check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup node + uses: ./.github/actions/setup-node + - name: Install dependencies + run: npm ci + - name: Test + run: npm run check:external-urls diff --git a/README.md b/README.md index 7851a002..40de5dd2 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,12 @@ src="https://github.com/undergroundwires/privacy.sexy/workflows/checks.desktop-runtime-errors/badge.svg" /> + + Status of external URL checks +
diff --git a/docs/development.md b/docs/development.md index beae74e2..3fa43413 100644 --- a/docs/development.md +++ b/docs/development.md @@ -21,7 +21,10 @@ You could run other types of tests as well, but they may take longer time and ov - Run end-to-end (e2e) tests: - `npm run test:cy:open`: Run tests interactively using the development server with hot-reloading. - `npm run test:cy:run`: Run tests on the production build in a headless mode. -- Run runtime checks for packaged desktop applications: `npm run check:desktop`, see its [README.md](./../scripts/check-desktop-runtime-errors/README.md). +- Run checks: + - `npm run check:desktop`: Run runtime checks for packaged desktop applications ([README.md](./../tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/README.md)). + - You can set environment variables active its flags such as `BUILD=true SCREENSHOT=true npm run check:desktop` + - `npm run check:external-urls`: Test whether external URLs used in applications are alive. 📖 Read more about testing in [tests](./tests.md). diff --git a/package-lock.json b/package-lock.json index 49c5f3dd..51470310 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,6 @@ "start-server-and-test": "^2.0.0", "svgexport": "^0.4.2", "terser": "^5.19.2", - "ts-node": "^10.9.1", "tslib": "~2.4.0", "typescript": "~4.6.2", "vite": "^4.4.9", @@ -1790,28 +1789,6 @@ "node": ">=0.1.90" } }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, "node_modules/@cypress/request": { "version": "2.88.12", "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.12.tgz", @@ -3634,30 +3611,6 @@ "node": ">= 10" } }, - "node_modules/@tsconfig/node10": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", - "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", - "dev": true - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true - }, "node_modules/@types/ace": { "version": "0.0.48", "resolved": "https://registry.npmjs.org/@types/ace/-/ace-0.0.48.tgz", @@ -6117,12 +6070,6 @@ "buffer": "^5.1.0" } }, - "node_modules/create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true - }, "node_modules/cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -11238,12 +11185,6 @@ "node": ">=12" } }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true - }, "node_modules/map-age-cleaner": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", @@ -19661,64 +19602,6 @@ "utf8-byte-length": "^1.0.1" } }, - "node_modules/ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "dev": true, - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/ts-node/node_modules/arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true - }, - "node_modules/ts-node/node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -20482,12 +20365,6 @@ "vue-resize": "^1.0.1" } }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true - }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -21412,15 +21289,6 @@ "fd-slicer": "~1.1.0" } }, - "node_modules/yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true, - "engines": { - "node": ">=6" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -22641,27 +22509,6 @@ "dev": true, "optional": true }, - "@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "requires": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "dependencies": { - "@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "requires": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - } - } - }, "@cypress/request": { "version": "2.88.12", "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.12.tgz", @@ -23939,30 +23786,6 @@ "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", "dev": true }, - "@tsconfig/node10": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", - "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", - "dev": true - }, - "@tsconfig/node12": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true - }, - "@tsconfig/node14": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true - }, - "@tsconfig/node16": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true - }, "@types/ace": { "version": "0.0.48", "resolved": "https://registry.npmjs.org/@types/ace/-/ace-0.0.48.tgz", @@ -25837,12 +25660,6 @@ "buffer": "^5.1.0" } }, - "create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true - }, "cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -29752,12 +29569,6 @@ "@jridgewell/sourcemap-codec": "^1.4.15" } }, - "make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true - }, "map-age-cleaner": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", @@ -35744,41 +35555,6 @@ "utf8-byte-length": "^1.0.1" } }, - "ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "dev": true, - "requires": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "dependencies": { - "arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true - }, - "diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true - } - } - }, "tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -36339,12 +36115,6 @@ "vue-resize": "^1.0.1" } }, - "v8-compile-cache-lib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true - }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -36966,12 +36736,6 @@ "fd-slicer": "~1.1.0" } }, - "yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true - }, "yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index bf334999..d41a3e37 100644 --- a/package.json +++ b/package.json @@ -12,12 +12,12 @@ "preview": "vite preview", "test:unit": "vitest run --dir tests/unit", "test:integration": "vitest run --dir tests/integration", - "test:e2e": "vue-cli-service test:e2e", "test:cy:run": "start-server-and-test \"vite build && vite preview --port 7070\" http://localhost:7070 \"cypress run --config baseUrl=http://localhost:7070\"", "test:cy:open": "start-server-and-test \"vite --port 7070 --mode production\" http://localhost:7070 \"cypress open --config baseUrl=http://localhost:7070\"", "lint": "npm run lint:md && npm run lint:md:consistency && npm run lint:md:relative-urls && npm run lint:eslint && npm run lint:yaml", "icons:build": "node scripts/logo-update.js", - "check:desktop": "ts-node --experimentalSpecifierResolution node --esm scripts/check-desktop-runtime-errors/index.ts", + "check:desktop": "vitest run --dir tests/checks/desktop-runtime-errors --environment node", + "check:external-urls": "vitest run --dir tests/checks/external-urls --environment node", "electron:dev": "electron-vite dev", "electron:preview": "electron-vite preview", "electron:prebuild": "electron-vite build", @@ -86,7 +86,6 @@ "start-server-and-test": "^2.0.0", "svgexport": "^0.4.2", "terser": "^5.19.2", - "ts-node": "^10.9.1", "tslib": "~2.4.0", "typescript": "~4.6.2", "vite": "^4.4.9", diff --git a/scripts/check-desktop-runtime-errors/cli-args.ts b/scripts/check-desktop-runtime-errors/cli-args.ts deleted file mode 100644 index 91aa247b..00000000 --- a/scripts/check-desktop-runtime-errors/cli-args.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { log } from './utils/log'; - -const PROCESS_ARGUMENTS: string[] = process.argv.slice(2); - -export enum CommandLineFlag { - ForceRebuild, - TakeScreenshot, -} - -export const COMMAND_LINE_FLAGS = Object.freeze({ - [CommandLineFlag.ForceRebuild]: '--build', - [CommandLineFlag.TakeScreenshot]: '--screenshot', -}); - -export function logCurrentArgs(): void { - if (!PROCESS_ARGUMENTS.length) { - log('No additional arguments provided.'); - return; - } - log(`Arguments: ${PROCESS_ARGUMENTS.join(', ')}`); -} - -export function hasCommandLineFlag(flag: CommandLineFlag): boolean { - return PROCESS_ARGUMENTS.includes(COMMAND_LINE_FLAGS[flag]); -} diff --git a/scripts/check-desktop-runtime-errors/README.md b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/README.md similarity index 100% rename from scripts/check-desktop-runtime-errors/README.md rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/README.md diff --git a/scripts/check-desktop-runtime-errors/app/app-logs.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/app-logs.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/app-logs.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/app-logs.ts diff --git a/scripts/check-desktop-runtime-errors/app/check-for-errors.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/check-for-errors.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/check-for-errors.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/check-for-errors.ts diff --git a/scripts/check-desktop-runtime-errors/app/error-ignore-patterns.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/error-ignore-patterns.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/error-ignore-patterns.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/error-ignore-patterns.ts diff --git a/scripts/check-desktop-runtime-errors/app/extractors/common/app-artifact-locator.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/common/app-artifact-locator.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/extractors/common/app-artifact-locator.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/common/app-artifact-locator.ts diff --git a/scripts/check-desktop-runtime-errors/app/extractors/common/extraction-result.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/common/extraction-result.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/extractors/common/extraction-result.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/common/extraction-result.ts diff --git a/scripts/check-desktop-runtime-errors/app/extractors/linux.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/linux.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/extractors/linux.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/linux.ts diff --git a/scripts/check-desktop-runtime-errors/app/extractors/macos.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/macos.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/extractors/macos.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/macos.ts diff --git a/scripts/check-desktop-runtime-errors/app/extractors/windows.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/windows.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/extractors/windows.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/extractors/windows.ts diff --git a/scripts/check-desktop-runtime-errors/app/runner.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/runner.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/runner.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/runner.ts diff --git a/scripts/check-desktop-runtime-errors/app/system-capture/screen-capture.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/system-capture/screen-capture.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/system-capture/screen-capture.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/system-capture/screen-capture.ts diff --git a/scripts/check-desktop-runtime-errors/app/system-capture/window-title-capture.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/system-capture/window-title-capture.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/app/system-capture/window-title-capture.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/system-capture/window-title-capture.ts diff --git a/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/cli-args.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/cli-args.ts new file mode 100644 index 00000000..dd85c150 --- /dev/null +++ b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/cli-args.ts @@ -0,0 +1,36 @@ +import { log } from './utils/log'; + +export enum CommandLineFlag { + ForceRebuild, + TakeScreenshot, +} + +export const COMMAND_LINE_FLAGS: { + readonly [key in CommandLineFlag]: string; +} = Object.freeze({ + [CommandLineFlag.ForceRebuild]: '--build', + [CommandLineFlag.TakeScreenshot]: '--screenshot', +}); + +export function logCurrentArgs(): void { + const processArguments = getProcessArguments(); + if (!processArguments.length) { + log('No additional arguments provided.'); + return; + } + log(`Arguments: ${processArguments.join(', ')}`); +} + +export function hasCommandLineFlag(flag: CommandLineFlag): boolean { + return getProcessArguments() + .includes(COMMAND_LINE_FLAGS[flag]); +} + +/* + Fetches process arguments dynamically each time the function is called. + This design allows for runtime modifications to process.argv, supporting scenarios + where the command-line arguments might be altered dynamically. +*/ +function getProcessArguments(): string[] { + return process.argv.slice(2); +} diff --git a/scripts/check-desktop-runtime-errors/config.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/config.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/config.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/config.ts diff --git a/scripts/check-desktop-runtime-errors/index.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/index.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/index.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/index.ts diff --git a/scripts/check-desktop-runtime-errors/main.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/main.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/main.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/main.ts diff --git a/scripts/check-desktop-runtime-errors/utils/io.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/io.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/io.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/io.ts diff --git a/scripts/check-desktop-runtime-errors/utils/log.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/log.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/log.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/log.ts diff --git a/scripts/check-desktop-runtime-errors/utils/npm.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/npm.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/npm.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/npm.ts diff --git a/scripts/check-desktop-runtime-errors/utils/platform.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/platform.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/platform.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/platform.ts diff --git a/scripts/check-desktop-runtime-errors/utils/run-command.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/run-command.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/run-command.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/run-command.ts diff --git a/scripts/check-desktop-runtime-errors/utils/sleep.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/sleep.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/sleep.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/sleep.ts diff --git a/scripts/check-desktop-runtime-errors/utils/text.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/text.ts similarity index 100% rename from scripts/check-desktop-runtime-errors/utils/text.ts rename to tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/text.ts diff --git a/tests/checks/desktop-runtime-errors/main.spec.ts b/tests/checks/desktop-runtime-errors/main.spec.ts new file mode 100644 index 00000000..e7f006f7 --- /dev/null +++ b/tests/checks/desktop-runtime-errors/main.spec.ts @@ -0,0 +1,40 @@ +import { test } from 'vitest'; +import { main } from './check-desktop-runtime-errors/main'; +import { COMMAND_LINE_FLAGS, CommandLineFlag } from './check-desktop-runtime-errors/cli-args'; + +test('should have no desktop runtime errors', async () => { + // arrange + setCommandLineFlagsFromEnvironmentVariables(); + let exitCode: number; + global.process.exit = (code?: number): never => { + exitCode = code; + return undefined as never; + }; + // act + await main(); + // assert + expect(exitCode).to.equal(0); +}, { + timeout: 60 /* minutes */ * 10000, +}); + +/* + Map environment variables to CLI arguments for compatibility with Vitest. +*/ +function setCommandLineFlagsFromEnvironmentVariables() { + const flagEnvironmentVariableKeyMappings: { + readonly [key in CommandLineFlag]: string; + } = { + [CommandLineFlag.ForceRebuild]: 'BUILD', + [CommandLineFlag.TakeScreenshot]: 'SCREENSHOT', + }; + Object.entries(flagEnvironmentVariableKeyMappings) + .forEach(([flag, environmentVariableKey]) => { + if (process.env[environmentVariableKey] !== undefined) { + process.argv = [ + ...process.argv, + COMMAND_LINE_FLAGS[flag], + ]; + } + }); +} diff --git a/tests/integration/application/collections/StatusChecker/BatchStatusChecker.ts b/tests/checks/external-urls/StatusChecker/BatchStatusChecker.ts similarity index 97% rename from tests/integration/application/collections/StatusChecker/BatchStatusChecker.ts rename to tests/checks/external-urls/StatusChecker/BatchStatusChecker.ts index 8631ed92..bc2a8ba6 100644 --- a/tests/integration/application/collections/StatusChecker/BatchStatusChecker.ts +++ b/tests/checks/external-urls/StatusChecker/BatchStatusChecker.ts @@ -32,6 +32,7 @@ const DefaultOptions: IBatchRequestOptions = { }, requestOptions: { retryExponentialBaseInMs: 5 /* sec */ * 1000, + requestTimeoutInMs: 60 /* sec */ * 1000, additionalHeaders: {}, }, }; diff --git a/tests/integration/application/collections/StatusChecker/ExponentialBackOffRetryHandler.ts b/tests/checks/external-urls/StatusChecker/ExponentialBackOffRetryHandler.ts similarity index 100% rename from tests/integration/application/collections/StatusChecker/ExponentialBackOffRetryHandler.ts rename to tests/checks/external-urls/StatusChecker/ExponentialBackOffRetryHandler.ts diff --git a/tests/integration/application/collections/StatusChecker/FetchFollow.ts b/tests/checks/external-urls/StatusChecker/FetchFollow.ts similarity index 86% rename from tests/integration/application/collections/StatusChecker/FetchFollow.ts rename to tests/checks/external-urls/StatusChecker/FetchFollow.ts index f0ea4fd6..3d16b856 100644 --- a/tests/integration/application/collections/StatusChecker/FetchFollow.ts +++ b/tests/checks/external-urls/StatusChecker/FetchFollow.ts @@ -1,18 +1,20 @@ -import fetch from 'cross-fetch'; +import { fetchWithTimeout } from './FetchWithTimeout'; export function fetchFollow( url: string, + timeoutInMs: number, fetchOptions: RequestInit, followOptions: IFollowOptions, ): Promise { followOptions = { ...DefaultOptions, ...followOptions }; if (followRedirects(followOptions)) { - return fetch(url, fetchOptions); + return fetchWithTimeout(url, timeoutInMs, fetchOptions); } fetchOptions = { ...fetchOptions, redirect: 'manual' /* handled manually */ }; const cookies = new CookieStorage(followOptions.enableCookies); return followRecursivelyWithCookies( url, + timeoutInMs, fetchOptions, followOptions.maximumRedirectFollowDepth, cookies, @@ -33,12 +35,17 @@ const DefaultOptions: IFollowOptions = { async function followRecursivelyWithCookies( url: string, + timeoutInMs: number, options: RequestInit, followDepth: number, cookies: CookieStorage, ): Promise { options = updateCookieHeader(cookies, options); - const response = await fetch(url, options); + const response = await fetchWithTimeout( + url, + timeoutInMs, + options, + ); if (!isRedirect(response.status)) { return response; } @@ -49,7 +56,7 @@ async function followRecursivelyWithCookies( const cookieHeader = response.headers.get('set-cookie'); cookies.addHeader(cookieHeader); const nextUrl = response.headers.get('location'); - return followRecursivelyWithCookies(nextUrl, options, newFollowDepth, cookies); + return followRecursivelyWithCookies(nextUrl, timeoutInMs, options, newFollowDepth, cookies); } function isRedirect(code: number): boolean { diff --git a/tests/checks/external-urls/StatusChecker/FetchWithTimeout.ts b/tests/checks/external-urls/StatusChecker/FetchWithTimeout.ts new file mode 100644 index 00000000..f4d480f3 --- /dev/null +++ b/tests/checks/external-urls/StatusChecker/FetchWithTimeout.ts @@ -0,0 +1,16 @@ +import fetch from 'cross-fetch'; + +export async function fetchWithTimeout( + url: string, + timeoutInMs: number, + init?: RequestInit, +): Promise { + const controller = new AbortController(); + const options: RequestInit = { + ...(init ?? {}), + signal: controller.signal, + }; + const promise = fetch(url, options); + const timeout = setTimeout(() => controller.abort(), timeoutInMs); + return promise.finally(() => clearTimeout(timeout)); +} diff --git a/tests/integration/application/collections/StatusChecker/IUrlStatus.ts b/tests/checks/external-urls/StatusChecker/IUrlStatus.ts similarity index 100% rename from tests/integration/application/collections/StatusChecker/IUrlStatus.ts rename to tests/checks/external-urls/StatusChecker/IUrlStatus.ts diff --git a/tests/checks/external-urls/StatusChecker/README.md b/tests/checks/external-urls/StatusChecker/README.md new file mode 100644 index 00000000..8b30465f --- /dev/null +++ b/tests/checks/external-urls/StatusChecker/README.md @@ -0,0 +1,111 @@ +# status-checker + +A CLI and SDK for checking the availability of external URLs. + +🧐 Why? + +- 🏃 **Fast**: Batch checks the statuses of URLs in parallel. +- 🤖 **Easy-to-Use**: Zero-touch startup with pre-configured settings for reliable results, yet customizable. +- 🤞 **Reliable**: Mimics real web browser behavior by following redirects and maintaining cookie storage. + +🍭 Additional features + +- 😇 **Rate Limiting**: Queues requests by domain to be polite. +- 🔁 **Retries**: Implements retry pattern with exponential back-off. +- ⌚ **Timeouts**: Configurable timeout for each request. +- 🎭️ **User-Agent Rotation**: Change user agents for each request. + +## CLI + +Coming soon 🚧 + +## Programmatic usage + +The SDK supports both Node.js and browser environments. + +### `getUrlStatusesInParallel` + +```js +// Simple example +const statuses = await getUrlStatusesInParallel([ 'https://privacy.sexy', /* ... */ ]); +if(statuses.all((r) => r.code === 200)) { + console.log('All URLs are alive!'); +} else { + console.log('Dead URLs:', statuses.filter((r) => r.code !== 200).map((r) => r.url)); +} + +// Fastest configuration +const statuses = await getUrlStatusesInParallel([ 'https://privacy.sexy', /* ... */ ], { + domainOptions: { + sameDomainParallelize: false, + } +}); +``` + +#### Batch request options + +- `domainOptions`: + - **`sameDomainParallelize`**, (*boolean*), default: `false` + - Determines if requests to the same domain will be parallelized. + - Setting to `false` makes all requests parallel. + - Setting to `true` queues requests for each unique domain while parallelizing across different domains. + - Requests to different domains are always parallelized regardless of this option. + - 💡 This helps to avoid `429 Too Many Requests` and be nice to websites + - **`sameDomainDelayInMs`** (*number*), default: `3000` (3 seconds) + - Sets the delay between requests to the same domain. +- `requestOptions` (*object*): See [request options](#request-options). + +### `getUrlStatus` + +Check the availability of a single URL. + +```js +// Simple example +const status = await getUrlStatus('https://privacy.sexy'); +console.log(`Status code: ${status.code}`); +``` + +#### Request options + +- **`retryExponentialBaseInMs`** (*number*), default: `5000` (5 seconds) + - Base time for the exponential back-off calculation for retries. + - The longer the base time, the greater the intervals between retries. +- **`additionalHeaders`** (*object*), default: `false` + - Additional HTTP headers to send along with the default headers. Overrides default headers if specified. +- **`followOptions`** (*object*): See [follow options](#follow-options). +- **`requestTimeoutInMs`** (*number*), default: `60000` (60 seconds) + - Time limit to abort the request if no response is received within the specified time frame. + +### `fetchFollow` + +Follows `3XX` redirects while preserving cookies. + +Same fetch API except third parameter that specifies [follow options](#follow-options), `redirect: 'follow' | 'manual' | 'error'` is discarded in favor of the third parameter. + +```js +const status = await fetchFollow('https://privacy.sexy', { + // First argument is same options as fetch API, except `redirect` options + // that's discarded in favor of next argument follow options + headers: { + 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0' + }, + }, { + // Second argument sets the redirect behavior + followRedirects: true, + maximumRedirectFollowDepth: 20, + enableCookies: true, + } +); +console.log(`Status code: ${status.code}`); +``` + +#### Follow options + +- **`followRedirects`** (*boolean*), default: `true` + - Determines whether or not to follow redirects with `3XX` response codes. +- **`maximumRedirectFollowDepth`** (*boolean*), default: `20` + - Specifies the maximum number of sequential redirects that the function will follow. + - 💡 Helps to solve maximum redirect reached errors. +- **`enableCookies`** (*boolean*), default: `true` + - Enables cookie storage to facilitate seamless navigation through login or other authentication challenges. + - 💡 Helps to over-come sign-in challenges with callbacks. diff --git a/tests/integration/application/collections/StatusChecker/Requestor.ts b/tests/checks/external-urls/StatusChecker/Requestor.ts similarity index 60% rename from tests/integration/application/collections/StatusChecker/Requestor.ts rename to tests/checks/external-urls/StatusChecker/Requestor.ts index d57687ce..10ae67b0 100644 --- a/tests/integration/application/collections/StatusChecker/Requestor.ts +++ b/tests/checks/external-urls/StatusChecker/Requestor.ts @@ -1,6 +1,7 @@ import { retryWithExponentialBackOff } from './ExponentialBackOffRetryHandler'; import { IUrlStatus } from './IUrlStatus'; import { fetchFollow, IFollowOptions } from './FetchFollow'; +import { getRandomUserAgent } from './UserAgents'; export function getUrlStatus( url: string, @@ -12,7 +13,12 @@ export function getUrlStatus( console.log('Requesting', url); let result: IUrlStatus; try { - const response = await fetchFollow(url, fetchOptions, options.followOptions); + const response = await fetchFollow( + url, + options.requestTimeoutInMs, + fetchOptions, + options.followOptions, + ); result = { url, code: response.status }; } catch (err) { result = { url, error: JSON.stringify(err, null, '\t') }; @@ -26,32 +32,38 @@ export interface IRequestOptions { additionalHeaders?: Record; additionalHeadersUrlIgnore?: string[]; followOptions?: IFollowOptions; + requestTimeoutInMs: number; } const DefaultOptions: IRequestOptions = { retryExponentialBaseInMs: 5000, additionalHeaders: {}, additionalHeadersUrlIgnore: [], + requestTimeoutInMs: 60 /* seconds */ * 1000, }; function getFetchOptions(url: string, options: IRequestOptions): RequestInit { const additionalHeaders = options.additionalHeadersUrlIgnore - .some((ignorePattern) => url.match(ignorePattern)) + .some((ignorePattern) => url.startsWith(ignorePattern)) ? {} : options.additionalHeaders; return { - method: 'GET', - headers: { ...DefaultHeaders, ...additionalHeaders }, + method: 'HEAD', + headers: { + ...getDefaultHeaders(), + ...additionalHeaders, + }, }; } -const DefaultHeaders: Record = { - /* Chrome on macOS */ - 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36', - 'upgrade-insecure-requests': '1', - connection: 'keep-alive', - accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'accept-encoding': 'gzip, deflate, br', - 'cache-control': 'max-age=0', - 'accept-language': 'en-US,en;q=0.9', -}; +function getDefaultHeaders(): Record { + return { + 'user-agent': getRandomUserAgent(), + 'upgrade-insecure-requests': '1', + connection: 'keep-alive', + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', + 'accept-encoding': 'gzip, deflate, br', + 'cache-control': 'max-age=0', + 'accept-language': 'en-US,en;q=0.9', + }; +} diff --git a/tests/integration/application/collections/StatusChecker/UrlPerDomainGrouper.ts b/tests/checks/external-urls/StatusChecker/UrlPerDomainGrouper.ts similarity index 100% rename from tests/integration/application/collections/StatusChecker/UrlPerDomainGrouper.ts rename to tests/checks/external-urls/StatusChecker/UrlPerDomainGrouper.ts diff --git a/tests/checks/external-urls/StatusChecker/UserAgents.ts b/tests/checks/external-urls/StatusChecker/UserAgents.ts new file mode 100644 index 00000000..1c389b77 --- /dev/null +++ b/tests/checks/external-urls/StatusChecker/UserAgents.ts @@ -0,0 +1,75 @@ +export function getRandomUserAgent(): string { + return UserAgents[Math.floor(Math.random() * UserAgents.length)]; +} + +const UserAgents = [ + // Chrome + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537', + + // Firefox + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15', + + // Safari + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Safari/604.1', + + // Internet Explorer + 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko', + + // Edge + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3 Edge/15.0', + + // Opera + 'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14', + + // iOS Devices + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/18.2b11866 Mobile/16B91 Safari/605.1.15', + 'Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1', + + // Android Devices + 'Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.3', + + // Other Devices/Browsers + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.3', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Safari/605.1.15', + 'Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.3 Edge/15.0', + 'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0', + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0', + 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.3', + 'Mozilla/5.0 (Linux; Android 7.0; SM-G930F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.3', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.3', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1', + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.3 OPR/53.0.2907.99', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2)', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20120121 Firefox/46.0', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0)', + 'Mozilla/5.0 (Windows NT 5.1; rv:36.0) Gecko/20100101 Firefox/36.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0', + 'Mozilla/5.0 (X11; Linux i686; rv:30.0) Gecko/20100101 Firefox/30.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:28.0) Gecko/20100101 Firefox/28.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/20161202 Firefox/21.0.1', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:20.0) Gecko/20100101 Firefox/20.0', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0', + 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0', + 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0', + 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.3', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.3', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', + 'Mozilla/5.0 (X11; CrOS x86_64 4319.74.0) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', + 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3', +]; diff --git a/tests/checks/external-urls/main.spec.ts b/tests/checks/external-urls/main.spec.ts new file mode 100644 index 00000000..fd1558c7 --- /dev/null +++ b/tests/checks/external-urls/main.spec.ts @@ -0,0 +1,50 @@ +import { test, expect } from 'vitest'; +import { parseApplication } from '@/application/Parser/ApplicationParser'; +import { IApplication } from '@/domain/IApplication'; +import { IUrlStatus } from './StatusChecker/IUrlStatus'; +import { getUrlStatusesInParallel, IBatchRequestOptions } from './StatusChecker/BatchStatusChecker'; + +const app = parseApplication(); +const urls = collectUniqueUrls(app); +const requestOptions: IBatchRequestOptions = { + domainOptions: { + sameDomainParallelize: false, // be nice to our external servers + sameDomainDelayInMs: 5 /* sec */ * 1000, + }, + requestOptions: { + retryExponentialBaseInMs: 3 /* sec */ * 1000, + requestTimeoutInMs: 60 /* sec */ * 1000, + additionalHeaders: { referer: app.info.homepage }, + }, +}; +const testTimeoutInMs = urls.length * 60 /* seconds */ * 1000; + +test(`all URLs (${urls.length}) should be alive`, async () => { + const results = await getUrlStatusesInParallel(urls, requestOptions); + const deadUrls = results.filter((r) => r.code !== 200); + expect(deadUrls).to.have.lengthOf(0, printUrls(deadUrls)); +}, testTimeoutInMs); + +function collectUniqueUrls(application: IApplication): string[] { + return [ // Get all nodes + ...application.collections.flatMap((c) => c.getAllCategories()), + ...application.collections.flatMap((c) => c.getAllScripts()), + ] + // Get all docs + .flatMap((documentable) => documentable.docs) + // Parse all URLs + .flatMap((docString) => docString.match(/(https?:\/\/[^\s`"<>()]+)/g) || []) + // Remove duplicates + .filter((url, index, array) => array.indexOf(url) === index); +} + +function printUrls(statuses: IUrlStatus[]): string { + /* eslint-disable prefer-template */ + return '\n' + + statuses.map((status) => `- ${status.url}\n` + + (status.code ? `\tResponse code: ${status.code}` : '') + + (status.error ? `\tError: ${status.error}` : '')) + .join('\n') + + '\n'; + /* eslint-enable prefer-template */ +} diff --git a/tests/integration/application/collections/NoDeadDocumentationUrls.spec.ts b/tests/integration/application/collections/NoDeadDocumentationUrls.spec.ts deleted file mode 100644 index 68fbc98c..00000000 --- a/tests/integration/application/collections/NoDeadDocumentationUrls.spec.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { parseApplication } from '@/application/Parser/ApplicationParser'; -import { IApplication } from '@/domain/IApplication'; -import { IUrlStatus } from './StatusChecker/IUrlStatus'; -import { getUrlStatusesInParallel, IBatchRequestOptions } from './StatusChecker/BatchStatusChecker'; - -describe('collections', () => { - // arrange - const app = parseApplication(); - const urls = collectUniqueUrls(app); - const options: IBatchRequestOptions = { - domainOptions: { - sameDomainParallelize: true, // no need to be so nice until sources start failing - // sameDomainDelayInMs: 2 /* sec */ * 1000, - }, - requestOptions: { - retryExponentialBaseInMs: 3 /* sec */ * 1000, - additionalHeaders: { referer: app.info.homepage }, - additionalHeadersUrlIgnore: [ - 'http://batcmd.com/', // Otherwise it responds with 403 - ], - }, - }; - const testTimeoutInMs = urls.length * 60 /* minutes */ * 1000; - it('have no dead urls', async () => { - // act - const results = await getUrlStatusesInParallel(urls, options); - // assert - const deadUrls = results.filter((r) => r.code !== 200); - expect(deadUrls).to.have.lengthOf(0, printUrls(deadUrls)); - }, testTimeoutInMs); -}); - -function collectUniqueUrls(app: IApplication): string[] { - return [ // Get all nodes - ...app.collections.flatMap((c) => c.getAllCategories()), - ...app.collections.flatMap((c) => c.getAllScripts()), - ] - // Get all docs - .flatMap((documentable) => documentable.docs) - // Parse all URLs - .flatMap((docString) => docString.match(/(https?:\/\/[^\s]+)/g) || []) - // Remove duplicates - .filter((url, index, array) => array.indexOf(url) === index); -} - -function printUrls(statuses: IUrlStatus[]): string { - /* eslint-disable prefer-template */ - return '\n' - + statuses.map((status) => `- ${status.url}\n` - + (status.code ? `\tResponse code: ${status.code}` : '') - + (status.error ? `\tError: ${status.error}` : '')) - .join('\n') - + '\n'; - /* eslint-enable prefer-template */ -} diff --git a/tests/integration/application/collections/StatusChecker/README.md b/tests/integration/application/collections/StatusChecker/README.md deleted file mode 100644 index 4cb9bafe..00000000 --- a/tests/integration/application/collections/StatusChecker/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# status-checker - -CLI and SDK to check whether an external URL is alive. - -🧐 Why? - -- 🏃🏻 Batch checking status of URLs in parallel. -- 🤖 Zero-touch start, pre-configured for reliable results, still configurable. -- 🤞 Reliable, mimics a real web browser by following redirect, and cookie storage. - -🍭 Sweets such as - -- 😇 Queueing requests by domain to be nice to them -- 🔁 Retry pattern with exponential back-off - -## CLI - -Coming soon 🚧 - -## Programmatic usage - -Programmatic usage is supported both on Node.js and browser. - -### `getUrlStatusesInParallel` - -```js -// Simple example -const statuses = await getUrlStatusesInParallel([ 'https://privacy.sexy', /* ... */ ]); -if(statuses.all((r) => r.code === 200)) { - console.log('All URLs are alive!'); -} else { - console.log('Dead URLs:', statuses.filter((r) => r.code !== 200).map((r) => r.url)); -} - -// Fastest configuration -const statuses = await getUrlStatusesInParallel([ 'https://privacy.sexy', /* ... */ ], { - domainOptions: { - sameDomainParallelize: false, - } -}); -``` - -#### Batch request options - -- `domainOptions`: - - **`sameDomainParallelize`**, (*boolean*), default: `false` - - Determines whether the requests to URLs under same domain will be parallelize. - - Setting `false` parallelizes all requests. - - Setting `true` sends requests in queue for each unique domain, still parallelizing for different domains. - - Requests to different domains are always parallelized regardless of this option. - - 💡 This helps to avoid `429 Too Many Requests` and be nice to websites - - **`sameDomainDelayInMs`** (*boolean*), default: `3000` (3 seconds) - - Sets delay between requests to same host (domain) if same domain parallelization is disabled. -- `requestOptions` (*object*): See [request options](#request-options). - -### `getUrlStatus` - -Checks whether single URL is dead or alive. - -```js -// Simple example -const status = await getUrlStatus('https://privacy.sexy'); -console.log(`Status code: ${status.code}`); -``` - -#### Request options - -- **`retryExponentialBaseInMs`** (*boolean*), default: `5000` (5 seconds) - - The based time that's multiplied by exponential value for exponential backoff and retry calculations - - The longer it is, the longer the delay between retries are. -- **`additionalHeaders`** (*boolean*), default: `false` - - Additional headers that will be sent alongside default headers mimicking browser. - - If default header are specified, additional headers override defaults. -- **`followOptions`** (*object*): See [follow options](#follow-options). - -### `fetchFollow` - -Gets response from single URL by following `3XX` redirect targets by sending necessary cookies. - -Same fetch API except third parameter that specifies [follow options](#follow-options), `redirect: 'follow' | 'manual' | 'error'` is discarded in favor of the third parameter. - -```js -const status = await fetchFollow('https://privacy.sexy', { - // First argument is same options as fetch API, except `redirect` options - // that's discarded in favor of next argument follow options - headers: { - 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0' - }, - }, { - // Second argument sets the redirect behavior - followRedirects: true, - maximumRedirectFollowDepth: 20, - enableCookies: true, - } -); -console.log(`Status code: ${status.code}`); -``` - -#### Follow options - -- **`followRedirects`** (*boolean*), default: `true` - - Determines whether redirects with `3XX` response code will be followed. -- **`maximumRedirectFollowDepth`** (*boolean*), default: `20` - - Determines maximum consequent redirects that will be followed. - - 💡 Helps to solve maximum redirect reached errors. -- **`enableCookies`** (*boolean*), default: `true` - - Saves cookies requested to store by webpages and sends them when redirected. - - 💡 Helps to over-come sign-in challenges with callbacks.