From 4beb1bb5748a60886210187ca3cdc7f4b41067c0 Mon Sep 17 00:00:00 2001 From: undergroundwires Date: Tue, 5 Sep 2023 13:39:15 +0200 Subject: [PATCH] Introduce retry mechanism for npm install in CI/CD This commit addresses occasional pipeline failures caused by transient network errors during dependency installation with `npm ci`. It centralizes the logic for installing npm dependencies and introduces a retry mechanism. The new approach will attempt `npm ci` up to 5 times with a 5-second interval between each attempt, thereby increasing the resilience of CI/CD pipelines. This commit adds a new script `npm-install.js` with `npm run install-deps` command to centralize npm dependency installation process throughout the project. Separate testing of scripts to a separate workflow. It removes unused `install` dependency from `package.json`. --- .../npm-install-dependencies/action.yml | 11 + .github/workflows/checks.build.yaml | 24 +-- .../checks.desktop-runtime-errors.yaml | 2 +- .github/workflows/checks.external-urls.yaml | 14 +- .github/workflows/checks.quality.yaml | 14 +- .github/workflows/checks.scripts.yaml | 55 +++++ .github/workflows/release.desktop.yaml | 2 +- .github/workflows/release.site.yaml | 5 +- .github/workflows/tests.e2e.yaml | 2 +- .github/workflows/tests.integration.yaml | 2 +- .github/workflows/tests.unit.yaml | 2 +- README.md | 6 + docs/development.md | 10 +- package-lock.json | 14 -- package.json | 2 +- scripts/fresh-npm-install.sh | 95 --------- scripts/npm-install.js | 199 ++++++++++++++++++ .../check-desktop-runtime-errors/utils/npm.ts | 44 +--- 18 files changed, 313 insertions(+), 190 deletions(-) create mode 100644 .github/actions/npm-install-dependencies/action.yml create mode 100644 .github/workflows/checks.scripts.yaml delete mode 100755 scripts/fresh-npm-install.sh create mode 100644 scripts/npm-install.js diff --git a/.github/actions/npm-install-dependencies/action.yml b/.github/actions/npm-install-dependencies/action.yml new file mode 100644 index 00000000..a9a1b388 --- /dev/null +++ b/.github/actions/npm-install-dependencies/action.yml @@ -0,0 +1,11 @@ +inputs: + working-directory: + required: false + default: '.' +runs: + using: composite + steps: + - + name: Run `npm ci` with retries + shell: bash + run: npm run install-deps -- --ci --root-directory "${{ inputs.working-directory }}" diff --git a/.github/workflows/checks.build.yaml b/.github/workflows/checks.build.yaml index ec657dd1..47cfaa8c 100644 --- a/.github/workflows/checks.build.yaml +++ b/.github/workflows/checks.build.yaml @@ -27,7 +27,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Build web run: npm run build -- --mode ${{ matrix.mode }} @@ -55,7 +55,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Prebuild desktop run: npm run electron:prebuild -- --mode ${{ matrix.mode }} @@ -68,23 +68,3 @@ jobs: - name: Verify bundled desktop build artifacts run: npm run check:verify-build-artifacts -- --electron-bundled - - create-icons: - strategy: - matrix: - os: [ macos, ubuntu, windows ] - fail-fast: false # Allows to see results from other combinations - runs-on: ${{ matrix.os }}-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup node - uses: ./.github/actions/setup-node - - - name: Install dependencies - run: npm ci - - - name: Create icons - run: npm run icons:build diff --git a/.github/workflows/checks.desktop-runtime-errors.yaml b/.github/workflows/checks.desktop-runtime-errors.yaml index f1c82040..97fedca2 100644 --- a/.github/workflows/checks.desktop-runtime-errors.yaml +++ b/.github/workflows/checks.desktop-runtime-errors.yaml @@ -21,7 +21,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Configure Ubuntu if: matrix.os == 'ubuntu' diff --git a/.github/workflows/checks.external-urls.yaml b/.github/workflows/checks.external-urls.yaml index baca35a0..1f0ffd37 100644 --- a/.github/workflows/checks.external-urls.yaml +++ b/.github/workflows/checks.external-urls.yaml @@ -8,11 +8,15 @@ jobs: run-check: runs-on: ubuntu-latest steps: - - name: Checkout + - + name: Checkout uses: actions/checkout@v2 - - name: Setup node + - + name: Setup node uses: ./.github/actions/setup-node - - name: Install dependencies - run: npm ci - - name: Test + - + name: Install dependencies + uses: ./.github/actions/npm-install-dependencies + - + name: Test run: npm run check:external-urls diff --git a/.github/workflows/checks.quality.yaml b/.github/workflows/checks.quality.yaml index 0af7bdee..77555b30 100644 --- a/.github/workflows/checks.quality.yaml +++ b/.github/workflows/checks.quality.yaml @@ -16,11 +16,15 @@ jobs: os: [ macos, ubuntu, windows ] fail-fast: false # Still interested to see results from other combinations steps: - - name: Checkout + - + name: Checkout uses: actions/checkout@v2 - - name: Setup node + - + name: Setup node uses: ./.github/actions/setup-node - - name: Install dependencies - run: npm ci - - name: Lint + - + name: Install dependencies + uses: ./.github/actions/npm-install-dependencies + - + name: Lint run: ${{ matrix.lint-command }} diff --git a/.github/workflows/checks.scripts.yaml b/.github/workflows/checks.scripts.yaml new file mode 100644 index 00000000..91351427 --- /dev/null +++ b/.github/workflows/checks.scripts.yaml @@ -0,0 +1,55 @@ +name: checks.scripts + +on: + push: + pull_request: + +jobs: + icons-build: + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + os: [ macos, ubuntu, windows ] + fail-fast: false # Still interested to see results from other combinations + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Setup node + uses: ./.github/actions/setup-node + - + name: Install dependencies + uses: ./.github/actions/npm-install-dependencies + - + name: Create icons + run: npm run icons:build + + install-deps: + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + install-deps-before: [true, false] + install-command: + - npm run install-deps + - npm run install-deps -- --no-errors + - npm run install-deps -- --ci + - npm run install-deps -- --fresh --non-deterministic + - npm run install-deps -- --fresh + - npm run install-deps -- --non-deterministic + os: [ macos, ubuntu, windows ] + fail-fast: false # Still interested to see results from other combinations + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Setup node + uses: ./.github/actions/setup-node + - + name: Install dependencies + if: matrix.install-deps-before == true + uses: ./.github/actions/npm-install-dependencies + - + name: Run install-deps + run: ${{ matrix.install-command }} diff --git a/.github/workflows/release.desktop.yaml b/.github/workflows/release.desktop.yaml index b113b1b1..d505dd17 100644 --- a/.github/workflows/release.desktop.yaml +++ b/.github/workflows/release.desktop.yaml @@ -26,7 +26,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Run unit tests run: npm run test:unit diff --git a/.github/workflows/release.site.yaml b/.github/workflows/release.site.yaml index def57733..c7634a29 100644 --- a/.github/workflows/release.site.yaml +++ b/.github/workflows/release.site.yaml @@ -84,8 +84,9 @@ jobs: uses: ./app/.github/actions/setup-node - name: "App: Install dependencies" - run: npm ci - working-directory: app + uses: ./.github/actions/npm-install-dependencies + with: + working-directory: app - name: "App: Run unit tests" run: npm run test:unit diff --git a/.github/workflows/tests.e2e.yaml b/.github/workflows/tests.e2e.yaml index 2ce36c5a..e5313277 100644 --- a/.github/workflows/tests.e2e.yaml +++ b/.github/workflows/tests.e2e.yaml @@ -20,7 +20,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Run e2e tests run: npm run test:cy:run diff --git a/.github/workflows/tests.integration.yaml b/.github/workflows/tests.integration.yaml index d60c7b64..eb4193ee 100644 --- a/.github/workflows/tests.integration.yaml +++ b/.github/workflows/tests.integration.yaml @@ -22,7 +22,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Run integration tests run: npm run test:integration diff --git a/.github/workflows/tests.unit.yaml b/.github/workflows/tests.unit.yaml index d1aad9e6..ff217091 100644 --- a/.github/workflows/tests.unit.yaml +++ b/.github/workflows/tests.unit.yaml @@ -20,7 +20,7 @@ jobs: uses: ./.github/actions/setup-node - name: Install dependencies - run: npm ci + uses: ./.github/actions/npm-install-dependencies - name: Run unit tests run: npm run test:unit diff --git a/README.md b/README.md index 40de5dd2..cd5ba318 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,12 @@ src="https://github.com/undergroundwires/privacy.sexy/workflows/checks.desktop-runtime-errors/badge.svg" /> + + Status of script checks + Status of external URL checks15.x. -- Install dependencies using `npm install`. +- Install Node >16.x. +- Install dependencies using `npm install` (or [`npm run install-deps`](#utility-scripts) for more options). ### Testing @@ -73,9 +73,9 @@ See [ci-cd.md](./ci-cd.md) for more information. #### Utility scripts -- [**`./scripts/fresh-npm-install.sh`**](../scripts/fresh-npm-install.sh): - - Run fresh NPM install. - - This script provides a clean NPM install, removing existing node modules and optionally the package-lock.json (when run with -n), then installs dependencies and runs unit tests. +- [**`npm run install-deps [-- ]`**](../scripts/npm-install.js): + - Manages NPM dependency installation, it offers capabilities like doing a fresh install, retries on network errors, and other features. + - For example, you can run `npm run install-deps -- --fresh` to do clean installation of dependencies. - [**`./scripts/configure-vscode.sh`**](../scripts/configure-vscode.sh): - This script checks and sets the necessary configurations for VSCode in `settings.json` file. diff --git a/package-lock.json b/package-lock.json index 51470310..a18d8fea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,6 @@ "cross-fetch": "^4.0.0", "electron-progressbar": "^2.1.0", "file-saver": "^2.0.5", - "install": "^0.13.0", "liquor-tree": "^0.2.70", "markdown-it": "^13.0.1", "npm": "^9.8.1", @@ -9425,14 +9424,6 @@ "node": ">=10" } }, - "node_modules/install": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", - "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/internal-slot": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", @@ -28223,11 +28214,6 @@ "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", "dev": true }, - "install": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", - "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==" - }, "internal-slot": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", diff --git a/package.json b/package.json index e0c85f3a..2e55b21d 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test:cy:run": "start-server-and-test \"vite build && vite preview --port 7070\" http://localhost:7070 \"cypress run --config baseUrl=http://localhost:7070\"", "test:cy:open": "start-server-and-test \"vite --port 7070 --mode production\" http://localhost:7070 \"cypress open --config baseUrl=http://localhost:7070\"", "lint": "npm run lint:md && npm run lint:md:consistency && npm run lint:md:relative-urls && npm run lint:eslint && npm run lint:yaml", + "install-deps": "node scripts/npm-install.js", "icons:build": "node scripts/logo-update.js", "check:desktop": "vitest run --dir tests/checks/desktop-runtime-errors --environment node", "check:external-urls": "vitest run --dir tests/checks/external-urls --environment node", @@ -43,7 +44,6 @@ "cross-fetch": "^4.0.0", "electron-progressbar": "^2.1.0", "file-saver": "^2.0.5", - "install": "^0.13.0", "liquor-tree": "^0.2.70", "markdown-it": "^13.0.1", "npm": "^9.8.1", diff --git a/scripts/fresh-npm-install.sh b/scripts/fresh-npm-install.sh deleted file mode 100755 index 194bea2a..00000000 --- a/scripts/fresh-npm-install.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -# Description: -# This script ensures npm is available, removes existing node modules, optionally -# removes package-lock.json (when -n flag is used), installs dependencies and runs unit tests. -# Usage: -# ./fresh-npm-install.sh # Regular execution -# ./fresh-npm-install.sh -n # Non-deterministic mode (removes package-lock.json) - -declare NON_DETERMINISTIC_FLAG=0 - - -main() { - parse_args "$@" - ensure_npm_is_available - ensure_npm_root - remove_existing_modules - if [[ $NON_DETERMINISTIC_FLAG -eq 1 ]]; then - remove_package_lock_json - fi - install_dependencies - run_unit_tests -} - -ensure_npm_is_available() { - if ! command -v npm &> /dev/null; then - log::fatal 'npm could not be found, please install it first.' - fi -} - -ensure_npm_root() { - if [ ! -f package.json ]; then - log::fatal 'Current directory is not a npm root. Please run the script in a npm root directory.' - fi -} - -remove_existing_modules() { - if [ -d ./node_modules ]; then - log::info 'Removing existing node modules...' - if ! rm -rf ./node_modules; then - log::fatal 'Could not remove existing node modules.' - fi - fi -} - -install_dependencies() { - log::info 'Installing dependencies...' - if ! npm install; then - log::fatal 'Failed to install dependencies.' - fi -} - -remove_package_lock_json() { - if [ -f ./package-lock.json ]; then - log::info 'Removing package-lock.json...' - if ! rm -rf ./package-lock.json; then - log::fatal 'Could not remove package-lock.json.' - fi - fi -} - -run_unit_tests() { - log::info 'Running unit tests...' - if ! npm run test:unit; then - pwd - log::fatal 'Failed to run unit tests.' - fi -} - -log::info() { - local -r message="$1" - echo "📣 ${message}" -} - -log::fatal() { - local -r message="$1" - echo "❌ ${message}" >&2 - exit 1 -} - -parse_args() { - while getopts "n" opt; do - case ${opt} in - n) - NON_DETERMINISTIC_FLAG=1 - ;; - \?) - echo "Invalid option: $OPTARG" 1>&2 - exit 1 - ;; - esac - done -} - -main "$1" diff --git a/scripts/npm-install.js b/scripts/npm-install.js new file mode 100644 index 00000000..9b346ac9 --- /dev/null +++ b/scripts/npm-install.js @@ -0,0 +1,199 @@ +/* +Description: + This script manages NPM dependencies for a project. + It offers capabilities like doing a fresh install, retries on network errors, and other features. + +Usage: + npm run install-deps [-- ] + node scripts/npm-install.js [options] + +Options: + --root-directory + Specifies the root directory where package.json resides + Defaults to the current working directory. + Example: npm run install-deps -- --root-directory /your/path/here + + --no-errors + Ignores errors and continues the execution. + Example: npm run install-deps -- --no-errors + + --ci + Uses 'npm ci' for dependency installation instead of 'npm install'. + Example: npm run install-deps -- --ci + + --fresh + Removes the existing node_modules directory before installing dependencies. + Example: npm run install-deps -- --fresh + + --non-deterministic + Removes package-lock.json for a non-deterministic installation. + Example: npm run install-deps -- --non-deterministic + +Note: + + Flags can be combined as needed. + Example: npm run install-deps -- --fresh --non-deterministic +*/ + +import { exec } from 'child_process'; +import { resolve } from 'path'; +import { access, rm, unlink } from 'fs/promises'; +import { constants } from 'fs'; + +const MAX_RETRIES = 5; +const RETRY_DELAY_IN_MS = 5 /* seconds */ * 1000; +const ARG_NAMES = { + rootDirectory: '--root-directory', + ignoreErrors: '--no-errors', + ci: '--ci', + fresh: '--fresh', + nonDeterministic: '--non-deterministic', +}; + +async function main() { + const options = getOptions(); + console.log('Options:', options); + await ensureNpmRootDirectory(options.rootDirectory); + await ensureNpmIsAvailable(); + if (options.fresh) { + await removeNodeModules(options.rootDirectory); + } + if (options.nonDeterministic) { + await removePackageLockJson(options.rootDirectory); + } + const command = buildCommand(options.ci, options.outputErrors); + console.log('Starting dependency installation...'); + const exitCode = await executeWithRetry( + command, + options.workingDirectory, + MAX_RETRIES, + RETRY_DELAY_IN_MS, + ); + if (exitCode === 0) { + console.log('🎊 Installed dependencies...'); + } else { + console.error(`💀 Failed to install dependencies, exit code: ${exitCode}`); + } + process.exit(exitCode); +} + +async function removeNodeModules(workingDirectory) { + const nodeModulesDirectory = resolve(workingDirectory, 'node_modules'); + if (await exists('./node_modules')) { + console.log('Removing node_modules...'); + await rm(nodeModulesDirectory, { recursive: true }); + } +} + +async function removePackageLockJson(workingDirectory) { + const packageLockJsonFile = resolve(workingDirectory, 'package-lock.json'); + if (await exists(packageLockJsonFile)) { + console.log('Removing package-lock.json...'); + await unlink(packageLockJsonFile); + } +} + +async function ensureNpmIsAvailable() { + const exitCode = await executeCommand('npm --version'); + if (exitCode !== 0) { + throw new Error('`npm` in not available!'); + } +} + +async function ensureNpmRootDirectory(workingDirectory) { + const packageJsonPath = resolve(workingDirectory, 'package.json'); + if (!await exists(packageJsonPath)) { + throw new Error(`Not an NPM project root: ${workingDirectory}`); + } +} + +function buildCommand(ci, outputErrors) { + const baseCommand = ci ? 'npm ci' : 'npm install'; + if (!outputErrors) { + return `${baseCommand} --loglevel=error`; + } + return baseCommand; +} + +function getOptions() { + const processArgs = process.argv.slice(2); // Slice off the node and script name + return { + rootDirectory: processArgs.includes('--root-directory') ? processArgs[processArgs.indexOf('--root-directory') + 1] : process.cwd(), + outputErrors: !processArgs.includes(ARG_NAMES.ignoreErrors), + ci: processArgs.includes(ARG_NAMES.ci), + fresh: processArgs.includes(ARG_NAMES.fresh), + nonDeterministic: processArgs.includes(ARG_NAMES.nonDeterministic), + }; +} + +async function executeWithRetry( + command, + workingDirectory, + maxRetries, + retryDelayInMs, + currentAttempt = 1, +) { + const statusCode = await executeCommand(command, workingDirectory, true, true); + if (statusCode === 0 || currentAttempt >= maxRetries) { + return statusCode; + } + + console.log(`⚠️🔄 Attempt ${currentAttempt} failed. Retrying in ${retryDelayInMs / 1000} seconds...`); + await sleep(retryDelayInMs); + + const retryResult = await executeWithRetry( + command, + workingDirectory, + maxRetries, + retryDelayInMs, + currentAttempt + 1, + ); + return retryResult; +} + +async function executeCommand( + command, + workingDirectory = process.cwd(), + logStdout = false, + logCommand = false, +) { + if (logCommand) { + console.log(`▶️ Executing command "${command}" at "${workingDirectory}"`); + } + const process = exec( + command, + { + cwd: workingDirectory, + }, + ); + if (logStdout) { + process.stdout.on('data', (data) => { + console.log(data.toString()); + }); + } + process.stderr.on('data', (data) => { + console.error(data.toString()); + }); + return new Promise((resolve) => { + process.on('exit', (code) => { + resolve(code); + }); + }); +} + +function sleep(milliseconds) { + return new Promise((resolve) => { + setTimeout(resolve, milliseconds); + }); +} + +async function exists(path) { + try { + await access(path, constants.F_OK); + return true; + } catch { + return false; + } +} + +await main(); diff --git a/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/npm.ts b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/npm.ts index 391d67ff..22fe388b 100644 --- a/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/npm.ts +++ b/tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/npm.ts @@ -1,13 +1,8 @@ import { join } from 'path'; import { rm, readFile } from 'fs/promises'; import { exists, isDirMissingOrEmpty } from './io'; -import { CommandResult, runCommand } from './run-command'; +import { runCommand } from './run-command'; import { LogLevel, die, log } from './log'; -import { sleep } from './sleep'; -import type { ExecOptions } from 'child_process'; - -const NPM_INSTALL_MAX_RETRIES = 3; -const NPM_INSTALL_RETRY_DELAY_MS = 5 /* seconds */ * 1000; export async function ensureNpmProjectDir(projectDir: string): Promise { if (!projectDir) { throw new Error('missing project directory'); } @@ -20,13 +15,16 @@ export async function npmInstall(projectDir: string): Promise { if (!projectDir) { throw new Error('missing project directory'); } const npmModulesPath = join(projectDir, 'node_modules'); if (!await isDirMissingOrEmpty(npmModulesPath)) { - log(`Directory "${npmModulesPath}" exists and has content. Skipping \`npm install\`.`); + log(`Directory "${npmModulesPath}" exists and has content. Skipping installing dependencies.`); return; } log('Starting dependency installation...'); - const { error } = await executeWithRetry('npm install --loglevel=error', { - cwd: projectDir, - }, NPM_INSTALL_MAX_RETRIES, NPM_INSTALL_RETRY_DELAY_MS); + const { error } = await runCommand( + `npm run install-deps -- --no-errors --root-directory ${projectDir}`, + { + cwd: projectDir, + }, + ); if (error) { die(error); } @@ -103,29 +101,3 @@ async function readPackageJsonContents(projectDir: string): Promise { return die(`Error detail: ${error}`); } } - -async function executeWithRetry( - command: string, - options: ExecOptions, - maxRetries: number, - retryDelayInMs: number, - currentAttempt = 1, -): Promise { - const result = await runCommand(command, options); - - if (!result.error || currentAttempt >= maxRetries) { - return result; - } - - log(`Attempt ${currentAttempt} failed. Retrying in ${retryDelayInMs / 1000} seconds...`); - await sleep(retryDelayInMs); - - const retryResult = await executeWithRetry( - command, - options, - maxRetries, - retryDelayInMs, - currentAttempt + 1, - ); - return retryResult; -}