ci/cd: trigger URL checks more, and limit amount

Fix all URL checks failing in GitHub runner due to:

- Missing Happy Eyeballs in Node.js
  nodejs/undici$1531
  nodejs/node$41625
- Missing IPv6 support in GitHub runners:
  actions/runner$3138
  actions/runner-images$668

Tried (did not work):

1)

```
import dns from 'dns';
dns.setDefaultResultOrder('ipv4first');
```

2) Bumping node to v20.

3) TODO: Try autoSelectFamily

- Or is it due too to many max connections? Test this.
  Mentioned in comment nodejs/node$41625.

Key changes:

- Run URL checks more frequently on every change.
- Introduce environment variable to randomly select and limit URLs
  tested, this way the tests will provide quicker feedback on code
  changes.

Other supporting changes:

- Log more information about test before running the test to enable
  easier troubleshooting.
- Move shuffle function for arrays for reusability and missing tests.

.github/workflows/checks.external-urls.yaml

@@ -1,6 +1,7 @@
 name: checks.external-urls
 
 on:
+  push:
   schedule:
     - cron: '0 0 * * 0' # at 00:00 on every Sunday
 
@@ -20,3 +21,7 @@ jobs:
       -
         name: Test
         run: npm run check:external-urls
+        env:
+          RANDOMIZED_URL_CHECK_LIMIT: "${{ github.event_name == 'push' && '10' || '' }}"
+          # - Scheduled checks has no limits, ensuring thorough testing.
+          # - For push events, triggered by code changes, the amount of URLs are limited to provide quick feedback.

package.json

@@ -82,7 +82,7 @@
     "tslib": "^2.6.2",
     "typescript": "^5.3.3",
     "vite": "^5.1.6",
-    "vitest": "^0.34.6",
+    "vitest": "^1.3.1",
     "vue-tsc": "^1.8.19",
     "yaml-lint": "^1.7.0"
   },

src/application/Common/Shuffle.ts

@@ -0,0 +1,12 @@
+/*
+  Shuffle an array of strings, returning a new array with elements in random order.
+  Uses the Fisher-Yates (or Durstenfeld) algorithm.
+*/
+export function shuffle<T>(array: readonly T[]): T[] {
+  const shuffledArray = [...array];
+  for (let i = array.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffledArray[i], shuffledArray[j]] = [shuffledArray[j], shuffledArray[i]];
+  }
+  return shuffledArray;
+}

src/application/collections/windows.yaml

@@ -1174,7 +1174,7 @@ actions:
                                         parameters:
                                             fileGlob: '%SYSTEMROOT%\Logs\DISM\DISM.log'
                             -
-                                name: Clear Windows update files # Marked: stop-service-do-stuff-restart-service
+                                name: Clear Windows update files
                                 docs: |-
                                     This script clears the contents of the `%SYSTEMROOT%\SoftwareDistribution\` directory.
                                     This action is sometimes called *resetting the Windows Update Agent* or *resetting Windows Update components* by Microsoft [1].
@@ -1203,18 +1203,22 @@ actions:
                                     [8]: https://web.archive.org/web/20231027190213/https://support.microsoft.com/en-us/windows/troubleshoot-problems-updating-windows-188c2b0f-10a7-d72f-65b8-32d177eb136c#WindowsVersion=Windows_11 "Troubleshoot problems updating Windows - Microsoft Support | support.microsoft.com"
                                     [9]: https://web.archive.org/web/20231027190503/https://learn.microsoft.com/en-us/troubleshoot/mem/configmgr/update-management/troubleshoot-software-update-scan-failures "Troubleshoot software update scan failures - Configuration Manager | Microsoft Learn | learn.microsoft.com"
                                     [10]: https://web.archive.org/web/20231029172022/https://support.microsoft.com/en-us/topic/you-receive-an-administrators-only-error-message-in-windows-xp-when-you-try-to-visit-the-windows-update-web-site-or-the-microsoft-update-web-site-d2c732b6-21e0-a2ce-8d18-303ed71736c9 'You receive an "Administrators only" error message in Windows XP when you try to visit the Windows Update Web site or the Microsoft Update Web site - Microsoft Support | support.microsoft.com'
-                                code: |- # `sc queryex` output is the same in every OS language
-                                    setlocal EnableDelayedExpansion
-                                        SET /A wuau_service_running=0
-                                        SC queryex "wuauserv"|Find "STATE"|Find /v "RUNNING">Nul||(
-                                            SET /A wuau_service_running=1
-                                            net stop wuauserv
-                                        )
-                                        del /q /s /f "%SYSTEMROOT%\SoftwareDistribution\*"
-                                        IF !wuau_service_running! == 1 (
-                                            net start wuauserv
-                                        )
-                                    endlocal
+                                call:
+                                    -
+                                        function: StopService
+                                        parameters:
+                                            serviceName: wuauserv
+                                            waitUntilStopped: true
+                                            serviceRestartStateFile: '%APPDATA%\privacy.sexy-wuauserv' # Marked: refactor-with-variables (app dir should be unified, not using %TEMP% as it can be cleaned during operation)
+                                    -
+                                        function: ClearDirectoryContents
+                                        parameters:
+                                            directoryGlob: '%SYSTEMROOT%\SoftwareDistribution'
+                                    -
+                                        function: StartService
+                                        parameters:
+                                            serviceName: wuauserv
+                                            serviceRestartStateFile: '%APPDATA%\privacy.sexy-wuauserv' # Marked: refactor-with-variables (app dir should be unified, not using %TEMP% as it can be cleaned during operation)
                             -
                                 name: Clear Common Language Runtime system logs
                                 recommend: standard
@@ -1251,7 +1255,7 @@ actions:
                                     parameters:
                                         directoryGlob: '%SYSTEMROOT%\System32\LogFiles\setupcln'
                     -
-                        name: Clear diagnostics tracking logs # Marked: stop-service-do-stuff-restart-service ("DiagTrack")
+                        name: Clear diagnostics tracking logs
                         recommend: standard
                         docs: |-
                             This script deletes primary telemetry files in Windows.
@@ -1286,6 +1290,12 @@ actions:
                             [6]: https://web.archive.org/web/20231027164510/https://learn.microsoft.com/en-us/windows/win32/etw/configuring-and-starting-an-autologger-session "Configuring and Starting an AutoLogger Session - Win32 apps | Microsoft Learn | learn.microsoft.com"
                             [7]: https://web.archive.org/web/20240217185108/https://learn.microsoft.com/en-us/windows/privacy/configure-windows-diagnostic-data-in-your-organization "Configure Windows diagnostic data in your organization (Windows 10 and Windows 11) - Windows Privacy | Microsoft Learn | learn.microsoft.com"
                         call:
+                            -
+                                function: StopService
+                                parameters:
+                                    serviceName: DiagTrack
+                                    waitUntilStopped: true
+                                    serviceRestartStateFile: '%APPDATA%\privacy.sexy-DiagTrack' # Marked: refactor-with-variables (app dir should be unified, not using %TEMP% as it can be cleaned during operation)
                             -
                                 function: DeleteFiles
                                 parameters:
@@ -1296,6 +1306,11 @@ actions:
                                 parameters:
                                     fileGlob: '%PROGRAMDATA%\Microsoft\Diagnosis\ETLLogs\ShutdownLogger\AutoLogger-Diagtrack-Listener.etl'
                                     grantPermissions: true
+                            -
+                                function: StartService
+                                parameters:
+                                    serviceName: DiagTrack
+                                    serviceRestartStateFile: '%APPDATA%\privacy.sexy-DiagTrack' # Marked: refactor-with-variables (app dir should be unified, not using %TEMP% as it can be cleaned during operation)
                     -
                         name: Clear event logs in Event Viewer application
                         docs: https://serverfault.com/questions/407838/do-windows-events-from-the-windows-event-log-have-sensitive-information
@@ -1452,7 +1467,7 @@ actions:
                 recommend: standard
                 code: dism /online /Remove-DefaultAppAssociations
             -
-                name: Clear System Resource Usage Monitor (SRUM) data # Marked: stop-service-do-stuff-restart-service
+                name: Clear System Resource Usage Monitor (SRUM) data
                 recommend: standard
                 docs: |-
                     This script deletes the Windows System Resource Usage Monitor (SRUM) database file.
@@ -1472,47 +1487,25 @@ actions:
                     [5]: https://web.archive.org/web/20231008135321/https://devblogs.microsoft.com/sustainable-software/measuring-your-application-power-and-carbon-impact-part-1/ "Measuring Your Application Power and Carbon Impact (Part 1) - Sustainable Software | devblogs.microsoft.com"
                     [6]: https://web.archive.org/web/20231008135333/https://www.sciencedirect.com/science/article/abs/pii/S1742287615000031 "Forensic implications of System Resource Usage Monitor (SRUM) data in Windows 8 | Yogesh Khatri | sciencedirect.com"
                 call:
-                    function: RunPowerShell
-                    parameters:
+                    -
                         # If the service is not stopped, following error is thrown:
                         #   Failed to delete SRUM database file at: "C:\Windows\System32\sru\SRUDB.dat". Error Details: The process cannot access
-                        #   the file 'C:\Windows\System32\sru\SRUDB.dat' because it is being used by another process.
-                        code: |-
-                            $srumDatabaseFilePath = "$env:WINDIR\System32\sru\SRUDB.dat"
-                            if (!(Test-Path -Path $srumDatabaseFilePath)) {
-                                Write-Output "Skipping, SRUM database file not found at `"$srumDatabaseFilePath`". No actions are required."
-                                exit 0
-                            }
-                            $dps = Get-Service -Name 'DPS' -ErrorAction Ignore
-                            $isDpsInitiallyRunning = $false
-                            if ($dps) {
-                                $isDpsInitiallyRunning = $dps.Status -eq [System.ServiceProcess.ServiceControllerStatus]::Running
-                                if ($isDpsInitiallyRunning) {
-                                    Write-Output "Stopping the Diagnostic Policy Service (DPS) to delete the SRUM database file."
-                                    $dps | Stop-Service -Force
-                                    $dps.WaitForStatus([System.ServiceProcess.ServiceControllerStatus]::Stopped)
-                                    Write-Output "Successfully stopped Diagnostic Policy Service (DPS)."
-                                }
-                            } else {
-                                Write-Output "Diagnostic Policy Service (DPS) not found. Proceeding without stopping the service."
-                            }
-                            try {
-                                Remove-Item -Path $srumDatabaseFilePath -Force -ErrorAction Stop
-                                Write-Output "Successfully deleted the SRUM database file at `"$srumDatabaseFilePath`"."
-                            } catch {
-                                throw "Failed to delete SRUM database file at: `"$srumDatabaseFilePath`". Error Details: $($_.Exception.Message)"
-                            } finally {
-                                if ($isDpsInitiallyRunning) {
-                                    try {
-                                        if ((Get-Service -Name 'DPS').Status -ne [System.ServiceProcess.ServiceControllerStatus]::Running) {
-                                            Write-Output "Restarting the Diagnostic Policy Service (DPS)."
-                                            $dps | Start-Service
-                                        }
-                                    } catch {
-                                        throw "Failed to restart the Diagnostic Policy Service (DPS). Error Details: $($_.Exception.Message)"
-                                    }
-                                }
-                            }
+                        #   the file 'C:\Windows\System32\sru\SRUDB.dat' because it is being used by another proces
+                        function: StopService
+                        parameters:
+                            serviceName: DPS
+                            waitUntilStopped: true
+                            serviceRestartStateFile: '%APPDATA%\privacy.sexy-DPS' # Marked: refactor-with-variables (app dir should be unified, not using %TEMP% as it can be cleaned during operation)
+                    -
+                        function: DeleteFiles
+                        parameters:
+                            fileGlob: '%WINDIR%\System32\sru\SRUDB.dat'
+                            grantPermissions: true
+                    -
+                        function: StartService
+                        parameters:
+                            serviceName: DPS
+                            serviceRestartStateFile: '%APPDATA%\privacy.sexy-DPS' # Marked: refactor-with-variables (app dir should be unified, not using %TEMP% as it can be cleaned during operation)
             -
                 name: Clear previous Windows installations
                 call:   
@@ -15882,10 +15875,12 @@ actions:
         category: Advanced settings
         children:
             -
-                name: Set NTP (time) server to `pool.ntp.org` # Marked: stop-service-do-stuff-restart-service
+                name: Set NTP (time) server to `pool.ntp.org`
                 docs: https://www.pool.ntp.org/en/use.html
                 recommend: strict
                  # `sc queryex` output is same in every OS language
+                 # Marked: refactor-with-revert-call, refactor-with-variables
+                 #  This would allow re-using `StartService` and `StopService`
                 code: |-
                     :: Configure time source
                     w32tm /config /syncfromflags:manual /manualpeerlist:"0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org 3.pool.ntp.org"
@@ -15904,7 +15899,7 @@ actions:
                     SC queryex "w32time"|Find "STATE"|Find /v "RUNNING">Nul||(
                         net stop w32time
                     )
-                    :: Start time servie and sync now
+                    :: Start time service and sync now
                     net start w32time
                     w32tm /config /update
                     w32tm /resync
@@ -16717,6 +16712,8 @@ functions:
             - name: defaultStartupMode # Allowed values: Boot | System | Automatic | Manual
         call:
             function: RunPowerShell
+            # Marked: refactor-with-revert-call, refactor-with-variables
+            #   Implementation of those should share similar code: `DisableService`, `StopService`, `StartService`, `DisableServiceInRegistry`
             parameters:
                 code: |- # We do registry way because GUI, "sc config" or "Set-Service" won't not work
                     $serviceQuery = '{{ $serviceName }}'
@@ -16938,6 +16935,123 @@ functions:
                         }
                         exit 1
                     }
+    -
+        name: StopService
+        parameters:
+            - name: serviceName
+            - name: serviceRestartStateFile     # This file is created only if the service is successfully stopped.
+              optional: true
+            - name: waitUntilStopped            # Makes the script wait until the service is stopped
+              optional: true
+        call:
+            -
+                function: Comment
+                parameters:
+                    codeComment: >-
+                        Stop service: {{ $serviceName }}
+                        {{ with $serviceRestartStateFile }}(with state flag){{ end }}
+                        {{ with $waitUntilStopped }}(wait until stopped){{ end }}
+            -
+                function: RunPowerShell
+                parameters:
+                    # Marked: refactor-with-variables
+                    #   Implementation of those should share similar code: `DisableService`, `StopService`, `StartService`, `DisableServiceInRegistry`
+                    code: |-
+                        $serviceName = '{{ $serviceName }}'
+                        Write-Host "Stopping service: `"$serviceName`"."
+                        $service = Get-Service -Name $serviceName -ErrorAction SilentlyContinue
+                        if (!$service) {
+                            Write-Host "Skipping, service `"$serviceName`" could not be not found, no need to stop it."
+                            exit 0
+                        }
+                        if ($service.Status -ne [System.ServiceProcess.ServiceControllerStatus]::Running) {
+                            Write-Host "Skipping, `"$serviceName`" is not running, no need to stop."
+                            exit 0
+                        }
+                        Write-Host "`"$serviceName`" is running, stopping it."
+                        try {
+                            $service | Stop-Service -Force -ErrorAction Stop
+                            {{ with $waitUntilStopped }}
+                                $service.WaitForStatus([System.ServiceProcess.ServiceControllerStatus]::Stopped)
+                            {{ end }}
+                        } catch {
+                            throw "Failed to stop the service `"$serviceName`": $_"
+                        }
+                        Write-Host "Successfully stopped the service: `"$serviceName`"."
+                        {{ with $serviceRestartStateFile }}
+                            $stateFilePath = '{{ . }}'
+                            $expandedStateFilePath = [System.Environment]::ExpandEnvironmentVariables($stateFilePath)
+                            if (Test-Path -Path $expandedStateFilePath) {
+                                Write-Host "Skipping creating a service state file, it already exists: `"$expandedStateFilePath`"."
+                            } else {
+                                # Ensure the directory exists
+                                $parentDirectory = [System.IO.Path]::GetDirectoryName($expandedStateFilePath)
+                                if (-not (Test-Path $parentDirectory -PathType Container)) {
+                                    try {
+                                        New-Item -ItemType Directory -Path $parentDirectory -Force -ErrorAction Stop | Out-Null
+                                    }  catch {
+                                        Write-Warning "Failed to create parent directory of service state file `"$parentDirectory`": $_"
+                                    }
+                                }
+                                # Create the state file
+                                try {
+                                    New-Item -ItemType File -Path $expandedStateFilePath -Force -ErrorAction Stop | Out-Null
+                                    Write-Host 'The service will be started again.'
+                                } catch {
+                                    Write-Warning "Failed to create service state file `"$expandedStateFilePath`": $_"
+                                }
+                            }
+                        {{ end }}
+    -
+        name: StartService
+        parameters:
+            - name: serviceName
+            - name: serviceRestartStateFile     # Used for "check and delete": Starts the service only if file exists, always deletes the file.
+              optional: true
+        call:
+            -
+                function: Comment
+                parameters:
+                    codeComment: >-
+                        Start service: {{ $serviceName }}
+                        {{ with $serviceRestartStateFile }}(with state flag){{ end }}
+            -
+                function: RunPowerShell
+                parameters:
+                    # Marked: refactor-with-variables
+                    #   Implementation of those should share similar code: `DisableService`, `StopService`, `StartService`, `DisableServiceInRegistry`
+                    code: |-
+                        $serviceName = '{{ $serviceName }}'
+                        {{ with $serviceRestartStateFile }}
+                            $stateFilePath = '{{ . }}'
+                            $expandedStateFilePath = [System.Environment]::ExpandEnvironmentVariables($stateFilePath)
+                            if (-not (Test-Path -Path $expandedStateFilePath)) {
+                                Write-Host "Skipping starting the service: It was not running before."
+                            } else {
+                                try {
+                                    Remove-Item -Path $expandedStateFilePath -Force -ErrorAction Stop
+                                    Write-Host 'The service is expected to be started.'
+                                } catch {
+                                    Write-Warning "Failed to delete the service state file `"$expandedStateFilePath`": $_"
+                                }
+                            }
+                        {{ end }}
+                        $service = Get-Service -Name $serviceName -ErrorAction SilentlyContinue
+                        if (!$service) {
+                            throw "Failed to start service `"$serviceName`": Service not found."
+                        }
+                        if ($service.Status -eq [System.ServiceProcess.ServiceControllerStatus]::Running) {
+                            Write-Host "Skipping, `"$serviceName`" is already running, no need to start."
+                            exit 0
+                        }
+                        Write-Host "`"$serviceName`" is not running, starting it."
+                        try {
+                            $service | Start-Service -ErrorAction Stop
+                            Write-Host "Successfully started the service: `"$serviceName`"."
+                        } catch {
+                            Write-Warning "Failed to start the service: `"$serviceName`"."
+                            exit 1
+                        }
     -
         name: DisableService
         parameters:
@@ -16950,6 +17064,8 @@ functions:
                     codeComment: "Disable service(s): `{{ $serviceName }}`"
                     revertCodeComment: "Restore service(s) to default state: `{{ $serviceName }}`"
             -
+                # Marked: refactor-with-revert-call, refactor-with-variables
+                #   Implementation of those should share similar code: `DisableService`, `StopService`, `StartService`, `DisableServiceInRegistry`
                 function: RunPowerShell
                 # Careful with Set-Service cmdlet:
                 #   1. It exits with positive code even if service is disabled
@@ -16959,7 +17075,7 @@ functions:
                 #           So "Disabled", "Automatic" and "Manual" are only consistent ones.
                 #       Read more:
                 #           https://github.com/PowerShell/PowerShell/blob/v7.2.0/src/Microsoft.PowerShell.Commands.Management/commands/management/Service.cs#L2966-L2978
-                #           https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.management/set-service?view=powershell-7.1
+                #           https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.management/set-service?view=powershell-7.4
                 parameters:
                     code: |-
                         $serviceName = '{{ $serviceName }}'
@@ -16982,7 +17098,6 @@ functions:
                         } else {
                             Write-Host "`"$serviceName`" is not running, no need to stop."
                         }
-
                         # -- 3. Skip if already disabled
                         $startupType = $service.StartType # Does not work before .NET 4.6.1
                         if(!$startupType) {

src/infrastructure/Threading/AsyncSleep.ts

@@ -1,7 +1,10 @@
 export type SchedulerCallbackType = (...args: unknown[]) => void;
 export type SchedulerType = (callback: SchedulerCallbackType, ms: number) => void;
 
-export function sleep(time: number, scheduler: SchedulerType = setTimeout) {
+export function sleep(
+  time: number,
+  scheduler: SchedulerType = setTimeout,
+): Promise<void> {
   return new Promise((resolve) => {
     scheduler(() => resolve(undefined), time);
   });

tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/check-for-errors.ts

@@ -1,4 +1,4 @@
-import { splitTextIntoLines, indentText } from '../utils/text';
+import { indentText, splitTextIntoLines } from '@tests/shared/Text';
 import { log, die } from '../utils/log';
 import { readAppLogFile } from './app-logs';
 import { STDERR_IGNORE_PATTERNS } from './error-ignore-patterns';

tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/app/system-capture/window-title-capture.ts

@@ -1,7 +1,7 @@
+import { filterEmpty } from '@tests/shared/Text';
 import { runCommand } from '../../utils/run-command';
 import { log, LogLevel } from '../../utils/log';
 import { SupportedPlatform, CURRENT_PLATFORM } from '../../utils/platform';
-import { filterEmpty } from '../../utils/text';
 
 export async function captureWindowTitles(processId: number) {
   if (!processId) { throw new Error('Missing process ID.'); }

tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/main.ts

@@ -1,3 +1,4 @@
+import { indentText } from '@tests/shared/Text';
 import { logCurrentArgs, CommandLineFlag, hasCommandLineFlag } from './cli-args';
 import { log, die } from './utils/log';
 import { ensureNpmProjectDir, npmInstall, npmBuild } from './utils/npm';
@@ -15,7 +16,6 @@ import {
   APP_EXECUTION_DURATION_IN_SECONDS,
   SCREENSHOT_PATH,
 } from './config';
-import { indentText } from './utils/text';
 import type { ExtractionResult } from './app/extractors/common/extraction-result';
 
 export async function main(): Promise<void> {

tests/checks/desktop-runtime-errors/check-desktop-runtime-errors/utils/run-command.ts

@@ -1,5 +1,6 @@
-import { exec, type ExecOptions, type ExecException } from 'node:child_process';
-import { indentText } from './text';
+import { exec } from 'child_process';
+import { indentText } from '@tests/shared/Text';
+import type { ExecOptions, ExecException } from 'child_process';
 
 const TIMEOUT_IN_SECONDS = 180;
 const MAX_OUTPUT_BUFFER_SIZE = 1024 * 1024; // 1 MB

tests/checks/external-urls/DocumentationUrlExtractor.ts

@@ -0,0 +1,69 @@
+import type { IApplication } from '@/domain/IApplication';
+import type { TestExecutionDetailsLogger } from './TestExecutionDetailsLogger';
+
+interface UrlExtractionContext {
+  readonly logger: TestExecutionDetailsLogger;
+  readonly application: IApplication;
+  readonly urlExclusionPatterns: readonly RegExp[];
+}
+
+export function extractDocumentationUrls(
+  context: UrlExtractionContext,
+): string[] {
+  const urlsInApplication = extractUrlsFromApplication(context.application);
+  context.logger.logLabeledInformation(
+    'Extracted URLs from application',
+    urlsInApplication.length.toString(),
+  );
+  const uniqueUrls = filterDuplicateUrls(urlsInApplication);
+  context.logger.logLabeledInformation(
+    'Unique URLs after deduplication',
+    `${uniqueUrls.length} (duplicates removed)`,
+  );
+  context.logger.logLabeledInformation(
+    'Exclusion patterns for URLs',
+    context.urlExclusionPatterns.length === 0
+      ? 'None (all URLs included)'
+      : context.urlExclusionPatterns.map((pattern, index) => `${index + 1}) ${pattern.toString()}`).join('\n'),
+  );
+  const includedUrls = filterUrlsExcludingPatterns(uniqueUrls, context.urlExclusionPatterns);
+  context.logger.logLabeledInformation(
+    'URLs extracted for testing',
+    `${includedUrls.length} (after applying exclusion patterns; ${uniqueUrls.length - includedUrls.length} URLs ignored)`,
+  );
+  return includedUrls;
+}
+
+function extractUrlsFromApplication(application: IApplication): string[] {
+  return [ // Get all executables
+    ...application.collections.flatMap((c) => c.getAllCategories()),
+    ...application.collections.flatMap((c) => c.getAllScripts()),
+  ]
+    // Get all docs
+    .flatMap((documentable) => documentable.docs)
+    // Parse all URLs
+    .flatMap((docString) => extractUrlsExcludingCodeBlocks(docString));
+}
+
+function filterDuplicateUrls(urls: readonly string[]): string[] {
+  return urls.filter((url, index, array) => array.indexOf(url) === index);
+}
+
+function filterUrlsExcludingPatterns(
+  urls: readonly string[],
+  patterns: readonly RegExp[],
+): string[] {
+  return urls.filter((url) => !patterns.some((pattern) => pattern.test(url)));
+}
+
+function extractUrlsExcludingCodeBlocks(textWithInlineCode: string): string[] {
+  /*
+    Matches URLs:
+    - Excludes inline code blocks as they may contain URLs not intended for user interaction
+      and not guaranteed to support expected HTTP methods, leading to false-negatives.
+    - Supports URLs containing parentheses, avoiding matches within code that might not represent
+      actual links.
+  */
+  const nonCodeBlockUrlRegex = /(?<!`)(https?:\/\/[^\s`"<>()]+(?:\([^\s`"<>()]*\))?[^\s`"<>()]*)/g;
+  return textWithInlineCode.match(nonCodeBlockUrlRegex) || [];
+}

tests/checks/external-urls/StatusChecker/BatchStatusChecker.ts

@@ -1,64 +1,65 @@
 import { sleep } from '@/infrastructure/Threading/AsyncSleep';
-import { getUrlStatus, type IRequestOptions } from './Requestor';
-import { groupUrlsByDomain } from './UrlPerDomainGrouper';
-import type { IUrlStatus } from './IUrlStatus';
+import { getUrlStatus, type RequestOptions } from './Requestor';
+import { groupUrlsByDomain } from './UrlDomainProcessing';
+import type { FollowOptions } from './FetchFollow';
+import type { UrlStatus } from './UrlStatus';
 
 export async function getUrlStatusesInParallel(
   urls: string[],
-  options?: IBatchRequestOptions,
-): Promise<IUrlStatus[]> {
-  // urls = [ 'https://privacy.sexy' ]; // Here to comment out when testing
+  options?: BatchRequestOptions,
+): Promise<UrlStatus[]> {
+  // urls = ['https://privacy.sexy']; // Comment out this line to use a hardcoded URL for testing.
   const uniqueUrls = Array.from(new Set(urls));
-  const defaultedOptions = { ...DefaultOptions, ...options };
-  console.log('Options: ', defaultedOptions);
-  const results = await request(uniqueUrls, defaultedOptions);
+  const defaultedDomainOptions: Required<DomainOptions> = {
+    ...DefaultDomainOptions,
+    ...options?.domainOptions,
+  };
+  console.log('Batch request options applied:', defaultedDomainOptions);
+  const results = await request(uniqueUrls, defaultedDomainOptions, options);
   return results;
 }
 
-export interface IBatchRequestOptions {
-  domainOptions?: IDomainOptions;
-  requestOptions?: IRequestOptions;
+export interface BatchRequestOptions {
+  readonly domainOptions?: Partial<DomainOptions>;
+  readonly requestOptions?: Partial<RequestOptions>;
+  readonly followOptions?: Partial<FollowOptions>;
 }
 
-interface IDomainOptions {
-  sameDomainParallelize?: boolean;
-  sameDomainDelayInMs?: number;
+interface DomainOptions {
+  readonly sameDomainParallelize?: boolean;
+  readonly sameDomainDelayInMs?: number;
 }
 
-const DefaultOptions: Required<IBatchRequestOptions> = {
-  domainOptions: {
-    sameDomainParallelize: false,
-    sameDomainDelayInMs: 3 /* sec */ * 1000,
-  },
-  requestOptions: {
-    retryExponentialBaseInMs: 5 /* sec */ * 1000,
-    requestTimeoutInMs: 60 /* sec */ * 1000,
-    additionalHeaders: {},
-  },
+const DefaultDomainOptions: Required<DomainOptions> = {
+  sameDomainParallelize: false,
+  sameDomainDelayInMs: 3 /* sec */ * 1000,
 };
 
 function request(
   urls: string[],
-  options: Required<IBatchRequestOptions>,
-): Promise<IUrlStatus[]> {
-  if (!options.domainOptions.sameDomainParallelize) {
+  domainOptions: Required<DomainOptions>,
+  options?: BatchRequestOptions,
+): Promise<UrlStatus[]> {
+  if (!domainOptions.sameDomainParallelize) {
     return runOnEachDomainWithDelay(
       urls,
-      (url) => getUrlStatus(url, options.requestOptions),
-      options.domainOptions.sameDomainDelayInMs,
+      (url) => getUrlStatus(url, options?.requestOptions, options?.followOptions),
+      domainOptions.sameDomainDelayInMs,
     );
   }
-  return Promise.all(urls.map((url) => getUrlStatus(url, options.requestOptions)));
+  return Promise.all(
+    urls.map((url) => getUrlStatus(url, options?.requestOptions, options?.followOptions)),
+  );
 }
 
 async function runOnEachDomainWithDelay(
   urls: string[],
-  action: (url: string) => Promise<IUrlStatus>,
+  action: (url: string) => Promise<UrlStatus>,
   delayInMs: number | undefined,
-): Promise<IUrlStatus[]> {
+): Promise<UrlStatus[]> {
   const grouped = groupUrlsByDomain(urls);
   const tasks = grouped.map(async (group) => {
-    const results = new Array<IUrlStatus>();
+    const results = new Array<UrlStatus>();
     /* eslint-disable no-await-in-loop */
     for (const url of group) {
       const status = await action(url);

tests/checks/external-urls/StatusChecker/ExponentialBackOffRetryHandler.ts

@@ -1,27 +1,33 @@
 import { sleep } from '@/infrastructure/Threading/AsyncSleep';
-import type { IUrlStatus } from './IUrlStatus';
+import { indentText } from '@tests/shared/Text';
+import { type UrlStatus, formatUrlStatus } from './UrlStatus';
 
 const DefaultBaseRetryIntervalInMs = 5 /* sec */ * 1000;
 
 export async function retryWithExponentialBackOff(
-  action: () => Promise<IUrlStatus>,
+  action: () => Promise<UrlStatus>,
   baseRetryIntervalInMs: number = DefaultBaseRetryIntervalInMs,
   currentRetry = 1,
-): Promise<IUrlStatus> {
+): Promise<UrlStatus> {
   const maxTries = 3;
   const status = await action();
   if (shouldRetry(status)) {
     if (currentRetry <= maxTries) {
       const exponentialBackOffInMs = getRetryTimeoutInMs(currentRetry, baseRetryIntervalInMs);
-      console.log(`Retrying (${currentRetry}) in ${exponentialBackOffInMs / 1000} seconds`, status);
+      console.log([
+        `Attempt ${currentRetry}: Retrying in ${exponentialBackOffInMs / 1000} seconds.`,
+        'Details:',
+        indentText(formatUrlStatus(status)),
+      ].join('\n'));
       await sleep(exponentialBackOffInMs);
       return retryWithExponentialBackOff(action, baseRetryIntervalInMs, currentRetry + 1);
     }
+    console.warn('💀 All retry attempts failed. Final failure to retrieve URL:', indentText(formatUrlStatus(status)));
   }
   return status;
 }
 
-function shouldRetry(status: IUrlStatus) {
+function shouldRetry(status: UrlStatus): boolean {
   if (status.error) {
     return true;
   }
@@ -32,14 +38,14 @@ function shouldRetry(status: IUrlStatus) {
     || status.code === 429; // Too Many Requests
 }
 
-function isTransientError(statusCode: number) {
+function isTransientError(statusCode: number): boolean {
   return statusCode >= 500 && statusCode <= 599;
 }
 
 function getRetryTimeoutInMs(
   currentRetry: number,
   baseRetryIntervalInMs: number = DefaultBaseRetryIntervalInMs,
-) {
+): number {
   const retryRandomFactor = 0.5; // Retry intervals are between 50% and 150%
   // of the exponentially increasing base amount
   const minRandom = 1 - retryRandomFactor;

tests/checks/external-urls/StatusChecker/FetchFollow.ts

@@ -1,19 +1,22 @@
+import { indentText } from '@tests/shared/Text';
 import { fetchWithTimeout } from './FetchWithTimeout';
+import { getDomainFromUrl } from './UrlDomainProcessing';
 
 export function fetchFollow(
   url: string,
   timeoutInMs: number,
-  fetchOptions: RequestInit,
-  followOptions: IFollowOptions | undefined,
+  fetchOptions?: Partial<RequestInit>,
+  followOptions?: Partial<FollowOptions>,
 ): Promise<Response> {
-  const defaultedFollowOptions = {
+  const defaultedFollowOptions: Required<FollowOptions> = {
     ...DefaultFollowOptions,
     ...followOptions,
   };
-  if (followRedirects(defaultedFollowOptions)) {
+  console.log(indentText(`Follow options: ${JSON.stringify(defaultedFollowOptions)}`));
+  if (!followRedirects(defaultedFollowOptions)) {
     return fetchWithTimeout(url, timeoutInMs, fetchOptions);
   }
-  fetchOptions = { ...fetchOptions, redirect: 'manual' /* handled manually */ };
+  fetchOptions = { ...fetchOptions, redirect: 'manual' /* handled manually */, mode: 'cors' };
   const cookies = new CookieStorage(defaultedFollowOptions.enableCookies);
   return followRecursivelyWithCookies(
     url,
@@ -24,13 +27,13 @@ export function fetchFollow(
   );
 }
 
-export interface IFollowOptions {
-  followRedirects?: boolean;
-  maximumRedirectFollowDepth?: number;
-  enableCookies?: boolean;
+export interface FollowOptions {
+  readonly followRedirects?: boolean;
+  readonly maximumRedirectFollowDepth?: number;
+  readonly enableCookies?: boolean;
 }
 
-export const DefaultFollowOptions: Required<IFollowOptions> = {
+const DefaultFollowOptions: Required<FollowOptions> = {
   followRedirects: true,
   maximumRedirectFollowDepth: 20,
   enableCookies: true,
@@ -64,6 +67,10 @@ async function followRecursivelyWithCookies(
   if (cookieHeader) {
     cookies.addHeader(cookieHeader);
   }
+  options.headers = {
+    ...options.headers,
+    Host: getDomainFromUrl(nextUrl),
+  };
   return followRecursivelyWithCookies(nextUrl, timeoutInMs, options, newFollowDepth, cookies);
 }
 
@@ -77,7 +84,7 @@ class CookieStorage {
   constructor(private readonly enabled: boolean) {
   }
 
-  public hasAny() {
+  public hasAny(): boolean {
     return this.enabled && this.cookies.length > 0;
   }
 
@@ -88,17 +95,17 @@ class CookieStorage {
     this.cookies.push(header);
   }
 
-  public getHeader() {
+  public getHeader(): string {
     return this.cookies.join(' ; ');
   }
 }
 
-function followRedirects(options: IFollowOptions) {
-  if (!options.followRedirects) {
+function followRedirects(options: FollowOptions): boolean {
+  if (options.followRedirects !== true) {
     return false;
   }
-  if (options.maximumRedirectFollowDepth === 0) {
-    return false;
+  if (options.maximumRedirectFollowDepth === undefined || options.maximumRedirectFollowDepth <= 0) {
+    throw new Error('Invalid followRedirects configuration: maximumRedirectFollowDepth must be a positive integer');
   }
   return true;
 }

tests/checks/external-urls/StatusChecker/FetchWithTimeout.ts

@@ -2,13 +2,13 @@ export async function fetchWithTimeout(
   url: string,
   timeoutInMs: number,
   init?: RequestInit,
-): Promise<Response> {
-  const controller = new AbortController();
+): ReturnType<typeof fetch> {
   const options: RequestInit = {
     ...(init ?? {}),
-    signal: controller.signal,
+    signal: AbortSignal.timeout(timeoutInMs),
   };
-  const promise = fetch(url, options);
-  const timeout = setTimeout(() => controller.abort(), timeoutInMs);
-  return promise.finally(() => clearTimeout(timeout));
+  return fetch(
+    url,
+    options,
+  );
 }

tests/checks/external-urls/StatusChecker/IUrlStatus.ts

@@ -1,5 +0,0 @@
-export interface IUrlStatus {
-  url: string;
-  error?: string;
-  code?: number;
-}

tests/checks/external-urls/StatusChecker/README.md

@@ -13,7 +13,10 @@ A CLI and SDK for checking the availability of external URLs.
 - 😇 **Rate Limiting**: Queues requests by domain to be polite.
 - 🔁 **Retries**: Implements retry pattern with exponential back-off.
 - ⌚ **Timeouts**: Configurable timeout for each request.
-- 🎭️ **User-Agent Rotation**: Change user agents for each request.
+- 🎭️ **Impersonation**: Impersonate different browsers for each request.
+  - **🌐 User-Agent Rotation**: Change user agents.
+  - **🔑 TLS Handshakes**: Perform TLS and HTTP handshakes that are identical to that of a real browser.
+- 🫙 **Cookie jar**: Preserve cookies during redirects to mimic real browser.
 
 ## CLI
 
@@ -54,6 +57,7 @@ const statuses = await getUrlStatusesInParallel([ 'https://privacy.sexy', /* ...
   - **`sameDomainDelayInMs`** (*number*), default: `3000` (3 seconds)
     - Sets the delay between requests to the same domain.
 - `requestOptions` (*object*): See [request options](#request-options).
+- `followOptions` (*object*): See [follow options](#follow-options).
 
 ### `getUrlStatus`
 
@@ -72,7 +76,6 @@ console.log(`Status code: ${status.code}`);
   - The longer the base time, the greater the intervals between retries.
 - **`additionalHeaders`** (*object*), default: `false`
   - Additional HTTP headers to send along with the default headers. Overrides default headers if specified.
-- **`followOptions`** (*object*): See [follow options](#follow-options).
 - **`requestTimeoutInMs`**  (*number*), default: `60000` (60 seconds)
   - Time limit to abort the request if no response is received within the specified time frame.
 
@@ -83,19 +86,7 @@ Follows `3XX` redirects while preserving cookies.
 Same fetch API except third parameter that specifies [follow options](#follow-options), `redirect: 'follow' | 'manual' | 'error'` is discarded in favor of the third parameter.
 
 ```js
-const status = await fetchFollow('https://privacy.sexy', {
-        // First argument is same options as fetch API, except `redirect` options
-        // that's discarded in favor of next argument follow options
-        headers: {
-            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'
-        },
-    }, {
-        // Second argument sets the redirect behavior
-        followRedirects: true,
-        maximumRedirectFollowDepth: 20,
-        enableCookies: true,
-    }
-);
+const status = await fetchFollow('https://privacy.sexy', 1000 /* timeout in milliseconds */);
 console.log(`Status code: ${status.code}`); 
 ```
 
@@ -109,3 +100,10 @@ console.log(`Status code: ${status.code}`);
 - **`enableCookies`** (*boolean*), default: `true`
   - Enables cookie storage to facilitate seamless navigation through login or other authentication challenges.
   - 💡 Helps to over-come sign-in challenges with callbacks.
+- **`forceHttpGetForUrlPatterns`** (*array*), default: `[]`
+  - Specifies URL patterns that should always use an HTTP GET request instead of the default HTTP HEAD.
+  - This is useful for websites that do not respond to HEAD requests, such as those behind certain CDN or web application firewalls.
+  - Provide patterns as regular expressions (`RegExp`), allowing them to match any part of a URL.
+  - Examples:
+    - To match any URL starting with "https://example.com/api": `/^https:\/\/example\.com\/api/`
+    - To match any domain ending with "cloudflare.com": `/^https:\/\/.*\.cloudflare\.com\//`

tests/checks/external-urls/StatusChecker/Requestor.ts

@@ -1,70 +1,123 @@
+import { indentText } from '@tests/shared/Text';
 import { retryWithExponentialBackOff } from './ExponentialBackOffRetryHandler';
-import { fetchFollow, type IFollowOptions, DefaultFollowOptions } from './FetchFollow';
+import { fetchFollow, type FollowOptions } from './FetchFollow';
 import { getRandomUserAgent } from './UserAgents';
-import type { IUrlStatus } from './IUrlStatus';
+import { getDomainFromUrl } from './UrlDomainProcessing';
+import { randomizeTlsFingerprint, getTlsContextInfo } from './TlsFingerprintRandomizer';
+import type { UrlStatus } from './UrlStatus';
 
 export function getUrlStatus(
   url: string,
-  options: IRequestOptions = DefaultOptions,
-): Promise<IUrlStatus> {
-  const defaultedOptions = { ...DefaultOptions, ...options };
-  const fetchOptions = getFetchOptions(url, defaultedOptions);
-  return retryWithExponentialBackOff(async () => {
-    console.log('Requesting', url);
-    let result: IUrlStatus;
-    try {
-      const response = await fetchFollow(
-        url,
-        defaultedOptions.requestTimeoutInMs,
-        fetchOptions,
-        defaultedOptions.followOptions,
-      );
-      result = { url, code: response.status };
-    } catch (err) {
-      result = { url, error: JSON.stringify(err, null, '\t') };
-    }
-    return result;
-  }, defaultedOptions.retryExponentialBaseInMs);
+  requestOptions?: Partial<RequestOptions>,
+  followOptions?: Partial<FollowOptions>,
+): Promise<UrlStatus> {
+  const defaultedOptions = getDefaultedRequestOptions(requestOptions);
+  if (defaultedOptions.randomizeTlsFingerprint) {
+    randomizeTlsFingerprint();
+  }
+  return fetchUrlStatusWithRetry(url, defaultedOptions, followOptions);
 }
 
-export interface IRequestOptions {
+export interface RequestOptions {
   readonly retryExponentialBaseInMs?: number;
   readonly additionalHeaders?: Record<string, string>;
   readonly additionalHeadersUrlIgnore?: string[];
-  readonly followOptions?: IFollowOptions;
   readonly requestTimeoutInMs: number;
+  readonly randomizeTlsFingerprint: boolean;
+  readonly forceHttpGetForUrlPatterns: RegExp[];
 }
 
-const DefaultOptions: Required<IRequestOptions> = {
-  retryExponentialBaseInMs: 5000,
+const DefaultOptions: Required<RequestOptions> = {
+  retryExponentialBaseInMs: 5 /* sec */ * 1000,
   additionalHeaders: {},
   additionalHeadersUrlIgnore: [],
   requestTimeoutInMs: 60 /* seconds */ * 1000,
-  followOptions: DefaultFollowOptions,
+  randomizeTlsFingerprint: true,
+  forceHttpGetForUrlPatterns: [],
 };
 
-function getFetchOptions(url: string, options: Required<IRequestOptions>): RequestInit {
+function fetchUrlStatusWithRetry(
+  url: string,
+  requestOptions: Required<RequestOptions>,
+  followOptions?: Partial<FollowOptions>,
+): Promise<UrlStatus> {
+  const fetchOptions = getFetchOptions(url, requestOptions);
+  return retryWithExponentialBackOff(async () => {
+    console.log(`🚀 Initiating request for URL: ${url}`);
+    console.log(indentText([
+      `HTTP method: ${fetchOptions.method}`,
+      `Request options: ${JSON.stringify(requestOptions)}`,
+    ].join('\n')));
+    let result: UrlStatus;
+    try {
+      const response = await fetchFollow(
+        url,
+        requestOptions.requestTimeoutInMs,
+        fetchOptions,
+        followOptions,
+      );
+      result = { url, code: response.status };
+    } catch (err) {
+      result = {
+        url,
+        error: [
+          'Error:', indentText(JSON.stringify(err, null, '\t') || err.toString()),
+          'Fetch options:', indentText(JSON.stringify(fetchOptions, null, '\t')),
+          'Request options:', indentText(JSON.stringify(requestOptions, null, '\t')),
+          'TLS:', indentText(getTlsContextInfo()),
+        ].join('\n'),
+      };
+    }
+    return result;
+  }, requestOptions.retryExponentialBaseInMs);
+}
+
+function getFetchOptions(url: string, options: Required<RequestOptions>): RequestInit {
   const additionalHeaders = options.additionalHeadersUrlIgnore
     .some((ignorePattern) => url.startsWith(ignorePattern))
     ? {}
     : options.additionalHeaders;
   return {
-    method: 'HEAD',
+    method: getHttpMethod(url, options),
     headers: {
-      ...getDefaultHeaders(),
+      ...getDefaultHeaders(url),
       ...additionalHeaders,
     },
+    redirect: 'manual', // Redirects are handled manually, automatic redirects do not work with Host header
   };
 }
 
-function getDefaultHeaders(): Record<string, string> {
+function getHttpMethod(url: string, options: Required<RequestOptions>): 'HEAD' | 'GET' {
+  if (options.forceHttpGetForUrlPatterns.some((pattern) => url.match(pattern))) {
+    return 'GET';
+  }
+  // By default fetch only headers without the full response body for better speed
+  return 'HEAD';
+}
+
+function getDefaultHeaders(url: string): Record<string, string> {
   return {
-    'user-agent': getRandomUserAgent(),
-    'upgrade-insecure-requests': '1',
-    connection: 'keep-alive',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
-    'accept-encoding': 'gzip, deflate, br',
-    'cache-control': 'max-age=0',
-    'accept-language': 'en-US,en;q=0.9',
+    // Needed for websites that filter out non-browser user agents.
+    'User-Agent': getRandomUserAgent(),
+
+    // Required for some websites, especially those behind proxies, to correctly handle the request.
+    Host: getDomainFromUrl(url),
+
+    // The following mimic a real browser request to improve compatibility with most web servers.
+    'Upgrade-Insecure-Requests': '1',
+    Connection: 'keep-alive',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Cache-Control': 'max-age=0',
+    'Accept-Language': 'en-US,en;q=0.9',
+  };
+}
+
+function getDefaultedRequestOptions(
+  options?: Partial<RequestOptions>,
+): Required<RequestOptions> {
+  return {
+    ...DefaultOptions,
+    ...options,
   };
 }

tests/checks/external-urls/StatusChecker/TlsFingerprintRandomizer.ts

@@ -0,0 +1,69 @@
+/**
+ * Modifies the TLS fingerprint of Node.js HTTP client to circumvent TLS fingerprinting blocks.
+ * TLS fingerprinting is a technique used to identify clients based on the unencrypted data sent
+ * during the TLS handshake, used for blocking or identifying non-browser clients like debugging
+ * proxies or automated scripts.
+ *
+ * However, Node.js's HTTP client does not fully support all methods required for impersonating a
+ * browser's TLS fingerprint, as reported in https://github.com/nodejs/undici/issues/1983.
+ * While this implementation can alter the TLS fingerprint by randomizing the cipher suite order,
+ * it may not perfectly mimic specific browser fingerprints due to limitations in the TLS
+ * implementation of Node.js.
+ *
+ * For more detailed information, visit:
+ * - https://archive.today/2024.03.13-102042/https://httptoolkit.com/blog/tls-fingerprinting-node-js/
+ * - https://check.ja3.zone/ (To check your tool's or browser's fingerprint)
+ * - https://github.com/lwthiker/curl-impersonate (A solution for curl)
+ * - https://github.com/depicts/got-tls (Cipher manipulation support for Node.js)
+ */
+
+import { constants } from 'crypto';
+import tls from 'tls';
+import { indentText } from '@tests/shared/Text';
+
+export function randomizeTlsFingerprint() {
+  tls.DEFAULT_CIPHERS = getShuffledCiphers().join(':');
+  console.log(indentText(
+    `TLS context:\n${indentText([
+      'Original ciphers:', indentText(constants.defaultCipherList),
+      'Current ciphers:', indentText(getTlsContextInfo()),
+    ].join('\n'))}`,
+  ));
+}
+
+export function getTlsContextInfo(): string {
+  return [
+    `Ciphers: ${tls.DEFAULT_CIPHERS}`,
+    `Minimum TLS protocol version: ${tls.DEFAULT_MIN_VERSION}`,
+    `Node fingerprint: ${constants.defaultCoreCipherList === tls.DEFAULT_CIPHERS ? 'Visible' : 'Masked'}`,
+  ].join('\n');
+}
+
+/**
+ * Shuffles the order of TLS ciphers, excluding the top 3 most important ciphers to maintain
+ * security preferences. This approach modifies the default cipher list of Node.js to create a
+ * unique TLS fingerprint, thus helping to circumvent detection mechanisms based on static
+ * fingerprinting. It leverages randomness in the cipher order as a simple method to generate a
+ * new, unique TLS fingerprint which is not easily identifiable. The technique is based on altering
+ * parameters used in the TLS handshake process, particularly the cipher suite order, to avoid
+ * matching known fingerprints that could identify the client as a Node.js application.
+ *
+ * For more details, refer to:
+ * - https://archive.today/2024.03.13-102234/https://getsetfetch.org/blog/tls-fingerprint.html
+ */
+export function getShuffledCiphers(): readonly string[] {
+  const nodeOrderedCipherList = constants.defaultCoreCipherList.split(':');
+  const totalTopCiphersToKeep = 3;
+  // Keep the most important ciphers in the same order
+  const fixedCiphers = nodeOrderedCipherList.slice(0, totalTopCiphersToKeep);
+  // Shuffle the rest
+  const shuffledCiphers = nodeOrderedCipherList.slice(totalTopCiphersToKeep)
+    .map((cipher) => ({ cipher, sort: Math.random() }))
+    .sort((a, b) => a.sort - b.sort)
+    .map(({ cipher }) => cipher);
+  const ciphers = [
+    ...fixedCiphers,
+    ...shuffledCiphers,
+  ];
+  return ciphers;
+}

tests/checks/external-urls/StatusChecker/UrlDomainProcessing.ts

@@ -2,18 +2,18 @@ export function groupUrlsByDomain(urls: string[]): string[][] {
   const domains = new Set<string>();
   const urlsWithDomain = urls.map((url) => ({
     url,
-    domain: extractDomain(url),
+    domain: getDomainFromUrl(url),
   }));
   for (const url of urlsWithDomain) {
     domains.add(url.domain);
   }
   return Array.from(domains).map((domain) => {
     return urlsWithDomain
-      .filter((url) => url.domain === domain)
+      .filter((url) => url.domain.toLowerCase() === domain.toLowerCase())
       .map((url) => url.url);
   });
 }
 
-function extractDomain(url: string): string {
-  return url.split('://')[1].split('/')[0].toLowerCase();
+export function getDomainFromUrl(url: string): string {
+  return new URL(url).host;
 }

tests/checks/external-urls/StatusChecker/UrlStatus.ts

@@ -0,0 +1,19 @@
+import { indentText } from '@tests/shared/Text';
+
+export interface UrlStatus {
+  readonly url: string;
+  readonly error?: string;
+  readonly code?: number;
+}
+
+export function formatUrlStatus(status: UrlStatus): string {
+  return [
+    `URL: ${status.url}`,
+    ...status.code !== undefined ? [
+      `Response code: ${status.code}`,
+    ] : [],
+    ...status.error ? [
+      `Error:\n${indentText(status.error)}`,
+    ] : [],
+  ].join('\n');
+}

tests/checks/external-urls/StatusChecker/UserAgents.ts

@@ -3,73 +3,28 @@ export function getRandomUserAgent(): string {
 }
 
 const UserAgents = [
-  // Chrome
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537',
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537',
-
-  // Firefox
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15',
-
-  // Safari
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Safari/604.1',
-
-  // Internet Explorer
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko',
-
-  // Edge
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3 Edge/15.0',
-
-  // Opera
-  'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14',
-
-  // iOS Devices
-  'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/18.2b11866 Mobile/16B91 Safari/605.1.15',
-  'Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1',
-
-  // Android Devices
-  'Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.3',
-
-  // Other Devices/Browsers
-  'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.3',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Safari/605.1.15',
-  'Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.3 Edge/15.0',
-  'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
-  'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0',
-  'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.3',
-  'Mozilla/5.0 (Linux; Android 7.0; SM-G930F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.3',
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.3',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15',
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
-  'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.3 OPR/53.0.2907.99',
-  'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2)',
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20120121 Firefox/46.0',
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0)',
-  'Mozilla/5.0 (Windows NT 5.1; rv:36.0) Gecko/20100101 Firefox/36.0',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0',
-  'Mozilla/5.0 (X11; Linux i686; rv:30.0) Gecko/20100101 Firefox/30.0',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:28.0) Gecko/20100101 Firefox/28.0',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/20161202 Firefox/21.0.1',
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:20.0) Gecko/20100101 Firefox/20.0',
-  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0',
-  'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0',
-  'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0',
-  'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.3',
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
-  'Mozilla/5.0 (X11; CrOS x86_64 4319.74.0) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
-  'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.3 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.3',
+  // Safari 17.1 - macOS and iPad
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
+  // Safari - iOS 17 - iPhone
+  'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+  // Safari - iOS 17 - iPad mini
+  'Mozilla/5.0 (iPad; CPU OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+  // Edge - macOS
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.51',
+  // Edge - Windows
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58',
+  // Edge - Android
+  'Mozilla/5.0 (Linux; Android 10; HD1913) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.43 Mobile Safari/537.36 EdgA/119.0.2151.92',
+  // Chrome - macOS
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+  // Chrome - Windows
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+  // Chrome - Android (Phone)
+  'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36',
+  // Firefox - macOS
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0',
+  // Firefox - Windows
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0,',
+  // Firefox - Android (Phone)
+  'Mozilla/5.0 (Android 14; Mobile; rv:109.0) Gecko/120.0 Firefox/120.0',
 ];

tests/checks/external-urls/TestExecutionDetailsLogger.ts

@@ -0,0 +1,26 @@
+import { indentText } from '@tests/shared/Text';
+
+export class TestExecutionDetailsLogger {
+  public logTestSectionStartDelimiter(): void {
+    this.logSectionDelimiterLine();
+  }
+
+  public logTestSectionEndDelimiter(): void {
+    this.logSectionDelimiterLine();
+  }
+
+  public logLabeledInformation(
+    label: string,
+    detailedInformation: string,
+  ): void {
+    console.log([
+      `${label}:`,
+      indentText(detailedInformation),
+    ].join('\n'));
+  }
+
+  private logSectionDelimiterLine(): void {
+    const horizontalLine = '─'.repeat(40);
+    console.log(horizontalLine);
+  }
+}

tests/checks/external-urls/main.spec.ts

@@ -1,50 +1,101 @@
 import { test, expect } from 'vitest';
 import { parseApplication } from '@/application/Parser/ApplicationParser';
-import type { IApplication } from '@/domain/IApplication';
-import { getUrlStatusesInParallel, type IBatchRequestOptions } from './StatusChecker/BatchStatusChecker';
-import type { IUrlStatus } from './StatusChecker/IUrlStatus';
+import { indentText } from '@tests/shared/Text';
+import { formatAssertionMessage } from '@tests/shared/FormatAssertionMessage';
+import { shuffle } from '@/application/Common/Shuffle';
+import { type UrlStatus, formatUrlStatus } from './StatusChecker/UrlStatus';
+import { getUrlStatusesInParallel, type BatchRequestOptions } from './StatusChecker/BatchStatusChecker';
+import { TestExecutionDetailsLogger } from './TestExecutionDetailsLogger';
+import { extractDocumentationUrls } from './DocumentationUrlExtractor';
 
+// arrange
+const logger = new TestExecutionDetailsLogger();
+logger.logTestSectionStartDelimiter();
 const app = parseApplication();
-const urls = collectUniqueUrls(app);
-const requestOptions: IBatchRequestOptions = {
+let urls = extractDocumentationUrls({
+  logger,
+  urlExclusionPatterns: [
+    /^https:\/\/archive\.ph/, // Drops HEAD/GET requests via fetch/curl, responding to Postman/Chromium.
+  ],
+  application: app,
+});
+urls = filterUrlsToEnvironmentCheckLimit(urls);
+logger.logLabeledInformation('URLs submitted for testing', urls.length.toString());
+const requestOptions: BatchRequestOptions = {
   domainOptions: {
-    sameDomainParallelize: false, // be nice to our external servers
+    sameDomainParallelize: false, // be nice to our third-party servers
     sameDomainDelayInMs: 5 /* sec */ * 1000,
   },
   requestOptions: {
     retryExponentialBaseInMs: 3 /* sec */ * 1000,
     requestTimeoutInMs: 60 /* sec */ * 1000,
     additionalHeaders: { referer: app.projectDetails.homepage },
+    randomizeTlsFingerprint: true,
+  },
+  followOptions: {
+    followRedirects: true,
+    enableCookies: true,
   },
 };
+logger.logLabeledInformation('HTTP request options', JSON.stringify(requestOptions, null, 2));
 const testTimeoutInMs = urls.length * 60 /* seconds */ * 1000;
-
+logger.logLabeledInformation('Scheduled test duration', convertMillisecondsToHumanReadableFormat(testTimeoutInMs));
+logger.logTestSectionEndDelimiter();
 test(`all URLs (${urls.length}) should be alive`, async () => {
+  // act
+  console.log('URLS', urls); // TODO: Delete
   const results = await getUrlStatusesInParallel(urls, requestOptions);
-  const deadUrls = results.filter((r) => r.code !== 200);
-  expect(deadUrls).to.have.lengthOf(0, printUrls(deadUrls));
+  // assert
+  const deadUrls = results.filter((r) => r.code === undefined || !isOkStatusCode(r.code));
+  expect(deadUrls).to.have.lengthOf(
+    0,
+    formatAssertionMessage([createReportForDeadUrlStatuses(deadUrls)]),
+  );
 }, testTimeoutInMs);
 
-function collectUniqueUrls(application: IApplication): string[] {
-  return [ // Get all nodes
-    ...application.collections.flatMap((c) => c.getAllCategories()),
-    ...application.collections.flatMap((c) => c.getAllScripts()),
-  ]
-    // Get all docs
-    .flatMap((documentable) => documentable.docs)
-    // Parse all URLs
-    .flatMap((docString) => docString.match(/(https?:\/\/[^\s`"<>()]+)/g) || [])
-    // Remove duplicates
-    .filter((url, index, array) => array.indexOf(url) === index);
+function isOkStatusCode(statusCode: number): boolean {
+  return statusCode >= 200 && statusCode < 300;
 }
 
-function printUrls(statuses: IUrlStatus[]): string {
-  /* eslint-disable prefer-template */
-  return '\n'
-    + statuses.map((status) => `- ${status.url}\n`
-        + (status.code ? `\tResponse code: ${status.code}` : '')
-        + (status.error ? `\tError: ${status.error}` : ''))
-      .join('\n')
-    + '\n';
-  /* eslint-enable prefer-template */
+function createReportForDeadUrlStatuses(deadUrlStatuses: readonly UrlStatus[]): string {
+  return `\n${deadUrlStatuses.map((status) => indentText(formatUrlStatus(status))).join('\n---\n')}\n`;
+}
+
+function filterUrlsToEnvironmentCheckLimit(originalUrls: string[]): string[] {
+  const { RANDOMIZED_URL_CHECK_LIMIT } = process.env;
+  logger.logLabeledInformation('URL check limit', RANDOMIZED_URL_CHECK_LIMIT || 'Unlimited');
+  if (RANDOMIZED_URL_CHECK_LIMIT !== undefined && RANDOMIZED_URL_CHECK_LIMIT !== '') {
+    const maxUrlsInTest = parseInt(RANDOMIZED_URL_CHECK_LIMIT, 10);
+    if (Number.isNaN(maxUrlsInTest)) {
+      throw new Error(`Invalid URL limit: ${RANDOMIZED_URL_CHECK_LIMIT}`);
+    }
+    if (maxUrlsInTest < originalUrls.length) {
+      return shuffle(originalUrls).slice(0, maxUrlsInTest);
+    }
+  }
+  return originalUrls;
+}
+
+function convertMillisecondsToHumanReadableFormat(milliseconds: number): string {
+  const timeParts: string[] = [];
+  const addTimePart = (amount: number, label: string) => {
+    if (amount === 0) {
+      return;
+    }
+    timeParts.push(`${amount} ${label}`);
+  };
+
+  const hours = milliseconds / (1000 * 60 * 60);
+  const absoluteHours = Math.floor(hours);
+  addTimePart(absoluteHours, 'hours');
+
+  const minutes = (hours - absoluteHours) * 60;
+  const absoluteMinutes = Math.floor(minutes);
+  addTimePart(absoluteMinutes, 'minutes');
+
+  const seconds = (minutes - absoluteMinutes) * 60;
+  const absoluteSeconds = Math.floor(seconds);
+  addTimePart(absoluteSeconds, 'seconds');
+
+  return timeParts.join(', ');
 }

tests/unit/application/Common/Shuffle.spec.ts

@@ -0,0 +1,52 @@
+import { describe, it, expect } from 'vitest';
+import { shuffle } from '@/application/Common/Shuffle';
+
+describe('Shuffle', () => {
+  describe('shuffle', () => {
+    it('returns a new array', () => {
+      // arrange
+      const inputArray = ['a', 'b', 'c', 'd'];
+      // act
+      const result = shuffle(inputArray);
+      // assert
+      expect(result).not.to.equal(inputArray);
+    });
+
+    it('returns an array of the same length', () => {
+      // arrange
+      const inputArray = ['a', 'b', 'c', 'd'];
+      // act
+      const result = shuffle(inputArray);
+      // assert
+      expect(result.length).toBe(inputArray.length);
+    });
+
+    it('contains the same elements', () => {
+      // arrange
+      const inputArray = ['a', 'b', 'c', 'd'];
+      // act
+      const result = shuffle(inputArray);
+      // assert
+      expect(result).to.have.members(inputArray);
+    });
+
+    it('does not modify the input array', () => {
+      // arrange
+      const inputArray = ['a', 'b', 'c', 'd'];
+      const inputArrayCopy = [...inputArray];
+      // act
+      shuffle(inputArray);
+      // assert
+      expect(inputArray).to.deep.equal(inputArrayCopy);
+    });
+
+    it('handles an empty array correctly', () => {
+      // arrange
+      const inputArray: string[] = [];
+      // act
+      const result = shuffle(inputArray);
+      // assert
+      expect(result).have.lengthOf(0);
+    });
+  });
+});

tests/unit/application/Parser/Script/Compiler/Function/Call/Argument/FunctionCallArgumentCollection.spec.ts

@@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest';
 import { FunctionCallArgumentCollection } from '@/application/Parser/Script/Compiler/Function/Call/Argument/FunctionCallArgumentCollection';
 import { FunctionCallArgumentStub } from '@tests/unit/shared/Stubs/FunctionCallArgumentStub';
 import { itEachAbsentStringValue } from '@tests/unit/shared/TestCases/AbsentTests';
+import type { IFunctionCallArgument } from '@/application/Parser/Script/Compiler/Function/Call/Argument/IFunctionCallArgument';
 
 describe('FunctionCallArgumentCollection', () => {
   describe('addArgument', () => {
@@ -20,21 +21,25 @@ describe('FunctionCallArgumentCollection', () => {
     });
   });
   describe('getAllParameterNames', () => {
-    it('returns as expected', () => {
+    describe('returns as expected', () => {
       // arrange
-      const testCases = [{
-        name: 'no args',
+      const testCases: ReadonlyArray<{
+        readonly description: string;
+        readonly args: readonly IFunctionCallArgument[];
+        readonly expectedParameterNames: string[];
+      }> = [{
+        description: 'no args',
         args: [],
-        expected: [],
+        expectedParameterNames: [],
       }, {
-        name: 'with some args',
+        description: 'with some args',
         args: [
           new FunctionCallArgumentStub().withParameterName('a-param-name'),
           new FunctionCallArgumentStub().withParameterName('b-param-name')],
-        expected: ['a-param-name', 'b-param-name'],
+        expectedParameterNames: ['a-param-name', 'b-param-name'],
       }];
       for (const testCase of testCases) {
-        it(testCase.name, () => {
+        it(testCase.description, () => {
           const sut = new FunctionCallArgumentCollection();
           // act
           for (const arg of testCase.args) {
@@ -42,7 +47,7 @@ describe('FunctionCallArgumentCollection', () => {
           }
           const actual = sut.getAllParameterNames();
           // assert
-          expect(actual).to.equal(testCase.expected);
+          expect(actual).to.deep.equal(testCase.expectedParameterNames);
         });
       }
     });

tests/unit/application/Parser/Script/Compiler/Function/SharedFunction.spec.ts

@@ -25,7 +25,7 @@ describe('SharedFunction', () => {
           // assert
           expect(sut.name).equal(expected);
         });
-        it('throws when absent', () => {
+        describe('throws when absent', () => {
           itEachAbsentStringValue((absentValue) => {
             // arrange
             const expectedError = 'missing function name';

tests/unit/presentation/bootstrapping/DependencyProvider.spec.ts

@@ -4,6 +4,7 @@ import { InjectionKeys } from '@/presentation/injectionSymbols';
 import { provideDependencies, type VueDependencyInjectionApi } from '@/presentation/bootstrapping/DependencyProvider';
 import { ApplicationContextStub } from '@tests/unit/shared/Stubs/ApplicationContextStub';
 import { itIsSingleton } from '@tests/unit/shared/TestCases/SingletonTests';
+import type { IApplicationContext } from '@/application/Context/IApplicationContext';
 
 describe('DependencyProvider', () => {
   describe('provideDependencies', () => {
@@ -74,25 +75,25 @@ function createSingletonTests() {
       const registeredObject = api.inject(injectionKey);
       expect(registeredObject).to.be.instanceOf(Object);
     });
-    it('should return the same instance for singleton dependency', () => {
+    describe('should return the same instance for singleton dependency', () => {
+      // arrange
+      const singletonContext = new ApplicationContextStub();
+      const api = new VueDependencyInjectionApiStub();
+      new ProvideDependenciesBuilder()
+        .withContext(singletonContext)
+        .withApi(api)
+        .provideDependencies();
+      // act
+      const getRegisteredInstance = () => api.inject(injectionKey);
+      // assert
       itIsSingleton({
-        getter: () => {
-          // arrange
-          const api = new VueDependencyInjectionApiStub();
-          // act
-          new ProvideDependenciesBuilder()
-            .withApi(api)
-            .provideDependencies();
-          // expect
-          const registeredObject = api.inject(injectionKey);
-          return registeredObject;
-        },
+        getter: getRegisteredInstance,
       });
     });
   };
 }
 class ProvideDependenciesBuilder {
-  private context = new ApplicationContextStub();
+  private context: IApplicationContext = new ApplicationContextStub();
 
   private api: VueDependencyInjectionApi = new VueDependencyInjectionApiStub();
 
@@ -101,6 +102,11 @@ class ProvideDependenciesBuilder {
     return this;
   }
 
+  public withContext(context: IApplicationContext): this {
+    this.context = context;
+    return this;
+  }
+
   public provideDependencies() {
     return provideDependencies(this.context, this.api);
   }

tests/unit/presentation/components/Scripts/View/Tree/TreeView/TreeRoot/NodeCollection/TreeNodeInitializerAndUpdater.spec.ts

@@ -10,7 +10,7 @@ import type { TreeInputNodeData } from '@/presentation/components/Scripts/View/T
 
 describe('TreeNodeInitializerAndUpdater', () => {
   describe('updateRootNodes', () => {
-    it('should throw an error if no data is provided', () => {
+    describe('should throw an error if no data is provided', () => {
       itEachAbsentCollectionValue<TreeInputNodeData>((absentValue) => {
         // arrange
         const expectedError = 'missing data';

tests/unit/presentation/electron/main/IpcRegistration.spec.ts

@@ -13,7 +13,7 @@ import { ScriptDiagnosticsCollectorStub } from '../../../shared/Stubs/ScriptDiag
 
 describe('IpcRegistration', () => {
   describe('registerAllIpcChannels', () => {
-    it('registers all defined IPC channels', () => {
+    describe('registers all defined IPC channels', () => {
       Object.entries(IpcChannelDefinitions).forEach(([key, expectedChannel]) => {
         it(key, () => {
           // arrange