Refactor and improve external URL checks

- Move external URL checks to its own module under `tests/`. This
  separates them from integration test, addressing long runs and
  frequent failures that led to ignoring test results.
- Move `check-desktop-runtime-errors` to `tests/checks` to keep all
  test-related checks into one directory.
- Replace `ts-node` with `vite` for running
  `check-desktop-runtime-errors` to maintain a consistent execution
  environment across checks.
- Implement a timeout for each fetch call.
- Be nice to external sources, wait 5 seconds before sending another
  request to an URL under same domain. This solves rate-limiting issues.
- Instead of running test on every push/pull request, run them only
  weekly.
- Do not run tests on each commit/PR but only scheduled (weekly) to
  minimize noise.
- Fix URLs are not captured correctly inside backticks or parenthesis.

tests/checks/external-urls/StatusChecker/Requestor.ts

@@ -0,0 +1,69 @@
+import { retryWithExponentialBackOff } from './ExponentialBackOffRetryHandler';
+import { IUrlStatus } from './IUrlStatus';
+import { fetchFollow, IFollowOptions } from './FetchFollow';
+import { getRandomUserAgent } from './UserAgents';
+
+export function getUrlStatus(
+  url: string,
+  options: IRequestOptions = DefaultOptions,
+): Promise<IUrlStatus> {
+  options = { ...DefaultOptions, ...options };
+  const fetchOptions = getFetchOptions(url, options);
+  return retryWithExponentialBackOff(async () => {
+    console.log('Requesting', url);
+    let result: IUrlStatus;
+    try {
+      const response = await fetchFollow(
+        url,
+        options.requestTimeoutInMs,
+        fetchOptions,
+        options.followOptions,
+      );
+      result = { url, code: response.status };
+    } catch (err) {
+      result = { url, error: JSON.stringify(err, null, '\t') };
+    }
+    return result;
+  }, options.retryExponentialBaseInMs);
+}
+
+export interface IRequestOptions {
+  retryExponentialBaseInMs?: number;
+  additionalHeaders?: Record<string, string>;
+  additionalHeadersUrlIgnore?: string[];
+  followOptions?: IFollowOptions;
+  requestTimeoutInMs: number;
+}
+
+const DefaultOptions: IRequestOptions = {
+  retryExponentialBaseInMs: 5000,
+  additionalHeaders: {},
+  additionalHeadersUrlIgnore: [],
+  requestTimeoutInMs: 60 /* seconds */ * 1000,
+};
+
+function getFetchOptions(url: string, options: IRequestOptions): RequestInit {
+  const additionalHeaders = options.additionalHeadersUrlIgnore
+    .some((ignorePattern) => url.startsWith(ignorePattern))
+    ? {}
+    : options.additionalHeaders;
+  return {
+    method: 'HEAD',
+    headers: {
+      ...getDefaultHeaders(),
+      ...additionalHeaders,
+    },
+  };
+}
+
+function getDefaultHeaders(): Record<string, string> {
+  return {
+    'user-agent': getRandomUserAgent(),
+    'upgrade-insecure-requests': '1',
+    connection: 'keep-alive',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+    'accept-encoding': 'gzip, deflate, br',
+    'cache-control': 'max-age=0',
+    'accept-language': 'en-US,en;q=0.9',
+  };
+}