Add automated checks for desktop app runtime #233

- Add automation script for building, packaging, installing, executing
  and verifying Electron distrubtions across macOS, Ubuntu and Windows.
- Add GitHub workflow to run the script to test distributions using the
  script.
- Update README with new workflow status badge.
- Add application initialization log to desktop applications to be able
  to test against crashes before application initialization.

scripts/check-desktop-runtime-errors/app/app-logs.js

@@ -0,0 +1,55 @@
+import { unlink, readFile } from 'fs/promises';
+import { join } from 'path';
+import { log, die, LOG_LEVELS } from '../utils/log.js';
+import { exists } from '../utils/io.js';
+import { SUPPORTED_PLATFORMS, CURRENT_PLATFORM } from '../utils/platform.js';
+import { getAppName } from '../utils/npm.js';
+
+export async function clearAppLogFile(projectDir) {
+  if (!projectDir) { throw new Error('missing project directory'); }
+  const logPath = await determineLogPath(projectDir);
+  if (!logPath || !await exists(logPath)) {
+    log(`Skipping clearing logs, log file does not exist: ${logPath}.`);
+    return;
+  }
+  try {
+    await unlink(logPath);
+    log(`Successfully cleared the log file at: ${logPath}.`);
+  } catch (error) {
+    die(`Failed to clear the log file at: ${logPath}. Reason: ${error}`);
+  }
+}
+
+export async function readAppLogFile(projectDir) {
+  if (!projectDir) { throw new Error('missing project directory'); }
+  const logPath = await determineLogPath(projectDir);
+  if (!logPath || !await exists(logPath)) {
+    log(`No log file at: ${logPath}`, LOG_LEVELS.WARN);
+    return undefined;
+  }
+  const logContent = await readLogFile(logPath);
+  return logContent;
+}
+
+async function determineLogPath(projectDir) {
+  if (!projectDir) { throw new Error('missing project directory'); }
+  const appName = await getAppName(projectDir);
+  if (!appName) {
+    die('App name not found.');
+  }
+  const logFilePaths = {
+    [SUPPORTED_PLATFORMS.MAC]: () => join(process.env.HOME, 'Library', 'Logs', appName, 'main.log'),
+    [SUPPORTED_PLATFORMS.LINUX]: () => join(process.env.HOME, '.config', appName, 'logs', 'main.log'),
+    [SUPPORTED_PLATFORMS.WINDOWS]: () => join(process.env.USERPROFILE, 'AppData', 'Roaming', appName, 'logs', 'main.log'),
+  };
+  const logFilePath = logFilePaths[CURRENT_PLATFORM]?.();
+  if (!logFilePath) {
+    log(`Cannot determine log path, unsupported OS: ${CURRENT_PLATFORM}`, LOG_LEVELS.WARN);
+  }
+  return logFilePath;
+}
+
+async function readLogFile(logFilePath) {
+  const content = await readFile(logFilePath, 'utf-8');
+  return content?.trim().length > 0 ? content : undefined;
+}

scripts/check-desktop-runtime-errors/app/check-for-errors.js

@@ -0,0 +1,114 @@
+import { splitTextIntoLines, indentText } from '../utils/text.js';
+import { die } from '../utils/log.js';
+import { readAppLogFile } from './app-logs.js';
+
+const LOG_ERROR_MARKER = '[error]'; // from electron-log
+const ELECTRON_CRASH_TITLE = 'Error'; // Used by electron for early crashes
+const APP_INITIALIZED_MARKER = '[APP_INIT_SUCCESS]'; // Logged by application on successful initialization
+
+export async function checkForErrors(stderr, windowTitles, projectDir) {
+  if (!projectDir) { throw new Error('missing project directory'); }
+  const errors = await gatherErrors(stderr, windowTitles, projectDir);
+  if (errors.length) {
+    die(formatErrors(errors));
+  }
+}
+
+async function gatherErrors(stderr, windowTitles, projectDir) {
+  if (!projectDir) { throw new Error('missing project directory'); }
+  const logContent = await readAppLogFile(projectDir);
+  return [
+    verifyStdErr(stderr),
+    verifyApplicationInitializationLog(logContent),
+    verifyWindowTitle(windowTitles),
+    verifyErrorsInLogs(logContent),
+  ].filter(Boolean);
+}
+
+function formatErrors(errors) {
+  if (!errors || !errors.length) { throw new Error('missing errors'); }
+  return [
+    'Errors detected during execution:',
+    ...errors.map(
+      (error) => formatError(error),
+    ),
+  ].join('\n---\n');
+}
+
+function formatError(error) {
+  if (!error) { throw new Error('missing error'); }
+  if (!error.reason) { throw new Error(`missing reason, error (${typeof error}): ${JSON.stringify(error)}`); }
+  let message = `Reason: ${indentText(error.reason, 1)}`;
+  if (error.description) {
+    message += `\nDescription:\n${indentText(error.description, 2)}`;
+  }
+  return message;
+}
+
+function verifyApplicationInitializationLog(logContent) {
+  if (!logContent || !logContent.length) {
+    return describeError(
+      'Missing application logs',
+      'Application logs are empty not were not found.',
+    );
+  }
+  if (!logContent.includes(APP_INITIALIZED_MARKER)) {
+    return describeError(
+      'Unexpected application logs',
+      `Missing identifier "${APP_INITIALIZED_MARKER}" in application logs.`,
+    );
+  }
+  return undefined;
+}
+
+function verifyWindowTitle(windowTitles) {
+  const errorTitles = windowTitles.filter(
+    (title) => title.toLowerCase().includes(ELECTRON_CRASH_TITLE),
+  );
+  if (errorTitles.length) {
+    return describeError(
+      'Unexpected window title',
+      'One or more window titles suggest an error occurred in the application:'
+        + `\nError Titles: ${errorTitles.join(', ')}`
+        + `\nAll Titles: ${windowTitles.join(', ')}`,
+    );
+  }
+  return undefined;
+}
+
+function verifyStdErr(stderrOutput) {
+  if (stderrOutput && stderrOutput.length > 0) {
+    return describeError(
+      'Standard error stream (`stderr`) is not empty.',
+      stderrOutput,
+    );
+  }
+  return undefined;
+}
+
+function verifyErrorsInLogs(logContent) {
+  if (!logContent || !logContent.length) {
+    return undefined;
+  }
+  const logLines = getNonEmptyLines(logContent)
+    .filter((line) => line.includes(LOG_ERROR_MARKER));
+  if (!logLines.length) {
+    return undefined;
+  }
+  return describeError(
+    'Application log file',
+    logLines.join('\n'),
+  );
+}
+
+function describeError(reason, description) {
+  return {
+    reason,
+    description: `${description}\nThis might indicate an early crash or significant runtime issue.`,
+  };
+}
+
+function getNonEmptyLines(text) {
+  return splitTextIntoLines(text)
+    .filter((line) => line?.trim().length > 0);
+}

scripts/check-desktop-runtime-errors/app/extractors/linux.js

@@ -0,0 +1,34 @@
+import { access, chmod } from 'fs/promises';
+import { constants } from 'fs';
+import { findSingleFileByExtension } from '../../utils/io.js';
+import { log } from '../../utils/log.js';
+
+export async function prepareLinuxApp(desktopDistPath) {
+  const { absolutePath: appFile } = await findSingleFileByExtension(
+    'AppImage',
+    desktopDistPath,
+  );
+  await makeExecutable(appFile);
+  return {
+    appExecutablePath: appFile,
+  };
+}
+
+async function makeExecutable(appFile) {
+  if (!appFile) { throw new Error('missing file'); }
+  if (await isExecutable(appFile)) {
+    log('AppImage is already executable.');
+    return;
+  }
+  log('Making it executable...');
+  await chmod(appFile, 0o755);
+}
+
+async function isExecutable(file) {
+  try {
+    await access(file, constants.X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}

scripts/check-desktop-runtime-errors/app/extractors/macos.js

@@ -0,0 +1,66 @@
+import { runCommand } from '../../utils/run-command.js';
+import { findSingleFileByExtension, exists } from '../../utils/io.js';
+import { log, die, LOG_LEVELS } from '../../utils/log.js';
+
+export async function prepareMacOsApp(desktopDistPath) {
+  const { absolutePath: dmgPath } = await findSingleFileByExtension('dmg', desktopDistPath);
+  const { mountPath } = await mountDmg(dmgPath);
+  const appPath = await findMacAppExecutablePath(mountPath);
+  return {
+    appExecutablePath: appPath,
+    cleanup: async () => {
+      log('Cleaning up resources...');
+      await detachMount(mountPath);
+    },
+  };
+}
+
+async function mountDmg(dmgFile) {
+  const { stdout: hdiutilOutput, error } = await runCommand(`hdiutil attach '${dmgFile}'`);
+  if (error) {
+    die(`Failed to mount DMG file at ${dmgFile}.\n${error}`);
+  }
+  const mountPathMatch = hdiutilOutput.match(/\/Volumes\/[^\n]+/);
+  const mountPath = mountPathMatch ? mountPathMatch[0] : null;
+  return {
+    mountPath,
+  };
+}
+
+async function findMacAppExecutablePath(mountPath) {
+  const { stdout: findOutput, error } = await runCommand(
+    `find '${mountPath}' -maxdepth 1 -type d -name "*.app"`,
+  );
+  if (error) {
+    die(`Failed to find executable path at mount path ${mountPath}\n${error}`);
+  }
+  const appFolder = findOutput.trim();
+  const appName = appFolder.split('/').pop().replace('.app', '');
+  const appPath = `${appFolder}/Contents/MacOS/${appName}`;
+  if (await exists(appPath)) {
+    log(`Application is located at ${appPath}`);
+  } else {
+    die(`Application does not exist at ${appPath}`);
+  }
+  return appPath;
+}
+
+async function detachMount(mountPath, retries = 5) {
+  const { error } = await runCommand(`hdiutil detach '${mountPath}'`);
+  if (error) {
+    if (retries <= 0) {
+      log(`Failed to detach mount after multiple attempts: ${mountPath}\n${error}`, LOG_LEVELS.WARN);
+      return;
+    }
+    await sleep(500);
+    await detachMount(mountPath, retries - 1);
+    return;
+  }
+  log(`Successfully detached from ${mountPath}`);
+}
+
+function sleep(milliseconds) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, milliseconds);
+  });
+}

scripts/check-desktop-runtime-errors/app/extractors/windows.js

@@ -0,0 +1,38 @@
+import { mkdtemp, rmdir } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { findSingleFileByExtension, exists } from '../../utils/io.js';
+import { log, die } from '../../utils/log.js';
+import { runCommand } from '../../utils/run-command.js';
+
+export async function prepareWindowsApp(desktopDistPath) {
+  const workdir = await mkdtemp(join(tmpdir(), 'win-nsis-installation-'));
+  if (await exists(workdir)) {
+    log(`Temporary directory ${workdir} already exists, cleaning up...`);
+    await rmdir(workdir, { recursive: true });
+  }
+  const { appExecutablePath } = await installNsis(workdir, desktopDistPath);
+  return {
+    appExecutablePath,
+    cleanup: async () => {
+      log(`Cleaning up working directory ${workdir}...`);
+      await rmdir(workdir, { recursive: true });
+    },
+  };
+}
+
+async function installNsis(installationPath, desktopDistPath) {
+  const { absolutePath: installerPath } = await findSingleFileByExtension('exe', desktopDistPath);
+
+  log(`Silently installing contents of ${installerPath} to ${installationPath}...`);
+  const { error } = await runCommand(`"${installerPath}" /S /D=${installationPath}`);
+  if (error) {
+    die(`Failed to install.\n${error}`);
+  }
+
+  const { absolutePath: appExecutablePath } = await findSingleFileByExtension('exe', installationPath);
+
+  return {
+    appExecutablePath,
+  };
+}

scripts/check-desktop-runtime-errors/app/runner.js

@@ -0,0 +1,164 @@
+import { spawn } from 'child_process';
+import { log, LOG_LEVELS, die } from '../utils/log.js';
+import { captureScreen } from './system-capture/screen-capture.js';
+import { captureWindowTitles } from './system-capture/window-title-capture.js';
+
+const TERMINATION_GRACE_PERIOD_IN_SECONDS = 60;
+const TERMINATION_CHECK_INTERVAL_IN_MS = 1000;
+const WINDOW_TITLE_CAPTURE_INTERVAL_IN_MS = 100;
+
+export function runApplication(
+  appFile,
+  executionDurationInSeconds,
+  enableScreenshot,
+  screenshotPath,
+) {
+  if (!appFile) {
+    throw new Error('Missing app file');
+  }
+
+  logDetails(appFile, executionDurationInSeconds);
+
+  const processDetails = {
+    stderrData: '',
+    stdoutData: '',
+    explicitlyKilled: false,
+    windowTitles: [],
+    isCrashed: false,
+    isDone: false,
+    process: undefined,
+    resolve: () => { /* NOOP */ },
+  };
+
+  const process = spawn(appFile);
+  processDetails.process = process;
+
+  return new Promise((resolve) => {
+    processDetails.resolve = resolve;
+    handleTitleCapture(process.pid, processDetails);
+    handleProcessEvents(
+      processDetails,
+      enableScreenshot,
+      screenshotPath,
+      executionDurationInSeconds,
+    );
+  });
+}
+
+function logDetails(appFile, executionDurationInSeconds) {
+  log(
+    [
+      'Executing the app to check for errors...',
+      `Maximum execution time: ${executionDurationInSeconds}`,
+      `Application path: ${appFile}`,
+    ].join('\n\t'),
+  );
+}
+
+function handleTitleCapture(processId, processDetails) {
+  const capture = async () => {
+    const titles = await captureWindowTitles(processId);
+
+    (titles || []).forEach((title) => {
+      if (!title || !title.length) {
+        return;
+      }
+      if (!processDetails.windowTitles.includes(title)) {
+        log(`New window title captured: ${title}`);
+        processDetails.windowTitles.push(title);
+      }
+    });
+
+    if (!processDetails.isDone) {
+      setTimeout(capture, WINDOW_TITLE_CAPTURE_INTERVAL_IN_MS);
+    }
+  };
+
+  capture();
+}
+
+function handleProcessEvents(
+  processDetails,
+  enableScreenshot,
+  screenshotPath,
+  executionDurationInSeconds,
+) {
+  const { process } = processDetails;
+  process.stderr.on('data', (data) => {
+    processDetails.stderrData += data.toString();
+  });
+  process.stdout.on('data', (data) => {
+    processDetails.stdoutData += data.toString();
+  });
+
+  process.on('error', (error) => {
+    die(`An issue spawning the child process: ${error}`, LOG_LEVELS.ERROR);
+  });
+
+  process.on('exit', async (code) => {
+    await onProcessExit(code, processDetails, enableScreenshot, screenshotPath);
+  });
+
+  setTimeout(async () => {
+    await onExecutionLimitReached(process, processDetails, enableScreenshot, screenshotPath);
+  }, executionDurationInSeconds * 1000);
+}
+
+async function onProcessExit(code, processDetails, enableScreenshot, screenshotPath) {
+  log(`Application exited ${code === null || Number.isNaN(code) ? '.' : `with code ${code}`}`);
+
+  if (processDetails.explicitlyKilled) return;
+
+  processDetails.isCrashed = true;
+
+  if (enableScreenshot) {
+    await captureScreen(screenshotPath);
+  }
+
+  finishProcess(processDetails);
+}
+
+async function onExecutionLimitReached(process, processDetails, enableScreenshot, screenshotPath) {
+  if (enableScreenshot) {
+    await captureScreen(screenshotPath);
+  }
+
+  processDetails.explicitlyKilled = true;
+  await terminateGracefully(process);
+  finishProcess(processDetails);
+}
+
+function finishProcess(processDetails) {
+  processDetails.isDone = true;
+  processDetails.resolve({
+    stderr: processDetails.stderrData,
+    stdout: processDetails.stdoutData,
+    windowTitles: [...processDetails.windowTitles],
+    isCrashed: processDetails.isCrashed,
+  });
+}
+
+async function terminateGracefully(process) {
+  let elapsedSeconds = 0;
+  log('Attempting to terminate the process gracefully...');
+  process.kill('SIGTERM');
+
+  return new Promise((resolve) => {
+    const checkInterval = setInterval(() => {
+      elapsedSeconds += TERMINATION_CHECK_INTERVAL_IN_MS / 1000;
+
+      if (!process.killed) {
+        if (elapsedSeconds >= TERMINATION_GRACE_PERIOD_IN_SECONDS) {
+          process.kill('SIGKILL');
+          log('Process did not terminate gracefully within the grace period. Forcing termination.', LOG_LEVELS.WARN);
+          clearInterval(checkInterval);
+          resolve();
+        }
+      } else {
+        log('Process terminated gracefully.');
+        clearInterval(checkInterval);
+        resolve();
+      }
+    }, TERMINATION_CHECK_INTERVAL_IN_MS);
+  });
+}

scripts/check-desktop-runtime-errors/app/system-capture/screen-capture.js

@@ -0,0 +1,59 @@
+import { unlink } from 'fs/promises';
+import { runCommand } from '../../utils/run-command.js';
+import { log, LOG_LEVELS } from '../../utils/log.js';
+import { CURRENT_PLATFORM, SUPPORTED_PLATFORMS } from '../../utils/platform.js';
+import { exists } from '../../utils/io.js';
+
+export async function captureScreen(imagePath) {
+  if (!imagePath) {
+    throw new Error('Path for screenshot not provided');
+  }
+
+  if (await exists(imagePath)) {
+    log(`Screenshot file already exists at ${imagePath}. It will be overwritten.`, LOG_LEVELS.WARN);
+    unlink(imagePath);
+  }
+
+  const platformCommands = {
+    [SUPPORTED_PLATFORMS.MAC]: `screencapture -x ${imagePath}`,
+    [SUPPORTED_PLATFORMS.LINUX]: `import -window root ${imagePath}`,
+    [SUPPORTED_PLATFORMS.WINDOWS]: `powershell -NoProfile -EncodedCommand ${encodeForPowershell(getScreenshotPowershellScript(imagePath))}`,
+  };
+
+  const commandForPlatform = platformCommands[CURRENT_PLATFORM];
+
+  if (!commandForPlatform) {
+    log(`Screenshot capture not supported on: ${CURRENT_PLATFORM}`, LOG_LEVELS.WARN);
+    return;
+  }
+
+  log(`Capturing screenshot to ${imagePath} using command:\n\t> ${commandForPlatform}`);
+
+  const { error } = await runCommand(commandForPlatform);
+  if (error) {
+    log(`Failed to capture screenshot.\n${error}`, LOG_LEVELS.WARN);
+    return;
+  }
+  log(`Captured screenshot to ${imagePath}.`);
+}
+
+function getScreenshotPowershellScript(imagePath) {
+  return `
+    $ProgressPreference = 'SilentlyContinue' # Do not pollute stderr
+    Add-Type -AssemblyName System.Windows.Forms
+    $screenBounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+
+    $bmp = New-Object System.Drawing.Bitmap $screenBounds.Width, $screenBounds.Height
+    $graphics = [System.Drawing.Graphics]::FromImage($bmp)
+    $graphics.CopyFromScreen([System.Drawing.Point]::Empty, [System.Drawing.Point]::Empty, $screenBounds.Size)
+
+    $bmp.Save('${imagePath}')
+    $graphics.Dispose()
+    $bmp.Dispose()
+  `;
+}
+
+function encodeForPowershell(script) {
+  const buffer = Buffer.from(script, 'utf-16le');
+  return buffer.toString('base64');
+}

scripts/check-desktop-runtime-errors/app/system-capture/window-title-capture.js

@@ -0,0 +1,113 @@
+import { runCommand } from '../../utils/run-command.js';
+import { log, LOG_LEVELS } from '../../utils/log.js';
+import { SUPPORTED_PLATFORMS, CURRENT_PLATFORM } from '../../utils/platform.js';
+
+export async function captureWindowTitles(processId) {
+  if (!processId) { throw new Error('Missing process ID.'); }
+
+  const captureFunction = windowTitleCaptureFunctions[CURRENT_PLATFORM];
+  if (!captureFunction) {
+    log(`Cannot capture window title, unsupported OS: ${CURRENT_PLATFORM}`, LOG_LEVELS.WARN);
+    return undefined;
+  }
+
+  return captureFunction(processId);
+}
+
+const windowTitleCaptureFunctions = {
+  [SUPPORTED_PLATFORMS.MAC]: captureTitlesOnMac,
+  [SUPPORTED_PLATFORMS.LINUX]: captureTitlesOnLinux,
+  [SUPPORTED_PLATFORMS.WINDOWS]: captureTitlesOnWindows,
+};
+
+async function captureTitlesOnWindows(processId) {
+  if (!processId) { throw new Error('Missing process ID.'); }
+
+  const { stdout: tasklistOutput, error } = await runCommand(
+    `tasklist /FI "PID eq ${processId}" /fo list /v`,
+  );
+  if (error) {
+    log(`Failed to retrieve window title.\n${error}`, LOG_LEVELS.WARN);
+    return [];
+  }
+  const match = tasklistOutput.match(/Window Title:\s*(.*)/);
+  if (match && match[1]) {
+    const title = match[1].trim();
+    if (title === 'N/A') {
+      return [];
+    }
+    return [title];
+  }
+  return [];
+}
+
+async function captureTitlesOnLinux(processId) {
+  if (!processId) { throw new Error('Missing process ID.'); }
+
+  const { stdout: windowIdsOutput, error: windowIdError } = await runCommand(
+    `xdotool search --pid '${processId}'`,
+  );
+
+  if (windowIdError || !windowIdsOutput) {
+    return undefined;
+  }
+
+  const windowIds = windowIdsOutput.trim().split('\n');
+
+  const titles = await Promise.all(windowIds.map(async (windowId) => {
+    const { stdout: titleOutput, error: titleError } = await runCommand(
+      `xprop -id ${windowId} | grep "WM_NAME(STRING)" | cut -d '=' -f 2 | sed 's/^[[:space:]]*"\\(.*\\)"[[:space:]]*$/\\1/'`
+    );
+    if (titleError || !titleOutput) {
+      return undefined;
+    }
+    return titleOutput.trim();
+  }));
+
+  return titles.filter(Boolean);
+}
+
+let hasAssistiveAccessOnMac = true;
+
+async function captureTitlesOnMac(processId) {
+  if (!processId) { throw new Error('Missing process ID.'); }
+  if (!hasAssistiveAccessOnMac) {
+    return [];
+  }
+  const script = `
+      tell application "System Events"
+      try
+        set targetProcess to first process whose unix id is ${processId}
+      on error
+        return
+      end try
+      tell targetProcess
+        if (count of windows) > 0 then
+          set window_name to name of front window
+          return window_name
+        end if
+      end tell
+    end tell
+  `;
+  const argument = script.trim()
+    .split(/[\r\n]+/)
+    .map((line) => `-e '${line.trim()}'`)
+    .join(' ');
+
+  const { stdout: titleOutput, error } = await runCommand(`osascript ${argument}`);
+  if (error) {
+    let errorMessage = '';
+    if (error.includes('-25211')) {
+      errorMessage += 'Capturing window title requires assistive access. You do not have it.\n';
+      hasAssistiveAccessOnMac = false;
+    }
+    errorMessage += error;
+    log(errorMessage, LOG_LEVELS.WARN);
+    return [];
+  }
+  const title = titleOutput?.trim();
+  if (!title) {
+    return [];
+  }
+  return [title];
+}