67 lines
3.0 KiB
Python
67 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Run the test suites and report.
|
|
|
|
python3 test/run_all.py every headless suite (test/t_*.js), in parallel
|
|
python3 test/run_all.py pipes stone only suites whose name contains one of these words
|
|
python3 test/run_all.py --browser also browser tests (play.py, t_smooth.py, t_dev_console.py),
|
|
after rebuilding dist/; they run one at a time
|
|
python3 test/run_all.py --browser-only
|
|
|
|
Exit status 0 only if everything passed. Failing checks are printed with their suite.
|
|
The headless suites are deterministic: the same code gives the same output, line for line, so
|
|
saving the output before and after a refactor and diffing it proves nothing changed.
|
|
"""
|
|
import concurrent.futures as cf, pathlib, subprocess, sys, time
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
TEST = ROOT / 'test'
|
|
|
|
|
|
def run_node(path):
|
|
t0 = time.time()
|
|
p = subprocess.run(['node', str(path)], cwd=ROOT, capture_output=True, text=True, timeout=1800)
|
|
return path.stem, p.returncode, p.stdout + p.stderr, time.time() - t0
|
|
|
|
|
|
def run_py(name):
|
|
t0 = time.time()
|
|
p = subprocess.run([sys.executable, str(TEST / name)], cwd=ROOT, capture_output=True, text=True, timeout=1800)
|
|
return name, p.returncode, p.stdout + p.stderr, time.time() - t0
|
|
|
|
|
|
def main():
|
|
sys.path.insert(0, str(ROOT / 'tools'))
|
|
from browser import consume_gpu_option
|
|
consume_gpu_option()
|
|
args = [a for a in sys.argv[1:] if not a.startswith('--')]
|
|
browser = '--browser' in sys.argv or '--browser-only' in sys.argv
|
|
node = '--browser-only' not in sys.argv
|
|
results = []
|
|
if node:
|
|
suites = sorted(TEST.glob('t_*.js'))
|
|
if args: suites = [s for s in suites if any(a in s.stem for a in args)]
|
|
with cf.ThreadPoolExecutor(max_workers=8) as ex:
|
|
for r in ex.map(run_node, suites): results.append(r)
|
|
if browser:
|
|
subprocess.run([sys.executable, str(ROOT / 'build.py')], cwd=ROOT, check=True)
|
|
# one at a time: two headless browsers at once can starve each other and flake
|
|
for name in ['play.py', 't_smooth.py', 't_dev_console.py', 't_caps.py', 'firefox_smoke.py']:
|
|
results.append(run_py(name))
|
|
bad = 0
|
|
for name, code, out, secs in results:
|
|
fails = [l for l in out.splitlines() if l.startswith('FAIL') or 'Error' in l or l.startswith('FAILS: [') and l != 'FAILS: []']
|
|
passes = sum(1 for l in out.splitlines() if l.startswith('PASS'))
|
|
ok = code == 0 and not [l for l in fails if l.startswith('FAIL ') or 'Error' in l]
|
|
bad += not ok
|
|
print(f"{'ok ' if ok else 'FAIL'} {name:22s} {passes:3d} checks passed {secs:5.1f}s")
|
|
for line in out.splitlines():
|
|
if line.startswith('SKIP '): print(' ' + line)
|
|
if not ok:
|
|
for l in (fails or out.splitlines()[-15:]): print(' ' + l)
|
|
print('ALL PASSED' if not bad else f'{bad} SUITE(S) FAILED')
|
|
sys.exit(1 if bad else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|