219 lines
12 KiB
Python
219 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Profile a seeded world on the real GPU, with isolated HTTP serving. Times are milliseconds."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from browser import launch, gpu_option, GPU_ARGS, SOFT_ARGS
|
|
from use_venv import ensure_playwright
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def size(value):
|
|
try:
|
|
w, h = map(int, value.lower().split('x'))
|
|
if w < 320 or h < 240 or w > 8192 or h > 8192:
|
|
raise ValueError()
|
|
return w, h
|
|
except ValueError:
|
|
raise argparse.ArgumentTypeError('size must be WxH, between 320x240 and 8192x8192')
|
|
|
|
|
|
def quantiles(values):
|
|
if not values:
|
|
raise ValueError('No real-loop samples were collected')
|
|
ordered = sorted(values)
|
|
def percentile(p):
|
|
i = (len(ordered) - 1) * p
|
|
lo = math.floor(i)
|
|
return ordered[lo] + (ordered[math.ceil(i)] - ordered[lo]) * (i - lo)
|
|
return {name: percentile(p) for name, p in [('p50', .5), ('p95', .95), ('p99', .99)]}
|
|
|
|
|
|
def measurements(data, prefix=''):
|
|
"""Only timings, never counts, frame timestamps or hardware identifiers."""
|
|
for key, value in data.items():
|
|
path = f'{prefix}/{key}'
|
|
if isinstance(value, dict):
|
|
yield from measurements(value, path)
|
|
elif isinstance(value, (int, float)) and (
|
|
key.endswith('_ms') or key.startswith('ms_per_') or key in ('p50', 'p95', 'p99')
|
|
or prefix.endswith('/marks_ms')):
|
|
yield path, value
|
|
|
|
|
|
def report(result, baseline=None):
|
|
m = result['machine']
|
|
print(f"GPU: {m['renderer']}\nCores: {m['cores']} | isolated: {m['crossOriginIsolated']} | navigator.gpu: {m['webgpu']}")
|
|
print(f"Viewport {result['options']['viewport']} | world {result['options']['world']} | seed 424242 | ms (CPU wall time)")
|
|
print('\nScene Look Zoom Light Grains World Frame Light Prep Colour Upload Blit Overlays Scale')
|
|
for scene, data in result['scenes'].items():
|
|
for key, view in data['views'].items():
|
|
look, zoom = key.split('/')
|
|
for light, r in view['lighting'].items():
|
|
p = r['marks_ms']
|
|
print(f"{scene:8} {look:6} {zoom:>4} {light:>5} {view['grains']:6d} "
|
|
f"{r['drawWorld_ms']:7.2f} {r.get('frame_ms', 0):6.2f} {p['light']:6.2f} {p['prepare']:5.2f} {p['color']:6.2f} "
|
|
f"{p['upload']:6.2f} {p['blit']:6.2f} {r['overlay_ms']:8.3f} {r['scale']:5.2f}")
|
|
print('\nSimulation: average ms/tick (periodic work amortized over 240 ticks)')
|
|
names = list(next(iter(result['scenes'].values()))['sim']['steps'])
|
|
print(f"{'Step':24}" + ''.join(f'{s:>12}' for s in result['scenes']))
|
|
for name in ['TOTAL', *names, 'unattributed']:
|
|
values = []
|
|
for d in result['scenes'].values():
|
|
sim = d['sim']
|
|
values.append(sim['total_ms'] if name == 'TOTAL' else sim['rest_ms'] if name == 'unattributed' else sim['steps'][name]['ms_per_tick'])
|
|
print(f'{name:24}' + ''.join(f'{v:12.4f}' for v in values))
|
|
for scene, d in result['scenes'].items():
|
|
s = d['sim']
|
|
print(f" {scene}: awake {s['start']['awake']} -> {s['end']['awake']}, hot {s['start']['hot']} -> {s['end']['hot']} / {s['end']['chunks']}")
|
|
print('\nReal loop: frame interval / CPU frame work / sim per frame, p50/p95/p99 ms')
|
|
for scene, d in result['scenes'].items():
|
|
for key, r in d['runs'].items():
|
|
triples = ['/'.join(f'{v:.2f}' for v in r[k].values()) for k in ('interval_ms', 'work_ms', 'sim_ms')]
|
|
print(f"{scene:8} {key:9} n={len(r['samples']):4} " + ' | '.join(f'{v:>20}' for v in triples) + f" | scale {r['scale']:.2f}")
|
|
if baseline:
|
|
old = dict(measurements(baseline['scenes']))
|
|
print('\nComparison: change in every matching timing (>10% marked REGRESSION; zero baselines are n/a)')
|
|
for key, value in measurements(result['scenes']):
|
|
before = old.get(key)
|
|
if before is None:
|
|
print(f'{key}: new measurement')
|
|
elif before == 0:
|
|
print(f'{key}: n/a (baseline zero)')
|
|
else:
|
|
delta = 100 * (value / before - 1)
|
|
print(f"{key}: {delta:+.1f}%{' REGRESSION' if delta > 10 else ''}")
|
|
print(f"\nFull run: {result['elapsed_seconds']:.1f} seconds")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument('--viewport', type=size, default=(1920, 1080))
|
|
ap.add_argument('--world', type=size, default=(2048, 1024))
|
|
ap.add_argument('--zooms', default='12,6,4,3,2')
|
|
ap.add_argument('--looks', default='pixel,smooth')
|
|
ap.add_argument('--scenes', default='settled,factory,heat')
|
|
ap.add_argument('--seconds', type=float, default=10)
|
|
ap.add_argument('--json', type=Path)
|
|
ap.add_argument('--compare', type=Path)
|
|
gpu_option(ap)
|
|
args = ap.parse_args()
|
|
try:
|
|
zooms = list(dict.fromkeys(float(z) for z in args.zooms.split(',')))
|
|
except ValueError:
|
|
ap.error('--zooms must be comma-separated numbers')
|
|
looks, scenes = args.looks.split(','), args.scenes.split(',')
|
|
if not zooms or any(not math.isfinite(z) or z < 2 or z > 12 for z in zooms):
|
|
ap.error('zooms must be between 2 and 12')
|
|
if not looks or set(looks) - {'pixel', 'smooth'}:
|
|
ap.error('looks must be pixel,smooth')
|
|
if not scenes or set(scenes) - {'settled', 'factory', 'heat'}:
|
|
ap.error('scenes must be settled,factory,heat')
|
|
if not math.isfinite(args.seconds) or args.seconds <= 0:
|
|
ap.error('--seconds must be positive')
|
|
if args.world[0] % 32 or args.world[1] % 32:
|
|
ap.error('world dimensions must be multiples of 32')
|
|
options = dict(viewport=list(args.viewport), world=list(args.world), zooms=zooms,
|
|
looks=looks, scenes=scenes, seconds=args.seconds)
|
|
baseline = json.loads(args.compare.read_text()) if args.compare else None
|
|
if baseline and (baseline.get('schema') != 1
|
|
or any(baseline['options'].get(k) != v for k, v in options.items() if k not in ('looks', 'scenes', 'zooms'))
|
|
or any(not set(options[k]).issubset(baseline['options'][k]) for k in ('looks', 'scenes', 'zooms'))):
|
|
ap.error('baseline schema/options differ; use a subset of the baseline matrix with the same dimensions and seconds')
|
|
ensure_playwright()
|
|
started = time.monotonic()
|
|
subprocess.run([sys.executable, str(ROOT / 'build.py')], cwd=ROOT, check=True, stdout=sys.stderr)
|
|
# Derive coverage from the source so a newly added sim step cannot silently disappear.
|
|
tick_source = (ROOT / 'src/sim/tick.js').read_text()
|
|
steps = list(dict.fromkeys(re.findall(r'\b([A-Za-z_]\w*)\(', tick_source)))
|
|
steps = [n for n in steps if n not in ('simTick', 'if', 'for')]
|
|
overlays = list(dict.fromkeys(re.findall(r'\b(draw[A-Z]\w*)\(', (ROOT / 'src/ui/loop.js').read_text())))
|
|
overlays.remove('drawWorld')
|
|
# boot's title draw is outside frame().
|
|
overlays.remove('drawTitle')
|
|
result = dict(schema=1, created_utc=datetime.now(timezone.utc).isoformat(), options=options,
|
|
gpu_args=GPU_ARGS if args.gpu == 'real' else SOFT_ARGS, scenes={}, sim_targets=steps, overlay_targets=overlays)
|
|
files = sorted((ROOT / 'src').rglob('*')) + [Path(__file__), ROOT / 'tools/browser.py', ROOT / 'tools/server.py',
|
|
ROOT / 'tools/perf-scenes.js', ROOT / 'tools/perf-measure.js']
|
|
result['source_sha256'] = hashlib.sha256(b''.join(p.read_bytes() for p in files if p.is_file())).hexdigest()
|
|
errors = []
|
|
with launch(gpu=args.gpu, viewport=dict(width=args.viewport[0], height=args.viewport[1])) as (browser, pg, url):
|
|
pg.on('pageerror', lambda e: errors.append(str(e)))
|
|
pg.add_init_script('''window.__perfStop = true;
|
|
const raf = window.requestAnimationFrame.bind(window), pending = new Set();
|
|
window.__perfHalt = () => { window.__perfStop = true; for (const id of pending) cancelAnimationFrame(id); pending.clear(); };
|
|
window.requestAnimationFrame = f => {
|
|
if (window.__perfStop) return 0;
|
|
const id = raf(t => { pending.delete(id); f(t); }); pending.add(id); return id;
|
|
};''')
|
|
pg.goto(url)
|
|
pg.evaluate('CAPS_CHECK')
|
|
if not pg.evaluate('GPU.ok'):
|
|
raise RuntimeError('WebGPU unavailable: ' + pg.evaluate('GPU.error'))
|
|
pg.add_script_tag(content=(ROOT / 'tools/perf-scenes.js').read_text())
|
|
pg.add_script_tag(content=(ROOT / 'tools/perf-measure.js').read_text())
|
|
result['machine'] = pg.evaluate('''() => {
|
|
return { renderer: 'WebGPU ' + CAPS.adapterInfo.vendor + ' ' + CAPS.adapterInfo.architecture,
|
|
vendor: CAPS.adapterInfo.vendor,
|
|
cores: navigator.hardwareConcurrency, crossOriginIsolated, webgpu: !!navigator.gpu,
|
|
adapter: CAPS.adapterInfo, timestamp_query: !!GPU.queries,
|
|
user_agent: navigator.userAgent, dpr: devicePixelRatio };
|
|
}''')
|
|
if not result['machine']['crossOriginIsolated']:
|
|
raise RuntimeError('Local server did not provide cross-origin isolation')
|
|
if args.gpu == 'real' and re.search('swiftshader|llvmpipe|softpipe|software', result['machine']['renderer'], re.I):
|
|
raise RuntimeError('Refusing a software-rendered baseline')
|
|
result['machine']['browser_version'] = browser.version
|
|
def setup(scene):
|
|
return pg.evaluate('([s,w,h]) => perfScene(s,w,h)', [scene, *args.world])
|
|
for scene in scenes:
|
|
print(f'Measuring {scene}...', file=sys.stderr, flush=True)
|
|
setup(scene)
|
|
sim = pg.evaluate('([names,s,w,h]) => perfSim(names,s,w,h)', [steps, scene, *args.world])
|
|
initial = setup(scene)
|
|
data = result['scenes'][scene] = dict(initial=initial, sim=sim, views={}, runs={})
|
|
for look in looks:
|
|
for zoom in zooms:
|
|
view = pg.evaluate('([l,z]) => perfView(l,z)', [look, zoom])
|
|
view['lighting'] = {}
|
|
for light in ['full', 'off']:
|
|
view['lighting'][light] = pg.evaluate('([l,o]) => perfRender(l,o)', [light, overlays])
|
|
if view['lighting']['full']['actual_look'] != look or view['lighting']['off']['actual_look'] != look:
|
|
raise RuntimeError('Look fell back while measuring')
|
|
if any(r['renderer'] != 'webgpu' for r in view['lighting'].values()):
|
|
raise RuntimeError('WebGPU fell back while measuring: ' + pg.evaluate('GPU.error'))
|
|
data['views'][f'{look}/{zoom:g}'] = view
|
|
for look in looks:
|
|
for zoom in [2, 6]:
|
|
setup(scene)
|
|
pg.evaluate('([l,z]) => perfView(l,z)', [look, zoom])
|
|
r = pg.evaluate('seconds => perfRun(seconds)', args.seconds)
|
|
if r['actual_look'] != look:
|
|
raise RuntimeError('Look fell back during real run')
|
|
if r['renderer'] != 'webgpu':
|
|
raise RuntimeError('WebGPU fell back during real run: ' + pg.evaluate('GPU.error'))
|
|
for key in ['interval', 'work', 'sim']:
|
|
r[key + '_ms'] = quantiles([s[key] for s in r['samples']])
|
|
data['runs'][f'{look}/{zoom}'] = r
|
|
print(f' {look}/{zoom}: {len(r["samples"])} frames', file=sys.stderr, flush=True)
|
|
browser.close()
|
|
if errors:
|
|
raise RuntimeError('Page errors: ' + '; '.join(errors))
|
|
result['elapsed_seconds'] = time.monotonic() - started
|
|
if args.json:
|
|
args.json.write_text(json.dumps(result, indent=2) + '\n')
|
|
report(result, baseline)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|