diff options
Diffstat (limited to 'tools/check_budgets.py')
| -rw-r--r-- | tools/check_budgets.py | 65 |
1 files changed, 38 insertions, 27 deletions
diff --git a/tools/check_budgets.py b/tools/check_budgets.py index 1e26037..8b55a2b 100644 --- a/tools/check_budgets.py +++ b/tools/check_budgets.py @@ -1,10 +1,16 @@ #!/usr/bin/env python3 +# Author: cirakg & claude ai +# Summary: Compare the app's debug diagnostics (start-up time, memory) with the budgets in tools/budgets.json. """Compare the app's debug diagnostics with the budgets in tools/budgets.json. -Debug builds print lines like - DIAG {"event":"first_frame","startup_ms":123,"mem_used":4567,"mem_total":786432} -(source/util/Diag.mc). app-diag.sh runs the app on each test device, saves the -console output to build/diag/<device>.log, then calls this script. +Two kinds of lines are read from build/diag/<device>.log: + + DIAG first_frame startup_ms=1 mem_used=6984 mem_total=781888 + printed by the debug build (source/util/Diag.mc) after its first frame; + HARNESS first_frame wall_ms=2310 + added by app-diag.sh: wall-clock time from launching the app with + monkeydo until that DIAG line appeared, which includes loading the + program (the in-app timer can't see that part). Usage: check_budgets.py {perf,mem} LOG [LOG ...] [--budgets FILE] [--summary OUT.json] Exit: 0 within budget, 1 over budget, 2 missing or unreadable diagnostics. @@ -14,38 +20,39 @@ from __future__ import annotations import argparse import json +import re import sys from pathlib import Path -from typing import Any EXIT_OK, EXIT_OVER, EXIT_MISSING = 0, 1, 2 +EVENT_RE = re.compile(r"\b(DIAG|HARNESS)\s+(\w+)((?:\s+\w+=-?\d+(?:\.\d+)?)+)") +PAIR_RE = re.compile(r"(\w+)=(-?\d+(?:\.\d+)?)") -def read_events(text: str) -> list[dict[str, Any]]: - """Every DIAG event in a console log, in order. Malformed lines are skipped.""" - events = [] - for line in text.splitlines(): - start = line.find("DIAG {") - if start < 0: - continue - try: - event = json.loads(line[start + len("DIAG ") :]) - except json.JSONDecodeError: - continue - if isinstance(event, dict): - events.append(event) + +def read_events(text: str) -> dict[str, dict[str, float]]: + """The first occurrence of each event, keyed as "DIAG.first_frame" etc.""" + events: dict[str, dict[str, float]] = {} + for match in EVENT_RE.finditer(text): + key = f"{match.group(1)}.{match.group(2)}" + if key not in events: + events[key] = {name: float(value) for name, value in PAIR_RE.findall(match.group(3))} return events -def measure(kind: str, events: list[dict[str, Any]]) -> dict[str, float]: - """The values a budget kind looks at, from the first first_frame event.""" - frame = next((e for e in events if e.get("event") == "first_frame"), None) +def measure(kind: str, events: dict[str, dict[str, float]]) -> dict[str, float]: + """The values one budget kind looks at; empty when the app never drew.""" + frame = events.get("DIAG.first_frame") if frame is None: return {} if kind == "perf": - return {"startup_ms": float(frame["startup_ms"])} - total = float(frame["mem_total"]) - return {"mem_used_fraction": float(frame["mem_used"]) / total if total else 1.0} + values = {"startup_ms": frame["startup_ms"]} + harness = events.get("HARNESS.first_frame") + if harness is not None: + values["first_frame_wall_ms"] = harness["wall_ms"] + return values + total = frame["mem_total"] + return {"mem_used_fraction": frame["mem_used"] / total if total else 1.0} def main(argv: list[str] | None = None) -> int: @@ -63,19 +70,23 @@ def main(argv: list[str] | None = None) -> int: device = log.stem try: values = measure(args.kind, read_events(log.read_text(encoding="utf-8", errors="replace"))) - except (OSError, KeyError, ValueError) as err: + except (OSError, KeyError) as err: print(f"[{device}] cannot read diagnostics: {err}") worst = max(worst, EXIT_MISSING) continue if not values: - print(f"[{device}] no DIAG first_frame line (did the app start? is this a debug build?)") + print(f"[{device}] no 'DIAG first_frame' line (did the app start? is this a debug build?)") worst = max(worst, EXIT_MISSING) continue summary[device] = values for name, limit in budgets.items(): + if name not in values: + print(f"[{device}] {name}: not measured") + worst = max(worst, EXIT_MISSING) + continue value = values[name] over = value > limit - print(f"[{device}] {name} = {value:.3g} (budget {limit:.3g}) {'OVER BUDGET' if over else 'ok'}") + print(f"[{device}] {name} = {value:.4g} (budget {limit:.4g}) {'OVER BUDGET' if over else 'ok'}") if over: worst = max(worst, EXIT_OVER) if args.summary: |