From 56e0c351bb95cfdf63aa76ab8c5d725a694bad63 Mon Sep 17 00:00:00 2001 From: cirakg Date: Wed, 21 Oct 2026 03:55:10 +0200 Subject: test(build): add make watchdog, the glance memory budget and the glance screenshot - make watchdog (in make ci): a stress build encodes, packs, turns into runs and draws the largest code of every 2D symbology at two steps per event on the Forerunner 255S and 70; it fails on a watchdog trip, and the last code left on the screen must decode (bitmap path on the 70). - make memtest checks the glance (DIAG glance, budget 70 % of 64 KB). - screenshots.sh launches the app again after the flow and compares the glance with the favourite's code (glance_code.png on 7 watches); the About goldens show 0.8.0. Co-Authored-By: Claude Opus 5.5 --- tools/budgets.json | 5 +-- tools/check_budgets.py | 8 ++++- tools/tests/test_check_budgets.py | 2 ++ tools/tests/test_ui_flow.py | 12 ++++++- tools/ui_flow.py | 74 ++++++++++++++++++++++++++++++++++----- 5 files changed, 88 insertions(+), 13 deletions(-) (limited to 'tools') diff --git a/tools/budgets.json b/tools/budgets.json index ba7432d..5e330c9 100644 --- a/tools/budgets.json +++ b/tools/budgets.json @@ -1,12 +1,13 @@ { "_author": "cirakg & claude ai", "_summary": "Simulator budgets checked by `make perf` and `make memtest` (README section 18).", - "_about": "Simulator timings depend on your PC. startup_ms (in-app, object creation to first full frame) measured 1-18 ms in M4; its budget leaves room for simulator jitter. first_frame_wall_ms includes the JVM start of monkeydo, loading the glance, the 2 s pause and the tap that opens the app (16-18 s in M4), so it only catches big regressions; set to about 2x. Memory is a hard limit: at most 70 % of the watch-app memory of the device.", + "_about": "Simulator timings depend on your PC. startup_ms (in-app, object creation to first full frame) measured 1-18 ms in M4; its budget leaves room for simulator jitter. first_frame_wall_ms includes the JVM start of monkeydo, loading the glance, the 2 s pause and the tap that opens the app (16-18 s in M4), so it only catches big regressions; set to about 2x. Memory is a hard limit: at most 70 % of the watch-app memory of the device, and 70 % of the glance's 64 KB.", "perf": { "startup_ms": 100, "first_frame_wall_ms": 36000 }, "mem": { - "mem_used_fraction": 0.7 + "mem_used_fraction": 0.7, + "glance_mem_used_fraction": 0.7 } } diff --git a/tools/check_budgets.py b/tools/check_budgets.py index 8b55a2b..8219a6d 100644 --- a/tools/check_budgets.py +++ b/tools/check_budgets.py @@ -7,6 +7,8 @@ Two kinds of lines are read from build/diag/.log: DIAG first_frame startup_ms=1 mem_used=6984 mem_total=781888 printed by the debug build (source/util/Diag.mc) after its first frame; + DIAG glance mem_used=12552 mem_total=65536 + printed by the glance after its first draw (the simulator starts there); HARNESS first_frame wall_ms=2310 added by app-diag.sh: wall-clock time from launching the app with monkeydo until that DIAG line appeared, which includes loading the @@ -52,7 +54,11 @@ def measure(kind: str, events: dict[str, dict[str, float]]) -> dict[str, float]: values["first_frame_wall_ms"] = harness["wall_ms"] return values total = frame["mem_total"] - return {"mem_used_fraction": frame["mem_used"] / total if total else 1.0} + values = {"mem_used_fraction": frame["mem_used"] / total if total else 1.0} + glance = events.get("DIAG.glance") + if glance is not None: + values["glance_mem_used_fraction"] = glance["mem_used"] / glance["mem_total"] if glance["mem_total"] else 1.0 + return values def main(argv: list[str] | None = None) -> int: diff --git a/tools/tests/test_check_budgets.py b/tools/tests/test_check_budgets.py index 2262a0b..28ba710 100644 --- a/tools/tests/test_check_budgets.py +++ b/tools/tests/test_check_budgets.py @@ -29,6 +29,8 @@ def test_measure_perf_and_memory() -> None: assert cb.measure("perf", events) == {"startup_ms": 12.0, "first_frame_wall_ms": 2310.0} assert cb.measure("mem", events) == {"mem_used_fraction": 100000 / 524288} assert cb.measure("perf", {}) == {} + with_glance = cb.read_events(f"DIAG glance mem_used=16384 mem_total=65536\n{FRAME}") + assert cb.measure("mem", with_glance) == {"mem_used_fraction": 100000 / 524288, "glance_mem_used_fraction": 0.25} def run(tmp_path: Path, kind: str, log_text: str, budgets: Mapping[str, Mapping[str, float]]) -> int: diff --git a/tools/tests/test_ui_flow.py b/tools/tests/test_ui_flow.py index 7e0d336..d98b4c2 100644 --- a/tools/tests/test_ui_flow.py +++ b/tools/tests/test_ui_flow.py @@ -1,5 +1,5 @@ # Author: cirakg & claude ai -# Summary: Tests for the UI flow driver's pure parts: device key layout, JSON lookup and the app-log reader. +# Summary: Tests for the UI flow driver's pure parts: key layout, JSON lookup, the app-log reader, barcode reading. from __future__ import annotations import json @@ -8,7 +8,9 @@ import time from pathlib import Path import pytest +import segno import ui_flow +from PIL import Image def write_sim(tmp_path: Path, config: dict[str, object]) -> Path: @@ -71,3 +73,11 @@ def test_app_log_latest_returns_the_last_match(tmp_path: Path) -> None: match = ui_flow.AppLog(log_path).latest(r"DIAG list_focus y=(\d+)") assert match is not None and match.group(1) == "42" assert ui_flow.AppLog(tmp_path / "missing.log").latest("x") is None + + +def test_barcodes_on_reads_a_drawn_code(tmp_path: Path) -> None: + path = tmp_path / "qr.png" + segno.make("OTPW stress", error="m").save(str(path), scale=4, border=4) + screen = Image.open(path).convert("RGB") + assert ui_flow.barcodes_on(screen) == ["OTPW stress"] + assert ui_flow.barcodes_on(Image.new("RGB", (100, 100), "white")) == [] diff --git a/tools/ui_flow.py b/tools/ui_flow.py index edfc9a4..a17d435 100644 --- a/tools/ui_flow.py +++ b/tools/ui_flow.py @@ -28,8 +28,11 @@ waiting for the PIN and confirmed from the menu (watches with buttons), the list, a TOTP entry, a HOTP entry and its next code, the PIN screen, a wrong PIN, a correct PIN opening a locked entry, auto-lock closing it, the menu, the About screen (watches with buttons; touch-only -menus have no focus to move), the lockout after 5 wrong PINs, and an EAN-13 -code whose screenshot must decode with zxing-cpp (README 12.4). +menus have no focus to move), the lockout after 5 wrong PINs, the barcodes +whose screenshots must decode with zxing-cpp (README 12.4). The glance with +the favourite's code is a separate launch (--shot glance_code, from +screenshots.sh): the simulator doesn't return to the glance when the app +exits. Each screenshot is compared with test/golden//.png like sim_screenshot.py does (or saved there with --approve). @@ -50,6 +53,7 @@ from pathlib import Path import sim_screenshot as shots import zxingcpp +from PIL import Image EXIT_OK, EXIT_DIFFERENT, EXIT_FLOW = 0, 1, 2 SCREEN_TIMEOUT_S = 20.0 @@ -90,6 +94,11 @@ class Key: hold: bool +def barcodes_on(screen: Image.Image) -> list[str]: + """The texts of every barcode zxing-cpp finds on a screenshot.""" + return [r.text for r in zxingcpp.read_barcodes(screen)] + + def load_keys(device_dir: Path) -> dict[str, Key]: """Button centres in device-drawing coordinates, by button id (enter, up, down, esc, menu).""" sim = json.loads((device_dir / "simulator.json").read_text(encoding="utf-8")) @@ -326,7 +335,7 @@ class Flow: return if expect_code is not None: # README 12.4 check 3: the code on the screenshot must decode. - read = [r.text for r in zxingcpp.read_barcodes(screen)] + read = barcodes_on(screen) if expect_code not in read: print(f"[{label}] the barcode didn't decode to {expect_code!r}: read {read}") self.results[name] = EXIT_DIFFERENT @@ -569,6 +578,50 @@ def open_from_glance(window: str, device_dir: Path, raw_dir: Path) -> None: sim.press("enter") +def decode_screen(window: str, device_dir: Path, out: Path, text: str) -> int: + """Screenshot the simulator; EXIT_OK when a barcode on it reads `text`.""" + name = device_dir.name + try: + sim = Simulator(window, device_dir, out / "raw") + screen = shots.extract_screen(sim.capture("decode"), device_dir, out / "decode.png") + except (FlowError, shots.ScreenshotError, subprocess.CalledProcessError, OSError) as err: + print(f"[{name}] no screenshot to decode: {err}") + return EXIT_FLOW + read = barcodes_on(screen) + if text not in read: + print(f"[{name}] the barcode on the screen didn't decode to {text!r}: read {read}") + return EXIT_DIFFERENT + print(f"[{name}] the barcode on the screen decodes to {text!r}") + return EXIT_OK + + +def open_app(window: str, device_dir: Path, out: Path) -> int: + """--open-from-glance: EXIT_OK once the input was sent.""" + try: + open_from_glance(window, device_dir, out / "raw") + except (FlowError, subprocess.CalledProcessError, OSError) as err: + print(f"[{device_dir.name}] could not open the app from its glance: {err}") + return EXIT_FLOW + return EXIT_OK + + +def single_shot(args: argparse.Namespace) -> int: + """Screenshot the simulator as args.shot and compare it with its golden image (or approve it).""" + window, device_dir, out, golden, name = args.window, args.device_dir, args.out, args.golden, args.shot + label = f"{device_dir.name}/{name}" + try: + sim = Simulator(window, device_dir, out / "raw") + screen = shots.extract_screen(sim.capture(name), device_dir, out / f"{name}.png") + except (FlowError, shots.ScreenshotError, subprocess.CalledProcessError, OSError) as err: + print(f"[{label}] {err}") + return EXIT_FLOW + if args.approve: + shots.save_png(screen, golden / f"{name}.png", f"Approved simulator screen '{name}' of {device_dir.name}") + print(f"[{label}] approved") + return EXIT_OK + return shots.check_golden(screen, label, out / f"{name}.png", golden / f"{name}.png") + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Scripted UI flow with screenshot checks.") parser.add_argument("--device-dir", type=Path, required=True) @@ -578,15 +631,18 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--golden", type=Path, help="directory of the approved images") parser.add_argument("--approve", action="store_true") parser.add_argument("--open-from-glance", action="store_true", help="only open the app from its glance") + parser.add_argument("--decode", metavar="TEXT", help="only check that the screen shows a barcode reading TEXT") + parser.add_argument("--shot", metavar="NAME", help="only screenshot the current screen as NAME (needs --golden)") args = parser.parse_args(argv) name = args.device_dir.name + if args.decode is not None: + return decode_screen(args.window, args.device_dir, args.out, args.decode) + if args.shot is not None: + if args.golden is None: + parser.error("--shot needs --golden") + return single_shot(args) if args.open_from_glance: - try: - open_from_glance(args.window, args.device_dir, args.out / "raw") - except (FlowError, subprocess.CalledProcessError, OSError) as err: - print(f"[{name}] could not open the app from its glance: {err}") - return EXIT_FLOW - return EXIT_OK + return open_app(args.window, args.device_dir, args.out) if args.log is None or args.golden is None: parser.error("--log and --golden are required for the flow") try: -- cgit v1.2.3