From 2292bf700d28ade39cf2d64b45509e6db21034bc Mon Sep 17 00:00:00 2001 From: cirakg Date: Sun, 18 Oct 2026 21:10:37 +0200 Subject: test(tools): add screenshot, budget and product checks; enforce file headers - sim_screenshot.py finds the device drawing in the simulator window, crops the screen and compares it with approved golden images. - check_budgets.py reads key=value DIAG/HARNESS lines (the simulator console escapes quotes, so JSON was dropped). - ciq_test_report.py accepts SDK 9.2.0's real summary line `PASSED (passed=N, failed=N, errors=N)`; a captured log is a fixture. - gen_products.py checks all manifest products are watches with API 5.0+ and at least 512 KB for apps. - lint_project.py rule HDR001: every file carries the Author/Summary header (JSON keys, PNG text chunks; generated package-lock exempt). - make_icon.py --all draws one icon per launcher size in use. - Golden screenshots for fr255s, venusq2, fr70 and fr965. - 44 tests, warnings as errors, ruff and mypy --strict clean. Co-Authored-By: Claude Opus 5.5 --- tools/tests/conftest.py | 2 + tools/tests/data/runnoevil_pass.log | 14 ++++++ tools/tests/test_check_budgets.py | 36 +++++++++----- tools/tests/test_check_docs.py | 2 + tools/tests/test_ciq_test_report.py | 43 +++++++++++------ tools/tests/test_gen_products.py | 63 +++++++++++++++++++++++++ tools/tests/test_lint_project.py | 43 ++++++++++++++++- tools/tests/test_make_icon.py | 2 + tools/tests/test_sim_screenshot.py | 94 +++++++++++++++++++++++++++++++++++++ 9 files changed, 273 insertions(+), 26 deletions(-) create mode 100644 tools/tests/data/runnoevil_pass.log create mode 100644 tools/tests/test_gen_products.py create mode 100644 tools/tests/test_sim_screenshot.py (limited to 'tools/tests') diff --git a/tools/tests/conftest.py b/tools/tests/conftest.py index 7054ecb..d4dae36 100644 --- a/tools/tests/conftest.py +++ b/tools/tests/conftest.py @@ -1,3 +1,5 @@ +# Author: cirakg & claude ai +# Summary: Shared pytest setup for the tool tests (import path, Hypothesis database). """Shared pytest setup for the tool tests. * Makes the modules in tools/ importable as top-level modules. diff --git a/tools/tests/data/runnoevil_pass.log b/tools/tests/data/runnoevil_pass.log new file mode 100644 index 0000000..82c3cf0 --- /dev/null +++ b/tools/tests/data/runnoevil_pass.log @@ -0,0 +1,14 @@ +# Author: cirakg & claude ai +# Summary: Real Run No Evil console output (SDK 9.2.0, fr255s, headless simulator), used as a parser fixture. +------------------------------------------------------------------------------ +Executing test smokeVersionIsSemver... +DEBUG (18:41): version 0.1.0 +PASS + +============================================================================== +RESULTS +Test: Status: +smokeVersionIsSemver PASS +Ran 1 test + +PASSED (passed=1, failed=0, errors=0) diff --git a/tools/tests/test_check_budgets.py b/tools/tests/test_check_budgets.py index 35cc2f8..2262a0b 100644 --- a/tools/tests/test_check_budgets.py +++ b/tools/tests/test_check_budgets.py @@ -1,30 +1,37 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/check_budgets.py (start-up time and memory budgets). """Tests for tools/check_budgets.py (perf and memory budgets).""" from __future__ import annotations import json +from collections.abc import Mapping from pathlib import Path import check_budgets as cb -FRAME = 'DIAG {"event":"first_frame","startup_ms":250,"mem_used":100000,"mem_total":524288}' +FRAME = "DIAG first_frame startup_ms=12 mem_used=100000 mem_total=524288" +WALL = "HARNESS first_frame wall_ms=2310" def test_reads_events_and_skips_noise() -> None: - text = f"Device Version 1\n{FRAME}\nDIAG {{not json\nsome DIAG line\n" - assert cb.read_events(text) == [ - {"event": "first_frame", "startup_ms": 250, "mem_used": 100000, "mem_total": 524288} - ] + text = f"Device Version 1\n{FRAME}\nDIAG broken\n{WALL}\nDIAG first_frame startup_ms=99 mem_used=1 mem_total=2\n" + events = cb.read_events(text) + # Only the first occurrence of each event counts. + assert events == { + "DIAG.first_frame": {"startup_ms": 12.0, "mem_used": 100000.0, "mem_total": 524288.0}, + "HARNESS.first_frame": {"wall_ms": 2310.0}, + } def test_measure_perf_and_memory() -> None: - events = cb.read_events(FRAME) - assert cb.measure("perf", events) == {"startup_ms": 250.0} + events = cb.read_events(f"{FRAME}\n{WALL}") + assert cb.measure("perf", events) == {"startup_ms": 12.0, "first_frame_wall_ms": 2310.0} assert cb.measure("mem", events) == {"mem_used_fraction": 100000 / 524288} - assert cb.measure("perf", []) == {} + assert cb.measure("perf", {}) == {} -def run(tmp_path: Path, kind: str, log_text: str, budgets: dict[str, dict[str, float]]) -> int: +def run(tmp_path: Path, kind: str, log_text: str, budgets: Mapping[str, Mapping[str, float]]) -> int: log = tmp_path / "fr255s.log" log.write_text(log_text, encoding="utf-8") budget_file = tmp_path / "budgets.json" @@ -33,13 +40,20 @@ def run(tmp_path: Path, kind: str, log_text: str, budgets: dict[str, dict[str, f def test_within_budget(tmp_path: Path) -> None: - assert run(tmp_path, "perf", FRAME, {"perf": {"startup_ms": 300}}) == cb.EXIT_OK - assert json.loads((tmp_path / "s.json").read_text()) == {"fr255s": {"startup_ms": 250.0}} + budgets = {"perf": {"startup_ms": 300, "first_frame_wall_ms": 5000}} + assert run(tmp_path, "perf", f"{FRAME}\n{WALL}", budgets) == cb.EXIT_OK + summary = json.loads((tmp_path / "s.json").read_text()) + assert summary == {"fr255s": {"startup_ms": 12.0, "first_frame_wall_ms": 2310.0}} def test_over_budget(tmp_path: Path) -> None: assert run(tmp_path, "mem", FRAME, {"mem": {"mem_used_fraction": 0.1}}) == cb.EXIT_OVER +def test_budgeted_value_not_measured(tmp_path: Path) -> None: + budgets = {"perf": {"first_frame_wall_ms": 5000}} + assert run(tmp_path, "perf", FRAME, budgets) == cb.EXIT_MISSING + + def test_missing_diagnostics(tmp_path: Path) -> None: assert run(tmp_path, "perf", "app crashed\n", {"perf": {"startup_ms": 300}}) == cb.EXIT_MISSING diff --git a/tools/tests/test_check_docs.py b/tools/tests/test_check_docs.py index 67404c9..a5e83d0 100644 --- a/tools/tests/test_check_docs.py +++ b/tools/tests/test_check_docs.py @@ -1,3 +1,5 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/check_docs.py, the Markdown link checker. """Tests for tools/check_docs.py, the offline Markdown link checker.""" from __future__ import annotations diff --git a/tools/tests/test_ciq_test_report.py b/tools/tests/test_ciq_test_report.py index cdd3901..8ed83cc 100644 --- a/tools/tests/test_ciq_test_report.py +++ b/tools/tests/test_ciq_test_report.py @@ -1,3 +1,5 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/ciq_test_report.py, the judge of Connect IQ unit-test logs. """Tests for tools/ciq_test_report.py, the judge of Connect IQ test logs.""" from __future__ import annotations @@ -9,10 +11,9 @@ import ciq_test_report as rep from hypothesis import given from hypothesis import strategies as st +DATA = Path(__file__).parent / "data" + PASSING = """\ -Device Version 0.1.0 -Device id 1 name "A garmin device" -Shell Version 0.1.0 ------------------------------------------------------------------------------ Executing test smokeVersionIsSemver... DEBUG (14:16): version 0.1.0 @@ -20,34 +21,47 @@ PASS ------------------------------------------------------------------------------ Executing test otherTest... PASS + ============================================================================== RESULTS -Test: Status: -smokeVersionIsSemver PASS -otherTest PASS +Test: Status: +smokeVersionIsSemver PASS +otherTest PASS Ran 2 tests -PASSED (failures=0, errors=0) -Connection Finished -Closing shell and port + +PASSED (passed=2, failed=0, errors=0) """ +def test_real_sdk_log_passes() -> None: + report = rep.parse((DATA / "runnoevil_pass.log").read_text(encoding="utf-8")) + assert report.results == {"smokeVersionIsSemver": "PASS"} + assert (report.ran, report.passed, report.failed, report.errors) == (1, 1, 0, 0) + assert report.problems(min_tests=1) == (rep.EXIT_PASS, []) + + def test_passing_log_passes() -> None: report = rep.parse(PASSING) assert report.results == {"smokeVersionIsSemver": "PASS", "otherTest": "PASS"} - assert report.ran == 2 assert report.output["smokeVersionIsSemver"] == ["DEBUG (14:16): version 0.1.0", "PASS"] assert report.problems(min_tests=1) == (rep.EXIT_PASS, []) +def test_documented_older_summary_format_passes() -> None: + text = PASSING.replace("PASSED (passed=2, failed=0, errors=0)", "PASSED (failures=0, errors=0)") + report = rep.parse(text) + assert report.passed is None + assert report.problems(min_tests=1)[0] == rep.EXIT_PASS + + def test_status_words_are_case_insensitive() -> None: text = PASSING.replace("PASS\n", "Pass\n").replace("PASSED (", "Passed (") assert rep.parse(text).problems(min_tests=1)[0] == rep.EXIT_PASS def test_failed_test_fails() -> None: - text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace( - "PASSED (failures=0", "FAILED (failures=1" + text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace( + "PASSED (passed=2, failed=0", "FAILED (passed=1, failed=1" ) code, reasons = rep.parse(text).problems(min_tests=1) assert code == rep.EXIT_FAIL @@ -55,10 +69,11 @@ def test_failed_test_fails() -> None: def test_error_row_fails_even_if_summary_claims_pass() -> None: - text = PASSING.replace("otherTest PASS", "otherTest ERROR") + text = PASSING.replace("otherTest PASS", "otherTest ERROR") code, reasons = rep.parse(text).problems(min_tests=1) assert code == rep.EXIT_FAIL assert "otherTest: ERROR" in reasons + assert any("passed=2 but 1 rows" in r for r in reasons) def test_log_without_results_is_incomplete() -> None: @@ -80,7 +95,7 @@ def test_ran_count_must_match_rows() -> None: def test_junit_output_is_valid(tmp_path: Path) -> None: log = tmp_path / "fr70.log" - log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL")) + log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL")) junit = tmp_path / "out" / "fr70.xml" assert rep.main([str(log), "--device", "fr70", "--junit", str(junit)]) == rep.EXIT_FAIL suite = ET.parse(junit).getroot() diff --git a/tools/tests/test_gen_products.py b/tools/tests/test_gen_products.py new file mode 100644 index 0000000..47424d2 --- /dev/null +++ b/tools/tests/test_gen_products.py @@ -0,0 +1,63 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/gen_products.py (the manifest product rule). +"""Tests for tools/gen_products.py, with synthetic device definitions.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import gen_products as gp + +MANIFEST = """ + {products} +""" + + +def add_device(devices: Path, product: str, api: list[str], memory: int, group: str = gp.WATCH_GROUP) -> None: + folder = devices / product + folder.mkdir(parents=True) + compiler = { + "displayName": product.upper(), + "webDocDeviceGroup": group, + "partNumbers": [{"connectIQVersion": v} for v in api], + "appTypes": [{"type": "watchApp", "memoryLimit": memory}, {"type": "glance", "memoryLimit": 65536}], + } + (folder / "compiler.json").write_text(json.dumps(compiler)) + + +def write_manifest(tmp_path: Path, products: list[str]) -> Path: + path = tmp_path / "manifest.xml" + path.write_text(MANIFEST.format(products="".join(f'' for p in products))) + return path + + +def test_qualifying_products_pass(tmp_path: Path) -> None: + devices = tmp_path / "Devices" + add_device(devices, "fr70", ["6.0.3"], 786432) + add_device(devices, "fr255s", ["5.2.0", "5.2.1"], 524288) + assert gp.check(write_manifest(tmp_path, ["fr70", "fr255s"]), devices) == gp.EXIT_OK + + +def test_each_rule_is_enforced(tmp_path: Path) -> None: + devices = tmp_path / "Devices" + add_device(devices, "old", ["4.2.0", "5.0.0"], 786432) # one firmware variant below 5.0 + add_device(devices, "small", ["6.0.0"], 131072) + add_device(devices, "bike", ["5.1.0"], 786432, group="Edge") + manifest = write_manifest(tmp_path, ["old", "small", "bike"]) + assert gp.check(manifest, devices) == gp.EXIT_UNQUALIFIED + + +def test_missing_device_files(tmp_path: Path) -> None: + devices = tmp_path / "Devices" + devices.mkdir() + assert gp.check(write_manifest(tmp_path, ["ghost"]), devices) == gp.EXIT_MISSING + + +def test_problem_messages() -> None: + device = gp.Device("x", "X", "Edge", (4, 2, 0), 131072) + assert device.problems() == [ + "group is 'Edge', not 'Watches/Wearables'", + "API 4.2.0 is below 5.0.0", + "watch apps get 128 KB, below 512 KB", + ] diff --git a/tools/tests/test_lint_project.py b/tools/tests/test_lint_project.py index be0151e..31f8125 100644 --- a/tools/tests/test_lint_project.py +++ b/tools/tests/test_lint_project.py @@ -1,11 +1,16 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/lint_project.py, the project policy linter. """Tests for tools/lint_project.py, the project policy linter.""" from __future__ import annotations +import json from pathlib import Path import lint_project as lp from conftest import PROJECT_ROOT +from PIL import Image +from PIL.PngImagePlugin import PngInfo GOOD_MANIFEST = "\n" @@ -19,7 +24,8 @@ def make_project(tmp_path: Path, files: dict[str, str]) -> Path: def codes(root: Path) -> list[tuple[str, str, int]]: - return [(str(p.path), p.code, p.line) for p in lp.lint(root)] + """Findings of the code rules (MC001-MC005) only.""" + return [(str(p.path), p.code, p.line) for p in sorted(lp.lint_code(root))] def test_clean_project_passes(tmp_path: Path) -> None: @@ -76,6 +82,41 @@ def test_manifest_permissions_are_rejected(tmp_path: Path) -> None: assert codes(root) == [("manifest.xml", "MC005", 2)] +HEADER = "# Author: cirakg & claude ai\n# Summary: A file with a proper header.\n" + + +def header_findings(root: Path) -> list[str]: + return sorted(str(p.path) for p in lp.lint_headers(root)) + + +def save_png(path: Path, author: str | None) -> None: + info = PngInfo() + if author is not None: + info.add_text("Author", author) + info.add_text("Description", "test image") + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (4, 4)).save(path, pnginfo=info) + + +def test_headers_are_required_in_every_file_type(tmp_path: Path) -> None: + files = { + "ok.sh": "#!/bin/sh\n" + HEADER, + "late.txt": "\n" * 20 + HEADER, # too far down + "none.md": "# Title\n", + "data/ok.json": json.dumps({"_author": lp.AUTHOR, "_summary": "x"}), + "data/bad.json": json.dumps({"author": "someone else", "description": "x"}), + "docker/docs/package-lock.json": "{}", # generated, exempt + "build/out.txt": "ignored folder", + } + for name, text in files.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + save_png(tmp_path / "img/ok.png", lp.AUTHOR) + save_png(tmp_path / "img/anon.png", None) + assert header_findings(tmp_path) == ["data/bad.json", "img/anon.png", "late.txt", "none.md"] + + def test_project_passes_its_own_rules() -> None: problems = lp.lint(PROJECT_ROOT) assert problems == [], "\n".join(map(str, problems)) diff --git a/tools/tests/test_make_icon.py b/tools/tests/test_make_icon.py index 66128c5..93d7c94 100644 --- a/tools/tests/test_make_icon.py +++ b/tools/tests/test_make_icon.py @@ -1,3 +1,5 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/make_icon.py, the launcher icon generator. """Tests for tools/make_icon.py (launcher icon generator).""" from __future__ import annotations diff --git a/tools/tests/test_sim_screenshot.py b/tools/tests/test_sim_screenshot.py new file mode 100644 index 0000000..7a1a087 --- /dev/null +++ b/tools/tests/test_sim_screenshot.py @@ -0,0 +1,94 @@ +# Author: cirakg & claude ai +# Summary: Tests for tools/sim_screenshot.py (finding, cropping and comparing simulator screens). +"""Tests for tools/sim_screenshot.py, using synthetic simulator windows.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import sim_screenshot as ss +from PIL import Image, ImageDraw + +DISPLAY = {"x": 20, "y": 30, "width": 60, "height": 60} + + +def make_device(tmp_path: Path, shape: str = "round") -> Path: + """A fake device folder: a striped bezel drawing and its simulator.json.""" + device_dir = tmp_path / "fakewatch" + device_dir.mkdir() + bezel = Image.new("RGBA", (100, 120), (0, 0, 0, 0)) + draw = ImageDraw.Draw(bezel) + for y in range(0, 120, 6): # stripes make the offset unambiguous + draw.rectangle((0, y, 99, y + 2), fill=(120 + y, 60, 200 - y, 255)) + draw.rectangle((DISPLAY["x"], DISPLAY["y"], DISPLAY["x"] + 59, DISPLAY["y"] + 59), fill=(0, 0, 0, 255)) + bezel.save(device_dir / "fake.png") + simulator = {"image": "fake.png", "display": {"location": DISPLAY, "shape": shape}} + (device_dir / "simulator.json").write_text(json.dumps(simulator)) + return device_dir + + +def make_window(device_dir: Path, offset: tuple[int, int], screen: Image.Image) -> Image.Image: + """A simulator window: grey background, the drawing at offset, the app's screen on top.""" + window = Image.new("RGB", (100 + offset[0], 150), (230, 230, 230)) + bezel = ss.open_image(device_dir / "fake.png").convert("RGBA") + window.paste(bezel, offset, bezel) + window.paste(screen, (offset[0] + DISPLAY["x"], offset[1] + DISPLAY["y"])) + return window + + +def app_screen(text_colour: tuple[int, int, int] = (255, 255, 255)) -> Image.Image: + screen = Image.new("RGB", (60, 60), (0, 0, 0)) + ImageDraw.Draw(screen).rectangle((20, 25, 40, 35), fill=text_colour) + return screen + + +def test_finds_offset_and_crops_screen(tmp_path: Path) -> None: + device_dir = make_device(tmp_path) + display = ss.load_display(device_dir) + window = make_window(device_dir, (1, 26), app_screen()) + bezel = ss.bezel_only(ss.open_image(device_dir / display.image), display) + dx, dy, score = ss.find_offset(window, bezel) + assert (dx, dy) == (1, 26) + assert score < 1.0 + screen = ss.crop_screen(window, (dx, dy), display) + assert screen.size == (60, 60) + assert screen.getpixel((30, 30)) == (255, 255, 255) + assert screen.getpixel((0, 0)) == (0, 0, 0) # outside the round display + + +def test_blank_screen_is_detected() -> None: + assert ss.is_blank(Image.new("RGB", (60, 60), (0, 0, 0))) + assert not ss.is_blank(app_screen()) + + +def test_compare_counts_differing_pixels() -> None: + fraction, _ = ss.compare(app_screen(), app_screen()) + assert fraction == 0 + fraction, overlay = ss.compare(app_screen((255, 255, 255)), app_screen((0, 0, 255))) + assert fraction == (21 * 11) / (60 * 60) + assert overlay.getpixel((30, 30)) == (255, 0, 0) + assert ss.compare(app_screen(), Image.new("RGB", (10, 10)))[0] == 1.0 + + +def test_main_flow_approve_then_compare(tmp_path: Path) -> None: + device_dir = make_device(tmp_path) + raw, out, golden = tmp_path / "raw.png", tmp_path / "out.png", tmp_path / "golden" / "fakewatch.png" + make_window(device_dir, (1, 26), app_screen()).save(raw) + args = ["--raw", str(raw), "--device-dir", str(device_dir), "--out", str(out), "--golden", str(golden)] + assert ss.main(args) == ss.EXIT_NO_GOLDEN + assert ss.main([*args, "--approve"]) == ss.EXIT_OK + assert ss.open_image(golden).info["Author"] == "cirakg & claude ai" + assert ss.main(args) == ss.EXIT_OK + + make_window(device_dir, (1, 26), app_screen((0, 255, 0))).save(raw) + assert ss.main(args) == ss.EXIT_DIFFERENT + assert (tmp_path / "out-diff.png").exists() + + +def test_missing_device_files_are_an_error(tmp_path: Path) -> None: + device_dir = make_device(tmp_path) + (device_dir / "fake.png").unlink() + raw = tmp_path / "raw.png" + Image.new("RGB", (100, 150)).save(raw) + assert ss.main(["--raw", str(raw), "--device-dir", str(device_dir), "--out", str(tmp_path / "o.png")]) == 2 -- cgit v1.2.3