summaryrefslogtreecommitdiff
path: root/tools/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tools/tests')
-rw-r--r--tools/tests/conftest.py2
-rw-r--r--tools/tests/data/runnoevil_pass.log14
-rw-r--r--tools/tests/test_check_budgets.py36
-rw-r--r--tools/tests/test_check_docs.py2
-rw-r--r--tools/tests/test_ciq_test_report.py43
-rw-r--r--tools/tests/test_gen_products.py63
-rw-r--r--tools/tests/test_lint_project.py43
-rw-r--r--tools/tests/test_make_icon.py2
-rw-r--r--tools/tests/test_sim_screenshot.py94
9 files changed, 273 insertions, 26 deletions
diff --git a/tools/tests/conftest.py b/tools/tests/conftest.py
index 7054ecb..d4dae36 100644
--- a/tools/tests/conftest.py
+++ b/tools/tests/conftest.py
@@ -1,3 +1,5 @@
+# Author: cirakg & claude ai
+# Summary: Shared pytest setup for the tool tests (import path, Hypothesis database).
"""Shared pytest setup for the tool tests.
* Makes the modules in tools/ importable as top-level modules.
diff --git a/tools/tests/data/runnoevil_pass.log b/tools/tests/data/runnoevil_pass.log
new file mode 100644
index 0000000..82c3cf0
--- /dev/null
+++ b/tools/tests/data/runnoevil_pass.log
@@ -0,0 +1,14 @@
+# Author: cirakg & claude ai
+# Summary: Real Run No Evil console output (SDK 9.2.0, fr255s, headless simulator), used as a parser fixture.
+------------------------------------------------------------------------------
+Executing test smokeVersionIsSemver...
+DEBUG (18:41): version 0.1.0
+PASS
+
+==============================================================================
+RESULTS
+Test: Status:
+smokeVersionIsSemver PASS
+Ran 1 test
+
+PASSED (passed=1, failed=0, errors=0)
diff --git a/tools/tests/test_check_budgets.py b/tools/tests/test_check_budgets.py
index 35cc2f8..2262a0b 100644
--- a/tools/tests/test_check_budgets.py
+++ b/tools/tests/test_check_budgets.py
@@ -1,30 +1,37 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/check_budgets.py (start-up time and memory budgets).
"""Tests for tools/check_budgets.py (perf and memory budgets)."""
from __future__ import annotations
import json
+from collections.abc import Mapping
from pathlib import Path
import check_budgets as cb
-FRAME = 'DIAG {"event":"first_frame","startup_ms":250,"mem_used":100000,"mem_total":524288}'
+FRAME = "DIAG first_frame startup_ms=12 mem_used=100000 mem_total=524288"
+WALL = "HARNESS first_frame wall_ms=2310"
def test_reads_events_and_skips_noise() -> None:
- text = f"Device Version 1\n{FRAME}\nDIAG {{not json\nsome DIAG line\n"
- assert cb.read_events(text) == [
- {"event": "first_frame", "startup_ms": 250, "mem_used": 100000, "mem_total": 524288}
- ]
+ text = f"Device Version 1\n{FRAME}\nDIAG broken\n{WALL}\nDIAG first_frame startup_ms=99 mem_used=1 mem_total=2\n"
+ events = cb.read_events(text)
+ # Only the first occurrence of each event counts.
+ assert events == {
+ "DIAG.first_frame": {"startup_ms": 12.0, "mem_used": 100000.0, "mem_total": 524288.0},
+ "HARNESS.first_frame": {"wall_ms": 2310.0},
+ }
def test_measure_perf_and_memory() -> None:
- events = cb.read_events(FRAME)
- assert cb.measure("perf", events) == {"startup_ms": 250.0}
+ events = cb.read_events(f"{FRAME}\n{WALL}")
+ assert cb.measure("perf", events) == {"startup_ms": 12.0, "first_frame_wall_ms": 2310.0}
assert cb.measure("mem", events) == {"mem_used_fraction": 100000 / 524288}
- assert cb.measure("perf", []) == {}
+ assert cb.measure("perf", {}) == {}
-def run(tmp_path: Path, kind: str, log_text: str, budgets: dict[str, dict[str, float]]) -> int:
+def run(tmp_path: Path, kind: str, log_text: str, budgets: Mapping[str, Mapping[str, float]]) -> int:
log = tmp_path / "fr255s.log"
log.write_text(log_text, encoding="utf-8")
budget_file = tmp_path / "budgets.json"
@@ -33,13 +40,20 @@ def run(tmp_path: Path, kind: str, log_text: str, budgets: dict[str, dict[str, f
def test_within_budget(tmp_path: Path) -> None:
- assert run(tmp_path, "perf", FRAME, {"perf": {"startup_ms": 300}}) == cb.EXIT_OK
- assert json.loads((tmp_path / "s.json").read_text()) == {"fr255s": {"startup_ms": 250.0}}
+ budgets = {"perf": {"startup_ms": 300, "first_frame_wall_ms": 5000}}
+ assert run(tmp_path, "perf", f"{FRAME}\n{WALL}", budgets) == cb.EXIT_OK
+ summary = json.loads((tmp_path / "s.json").read_text())
+ assert summary == {"fr255s": {"startup_ms": 12.0, "first_frame_wall_ms": 2310.0}}
def test_over_budget(tmp_path: Path) -> None:
assert run(tmp_path, "mem", FRAME, {"mem": {"mem_used_fraction": 0.1}}) == cb.EXIT_OVER
+def test_budgeted_value_not_measured(tmp_path: Path) -> None:
+ budgets = {"perf": {"first_frame_wall_ms": 5000}}
+ assert run(tmp_path, "perf", FRAME, budgets) == cb.EXIT_MISSING
+
+
def test_missing_diagnostics(tmp_path: Path) -> None:
assert run(tmp_path, "perf", "app crashed\n", {"perf": {"startup_ms": 300}}) == cb.EXIT_MISSING
diff --git a/tools/tests/test_check_docs.py b/tools/tests/test_check_docs.py
index 67404c9..a5e83d0 100644
--- a/tools/tests/test_check_docs.py
+++ b/tools/tests/test_check_docs.py
@@ -1,3 +1,5 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/check_docs.py, the Markdown link checker.
"""Tests for tools/check_docs.py, the offline Markdown link checker."""
from __future__ import annotations
diff --git a/tools/tests/test_ciq_test_report.py b/tools/tests/test_ciq_test_report.py
index cdd3901..8ed83cc 100644
--- a/tools/tests/test_ciq_test_report.py
+++ b/tools/tests/test_ciq_test_report.py
@@ -1,3 +1,5 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/ciq_test_report.py, the judge of Connect IQ unit-test logs.
"""Tests for tools/ciq_test_report.py, the judge of Connect IQ test logs."""
from __future__ import annotations
@@ -9,10 +11,9 @@ import ciq_test_report as rep
from hypothesis import given
from hypothesis import strategies as st
+DATA = Path(__file__).parent / "data"
+
PASSING = """\
-Device Version 0.1.0
-Device id 1 name "A garmin device"
-Shell Version 0.1.0
------------------------------------------------------------------------------
Executing test smokeVersionIsSemver...
DEBUG (14:16): version 0.1.0
@@ -20,34 +21,47 @@ PASS
------------------------------------------------------------------------------
Executing test otherTest...
PASS
+
==============================================================================
RESULTS
-Test: Status:
-smokeVersionIsSemver PASS
-otherTest PASS
+Test: Status:
+smokeVersionIsSemver PASS
+otherTest PASS
Ran 2 tests
-PASSED (failures=0, errors=0)
-Connection Finished
-Closing shell and port
+
+PASSED (passed=2, failed=0, errors=0)
"""
+def test_real_sdk_log_passes() -> None:
+ report = rep.parse((DATA / "runnoevil_pass.log").read_text(encoding="utf-8"))
+ assert report.results == {"smokeVersionIsSemver": "PASS"}
+ assert (report.ran, report.passed, report.failed, report.errors) == (1, 1, 0, 0)
+ assert report.problems(min_tests=1) == (rep.EXIT_PASS, [])
+
+
def test_passing_log_passes() -> None:
report = rep.parse(PASSING)
assert report.results == {"smokeVersionIsSemver": "PASS", "otherTest": "PASS"}
- assert report.ran == 2
assert report.output["smokeVersionIsSemver"] == ["DEBUG (14:16): version 0.1.0", "PASS"]
assert report.problems(min_tests=1) == (rep.EXIT_PASS, [])
+def test_documented_older_summary_format_passes() -> None:
+ text = PASSING.replace("PASSED (passed=2, failed=0, errors=0)", "PASSED (failures=0, errors=0)")
+ report = rep.parse(text)
+ assert report.passed is None
+ assert report.problems(min_tests=1)[0] == rep.EXIT_PASS
+
+
def test_status_words_are_case_insensitive() -> None:
text = PASSING.replace("PASS\n", "Pass\n").replace("PASSED (", "Passed (")
assert rep.parse(text).problems(min_tests=1)[0] == rep.EXIT_PASS
def test_failed_test_fails() -> None:
- text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace(
- "PASSED (failures=0", "FAILED (failures=1"
+ text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace(
+ "PASSED (passed=2, failed=0", "FAILED (passed=1, failed=1"
)
code, reasons = rep.parse(text).problems(min_tests=1)
assert code == rep.EXIT_FAIL
@@ -55,10 +69,11 @@ def test_failed_test_fails() -> None:
def test_error_row_fails_even_if_summary_claims_pass() -> None:
- text = PASSING.replace("otherTest PASS", "otherTest ERROR")
+ text = PASSING.replace("otherTest PASS", "otherTest ERROR")
code, reasons = rep.parse(text).problems(min_tests=1)
assert code == rep.EXIT_FAIL
assert "otherTest: ERROR" in reasons
+ assert any("passed=2 but 1 rows" in r for r in reasons)
def test_log_without_results_is_incomplete() -> None:
@@ -80,7 +95,7 @@ def test_ran_count_must_match_rows() -> None:
def test_junit_output_is_valid(tmp_path: Path) -> None:
log = tmp_path / "fr70.log"
- log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL"))
+ log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL"))
junit = tmp_path / "out" / "fr70.xml"
assert rep.main([str(log), "--device", "fr70", "--junit", str(junit)]) == rep.EXIT_FAIL
suite = ET.parse(junit).getroot()
diff --git a/tools/tests/test_gen_products.py b/tools/tests/test_gen_products.py
new file mode 100644
index 0000000..47424d2
--- /dev/null
+++ b/tools/tests/test_gen_products.py
@@ -0,0 +1,63 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/gen_products.py (the manifest product rule).
+"""Tests for tools/gen_products.py, with synthetic device definitions."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import gen_products as gp
+
+MANIFEST = """<iq:manifest version="3" xmlns:iq="http://www.garmin.com/xml/connectiq">
+ <iq:application id="x" type="watch-app"><iq:products>{products}</iq:products></iq:application>
+</iq:manifest>"""
+
+
+def add_device(devices: Path, product: str, api: list[str], memory: int, group: str = gp.WATCH_GROUP) -> None:
+ folder = devices / product
+ folder.mkdir(parents=True)
+ compiler = {
+ "displayName": product.upper(),
+ "webDocDeviceGroup": group,
+ "partNumbers": [{"connectIQVersion": v} for v in api],
+ "appTypes": [{"type": "watchApp", "memoryLimit": memory}, {"type": "glance", "memoryLimit": 65536}],
+ }
+ (folder / "compiler.json").write_text(json.dumps(compiler))
+
+
+def write_manifest(tmp_path: Path, products: list[str]) -> Path:
+ path = tmp_path / "manifest.xml"
+ path.write_text(MANIFEST.format(products="".join(f'<iq:product id="{p}"/>' for p in products)))
+ return path
+
+
+def test_qualifying_products_pass(tmp_path: Path) -> None:
+ devices = tmp_path / "Devices"
+ add_device(devices, "fr70", ["6.0.3"], 786432)
+ add_device(devices, "fr255s", ["5.2.0", "5.2.1"], 524288)
+ assert gp.check(write_manifest(tmp_path, ["fr70", "fr255s"]), devices) == gp.EXIT_OK
+
+
+def test_each_rule_is_enforced(tmp_path: Path) -> None:
+ devices = tmp_path / "Devices"
+ add_device(devices, "old", ["4.2.0", "5.0.0"], 786432) # one firmware variant below 5.0
+ add_device(devices, "small", ["6.0.0"], 131072)
+ add_device(devices, "bike", ["5.1.0"], 786432, group="Edge")
+ manifest = write_manifest(tmp_path, ["old", "small", "bike"])
+ assert gp.check(manifest, devices) == gp.EXIT_UNQUALIFIED
+
+
+def test_missing_device_files(tmp_path: Path) -> None:
+ devices = tmp_path / "Devices"
+ devices.mkdir()
+ assert gp.check(write_manifest(tmp_path, ["ghost"]), devices) == gp.EXIT_MISSING
+
+
+def test_problem_messages() -> None:
+ device = gp.Device("x", "X", "Edge", (4, 2, 0), 131072)
+ assert device.problems() == [
+ "group is 'Edge', not 'Watches/Wearables'",
+ "API 4.2.0 is below 5.0.0",
+ "watch apps get 128 KB, below 512 KB",
+ ]
diff --git a/tools/tests/test_lint_project.py b/tools/tests/test_lint_project.py
index be0151e..31f8125 100644
--- a/tools/tests/test_lint_project.py
+++ b/tools/tests/test_lint_project.py
@@ -1,11 +1,16 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/lint_project.py, the project policy linter.
"""Tests for tools/lint_project.py, the project policy linter."""
from __future__ import annotations
+import json
from pathlib import Path
import lint_project as lp
from conftest import PROJECT_ROOT
+from PIL import Image
+from PIL.PngImagePlugin import PngInfo
GOOD_MANIFEST = "<iq:manifest><iq:permissions/></iq:manifest>\n"
@@ -19,7 +24,8 @@ def make_project(tmp_path: Path, files: dict[str, str]) -> Path:
def codes(root: Path) -> list[tuple[str, str, int]]:
- return [(str(p.path), p.code, p.line) for p in lp.lint(root)]
+ """Findings of the code rules (MC001-MC005) only."""
+ return [(str(p.path), p.code, p.line) for p in sorted(lp.lint_code(root))]
def test_clean_project_passes(tmp_path: Path) -> None:
@@ -76,6 +82,41 @@ def test_manifest_permissions_are_rejected(tmp_path: Path) -> None:
assert codes(root) == [("manifest.xml", "MC005", 2)]
+HEADER = "# Author: cirakg & claude ai\n# Summary: A file with a proper header.\n"
+
+
+def header_findings(root: Path) -> list[str]:
+ return sorted(str(p.path) for p in lp.lint_headers(root))
+
+
+def save_png(path: Path, author: str | None) -> None:
+ info = PngInfo()
+ if author is not None:
+ info.add_text("Author", author)
+ info.add_text("Description", "test image")
+ path.parent.mkdir(parents=True, exist_ok=True)
+ Image.new("RGB", (4, 4)).save(path, pnginfo=info)
+
+
+def test_headers_are_required_in_every_file_type(tmp_path: Path) -> None:
+ files = {
+ "ok.sh": "#!/bin/sh\n" + HEADER,
+ "late.txt": "\n" * 20 + HEADER, # too far down
+ "none.md": "# Title\n",
+ "data/ok.json": json.dumps({"_author": lp.AUTHOR, "_summary": "x"}),
+ "data/bad.json": json.dumps({"author": "someone else", "description": "x"}),
+ "docker/docs/package-lock.json": "{}", # generated, exempt
+ "build/out.txt": "ignored folder",
+ }
+ for name, text in files.items():
+ path = tmp_path / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="utf-8")
+ save_png(tmp_path / "img/ok.png", lp.AUTHOR)
+ save_png(tmp_path / "img/anon.png", None)
+ assert header_findings(tmp_path) == ["data/bad.json", "img/anon.png", "late.txt", "none.md"]
+
+
def test_project_passes_its_own_rules() -> None:
problems = lp.lint(PROJECT_ROOT)
assert problems == [], "\n".join(map(str, problems))
diff --git a/tools/tests/test_make_icon.py b/tools/tests/test_make_icon.py
index 66128c5..93d7c94 100644
--- a/tools/tests/test_make_icon.py
+++ b/tools/tests/test_make_icon.py
@@ -1,3 +1,5 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/make_icon.py, the launcher icon generator.
"""Tests for tools/make_icon.py (launcher icon generator)."""
from __future__ import annotations
diff --git a/tools/tests/test_sim_screenshot.py b/tools/tests/test_sim_screenshot.py
new file mode 100644
index 0000000..7a1a087
--- /dev/null
+++ b/tools/tests/test_sim_screenshot.py
@@ -0,0 +1,94 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/sim_screenshot.py (finding, cropping and comparing simulator screens).
+"""Tests for tools/sim_screenshot.py, using synthetic simulator windows."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import sim_screenshot as ss
+from PIL import Image, ImageDraw
+
+DISPLAY = {"x": 20, "y": 30, "width": 60, "height": 60}
+
+
+def make_device(tmp_path: Path, shape: str = "round") -> Path:
+ """A fake device folder: a striped bezel drawing and its simulator.json."""
+ device_dir = tmp_path / "fakewatch"
+ device_dir.mkdir()
+ bezel = Image.new("RGBA", (100, 120), (0, 0, 0, 0))
+ draw = ImageDraw.Draw(bezel)
+ for y in range(0, 120, 6): # stripes make the offset unambiguous
+ draw.rectangle((0, y, 99, y + 2), fill=(120 + y, 60, 200 - y, 255))
+ draw.rectangle((DISPLAY["x"], DISPLAY["y"], DISPLAY["x"] + 59, DISPLAY["y"] + 59), fill=(0, 0, 0, 255))
+ bezel.save(device_dir / "fake.png")
+ simulator = {"image": "fake.png", "display": {"location": DISPLAY, "shape": shape}}
+ (device_dir / "simulator.json").write_text(json.dumps(simulator))
+ return device_dir
+
+
+def make_window(device_dir: Path, offset: tuple[int, int], screen: Image.Image) -> Image.Image:
+ """A simulator window: grey background, the drawing at offset, the app's screen on top."""
+ window = Image.new("RGB", (100 + offset[0], 150), (230, 230, 230))
+ bezel = ss.open_image(device_dir / "fake.png").convert("RGBA")
+ window.paste(bezel, offset, bezel)
+ window.paste(screen, (offset[0] + DISPLAY["x"], offset[1] + DISPLAY["y"]))
+ return window
+
+
+def app_screen(text_colour: tuple[int, int, int] = (255, 255, 255)) -> Image.Image:
+ screen = Image.new("RGB", (60, 60), (0, 0, 0))
+ ImageDraw.Draw(screen).rectangle((20, 25, 40, 35), fill=text_colour)
+ return screen
+
+
+def test_finds_offset_and_crops_screen(tmp_path: Path) -> None:
+ device_dir = make_device(tmp_path)
+ display = ss.load_display(device_dir)
+ window = make_window(device_dir, (1, 26), app_screen())
+ bezel = ss.bezel_only(ss.open_image(device_dir / display.image), display)
+ dx, dy, score = ss.find_offset(window, bezel)
+ assert (dx, dy) == (1, 26)
+ assert score < 1.0
+ screen = ss.crop_screen(window, (dx, dy), display)
+ assert screen.size == (60, 60)
+ assert screen.getpixel((30, 30)) == (255, 255, 255)
+ assert screen.getpixel((0, 0)) == (0, 0, 0) # outside the round display
+
+
+def test_blank_screen_is_detected() -> None:
+ assert ss.is_blank(Image.new("RGB", (60, 60), (0, 0, 0)))
+ assert not ss.is_blank(app_screen())
+
+
+def test_compare_counts_differing_pixels() -> None:
+ fraction, _ = ss.compare(app_screen(), app_screen())
+ assert fraction == 0
+ fraction, overlay = ss.compare(app_screen((255, 255, 255)), app_screen((0, 0, 255)))
+ assert fraction == (21 * 11) / (60 * 60)
+ assert overlay.getpixel((30, 30)) == (255, 0, 0)
+ assert ss.compare(app_screen(), Image.new("RGB", (10, 10)))[0] == 1.0
+
+
+def test_main_flow_approve_then_compare(tmp_path: Path) -> None:
+ device_dir = make_device(tmp_path)
+ raw, out, golden = tmp_path / "raw.png", tmp_path / "out.png", tmp_path / "golden" / "fakewatch.png"
+ make_window(device_dir, (1, 26), app_screen()).save(raw)
+ args = ["--raw", str(raw), "--device-dir", str(device_dir), "--out", str(out), "--golden", str(golden)]
+ assert ss.main(args) == ss.EXIT_NO_GOLDEN
+ assert ss.main([*args, "--approve"]) == ss.EXIT_OK
+ assert ss.open_image(golden).info["Author"] == "cirakg & claude ai"
+ assert ss.main(args) == ss.EXIT_OK
+
+ make_window(device_dir, (1, 26), app_screen((0, 255, 0))).save(raw)
+ assert ss.main(args) == ss.EXIT_DIFFERENT
+ assert (tmp_path / "out-diff.png").exists()
+
+
+def test_missing_device_files_are_an_error(tmp_path: Path) -> None:
+ device_dir = make_device(tmp_path)
+ (device_dir / "fake.png").unlink()
+ raw = tmp_path / "raw.png"
+ Image.new("RGB", (100, 150)).save(raw)
+ assert ss.main(["--raw", str(raw), "--device-dir", str(device_dir), "--out", str(tmp_path / "o.png")]) == 2
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-29 06:01:54 +0000