summaryrefslogtreecommitdiff
path: root/tools/tests/test_ciq_test_report.py
diff options
context:
space:
mode:
authorcirakg <cirakg@cirakg.xyz>2026-10-18 21:10:37 +0200
committercirakg <cirakg@cirakg.xyz>2026-10-18 21:10:37 +0200
commit2292bf700d28ade39cf2d64b45509e6db21034bc (patch)
treeef455320d39ebe51db236219dbbda2b148dcbe18 /tools/tests/test_ciq_test_report.py
parent85f1ecabd7f58f306b2a042376209ac8cf49ab9b (diff)
test(tools): add screenshot, budget and product checks; enforce file headers
- sim_screenshot.py finds the device drawing in the simulator window, crops the screen and compares it with approved golden images. - check_budgets.py reads key=value DIAG/HARNESS lines (the simulator console escapes quotes, so JSON was dropped). - ciq_test_report.py accepts SDK 9.2.0's real summary line `PASSED (passed=N, failed=N, errors=N)`; a captured log is a fixture. - gen_products.py checks all manifest products are watches with API 5.0+ and at least 512 KB for apps. - lint_project.py rule HDR001: every file carries the Author/Summary header (JSON keys, PNG text chunks; generated package-lock exempt). - make_icon.py --all draws one icon per launcher size in use. - Golden screenshots for fr255s, venusq2, fr70 and fr965. - 44 tests, warnings as errors, ruff and mypy --strict clean. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Diffstat (limited to 'tools/tests/test_ciq_test_report.py')
-rw-r--r--tools/tests/test_ciq_test_report.py43
1 files changed, 29 insertions, 14 deletions
diff --git a/tools/tests/test_ciq_test_report.py b/tools/tests/test_ciq_test_report.py
index cdd3901..8ed83cc 100644
--- a/tools/tests/test_ciq_test_report.py
+++ b/tools/tests/test_ciq_test_report.py
@@ -1,3 +1,5 @@
+# Author: cirakg & claude ai
+# Summary: Tests for tools/ciq_test_report.py, the judge of Connect IQ unit-test logs.
"""Tests for tools/ciq_test_report.py, the judge of Connect IQ test logs."""
from __future__ import annotations
@@ -9,10 +11,9 @@ import ciq_test_report as rep
from hypothesis import given
from hypothesis import strategies as st
+DATA = Path(__file__).parent / "data"
+
PASSING = """\
-Device Version 0.1.0
-Device id 1 name "A garmin device"
-Shell Version 0.1.0
------------------------------------------------------------------------------
Executing test smokeVersionIsSemver...
DEBUG (14:16): version 0.1.0
@@ -20,34 +21,47 @@ PASS
------------------------------------------------------------------------------
Executing test otherTest...
PASS
+
==============================================================================
RESULTS
-Test: Status:
-smokeVersionIsSemver PASS
-otherTest PASS
+Test: Status:
+smokeVersionIsSemver PASS
+otherTest PASS
Ran 2 tests
-PASSED (failures=0, errors=0)
-Connection Finished
-Closing shell and port
+
+PASSED (passed=2, failed=0, errors=0)
"""
+def test_real_sdk_log_passes() -> None:
+ report = rep.parse((DATA / "runnoevil_pass.log").read_text(encoding="utf-8"))
+ assert report.results == {"smokeVersionIsSemver": "PASS"}
+ assert (report.ran, report.passed, report.failed, report.errors) == (1, 1, 0, 0)
+ assert report.problems(min_tests=1) == (rep.EXIT_PASS, [])
+
+
def test_passing_log_passes() -> None:
report = rep.parse(PASSING)
assert report.results == {"smokeVersionIsSemver": "PASS", "otherTest": "PASS"}
- assert report.ran == 2
assert report.output["smokeVersionIsSemver"] == ["DEBUG (14:16): version 0.1.0", "PASS"]
assert report.problems(min_tests=1) == (rep.EXIT_PASS, [])
+def test_documented_older_summary_format_passes() -> None:
+ text = PASSING.replace("PASSED (passed=2, failed=0, errors=0)", "PASSED (failures=0, errors=0)")
+ report = rep.parse(text)
+ assert report.passed is None
+ assert report.problems(min_tests=1)[0] == rep.EXIT_PASS
+
+
def test_status_words_are_case_insensitive() -> None:
text = PASSING.replace("PASS\n", "Pass\n").replace("PASSED (", "Passed (")
assert rep.parse(text).problems(min_tests=1)[0] == rep.EXIT_PASS
def test_failed_test_fails() -> None:
- text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace(
- "PASSED (failures=0", "FAILED (failures=1"
+ text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace(
+ "PASSED (passed=2, failed=0", "FAILED (passed=1, failed=1"
)
code, reasons = rep.parse(text).problems(min_tests=1)
assert code == rep.EXIT_FAIL
@@ -55,10 +69,11 @@ def test_failed_test_fails() -> None:
def test_error_row_fails_even_if_summary_claims_pass() -> None:
- text = PASSING.replace("otherTest PASS", "otherTest ERROR")
+ text = PASSING.replace("otherTest PASS", "otherTest ERROR")
code, reasons = rep.parse(text).problems(min_tests=1)
assert code == rep.EXIT_FAIL
assert "otherTest: ERROR" in reasons
+ assert any("passed=2 but 1 rows" in r for r in reasons)
def test_log_without_results_is_incomplete() -> None:
@@ -80,7 +95,7 @@ def test_ran_count_must_match_rows() -> None:
def test_junit_output_is_valid(tmp_path: Path) -> None:
log = tmp_path / "fr70.log"
- log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL"))
+ log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL"))
junit = tmp_path / "out" / "fr70.xml"
assert rep.main([str(log), "--device", "fr70", "--junit", str(junit)]) == rep.EXIT_FAIL
suite = ET.parse(junit).getroot()