From 2071e2beda763137416ac386d4bb953711ec995d Mon Sep 17 00:00:00 2001
From: cirakg <cirakg@cirakg.xyz>
Date: Sun, 18 Oct 2026 20:28:09 +0200
Subject: test(tools): add test-log judge, docs and policy checks with tests

- ciq_test_report.py: decides pass/fail from Run No Evil output instead
  of monkeydo's unreliable exit code; writes JUnit XML.
- check_docs.py: offline check of relative links and GitHub anchors.
- lint_project.py: project rules (headers, no logging outside Diag,
  no secret-like literals, no network APIs or permissions).
- check_budgets.py with budgets.json: perf and memory gates.
- make_icon.py: reproducible launcher icon.
- 31 pytest tests, including Hypothesis fuzzing of the log parser and
  integration checks against the real docs and sources; ruff and mypy
  (strict) clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 tools/tests/test_ciq_test_report.py | 99 +++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 tools/tests/test_ciq_test_report.py

(limited to 'tools/tests/test_ciq_test_report.py')

diff --git a/tools/tests/test_ciq_test_report.py b/tools/tests/test_ciq_test_report.py
new file mode 100644
index 0000000..cdd3901
--- /dev/null
+++ b/tools/tests/test_ciq_test_report.py
@@ -0,0 +1,99 @@
+"""Tests for tools/ciq_test_report.py, the judge of Connect IQ test logs."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import ciq_test_report as rep
+from hypothesis import given
+from hypothesis import strategies as st
+
+PASSING = """\
+Device Version 0.1.0
+Device id 1 name "A garmin device"
+Shell Version 0.1.0
+------------------------------------------------------------------------------
+Executing test smokeVersionIsSemver...
+DEBUG (14:16): version 0.1.0
+PASS
+------------------------------------------------------------------------------
+Executing test otherTest...
+PASS
+==============================================================================
+RESULTS
+Test:                                     Status:
+smokeVersionIsSemver                      PASS
+otherTest                                 PASS
+Ran 2 tests
+PASSED (failures=0, errors=0)
+Connection Finished
+Closing shell and port
+"""
+
+
+def test_passing_log_passes() -> None:
+    report = rep.parse(PASSING)
+    assert report.results == {"smokeVersionIsSemver": "PASS", "otherTest": "PASS"}
+    assert report.ran == 2
+    assert report.output["smokeVersionIsSemver"] == ["DEBUG (14:16): version 0.1.0", "PASS"]
+    assert report.problems(min_tests=1) == (rep.EXIT_PASS, [])
+
+
+def test_status_words_are_case_insensitive() -> None:
+    text = PASSING.replace("PASS\n", "Pass\n").replace("PASSED (", "Passed (")
+    assert rep.parse(text).problems(min_tests=1)[0] == rep.EXIT_PASS
+
+
+def test_failed_test_fails() -> None:
+    text = PASSING.replace("otherTest                                 PASS", "otherTest  FAIL").replace(
+        "PASSED (failures=0", "FAILED (failures=1"
+    )
+    code, reasons = rep.parse(text).problems(min_tests=1)
+    assert code == rep.EXIT_FAIL
+    assert "otherTest: FAIL" in reasons
+
+
+def test_error_row_fails_even_if_summary_claims_pass() -> None:
+    text = PASSING.replace("otherTest                                 PASS", "otherTest  ERROR")
+    code, reasons = rep.parse(text).problems(min_tests=1)
+    assert code == rep.EXIT_FAIL
+    assert "otherTest: ERROR" in reasons
+
+
+def test_log_without_results_is_incomplete() -> None:
+    truncated = PASSING.partition("RESULTS")[0]
+    assert rep.parse(truncated).problems(min_tests=1)[0] == rep.EXIT_INCOMPLETE
+
+
+def test_too_few_tests_fails() -> None:
+    code, reasons = rep.parse(PASSING).problems(min_tests=3)
+    assert code == rep.EXIT_FAIL
+    assert any("expected at least 3" in r for r in reasons)
+
+
+def test_ran_count_must_match_rows() -> None:
+    code, reasons = rep.parse(PASSING.replace("Ran 2 tests", "Ran 3 tests")).problems(min_tests=1)
+    assert code == rep.EXIT_FAIL
+    assert any("'Ran 3'" in r for r in reasons)
+
+
+def test_junit_output_is_valid(tmp_path: Path) -> None:
+    log = tmp_path / "fr70.log"
+    log.write_text(PASSING.replace("otherTest                                 PASS", "otherTest  FAIL"))
+    junit = tmp_path / "out" / "fr70.xml"
+    assert rep.main([str(log), "--device", "fr70", "--junit", str(junit)]) == rep.EXIT_FAIL
+    suite = ET.parse(junit).getroot()
+    assert suite.get("tests") == "2"
+    assert suite.get("failures") == "1"
+    assert [c.get("name") for c in suite.iter("testcase")] == ["smokeVersionIsSemver", "otherTest"]
+
+
+def test_missing_log_is_incomplete(tmp_path: Path) -> None:
+    assert rep.main([str(tmp_path / "nope.log"), "--device", "x"]) == rep.EXIT_INCOMPLETE
+
+
+@given(st.text())
+def test_parser_never_crashes(text: str) -> None:
+    code, _ = rep.parse(text).problems(min_tests=1)
+    assert code in (rep.EXIT_PASS, rep.EXIT_FAIL, rep.EXIT_INCOMPLETE)
-- 
cgit v1.2.3