diff options
Diffstat (limited to 'tools/ciq_test_report.py')
| -rw-r--r-- | tools/ciq_test_report.py | 39 |
1 files changed, 27 insertions, 12 deletions
diff --git a/tools/ciq_test_report.py b/tools/ciq_test_report.py index b35095f..e69b93f 100644 --- a/tools/ciq_test_report.py +++ b/tools/ciq_test_report.py @@ -1,23 +1,30 @@ #!/usr/bin/env python3 +# Author: cirakg & claude ai +# Summary: Decide pass/fail of a Connect IQ unit-test run from its console log, and write JUnit XML. """Turn the console output of a Connect IQ unit-test run into a verdict. -`monkeydo app.prg <device> -t` prints Garmin's "Run No Evil" report: +`monkeydo app.prg <device> -t` prints Garmin's "Run No Evil" report. Captured +from SDK 9.2.0 (tools/tests/data/runnoevil_pass.log): Executing test smokeVersionIsSemver... - DEBUG (14:16): version 0.1.0 + DEBUG (18:41): version 0.1.0 PASS ============================================================================== RESULTS - Test: Status: - smokeVersionIsSemver PASS + Test: Status: + smokeVersionIsSemver PASS Ran 1 test - PASSED (failures=0, errors=0) + PASSED (passed=1, failed=0, errors=0) -monkeydo's exit code is reported to be non-zero even when every test passes, +Garmin's documentation shows the older summary form `PASSED (failures=0, +errors=0)`; both are accepted. + +monkeydo exits with 1 even when every test passes (verified with SDK 9.2.0), so this script decides instead. A run passes only when all of these hold: * the RESULTS section and the final PASSED/FAILED line are present (a crash or a watchdog kill leaves the log incomplete); - * the final line says PASSED with zero failures and zero errors; + * the final line says PASSED with zero failed and zero errors, and its + passed count (when present) equals the number of passing rows; * "Ran N tests" matches the number of per-test rows, and N >= --min-tests; * every per-test row says PASS. @@ -36,7 +43,8 @@ from xml.sax.saxutils import escape, quoteattr EXIT_PASS, EXIT_FAIL, EXIT_INCOMPLETE = 0, 1, 2 -SUMMARY_RE = re.compile(r"^(PASSED|FAILED)\s*\(\s*failures\s*=\s*(\d+)\s*,\s*errors\s*=\s*(\d+)\s*\)", re.I) +SUMMARY_RE = re.compile(r"^(PASSED|FAILED)\s*\(([^)]*)\)\s*$", re.I) +COUNT_RE = re.compile(r"(\w+)\s*=\s*(\d+)") RAN_RE = re.compile(r"^Ran\s+(\d+)\s+tests?\b", re.I) ROW_RE = re.compile(r"^(\S+)\s+(PASS|FAIL|ERROR)\s*$", re.I) EXECUTING_RE = re.compile(r"^Executing test\s+(\S+?)(?:\.\.\.|…)?\s*$") @@ -50,7 +58,8 @@ class Report: output: dict[str, list[str]] = field(default_factory=dict) # test name -> its console lines ran: int | None = None summary: str | None = None # PASSED or FAILED - failures: int = 0 + passed: int | None = None # absent in the older summary format + failed: int = 0 errors: int = 0 def problems(self, min_tests: int) -> tuple[int, list[str]]: @@ -58,8 +67,11 @@ class Report: if self.summary is None or self.ran is None: return EXIT_INCOMPLETE, ["log has no complete RESULTS section (crash, timeout or watchdog?)"] reasons = [] - if self.summary != "PASSED" or self.failures or self.errors: - reasons.append(f"summary says {self.summary} (failures={self.failures}, errors={self.errors})") + if self.summary != "PASSED" or self.failed or self.errors: + reasons.append(f"summary says {self.summary} (failed={self.failed}, errors={self.errors})") + passing_rows = sum(1 for status in self.results.values() if status == "PASS") + if self.passed is not None and self.passed != passing_rows: + reasons.append(f"summary says passed={self.passed} but {passing_rows} rows say PASS") if self.ran != len(self.results): reasons.append(f"'Ran {self.ran}' but {len(self.results)} result rows") if self.ran < min_tests: @@ -91,7 +103,10 @@ def parse(text: str) -> Report: report.ran = int(match.group(1)) elif match := SUMMARY_RE.match(line): report.summary = match.group(1).upper() - report.failures, report.errors = int(match.group(2)), int(match.group(3)) + counts = {key.lower(): int(value) for key, value in COUNT_RE.findall(match.group(2))} + report.passed = counts.get("passed") + report.failed = counts.get("failed", counts.get("failures", 0)) + report.errors = counts.get("errors", 0) return report |