1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
#!/usr/bin/env python3
# Author: cirakg & claude ai
# Summary: Decide pass/fail of a Connect IQ unit-test run from its console log, and write JUnit XML.
"""Turn the console output of a Connect IQ unit-test run into a verdict.
`monkeydo app.prg <device> -t` prints Garmin's "Run No Evil" report. Captured
from SDK 9.2.0 (tools/tests/data/runnoevil_pass.log):
Executing test smokeVersionIsSemver...
DEBUG (18:41): version 0.1.0
PASS
==============================================================================
RESULTS
Test: Status:
smokeVersionIsSemver PASS
Ran 1 test
PASSED (passed=1, failed=0, errors=0)
Garmin's documentation shows the older summary form `PASSED (failures=0,
errors=0)`; both are accepted.
monkeydo exits with 1 even when every test passes (verified with SDK 9.2.0),
so this script decides instead. A run passes only when all of these hold:
* the RESULTS section and the final PASSED/FAILED line are present (a crash
or a watchdog kill leaves the log incomplete);
* the final line says PASSED with zero failed and zero errors, and its
passed count (when present) equals the number of passing rows;
* "Ran N tests" matches the number of per-test rows, and N >= --min-tests;
* every per-test row says PASS.
Usage: ciq_test_report.py LOG --device ID [--junit OUT.xml] [--min-tests N]
Exit: 0 pass, 1 test failure, 2 log unreadable or incomplete.
"""
from __future__ import annotations
import argparse
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from xml.sax.saxutils import escape, quoteattr
EXIT_PASS, EXIT_FAIL, EXIT_INCOMPLETE = 0, 1, 2
SUMMARY_RE = re.compile(r"^(PASSED|FAILED)\s*\(([^)]*)\)\s*$", re.I)
COUNT_RE = re.compile(r"(\w+)\s*=\s*(\d+)")
RAN_RE = re.compile(r"^Ran\s+(\d+)\s+tests?\b", re.I)
ROW_RE = re.compile(r"^(\S+)\s+(PASS|FAIL|ERROR)\s*$", re.I)
EXECUTING_RE = re.compile(r"^Executing test\s+(\S+?)(?:\.\.\.|…)?\s*$")
@dataclass
class Report:
"""Everything parsed from one test log."""
results: dict[str, str] = field(default_factory=dict) # test name -> PASS/FAIL/ERROR
output: dict[str, list[str]] = field(default_factory=dict) # test name -> its console lines
ran: int | None = None
summary: str | None = None # PASSED or FAILED
passed: int | None = None # absent in the older summary format
failed: int = 0
errors: int = 0
def problems(self, min_tests: int) -> tuple[int, list[str]]:
"""Return (exit code, reasons) for this report."""
if self.summary is None or self.ran is None:
return EXIT_INCOMPLETE, ["log has no complete RESULTS section (crash, timeout or watchdog?)"]
reasons = []
if self.summary != "PASSED" or self.failed or self.errors:
reasons.append(f"summary says {self.summary} (failed={self.failed}, errors={self.errors})")
passing_rows = sum(1 for status in self.results.values() if status == "PASS")
if self.passed is not None and self.passed != passing_rows:
reasons.append(f"summary says passed={self.passed} but {passing_rows} rows say PASS")
if self.ran != len(self.results):
reasons.append(f"'Ran {self.ran}' but {len(self.results)} result rows")
if self.ran < min_tests:
reasons.append(f"only {self.ran} tests ran, expected at least {min_tests}")
reasons += [f"{name}: {status}" for name, status in self.results.items() if status != "PASS"]
return (EXIT_FAIL if reasons else EXIT_PASS), reasons
def parse(text: str) -> Report:
"""Parse a Run No Evil console log."""
report = Report()
in_results = False
current: str | None = None
for raw in text.splitlines():
line = raw.strip()
if not in_results:
match = EXECUTING_RE.match(line)
if match:
current = match.group(1)
report.output[current] = []
elif line == "RESULTS":
in_results = True
elif current is not None and line and not line.startswith(("---", "===")):
report.output[current].append(raw)
continue
if (match := ROW_RE.match(line)) and not line.lower().startswith("test:"):
report.results[match.group(1)] = match.group(2).upper()
elif match := RAN_RE.match(line):
report.ran = int(match.group(1))
elif match := SUMMARY_RE.match(line):
report.summary = match.group(1).upper()
counts = {key.lower(): int(value) for key, value in COUNT_RE.findall(match.group(2))}
report.passed = counts.get("passed")
report.failed = counts.get("failed", counts.get("failures", 0))
report.errors = counts.get("errors", 0)
return report
def junit_xml(report: Report, device: str) -> str:
"""A minimal JUnit XML document, readable by most CI dashboards."""
failed = sum(1 for status in report.results.values() if status != "PASS")
cases = []
for name, status in report.results.items():
body = "\n".join(report.output.get(name, []))
inner = f"<system-out>{escape(body)}</system-out>"
if status != "PASS":
inner = f"<failure message={quoteattr(status)}/>" + inner
cases.append(f" <testcase classname={quoteattr(device)} name={quoteattr(name)}>{inner}</testcase>")
return (
'<?xml version="1.0" encoding="UTF-8"?>\n'
f'<testsuite name={quoteattr("connectiq." + device)} tests="{len(report.results)}" failures="{failed}">\n'
+ "\n".join(cases)
+ "\n</testsuite>\n"
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Judge a Connect IQ unit-test log.")
parser.add_argument("log", type=Path)
parser.add_argument("--device", required=True)
parser.add_argument("--junit", type=Path)
parser.add_argument("--min-tests", type=int, default=1)
args = parser.parse_args(argv)
try:
text = args.log.read_text(encoding="utf-8", errors="replace")
except OSError as err:
print(f"[{args.device}] cannot read {args.log}: {err}")
return EXIT_INCOMPLETE
report = parse(text)
code, reasons = report.problems(args.min_tests)
if args.junit:
args.junit.parent.mkdir(parents=True, exist_ok=True)
args.junit.write_text(junit_xml(report, args.device), encoding="utf-8")
passed = sum(1 for status in report.results.values() if status == "PASS")
verdict = {EXIT_PASS: "PASS", EXIT_FAIL: "FAIL", EXIT_INCOMPLETE: "INCOMPLETE"}[code]
print(f"[{args.device}] {verdict}: {passed}/{len(report.results)} tests passed")
for reason in reasons:
print(f"[{args.device}] {reason}")
return code
if __name__ == "__main__":
sys.exit(main())
|
