1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
#!/usr/bin/env python3
"""Turn the console output of a Connect IQ unit-test run into a verdict.
`monkeydo app.prg <device> -t` prints Garmin's "Run No Evil" report:
Executing test smokeVersionIsSemver...
DEBUG (14:16): version 0.1.0
PASS
==============================================================================
RESULTS
Test: Status:
smokeVersionIsSemver PASS
Ran 1 test
PASSED (failures=0, errors=0)
monkeydo's exit code is reported to be non-zero even when every test passes,
so this script decides instead. A run passes only when all of these hold:
* the RESULTS section and the final PASSED/FAILED line are present (a crash
or a watchdog kill leaves the log incomplete);
* the final line says PASSED with zero failures and zero errors;
* "Ran N tests" matches the number of per-test rows, and N >= --min-tests;
* every per-test row says PASS.
Usage: ciq_test_report.py LOG --device ID [--junit OUT.xml] [--min-tests N]
Exit: 0 pass, 1 test failure, 2 log unreadable or incomplete.
"""
from __future__ import annotations
import argparse
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from xml.sax.saxutils import escape, quoteattr
EXIT_PASS, EXIT_FAIL, EXIT_INCOMPLETE = 0, 1, 2
SUMMARY_RE = re.compile(r"^(PASSED|FAILED)\s*\(\s*failures\s*=\s*(\d+)\s*,\s*errors\s*=\s*(\d+)\s*\)", re.I)
RAN_RE = re.compile(r"^Ran\s+(\d+)\s+tests?\b", re.I)
ROW_RE = re.compile(r"^(\S+)\s+(PASS|FAIL|ERROR)\s*$", re.I)
EXECUTING_RE = re.compile(r"^Executing test\s+(\S+?)(?:\.\.\.|…)?\s*$")
@dataclass
class Report:
"""Everything parsed from one test log."""
results: dict[str, str] = field(default_factory=dict) # test name -> PASS/FAIL/ERROR
output: dict[str, list[str]] = field(default_factory=dict) # test name -> its console lines
ran: int | None = None
summary: str | None = None # PASSED or FAILED
failures: int = 0
errors: int = 0
def problems(self, min_tests: int) -> tuple[int, list[str]]:
"""Return (exit code, reasons) for this report."""
if self.summary is None or self.ran is None:
return EXIT_INCOMPLETE, ["log has no complete RESULTS section (crash, timeout or watchdog?)"]
reasons = []
if self.summary != "PASSED" or self.failures or self.errors:
reasons.append(f"summary says {self.summary} (failures={self.failures}, errors={self.errors})")
if self.ran != len(self.results):
reasons.append(f"'Ran {self.ran}' but {len(self.results)} result rows")
if self.ran < min_tests:
reasons.append(f"only {self.ran} tests ran, expected at least {min_tests}")
reasons += [f"{name}: {status}" for name, status in self.results.items() if status != "PASS"]
return (EXIT_FAIL if reasons else EXIT_PASS), reasons
def parse(text: str) -> Report:
"""Parse a Run No Evil console log."""
report = Report()
in_results = False
current: str | None = None
for raw in text.splitlines():
line = raw.strip()
if not in_results:
match = EXECUTING_RE.match(line)
if match:
current = match.group(1)
report.output[current] = []
elif line == "RESULTS":
in_results = True
elif current is not None and line and not line.startswith(("---", "===")):
report.output[current].append(raw)
continue
if (match := ROW_RE.match(line)) and not line.lower().startswith("test:"):
report.results[match.group(1)] = match.group(2).upper()
elif match := RAN_RE.match(line):
report.ran = int(match.group(1))
elif match := SUMMARY_RE.match(line):
report.summary = match.group(1).upper()
report.failures, report.errors = int(match.group(2)), int(match.group(3))
return report
def junit_xml(report: Report, device: str) -> str:
"""A minimal JUnit XML document, readable by most CI dashboards."""
failed = sum(1 for status in report.results.values() if status != "PASS")
cases = []
for name, status in report.results.items():
body = "\n".join(report.output.get(name, []))
inner = f"<system-out>{escape(body)}</system-out>"
if status != "PASS":
inner = f"<failure message={quoteattr(status)}/>" + inner
cases.append(f" <testcase classname={quoteattr(device)} name={quoteattr(name)}>{inner}</testcase>")
return (
'<?xml version="1.0" encoding="UTF-8"?>\n'
f'<testsuite name={quoteattr("connectiq." + device)} tests="{len(report.results)}" failures="{failed}">\n'
+ "\n".join(cases)
+ "\n</testsuite>\n"
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Judge a Connect IQ unit-test log.")
parser.add_argument("log", type=Path)
parser.add_argument("--device", required=True)
parser.add_argument("--junit", type=Path)
parser.add_argument("--min-tests", type=int, default=1)
args = parser.parse_args(argv)
try:
text = args.log.read_text(encoding="utf-8", errors="replace")
except OSError as err:
print(f"[{args.device}] cannot read {args.log}: {err}")
return EXIT_INCOMPLETE
report = parse(text)
code, reasons = report.problems(args.min_tests)
if args.junit:
args.junit.parent.mkdir(parents=True, exist_ok=True)
args.junit.write_text(junit_xml(report, args.device), encoding="utf-8")
passed = sum(1 for status in report.results.values() if status == "PASS")
verdict = {EXIT_PASS: "PASS", EXIT_FAIL: "FAIL", EXIT_INCOMPLETE: "INCOMPLETE"}[code]
print(f"[{args.device}] {verdict}: {passed}/{len(report.results)} tests passed")
for reason in reasons:
print(f"[{args.device}] {reason}")
return code
if __name__ == "__main__":
sys.exit(main())
|
