1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
"""Tests for tools/ciq_test_report.py, the judge of Connect IQ test logs."""
from __future__ import annotations
import xml.etree.ElementTree as ET
from pathlib import Path
import ciq_test_report as rep
from hypothesis import given
from hypothesis import strategies as st
PASSING = """\
Device Version 0.1.0
Device id 1 name "A garmin device"
Shell Version 0.1.0
------------------------------------------------------------------------------
Executing test smokeVersionIsSemver...
DEBUG (14:16): version 0.1.0
PASS
------------------------------------------------------------------------------
Executing test otherTest...
PASS
==============================================================================
RESULTS
Test: Status:
smokeVersionIsSemver PASS
otherTest PASS
Ran 2 tests
PASSED (failures=0, errors=0)
Connection Finished
Closing shell and port
"""
def test_passing_log_passes() -> None:
report = rep.parse(PASSING)
assert report.results == {"smokeVersionIsSemver": "PASS", "otherTest": "PASS"}
assert report.ran == 2
assert report.output["smokeVersionIsSemver"] == ["DEBUG (14:16): version 0.1.0", "PASS"]
assert report.problems(min_tests=1) == (rep.EXIT_PASS, [])
def test_status_words_are_case_insensitive() -> None:
text = PASSING.replace("PASS\n", "Pass\n").replace("PASSED (", "Passed (")
assert rep.parse(text).problems(min_tests=1)[0] == rep.EXIT_PASS
def test_failed_test_fails() -> None:
text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace(
"PASSED (failures=0", "FAILED (failures=1"
)
code, reasons = rep.parse(text).problems(min_tests=1)
assert code == rep.EXIT_FAIL
assert "otherTest: FAIL" in reasons
def test_error_row_fails_even_if_summary_claims_pass() -> None:
text = PASSING.replace("otherTest PASS", "otherTest ERROR")
code, reasons = rep.parse(text).problems(min_tests=1)
assert code == rep.EXIT_FAIL
assert "otherTest: ERROR" in reasons
def test_log_without_results_is_incomplete() -> None:
truncated = PASSING.partition("RESULTS")[0]
assert rep.parse(truncated).problems(min_tests=1)[0] == rep.EXIT_INCOMPLETE
def test_too_few_tests_fails() -> None:
code, reasons = rep.parse(PASSING).problems(min_tests=3)
assert code == rep.EXIT_FAIL
assert any("expected at least 3" in r for r in reasons)
def test_ran_count_must_match_rows() -> None:
code, reasons = rep.parse(PASSING.replace("Ran 2 tests", "Ran 3 tests")).problems(min_tests=1)
assert code == rep.EXIT_FAIL
assert any("'Ran 3'" in r for r in reasons)
def test_junit_output_is_valid(tmp_path: Path) -> None:
log = tmp_path / "fr70.log"
log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL"))
junit = tmp_path / "out" / "fr70.xml"
assert rep.main([str(log), "--device", "fr70", "--junit", str(junit)]) == rep.EXIT_FAIL
suite = ET.parse(junit).getroot()
assert suite.get("tests") == "2"
assert suite.get("failures") == "1"
assert [c.get("name") for c in suite.iter("testcase")] == ["smokeVersionIsSemver", "otherTest"]
def test_missing_log_is_incomplete(tmp_path: Path) -> None:
assert rep.main([str(tmp_path / "nope.log"), "--device", "x"]) == rep.EXIT_INCOMPLETE
@given(st.text())
def test_parser_never_crashes(text: str) -> None:
code, _ = rep.parse(text).problems(min_tests=1)
assert code in (rep.EXIT_PASS, rep.EXIT_FAIL, rep.EXIT_INCOMPLETE)
|
