From 2071e2beda763137416ac386d4bb953711ec995d Mon Sep 17 00:00:00 2001 From: cirakg Date: Sun, 18 Oct 2026 20:28:09 +0200 Subject: test(tools): add test-log judge, docs and policy checks with tests - ciq_test_report.py: decides pass/fail from Run No Evil output instead of monkeydo's unreliable exit code; writes JUnit XML. - check_docs.py: offline check of relative links and GitHub anchors. - lint_project.py: project rules (headers, no logging outside Diag, no secret-like literals, no network APIs or permissions). - check_budgets.py with budgets.json: perf and memory gates. - make_icon.py: reproducible launcher icon. - 31 pytest tests, including Hypothesis fuzzing of the log parser and integration checks against the real docs and sources; ruff and mypy (strict) clean. Co-Authored-By: Claude Opus 5.5 --- tools/budgets.json | 9 +++ tools/check_budgets.py | 88 ++++++++++++++++++++++ tools/check_docs.py | 125 +++++++++++++++++++++++++++++++ tools/ciq_test_report.py | 145 ++++++++++++++++++++++++++++++++++++ tools/lint_project.py | 112 ++++++++++++++++++++++++++++ tools/make_icon.py | 68 +++++++++++++++++ tools/tests/conftest.py | 25 +++++++ tools/tests/test_check_budgets.py | 45 +++++++++++ tools/tests/test_check_docs.py | 68 +++++++++++++++++ tools/tests/test_ciq_test_report.py | 99 ++++++++++++++++++++++++ tools/tests/test_lint_project.py | 81 ++++++++++++++++++++ tools/tests/test_make_icon.py | 25 +++++++ 12 files changed, 890 insertions(+) create mode 100644 tools/budgets.json create mode 100644 tools/check_budgets.py create mode 100644 tools/check_docs.py create mode 100644 tools/ciq_test_report.py create mode 100644 tools/lint_project.py create mode 100644 tools/make_icon.py create mode 100644 tools/tests/conftest.py create mode 100644 tools/tests/test_check_budgets.py create mode 100644 tools/tests/test_check_docs.py create mode 100644 tools/tests/test_ciq_test_report.py create mode 100644 tools/tests/test_lint_project.py create mode 100644 tools/tests/test_make_icon.py diff --git a/tools/budgets.json b/tools/budgets.json new file mode 100644 index 0000000..615cd74 --- /dev/null +++ b/tools/budgets.json @@ -0,0 +1,9 @@ +{ + "_about": "Simulator budgets checked by `make perf` and `make memtest` (README section 18). Simulator timings depend on your PC, so perf budgets only catch big regressions; from M4 on they are set to about 2x the measured baseline. Memory is a hard limit: at most 70 % of the watch-app memory of the device.", + "perf": { + "startup_ms": 3000 + }, + "mem": { + "mem_used_fraction": 0.70 + } +} diff --git a/tools/check_budgets.py b/tools/check_budgets.py new file mode 100644 index 0000000..1e26037 --- /dev/null +++ b/tools/check_budgets.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Compare the app's debug diagnostics with the budgets in tools/budgets.json. + +Debug builds print lines like + DIAG {"event":"first_frame","startup_ms":123,"mem_used":4567,"mem_total":786432} +(source/util/Diag.mc). app-diag.sh runs the app on each test device, saves the +console output to build/diag/.log, then calls this script. + +Usage: check_budgets.py {perf,mem} LOG [LOG ...] [--budgets FILE] [--summary OUT.json] +Exit: 0 within budget, 1 over budget, 2 missing or unreadable diagnostics. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +EXIT_OK, EXIT_OVER, EXIT_MISSING = 0, 1, 2 + + +def read_events(text: str) -> list[dict[str, Any]]: + """Every DIAG event in a console log, in order. Malformed lines are skipped.""" + events = [] + for line in text.splitlines(): + start = line.find("DIAG {") + if start < 0: + continue + try: + event = json.loads(line[start + len("DIAG ") :]) + except json.JSONDecodeError: + continue + if isinstance(event, dict): + events.append(event) + return events + + +def measure(kind: str, events: list[dict[str, Any]]) -> dict[str, float]: + """The values a budget kind looks at, from the first first_frame event.""" + frame = next((e for e in events if e.get("event") == "first_frame"), None) + if frame is None: + return {} + if kind == "perf": + return {"startup_ms": float(frame["startup_ms"])} + total = float(frame["mem_total"]) + return {"mem_used_fraction": float(frame["mem_used"]) / total if total else 1.0} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Check diagnostics against budgets.") + parser.add_argument("kind", choices=("perf", "mem")) + parser.add_argument("logs", type=Path, nargs="+") + parser.add_argument("--budgets", type=Path, default=Path("tools/budgets.json")) + parser.add_argument("--summary", type=Path) + args = parser.parse_args(argv) + + budgets: dict[str, float] = json.loads(args.budgets.read_text(encoding="utf-8"))[args.kind] + worst = EXIT_OK + summary: dict[str, dict[str, float]] = {} + for log in args.logs: + device = log.stem + try: + values = measure(args.kind, read_events(log.read_text(encoding="utf-8", errors="replace"))) + except (OSError, KeyError, ValueError) as err: + print(f"[{device}] cannot read diagnostics: {err}") + worst = max(worst, EXIT_MISSING) + continue + if not values: + print(f"[{device}] no DIAG first_frame line (did the app start? is this a debug build?)") + worst = max(worst, EXIT_MISSING) + continue + summary[device] = values + for name, limit in budgets.items(): + value = values[name] + over = value > limit + print(f"[{device}] {name} = {value:.3g} (budget {limit:.3g}) {'OVER BUDGET' if over else 'ok'}") + if over: + worst = max(worst, EXIT_OVER) + if args.summary: + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return worst + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/check_docs.py b/tools/check_docs.py new file mode 100644 index 0000000..977bd11 --- /dev/null +++ b/tools/check_docs.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Check the project's Markdown files for broken internal links. + +What it checks, offline (`make docs` runs without network): + * relative links to files: the target must exist in the repository; + * anchors (`#section`), in the same file or another Markdown file: a heading + with that GitHub-style anchor must exist. +External links (http, https, mailto) are not fetched; the network is off. + +Usage: check_docs.py [ROOT] (ROOT defaults to the current directory) +Exit: 0 when every link resolves, 1 otherwise. Each problem is printed as + `file:line: message`. +""" + +from __future__ import annotations + +import re +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +# Folders that hold generated or third-party files, never project docs. +SKIP_DIRS = {".git", ".cache", "build", "node_modules"} + +# [text](target) or [text](target "title"); the target may be wrapped in <...>. +LINK_RE = re.compile(r"\[(?:[^\]\\]|\\.)*\]\(\s*(<[^>]*>|[^)\s]+)(?:\s+\"[^\"]*\")?\s*\)") +HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$") +FENCE_RE = re.compile(r"^\s*(```|~~~)") +INLINE_CODE_RE = re.compile(r"`[^`]*`") +EXTERNAL_PREFIXES = ("http://", "https://", "mailto:") + + +@dataclass(frozen=True) +class Problem: + """One broken link, reported as `file:line: message`.""" + + path: Path + line: int + message: str + + def __str__(self) -> str: + return f"{self.path}:{self.line}: {self.message}" + + +def slugify(heading: str) -> str: + """Return the anchor GitHub generates for a heading. + + GitHub lowercases the text, drops everything except letters, digits, + spaces, hyphens and underscores, then turns spaces into hyphens. Inline + markup is removed first: `code` keeps its text, [text](url) keeps `text`. + """ + text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", heading) + text = text.replace("`", "").replace("*", "").strip().lower() + text = re.sub(r"[^\w\- ]", "", text) + return text.replace(" ", "-") + + +def iter_lines_outside_code(text: str) -> Iterator[tuple[int, str]]: + """Yield (line number, line) for lines outside fenced code blocks.""" + in_fence = False + for number, line in enumerate(text.splitlines(), start=1): + if FENCE_RE.match(line): + in_fence = not in_fence + continue + if not in_fence: + yield number, line + + +def anchors_of(path: Path) -> set[str]: + """All heading anchors of a Markdown file, with GitHub's -1, -2 suffixes + for repeated headings.""" + seen: dict[str, int] = {} + anchors: set[str] = set() + for _, line in iter_lines_outside_code(path.read_text(encoding="utf-8")): + match = HEADING_RE.match(line) + if not match: + continue + slug = slugify(match.group(2)) + count = seen.get(slug, 0) + anchors.add(slug if count == 0 else f"{slug}-{count}") + seen[slug] = count + 1 + return anchors + + +def markdown_files(root: Path) -> list[Path]: + """Every Markdown file of the project, skipping generated folders.""" + return sorted(p for p in root.rglob("*.md") if not any(part in SKIP_DIRS for part in p.relative_to(root).parts)) + + +def check_file(path: Path, root: Path, cache: dict[Path, set[str]]) -> list[Problem]: + """Check every link of one Markdown file.""" + problems: list[Problem] = [] + rel = path.relative_to(root) + for number, line in iter_lines_outside_code(path.read_text(encoding="utf-8")): + for match in LINK_RE.finditer(INLINE_CODE_RE.sub("", line)): + target = match.group(1).strip("<>") + if target.startswith(EXTERNAL_PREFIXES): + continue + file_part, _, anchor = target.partition("#") + dest = path if not file_part else (path.parent / file_part).resolve() + if not dest.exists(): + problems.append(Problem(rel, number, f"missing file '{file_part}'")) + continue + if anchor and dest.suffix == ".md": + if dest not in cache: + cache[dest] = anchors_of(dest) + if anchor not in cache[dest]: + problems.append(Problem(rel, number, f"no heading for anchor '#{anchor}' in {dest.name}")) + return problems + + +def main(argv: list[str]) -> int: + root = Path(argv[1] if len(argv) > 1 else ".").resolve() + cache: dict[Path, set[str]] = {} + files = markdown_files(root) + problems = [p for f in files for p in check_file(f, root, cache)] + for problem in problems: + print(problem) + print(f"check_docs: {len(files)} Markdown files, {len(problems)} broken links") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/ciq_test_report.py b/tools/ciq_test_report.py new file mode 100644 index 0000000..b35095f --- /dev/null +++ b/tools/ciq_test_report.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Turn the console output of a Connect IQ unit-test run into a verdict. + +`monkeydo app.prg -t` prints Garmin's "Run No Evil" report: + + Executing test smokeVersionIsSemver... + DEBUG (14:16): version 0.1.0 + PASS + ============================================================================== + RESULTS + Test: Status: + smokeVersionIsSemver PASS + Ran 1 test + PASSED (failures=0, errors=0) + +monkeydo's exit code is reported to be non-zero even when every test passes, +so this script decides instead. A run passes only when all of these hold: + * the RESULTS section and the final PASSED/FAILED line are present (a crash + or a watchdog kill leaves the log incomplete); + * the final line says PASSED with zero failures and zero errors; + * "Ran N tests" matches the number of per-test rows, and N >= --min-tests; + * every per-test row says PASS. + +Usage: ciq_test_report.py LOG --device ID [--junit OUT.xml] [--min-tests N] +Exit: 0 pass, 1 test failure, 2 log unreadable or incomplete. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from xml.sax.saxutils import escape, quoteattr + +EXIT_PASS, EXIT_FAIL, EXIT_INCOMPLETE = 0, 1, 2 + +SUMMARY_RE = re.compile(r"^(PASSED|FAILED)\s*\(\s*failures\s*=\s*(\d+)\s*,\s*errors\s*=\s*(\d+)\s*\)", re.I) +RAN_RE = re.compile(r"^Ran\s+(\d+)\s+tests?\b", re.I) +ROW_RE = re.compile(r"^(\S+)\s+(PASS|FAIL|ERROR)\s*$", re.I) +EXECUTING_RE = re.compile(r"^Executing test\s+(\S+?)(?:\.\.\.|…)?\s*$") + + +@dataclass +class Report: + """Everything parsed from one test log.""" + + results: dict[str, str] = field(default_factory=dict) # test name -> PASS/FAIL/ERROR + output: dict[str, list[str]] = field(default_factory=dict) # test name -> its console lines + ran: int | None = None + summary: str | None = None # PASSED or FAILED + failures: int = 0 + errors: int = 0 + + def problems(self, min_tests: int) -> tuple[int, list[str]]: + """Return (exit code, reasons) for this report.""" + if self.summary is None or self.ran is None: + return EXIT_INCOMPLETE, ["log has no complete RESULTS section (crash, timeout or watchdog?)"] + reasons = [] + if self.summary != "PASSED" or self.failures or self.errors: + reasons.append(f"summary says {self.summary} (failures={self.failures}, errors={self.errors})") + if self.ran != len(self.results): + reasons.append(f"'Ran {self.ran}' but {len(self.results)} result rows") + if self.ran < min_tests: + reasons.append(f"only {self.ran} tests ran, expected at least {min_tests}") + reasons += [f"{name}: {status}" for name, status in self.results.items() if status != "PASS"] + return (EXIT_FAIL if reasons else EXIT_PASS), reasons + + +def parse(text: str) -> Report: + """Parse a Run No Evil console log.""" + report = Report() + in_results = False + current: str | None = None + for raw in text.splitlines(): + line = raw.strip() + if not in_results: + match = EXECUTING_RE.match(line) + if match: + current = match.group(1) + report.output[current] = [] + elif line == "RESULTS": + in_results = True + elif current is not None and line and not line.startswith(("---", "===")): + report.output[current].append(raw) + continue + if (match := ROW_RE.match(line)) and not line.lower().startswith("test:"): + report.results[match.group(1)] = match.group(2).upper() + elif match := RAN_RE.match(line): + report.ran = int(match.group(1)) + elif match := SUMMARY_RE.match(line): + report.summary = match.group(1).upper() + report.failures, report.errors = int(match.group(2)), int(match.group(3)) + return report + + +def junit_xml(report: Report, device: str) -> str: + """A minimal JUnit XML document, readable by most CI dashboards.""" + failed = sum(1 for status in report.results.values() if status != "PASS") + cases = [] + for name, status in report.results.items(): + body = "\n".join(report.output.get(name, [])) + inner = f"{escape(body)}" + if status != "PASS": + inner = f"" + inner + cases.append(f" {inner}") + return ( + '\n' + f'\n' + + "\n".join(cases) + + "\n\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Judge a Connect IQ unit-test log.") + parser.add_argument("log", type=Path) + parser.add_argument("--device", required=True) + parser.add_argument("--junit", type=Path) + parser.add_argument("--min-tests", type=int, default=1) + args = parser.parse_args(argv) + + try: + text = args.log.read_text(encoding="utf-8", errors="replace") + except OSError as err: + print(f"[{args.device}] cannot read {args.log}: {err}") + return EXIT_INCOMPLETE + + report = parse(text) + code, reasons = report.problems(args.min_tests) + if args.junit: + args.junit.parent.mkdir(parents=True, exist_ok=True) + args.junit.write_text(junit_xml(report, args.device), encoding="utf-8") + + passed = sum(1 for status in report.results.values() if status == "PASS") + verdict = {EXIT_PASS: "PASS", EXIT_FAIL: "FAIL", EXIT_INCOMPLETE: "INCOMPLETE"}[code] + print(f"[{args.device}] {verdict}: {passed}/{len(report.results)} tests passed") + for reason in reasons: + print(f"[{args.device}] {reason}") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/lint_project.py b/tools/lint_project.py new file mode 100644 index 0000000..f11d35e --- /dev/null +++ b/tools/lint_project.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Project-specific lint rules for the Monkey C sources and the manifest. + +The Monkey C compiler (strict type checking, `make lint`) catches language +errors. These rules catch project policy violations the compiler can't see: + + MC001 every .mc file starts with a comment header explaining the file + MC002 System.println only in source/util/Diag.mc (no logging, so no secret + can ever end up in a log) and in test code + MC003 no TODO / FIXME / XXX markers in source/ (finished code only) + MC004 no secret-looking string literals in source/ (long Base32 or hex + runs); real secrets never belong in code, test vectors live in test/. + A literal that is not a secret (for example the Base32 alphabet) + can be allowed with the comment `// lint: allow-literal` on its line. + MC005 no network: manifest.xml declares no permissions and no source file + imports or uses Toybox.Communications + +Usage: lint_project.py [ROOT] +Exit: 0 clean, 1 problems found (printed as file:line: CODE message). +""" + +from __future__ import annotations + +import re +import sys +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path + +DIAG_FILE = Path("source/util/Diag.mc") +BASE32_SECRET_RE = re.compile(r'"[^"\n]*[A-Z2-7]{16,}[^"\n]*"') +HEX_SECRET_RE = re.compile(r'"[^"\n]*[0-9a-fA-F]{32,}[^"\n]*"') +MARKER_RE = re.compile(r"\b(TODO|FIXME|XXX)\b") +PRINTLN_RE = re.compile(r"\bSystem\.println\s*\(") +COMMUNICATIONS_RE = re.compile(r"\bToybox\.Communications\b|\bCommunications\.") +PERMISSION_RE = re.compile(r" str: + return f"{self.path}:{self.line}: {self.code} {self.message}" + + +def strip_line_comment(line: str) -> str: + """Drop a trailing // comment (good enough: the rules only look at code).""" + in_string = False + for i, ch in enumerate(line): + if ch == '"' and (i == 0 or line[i - 1] != "\\"): + in_string = not in_string + elif not in_string and line.startswith("//", i): + return line[:i] + return line + + +def check_mc(path: Path, rel: Path) -> Iterable[Problem]: + """All Monkey C rules for one source or test file.""" + lines = path.read_text(encoding="utf-8").splitlines() + first = next((line for line in lines if line.strip()), "") + if not first.lstrip().startswith("//"): + yield Problem(rel, 1, "MC001", "file must start with a // comment header that explains it") + + in_source = rel.parts[0] == "source" + for number, line in enumerate(lines, start=1): + code = strip_line_comment(line) + if PRINTLN_RE.search(code) and in_source and rel != DIAG_FILE: + yield Problem(rel, number, "MC002", "System.println outside source/util/Diag.mc") + if in_source and MARKER_RE.search(line): + yield Problem(rel, number, "MC003", "unfinished-work marker in source/") + secret_like = BASE32_SECRET_RE.search(code) or HEX_SECRET_RE.search(code) + if in_source and secret_like and ALLOW_LITERAL not in line: + yield Problem(rel, number, "MC004", "secret-looking string literal in source/") + if COMMUNICATIONS_RE.search(code): + yield Problem(rel, number, "MC005", "network API (Toybox.Communications) is not allowed") + + +def check_manifest(path: Path, rel: Path) -> Iterable[Problem]: + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if PERMISSION_RE.search(line): + yield Problem(rel, number, "MC005", "manifest must not request permissions") + + +def lint(root: Path) -> list[Problem]: + problems: list[Problem] = [] + for folder in ("source", "test"): + for path in sorted((root / folder).rglob("*.mc")): + problems += check_mc(path, path.relative_to(root)) + manifest = root / "manifest.xml" + if manifest.exists(): + problems += check_manifest(manifest, manifest.relative_to(root)) + else: + problems.append(Problem(Path("manifest.xml"), 1, "MC005", "manifest.xml is missing")) + return sorted(problems) + + +def main(argv: list[str]) -> int: + root = Path(argv[1] if len(argv) > 1 else ".").resolve() + problems = lint(root) + for problem in problems: + print(problem) + print(f"lint_project: {len(problems)} problems") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/make_icon.py b/tools/make_icon.py new file mode 100644 index 0000000..6bb1cbf --- /dev/null +++ b/tools/make_icon.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Draw the app's launcher icon: a white padlock with three "code" dots on a +teal disc, on a transparent background. + +The icon is drawn at 8x the target size and scaled down, which gives smooth +edges at the small sizes watches use (35 to 70 px). + +Usage: make_icon.py --size 60 --out resources/drawables/launcher_icon.png +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from PIL import Image, ImageDraw + +TEAL = (14, 124, 134, 255) +WHITE = (255, 255, 255, 255) +SUPERSAMPLE = 8 + + +def draw_icon(size: int) -> Image.Image: + """Return the icon as an RGBA image of size x size pixels.""" + s = size * SUPERSAMPLE + img = Image.new("RGBA", (s, s), (0, 0, 0, 0)) + d = ImageDraw.Draw(img) + + margin = s * 0.02 + d.ellipse((margin, margin, s - margin, s - margin), fill=TEAL) + + # Lock body: a rounded rectangle in the lower middle. + body_w, body_h = s * 0.50, s * 0.36 + body_x0, body_y0 = (s - body_w) / 2, s * 0.44 + d.rounded_rectangle((body_x0, body_y0, body_x0 + body_w, body_y0 + body_h), radius=s * 0.06, fill=WHITE) + + # Shackle: the top half of an ellipse plus two short legs into the body. + stroke = int(s * 0.07) + sx0, sx1, sy0, sy1 = s * 0.34, s * 0.66, s * 0.20, s * 0.56 + d.arc((sx0, sy0, sx1, sy1), start=180, end=360, fill=WHITE, width=stroke) + leg_top, leg_bottom = (sy0 + sy1) / 2, body_y0 + s * 0.02 + d.rectangle((sx0, leg_top, sx0 + stroke, leg_bottom), fill=WHITE) + d.rectangle((sx1 - stroke, leg_top, sx1, leg_bottom), fill=WHITE) + + # Three dots on the body stand for a one-time code. + radius = s * 0.035 + center_y = body_y0 + body_h * 0.5 + for i in range(3): + center_x = s * (0.38 + 0.12 * i) + d.ellipse((center_x - radius, center_y - radius, center_x + radius, center_y + radius), fill=TEAL) + + return img.resize((size, size), Image.Resampling.LANCZOS) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--size", type=int, default=60, help="icon width and height in pixels") + parser.add_argument("--out", type=Path, required=True, help="PNG file to write") + args = parser.parse_args() + if not 16 <= args.size <= 256: + parser.error("--size must be between 16 and 256") + args.out.parent.mkdir(parents=True, exist_ok=True) + draw_icon(args.size).save(args.out, format="PNG", optimize=True) + print(f"wrote {args.out} ({args.size}x{args.size})") + + +if __name__ == "__main__": + main() diff --git a/tools/tests/conftest.py b/tools/tests/conftest.py new file mode 100644 index 0000000..7054ecb --- /dev/null +++ b/tools/tests/conftest.py @@ -0,0 +1,25 @@ +"""Shared pytest setup for the tool tests. + +* Makes the modules in tools/ importable as top-level modules. +* Keeps Hypothesis' example database in build/ (git-ignored) and disables its + per-example deadline, which is flaky on a busy machine. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from hypothesis import settings +from hypothesis.database import DirectoryBasedExampleDatabase + +TOOLS_DIR = Path(__file__).resolve().parents[1] +PROJECT_ROOT = TOOLS_DIR.parent +sys.path.insert(0, str(TOOLS_DIR)) + +settings.register_profile( + "project", + database=DirectoryBasedExampleDatabase(str(PROJECT_ROOT / "build" / ".hypothesis")), + deadline=None, +) +settings.load_profile("project") diff --git a/tools/tests/test_check_budgets.py b/tools/tests/test_check_budgets.py new file mode 100644 index 0000000..35cc2f8 --- /dev/null +++ b/tools/tests/test_check_budgets.py @@ -0,0 +1,45 @@ +"""Tests for tools/check_budgets.py (perf and memory budgets).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import check_budgets as cb + +FRAME = 'DIAG {"event":"first_frame","startup_ms":250,"mem_used":100000,"mem_total":524288}' + + +def test_reads_events_and_skips_noise() -> None: + text = f"Device Version 1\n{FRAME}\nDIAG {{not json\nsome DIAG line\n" + assert cb.read_events(text) == [ + {"event": "first_frame", "startup_ms": 250, "mem_used": 100000, "mem_total": 524288} + ] + + +def test_measure_perf_and_memory() -> None: + events = cb.read_events(FRAME) + assert cb.measure("perf", events) == {"startup_ms": 250.0} + assert cb.measure("mem", events) == {"mem_used_fraction": 100000 / 524288} + assert cb.measure("perf", []) == {} + + +def run(tmp_path: Path, kind: str, log_text: str, budgets: dict[str, dict[str, float]]) -> int: + log = tmp_path / "fr255s.log" + log.write_text(log_text, encoding="utf-8") + budget_file = tmp_path / "budgets.json" + budget_file.write_text(json.dumps(budgets), encoding="utf-8") + return cb.main([kind, str(log), "--budgets", str(budget_file), "--summary", str(tmp_path / "s.json")]) + + +def test_within_budget(tmp_path: Path) -> None: + assert run(tmp_path, "perf", FRAME, {"perf": {"startup_ms": 300}}) == cb.EXIT_OK + assert json.loads((tmp_path / "s.json").read_text()) == {"fr255s": {"startup_ms": 250.0}} + + +def test_over_budget(tmp_path: Path) -> None: + assert run(tmp_path, "mem", FRAME, {"mem": {"mem_used_fraction": 0.1}}) == cb.EXIT_OVER + + +def test_missing_diagnostics(tmp_path: Path) -> None: + assert run(tmp_path, "perf", "app crashed\n", {"perf": {"startup_ms": 300}}) == cb.EXIT_MISSING diff --git a/tools/tests/test_check_docs.py b/tools/tests/test_check_docs.py new file mode 100644 index 0000000..67404c9 --- /dev/null +++ b/tools/tests/test_check_docs.py @@ -0,0 +1,68 @@ +"""Tests for tools/check_docs.py, the offline Markdown link checker.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import check_docs as cd +import pytest +from conftest import PROJECT_ROOT +from hypothesis import given +from hypothesis import strategies as st + + +@pytest.mark.parametrize( + ("heading", "anchor"), + [ + ("12.4 How each symbology is proven scannable", "124-how-each-symbology-is-proven-scannable"), + ("3e. Writing values back: unknown", "3e-writing-values-back-unknown"), + ("1. Scope", "1-scope"), + ("`make sim` and noVNC", "make-sim-and-novnc"), + ("A — B", "a--b"), + ("[Link](x.md) text", "link-text"), + ], +) +def test_slugify_matches_github(heading: str, anchor: str) -> None: + assert cd.slugify(heading) == anchor + + +@given(st.text()) +def test_slugify_output_is_anchor_safe(heading: str) -> None: + assert re.fullmatch(r"[\w\-]*", cd.slugify(heading)) + + +def write(tmp_path: Path, name: str, text: str) -> Path: + path = tmp_path / name + path.write_text(text, encoding="utf-8") + return path + + +def test_repeated_headings_get_numbered_anchors(tmp_path: Path) -> None: + doc = write(tmp_path, "a.md", "# Notes\n\n## Notes\n\n```\n# not a heading\n```\n") + assert cd.anchors_of(doc) == {"notes", "notes-1"} + + +def test_broken_links_are_reported(tmp_path: Path) -> None: + write(tmp_path, "other.md", "# Present\n") + doc = write( + tmp_path, + "a.md", + "# Top\n" + "[ok](#top) [ok2](other.md#present) [web](https://example.com/x)\n" + "[missing file](nope.md)\n" + "[missing anchor](other.md#absent)\n" + "`[inline code](ignored.md)`\n" + "```\n[fenced](ignored.md)\n```\n", + ) + problems = cd.check_file(doc, tmp_path, {}) + assert [(p.line, p.message) for p in problems] == [ + (3, "missing file 'nope.md'"), + (4, "no heading for anchor '#absent' in other.md"), + ] + + +def test_project_docs_have_no_broken_links() -> None: + cache: dict[Path, set[str]] = {} + problems = [p for f in cd.markdown_files(PROJECT_ROOT) for p in cd.check_file(f, PROJECT_ROOT, cache)] + assert problems == [], "\n".join(map(str, problems)) diff --git a/tools/tests/test_ciq_test_report.py b/tools/tests/test_ciq_test_report.py new file mode 100644 index 0000000..cdd3901 --- /dev/null +++ b/tools/tests/test_ciq_test_report.py @@ -0,0 +1,99 @@ +"""Tests for tools/ciq_test_report.py, the judge of Connect IQ test logs.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +from pathlib import Path + +import ciq_test_report as rep +from hypothesis import given +from hypothesis import strategies as st + +PASSING = """\ +Device Version 0.1.0 +Device id 1 name "A garmin device" +Shell Version 0.1.0 +------------------------------------------------------------------------------ +Executing test smokeVersionIsSemver... +DEBUG (14:16): version 0.1.0 +PASS +------------------------------------------------------------------------------ +Executing test otherTest... +PASS +============================================================================== +RESULTS +Test: Status: +smokeVersionIsSemver PASS +otherTest PASS +Ran 2 tests +PASSED (failures=0, errors=0) +Connection Finished +Closing shell and port +""" + + +def test_passing_log_passes() -> None: + report = rep.parse(PASSING) + assert report.results == {"smokeVersionIsSemver": "PASS", "otherTest": "PASS"} + assert report.ran == 2 + assert report.output["smokeVersionIsSemver"] == ["DEBUG (14:16): version 0.1.0", "PASS"] + assert report.problems(min_tests=1) == (rep.EXIT_PASS, []) + + +def test_status_words_are_case_insensitive() -> None: + text = PASSING.replace("PASS\n", "Pass\n").replace("PASSED (", "Passed (") + assert rep.parse(text).problems(min_tests=1)[0] == rep.EXIT_PASS + + +def test_failed_test_fails() -> None: + text = PASSING.replace("otherTest PASS", "otherTest FAIL").replace( + "PASSED (failures=0", "FAILED (failures=1" + ) + code, reasons = rep.parse(text).problems(min_tests=1) + assert code == rep.EXIT_FAIL + assert "otherTest: FAIL" in reasons + + +def test_error_row_fails_even_if_summary_claims_pass() -> None: + text = PASSING.replace("otherTest PASS", "otherTest ERROR") + code, reasons = rep.parse(text).problems(min_tests=1) + assert code == rep.EXIT_FAIL + assert "otherTest: ERROR" in reasons + + +def test_log_without_results_is_incomplete() -> None: + truncated = PASSING.partition("RESULTS")[0] + assert rep.parse(truncated).problems(min_tests=1)[0] == rep.EXIT_INCOMPLETE + + +def test_too_few_tests_fails() -> None: + code, reasons = rep.parse(PASSING).problems(min_tests=3) + assert code == rep.EXIT_FAIL + assert any("expected at least 3" in r for r in reasons) + + +def test_ran_count_must_match_rows() -> None: + code, reasons = rep.parse(PASSING.replace("Ran 2 tests", "Ran 3 tests")).problems(min_tests=1) + assert code == rep.EXIT_FAIL + assert any("'Ran 3'" in r for r in reasons) + + +def test_junit_output_is_valid(tmp_path: Path) -> None: + log = tmp_path / "fr70.log" + log.write_text(PASSING.replace("otherTest PASS", "otherTest FAIL")) + junit = tmp_path / "out" / "fr70.xml" + assert rep.main([str(log), "--device", "fr70", "--junit", str(junit)]) == rep.EXIT_FAIL + suite = ET.parse(junit).getroot() + assert suite.get("tests") == "2" + assert suite.get("failures") == "1" + assert [c.get("name") for c in suite.iter("testcase")] == ["smokeVersionIsSemver", "otherTest"] + + +def test_missing_log_is_incomplete(tmp_path: Path) -> None: + assert rep.main([str(tmp_path / "nope.log"), "--device", "x"]) == rep.EXIT_INCOMPLETE + + +@given(st.text()) +def test_parser_never_crashes(text: str) -> None: + code, _ = rep.parse(text).problems(min_tests=1) + assert code in (rep.EXIT_PASS, rep.EXIT_FAIL, rep.EXIT_INCOMPLETE) diff --git a/tools/tests/test_lint_project.py b/tools/tests/test_lint_project.py new file mode 100644 index 0000000..be0151e --- /dev/null +++ b/tools/tests/test_lint_project.py @@ -0,0 +1,81 @@ +"""Tests for tools/lint_project.py, the project policy linter.""" + +from __future__ import annotations + +from pathlib import Path + +import lint_project as lp +from conftest import PROJECT_ROOT + +GOOD_MANIFEST = "\n" + + +def make_project(tmp_path: Path, files: dict[str, str]) -> Path: + for name, text in {"manifest.xml": GOOD_MANIFEST, **files}.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return tmp_path + + +def codes(root: Path) -> list[tuple[str, str, int]]: + return [(str(p.path), p.code, p.line) for p in lp.lint(root)] + + +def test_clean_project_passes(tmp_path: Path) -> None: + root = make_project( + tmp_path, + { + "source/util/Diag.mc": '// Diag\nSystem.println("DIAG {}");\n', + "source/otp/Base32.mc": ( + '// Base32\nconst ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; // lint: allow-literal\n' + ), + "test/T.mc": ( + "// T\n" + 'System.println("debug output in tests is allowed");\n' + 'const V = "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ";\n' + ), + }, + ) + assert codes(root) == [] + + +def test_each_rule_fires(tmp_path: Path) -> None: + root = make_project( + tmp_path, + { + "source/app/NoHeader.mc": "import Toybox.Lang;\n", + "source/app/Log.mc": '// Log\nSystem.println("secret?");\n', + "source/app/Todo.mc": "// Todo\nvar x = 1; // TODO finish\n", + "source/app/Secret.mc": ( + '// Secret\nconst S = "JBSWY3DPEHPK3PXPJBSWY3DP";\nconst H = "0123456789abcdef0123456789abcdef";\n' + ), + "source/app/Net.mc": "// Net\nimport Toybox.Communications;\n", + }, + ) + assert codes(root) == [ + ("source/app/Log.mc", "MC002", 2), + ("source/app/Net.mc", "MC005", 2), + ("source/app/NoHeader.mc", "MC001", 1), + ("source/app/Secret.mc", "MC004", 2), + ("source/app/Secret.mc", "MC004", 3), + ("source/app/Todo.mc", "MC003", 2), + ] + + +def test_println_in_comment_is_ignored(tmp_path: Path) -> None: + root = make_project(tmp_path, {"source/app/A.mc": "// A\nvar a = 1; // System.println(a)\n"}) + assert codes(root) == [] + + +def test_manifest_permissions_are_rejected(tmp_path: Path) -> None: + root = make_project( + tmp_path, + {"manifest.xml": '\n\n\n'}, + ) + assert codes(root) == [("manifest.xml", "MC005", 2)] + + +def test_project_passes_its_own_rules() -> None: + problems = lp.lint(PROJECT_ROOT) + assert problems == [], "\n".join(map(str, problems)) diff --git a/tools/tests/test_make_icon.py b/tools/tests/test_make_icon.py new file mode 100644 index 0000000..66128c5 --- /dev/null +++ b/tools/tests/test_make_icon.py @@ -0,0 +1,25 @@ +"""Tests for tools/make_icon.py (launcher icon generator).""" + +from __future__ import annotations + +import make_icon +from PIL import Image + + +def rgba(img: Image.Image, xy: tuple[int, int]) -> tuple[int, ...]: + """One RGBA pixel as a tuple (Pillow's getpixel returns a wider union type).""" + value = img.getpixel(xy) + assert isinstance(value, tuple) + return value + + +def test_icon_size_transparency_and_colours() -> None: + img = make_icon.draw_icon(60) + assert img.size == (60, 60) + assert img.mode == "RGBA" + # Corners are outside the disc, so fully transparent. + assert rgba(img, (0, 0))[3] == 0 + # The disc edge is teal, the lock body in the middle is white. + r, g, b, a = rgba(img, (30, 4)) + assert a > 200 and g > r and b > r + assert rgba(img, (22, 40))[:3] == (255, 255, 255) -- cgit v1.2.3