diff options
| author | cirakg <cirakg@cirakg.xyz> | 2026-10-21 03:55:10 +0200 |
|---|---|---|
| committer | cirakg <cirakg@cirakg.xyz> | 2026-10-21 03:55:10 +0200 |
| commit | 56e0c351bb95cfdf63aa76ab8c5d725a694bad63 (patch) | |
| tree | e99c7af6eca0656948279055c20a5f87d855d4c8 | |
| parent | 5e8ac1e2f5a660e55ea132996e1d5730cfa4fa64 (diff) | |
test(build): add make watchdog, the glance memory budget and the glance screenshot
- make watchdog (in make ci): a stress build encodes, packs, turns into
runs and draws the largest code of every 2D symbology at two steps per
event on the Forerunner 255S and 70; it fails on a watchdog trip, and
the last code left on the screen must decode (bitmap path on the 70).
- make memtest checks the glance (DIAG glance, budget 70 % of 64 KB).
- screenshots.sh launches the app again after the flow and compares the
glance with the favourite's code (glance_code.png on 7 watches); the
About goldens show 0.8.0.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
26 files changed, 425 insertions, 31 deletions
@@ -40,6 +40,9 @@ TEST_DEVICES ?= fr255s venusq2 fr70 fr965 # Extra screenshot-only devices (README 14): MIP with touch and buttons, # buttons-only AMOLED, the largest rectangle. SPOT_DEVICES ?= fenix7 instinct3amoled45mm venux1 +# `make watchdog`: one device per watchdog class (120,000 and 240,000 +# bytecodes per event). +WATCHDOG_DEVICES ?= fr255s fr70 # `make lock` only uses package releases published before this moment. LOCK_CUTOFF ?= 2026-10-01T00:00:00Z # Set to `nuke` to skip the confirmation prompt of `make nuke`. @@ -72,7 +75,7 @@ DOCS_RUN = $(DOCKER) run --rm --init --label project=$(PROJECT) \ --read-only --tmpfs /tmp:rw,exec,nosuid,size=1g --shm-size 512m \ -v "$(ROOT)":/work -w /work --network none $(DOCS_IMAGE) -.PHONY: help image docs-image sdk key build test test-tools sim screenshots screenshots-approve perf memtest icons vectors corpus codes spike spike-sim \ +.PHONY: help image docs-image sdk key build test test-tools sim screenshots screenshots-approve perf memtest watchdog icons vectors corpus codes spike spike-sim \ lint package docs ci shell clean nuke lock products \ check-docker dirs need-image need-docs-image @@ -177,6 +180,9 @@ perf: dirs need-image ## Start-up time on TEST_DEVICES against tools/budgets.jso memtest: dirs need-image ## Memory use on TEST_DEVICES against tools/budgets.json $(DEV_OFFLINE) app-diag.sh mem $(TEST_DEVICES) +watchdog: dirs need-image ## Largest 2D codes at twice the steps per event on WATCHDOG_DEVICES (no watchdog trip) + $(DEV_OFFLINE) app-diag.sh watchdog $(WATCHDOG_DEVICES) + lint: dirs need-image ## Strict type check (warnings fail), project rules, Python lint and types $(DEV_OFFLINE) lint.sh $(TEST_DEVICES) @@ -191,7 +197,7 @@ docs: dirs need-image need-docs-image ## Markdown lint, link check, Mermaid rend fi; \ done' -ci: lint test screenshots perf memtest package docs ## Everything above, in order; stops at the first failure +ci: lint test screenshots perf memtest watchdog package docs ## Everything above, in order; stops at the first failure @echo "make ci: all checks passed" # ---------------------------------------------------------------- cleanup diff --git a/docker/scripts/app-diag.sh b/docker/scripts/app-diag.sh index 804d68a..025fd49 100755 --- a/docker/scripts/app-diag.sh +++ b/docker/scripts/app-diag.sh @@ -1,21 +1,74 @@ #!/usr/bin/env bash # Author: cirakg & claude ai -# Summary: Run the debug app on each device in the headless simulator and check start-up time or memory budgets. +# Summary: Run the debug app on each device in the headless simulator and check start-up time, memory, or the watchdog stress test. # -# Usage: app-diag.sh perf|mem DEVICE [DEVICE ...] +# Usage: app-diag.sh perf|mem|watchdog DEVICE [DEVICE ...] # Output: build/diag/<DEVICE>.log (console with DIAG and HARNESS lines) and # build/diag/<perf|mem>-summary.json; verdict from tools/check_budgets.py. +# watchdog: the stress build (source/util/StressTest.mc) on each +# device, build/diag/watchdog-<DEVICE>.log; it fails when the +# simulator reports an error (such as "Watchdog Tripped"), the +# test doesn't finish, or its last code (left on the screen) +# doesn't decode. set -euo pipefail # shellcheck source=ciq-env.sh source "$(dirname "${BASH_SOURCE[0]}")/ciq-env.sh" -kind="${1:?usage: app-diag.sh perf|mem DEVICE [DEVICE ...]}" +kind="${1:?usage: app-diag.sh perf|mem|watchdog DEVICE [DEVICE ...]}" shift -[[ "${kind}" == perf || "${kind}" == mem ]] || die "unknown kind '${kind}' (use perf or mem)" -(($# > 0)) || die "usage: app-diag.sh perf|mem DEVICE [DEVICE ...]" +[[ "${kind}" == perf || "${kind}" == mem || "${kind}" == watchdog ]] || die "unknown kind '${kind}' (use perf, mem or watchdog)" +(($# > 0)) || die "usage: app-diag.sh perf|mem|watchdog DEVICE [DEVICE ...]" ciq_require_sdk ciq_require_key +# The stress test takes a few minutes per device (about 3,500 job steps at +# 50 ms on the 512 KB watches). +WATCHDOG_TIMEOUT_S=1200 +STRESS_ITEMS=10 + +if [[ "${kind}" == watchdog ]]; then + for device in "$@"; do + build-app.sh stress "${device}" + done + ciq_start_display + ciq_start_simulator + mkdir -p "${BUILD_DIR}/diag" + failed=0 + for device in "$@"; do + log="${BUILD_DIR}/diag/watchdog-${device}.log" + log "watchdog stress test on ${device}" + ciq_launch_app "${device}" "${log}" stress || log "no first frame on ${device}" + for _ in $(seq 1 "${WATCHDOG_TIMEOUT_S}"); do + grep -q "DIAG stress all done\|Error" "${log}" && break + kill -0 "${CIQ_MONKEYDO_PID}" 2>/dev/null || break + sleep 1 + done + # The last code stays on the screen; it must decode (on most watches + # it was drawn from a bitmap). + decoded=1 + if grep -aq "DIAG stress all done" "${log}"; then + sleep 1 + python3 /work/tools/ui_flow.py --decode "OTPW stress" --device-dir "${CIQ_HOME}/Devices/${device}" \ + --window "$(ciq_simulator_window)" --out "${BUILD_DIR}/diag/watchdog-${device}" >&2 || decoded=0 + fi + ciq_stop_app + grep -a "DIAG stress" "${log}" | sed "s/^/[${device}] /" >&2 || true + done_count="$(grep -ac "DIAG stress done name=" "${log}" || true)" + if grep -aq "Error" "${log}"; then + grep -a "Error" "${log}" | sed "s/^/[${device}] /" >&2 + failed=1 + elif ! grep -aq "DIAG stress all done" "${log}" || ((done_count != STRESS_ITEMS)); then + log "${device}: the stress test did not finish (${done_count} of ${STRESS_ITEMS} items)" + failed=1 + elif ((decoded == 0)); then + failed=1 + fi + done + ((failed == 0)) || die "watchdog stress test failed; logs in build/diag/" + log "watchdog stress test passed on: $*" + exit 0 +fi + for device in "$@"; do build-app.sh debug "${device}" done diff --git a/docker/scripts/build-app.sh b/docker/scripts/build-app.sh index 1a7d59a..5ead272 100755 --- a/docker/scripts/build-app.sh +++ b/docker/scripts/build-app.sh @@ -1,21 +1,22 @@ #!/usr/bin/env bash # Author: cirakg & claude ai -# Summary: Compile the watch app for one device: debug, release, unit-test or UI-test build. +# Summary: Compile the watch app for one device: debug, release, unit-test, UI-test or watchdog stress build. # -# Usage: build-app.sh debug|release|test|uitest DEVICE +# Usage: build-app.sh debug|release|test|uitest|stress DEVICE # Output: build/<mode>/<DEVICE>.prg, and the compiler log next to it. # # Policy (README section 17.7): strict type checking (-l 3), and any compiler # warning fails the build. UI-test builds add uitest.jungle (fixed clock, -# fake entries; see source/util/TestMode.mc). Test builds add test.jungle, so test code never -# reaches debug or release builds; release builds (-r) also drop every +# fake entries; see source/util/TestMode.mc), stress builds stress.jungle (the +# watchdog stress test, source/util/StressTest.mc). Test builds add +# test.jungle, so test code never reaches debug or release builds; release builds (-r) also drop every # (:debug) declaration, such as the diagnostics in source/util/Diag.mc. set -euo pipefail # shellcheck source=ciq-env.sh source "$(dirname "${BASH_SOURCE[0]}")/ciq-env.sh" -mode="${1:?usage: build-app.sh debug|release|test|uitest DEVICE}" -device="${2:?usage: build-app.sh debug|release|test|uitest DEVICE}" +mode="${1:?usage: build-app.sh debug|release|test|uitest|stress DEVICE}" +device="${2:?usage: build-app.sh debug|release|test|uitest|stress DEVICE}" ciq_require_sdk ciq_require_key @@ -29,11 +30,12 @@ case "${mode}" in debug) ;; release) args+=(-r) ;; uitest) jungles="${jungles};/work/uitest.jungle" ;; + stress) jungles="${jungles};/work/stress.jungle" ;; test) jungles="${jungles};/work/test.jungle" args+=(-t) ;; - *) die "unknown build mode '${mode}' (use debug, release, test or uitest)" ;; + *) die "unknown build mode '${mode}' (use debug, release, test, uitest or stress)" ;; esac out_dir="${BUILD_DIR}/${mode}" diff --git a/docker/scripts/screenshots.sh b/docker/scripts/screenshots.sh index 414af52..91477c4 100755 --- a/docker/scripts/screenshots.sh +++ b/docker/scripts/screenshots.sh @@ -9,7 +9,9 @@ # # Uses the UI-test build (uitest.jungle): fake entries, PIN 123456 and a # frozen clock, so every screenshot is reproducible. tools/ui_flow.py -# drives the app and checks the screens. +# drives the app and checks the screens. Then the app is launched again and +# its glance, which now shows the favourite's code (the first run sealed it), +# is checked as glance_code.png. set -euo pipefail # shellcheck source=ciq-env.sh source "$(dirname "${BASH_SOURCE[0]}")/ciq-env.sh" @@ -26,6 +28,34 @@ ciq_start_display ciq_start_simulator out="${BUILD_DIR}/screenshots" + +# A new launch starts in the glance; screenshot it before anything opens the app. +shoot_glance() { + local device="$1" glance_log="${out}/$1/glance.log" + : >"${glance_log}" + monkeydo "${BUILD_DIR}/uitest/${device}.prg" "${device}" >>"${glance_log}" 2>&1 & + CIQ_MONKEYDO_PID=$! + local shown=0 + for _ in $(seq 1 600); do + if grep -q "DIAG shown glance_code" "${glance_log}"; then + shown=1 + break + fi + sleep 0.05 + done + if ((shown == 0)); then + ciq_stop_app + log "${device}: the glance with the favourite's code didn't appear; see ${glance_log}" + return 1 + fi + sleep 1 + local status=0 + python3 /work/tools/ui_flow.py --shot glance_code --device-dir "${CIQ_HOME}/Devices/${device}" \ + --window "$(ciq_simulator_window)" --out "${out}/${device}" --golden "/work/test/golden/${device}" \ + "${approve[@]}" || status=1 + ciq_stop_app + return "${status}" +} mkdir -p "${out}" approve=() [[ "${APPROVE:-0}" == 1 ]] && approve=(--approve) @@ -43,6 +73,7 @@ for device in "$@"; do --window "$(ciq_simulator_window)" --out "${out}/${device}" --golden "/work/test/golden/${device}" \ "${approve[@]}" || failed=1 ciq_stop_app + shoot_glance "${device}" || failed=1 done ((failed == 0)) || die "screenshot checks failed; see build/screenshots/" diff --git a/monkey.jungle b/monkey.jungle index 623a55e..1da28dd 100644 --- a/monkey.jungle +++ b/monkey.jungle @@ -13,5 +13,7 @@ base.sourcePath = source base.resourcePath = resources # Code annotated (:uitest) only exists in the UI-test build (uitest.jungle), -# which swaps this for the (:no_uitest) alternatives (source/util/TestMode.mc). -base.excludeAnnotations = uitest +# which swaps this for the (:no_uitest) alternatives (source/util/TestMode.mc); +# (:stress) only in the watchdog stress build (stress.jungle, +# source/util/StressTest.mc). +base.excludeAnnotations = uitest;stress diff --git a/source/app/OtpWalletApp.mc b/source/app/OtpWalletApp.mc index 596ccc2..3fa42de 100644 --- a/source/app/OtpWalletApp.mc +++ b/source/app/OtpWalletApp.mc @@ -44,6 +44,9 @@ class OtpWalletApp extends Application.AppBase { // The list, or a full-screen message when the data can't be used. (:typecheck(disableGlanceCheck)) function getInitialView() as [Views] or [Views, InputDelegates] { + if (StressTest.ON) { + return StressTest.initialView(); // `make watchdog` only + } _started = true; AppState.start(); var error = AppState.startError; diff --git a/source/util/StressTest.mc b/source/util/StressTest.mc new file mode 100644 index 0000000..d1f99ea --- /dev/null +++ b/source/util/StressTest.mc @@ -0,0 +1,215 @@ +// Author: cirakg & claude ai +// Summary: Watchdog stress build: the largest code of every 2D symbology, encoded, packed, turned into runs and drawn, at twice the normal steps per event. +// +// Two versions of this module exist; each build keeps exactly one: +// * (:no_stress), in every normal build: ON is false. +// * (:stress), only with stress.jungle (`make watchdog`): the app skips +// the vault and runs StressTest.Job instead of the list. +// Connect IQ stops code that runs too long in one event ("Watchdog +// Tripped"), but only in the simulator's app runs, never in unit tests. So +// this build runs every 2D encoder, the cache packer and the run finder on +// the biggest inputs the app accepts, with TWO job steps per runner step: +// a pass shows that each step uses at most half of the watchdog budget +// (120,000 bytecodes per event on the 512 KB watches, 240,000 elsewhere, +// where the runner already does two steps per tick). Codes that fit the +// screen are then drawn the way CodeView draws them (RunsPainter: straight +// from the runs, or from a bitmap painted in steps), in one onUpdate. +// docker/scripts/app-diag.sh watchdog reads the DIAG stress lines. + +import Toybox.Graphics; +import Toybox.Lang; +import Toybox.System; +import Toybox.WatchUi; + +(:stress) +module StressTest { + const ON = true; + // Steps per runner step: the safety margin being tested. + const MARGIN = 2; + // The last code's text, which stays on the screen to be decoded. + const SCREEN_TEXT = "OTPW stress"; + + // One item: an encoder for the largest input of its kind, through + // Codes.DisplayJob (encode, pack, runs) like CodeView, MARGIN steps at a time. + class Item extends Jobs.Job { + var name as String; + var steps as Number = 0; + var job as Codes.DisplayJob; + var symbology as Number; + var painter as RunsPainter? = null; + // Highest memory use seen between steps. + var peak as Number = 0; + + function initialize(label as String, type as Number, encoder as Codes.MatrixJob) { + Job.initialize(); + name = label; + symbology = type; + job = new Codes.DisplayJob(encoder, null); + } + + // The display job, then (when there are runs) the painter. + function step(budget as Number) as Boolean { + for (var i = 0; i < MARGIN; i++) { + steps++; + var used = System.getSystemStats().usedMemory; + peak = used > peak ? used : peak; + var p = painter; + if (p != null) { + if (p.step(budget)) { + return true; + } + } else if (job.step(budget)) { + var runs = job.runs; + if (runs == null) { + return true; + } + painter = new RunsPainter(runs, Codes.minModulePx(symbology)); + } + } + return false; + } + } + + // The screen: progress while encoding, the runs once an item is done. + class StressView extends WatchUi.View { + var title as String = "Stress test"; + var painter as RunsPainter? = null; + private var _first as Boolean = true; + + function initialize() { + View.initialize(); + } + + function onUpdate(dc as Graphics.Dc) as Void { + dc.setColor(Graphics.COLOR_BLACK, Graphics.COLOR_WHITE); + dc.clear(); + var p = painter; + if (p != null) { + var drawn = p.draw(dc); + Diag.data("stress", "drawn name=" + title + " runs=" + p.runs.runs.size() + " direct=" + p.direct() + + " drawn=" + drawn); + painter = null; + } else { + dc.drawText(dc.getWidth() / 2, dc.getHeight() / 2, Graphics.FONT_XTINY, title, Graphics.TEXT_JUSTIFY_CENTER); + } + if (_first) { + _first = false; + Diag.firstFrame(); + start(); + } + } + } + + var view as StressView? = null; + var items as Array<Item> = [] as Array<Item>; + var current as Item? = null; + var runner as Jobs.JobRunner? = null; + + function initialView() as [WatchUi.Views] or [WatchUi.Views, WatchUi.InputDelegates] { + var v = new StressView(); + view = v; + return [v]; + } + + // The largest inputs: QR version 40 (bytes) and a long digit string at + // level H, Data Matrix 144 x 144 (ASCII) and Base256, Aztec 32 layers + // (text and binary), PDF417 as big as this screen takes; then the + // largest Data Matrix and QR that fit the screen, to be drawn (on most + // watches the QR is drawn from a bitmap, RunsPainter's other path). + function start() as Void { + var letters = repeat("abcdefghij", 290); + var digits = repeat("0123456789", 80); + var binary = repeat("é€", 120); + var screen = System.getDeviceSettings(); + var w = screen.screenWidth; + var h = screen.screenHeight; + var round = screen.screenShape == System.SCREEN_SHAPE_ROUND; + items = [ + new Item("qr_v40", Codes.QR, new Qr.QrJob(letters, 0, 0, -1)), + new Item("qr_digits_h", Codes.QR, new Qr.QrJob(digits, 3, 0, -1)), + new Item("dm_144", Codes.DATAMATRIX, new DataMatrix.DataMatrixJob(letters.substring(0, 1500) as String)), + new Item("dm_base256", Codes.DATAMATRIX, new DataMatrix.DataMatrixJob(binary)), + new Item("aztec_text", Codes.AZTEC, new Aztec.AztecJob(letters.substring(0, 1400) as String, 0)), + new Item("aztec_binary", Codes.AZTEC, new Aztec.AztecJob(binary.substring(0, 160) as String, 3)), + new Item("pdf417_text", Codes.PDF417, new Pdf417.Pdf417Job(letters.substring(0, w > 300 ? 900 : 250) as String, + 1, w, h, round)), + new Item("pdf417_digits", Codes.PDF417, new Pdf417.Pdf417Job(digits.substring(0, 400) as String, 1, w, h, round)), + new Item("dm_fit", Codes.DATAMATRIX, new DataMatrix.DataMatrixJob(letters.substring(0, largestDataMatrix(w, h, round)) + as String)), + // Last, so it stays on the screen: app-diag.sh decodes it. + new Item("qr_fit", Codes.QR, new Qr.QrJob(SCREEN_TEXT, 1, largestQrVersion(w, h, round), -1)) + ] as Array<Item>; + Diag.data("stress", "start used_kb=" + System.getSystemStats().usedMemory / 1024); + next(); + } + + function next() as Void { + if (items.size() == 0) { + Diag.data("stress", "all done"); + return; + } + var item = items[0]; + items = items.slice(1, null) as Array<Item>; + current = item; + (view as StressView).title = item.name; + WatchUi.requestUpdate(); + var r = new Jobs.JobRunner(item, 1, new Lang.Method(StressTest, :onItemDone)); + runner = r; + r.start(); + } + + function onItemDone() as Void { + var item = current as Item; + var runs = item.job.runs; + Diag.data("stress", "done name=" + item.name + " steps=" + item.steps + " peak_kb=" + item.peak / 1024 + + " total_kb=" + System.getSystemStats().totalMemory / 1024 + " error=" + item.job.error + + " size=" + (runs != null ? runs.width + "x" + runs.height : "-")); + (view as StressView).painter = item.painter; + WatchUi.requestUpdate(); + // The next item starts in its own event, after the draw. + var r = new Jobs.JobRunner(new Jobs.Job(), 1, new Lang.Method(StressTest, :next)); + runner = r; + r.start(); + } + + function repeat(text as String, times as Number) as String { + var out = ""; + for (var i = 0; i < times; i++) { + out += text; + } + return out; + } + + // The largest QR version that fits this screen at 2 px per module. + function largestQrVersion(w as Number, h as Number, round as Boolean) as Number { + for (var v = 40; v > 1; v--) { + var total = 17 + 4 * v + 8; + if (Codes.fits(total, total, 2, w, h, round)) { + return v; + } + } + return 1; + } + + // An ASCII payload length that fills the largest Data Matrix fitting + // this screen at 2 px per module. + function largestDataMatrix(w as Number, h as Number, round as Boolean) as Number { + var best = 1; + for (var i = 0; i < DataMatrix.SIZES.size(); i++) { + var s = DataMatrix.SIZES[i] as Array<Number>; + if (Codes.fits(s[0] + 2, s[0] + 2, 2, w, h, round)) { + best = s[3]; + } + } + return best; + } +} + +(:no_stress) +module StressTest { + const ON = false; + + function initialView() as [WatchUi.Views] or [WatchUi.Views, WatchUi.InputDelegates] { + return [new WatchUi.View()]; + } +} diff --git a/stress.jungle b/stress.jungle new file mode 100644 index 0000000..b1156cc --- /dev/null +++ b/stress.jungle @@ -0,0 +1,7 @@ +# Author: cirakg & claude ai +# Summary: Watchdog stress build: the largest 2D codes at twice the steps per event (source/util/StressTest.mc). +# +# Used on top of monkey.jungle by `build-app.sh stress` (make watchdog): it +# keeps the (:stress) code and drops the (:no_stress) alternative, so the app +# starts straight into StressTest instead of the list. Never shipped. +base.excludeAnnotations = uitest;no_stress diff --git a/test/golden/fenix7/about.png b/test/golden/fenix7/about.png Binary files differindex 0dbbb80..e58e178 100644 --- a/test/golden/fenix7/about.png +++ b/test/golden/fenix7/about.png diff --git a/test/golden/fenix7/glance_code.png b/test/golden/fenix7/glance_code.png Binary files differnew file mode 100644 index 0000000..0313cae --- /dev/null +++ b/test/golden/fenix7/glance_code.png diff --git a/test/golden/fr255s/about.png b/test/golden/fr255s/about.png Binary files differindex 51dc136..68bd5be 100644 --- a/test/golden/fr255s/about.png +++ b/test/golden/fr255s/about.png diff --git a/test/golden/fr255s/glance_code.png b/test/golden/fr255s/glance_code.png Binary files differnew file mode 100644 index 0000000..1009369 --- /dev/null +++ b/test/golden/fr255s/glance_code.png diff --git a/test/golden/fr70/about.png b/test/golden/fr70/about.png Binary files differindex 1765b65..c92b268 100644 --- a/test/golden/fr70/about.png +++ b/test/golden/fr70/about.png diff --git a/test/golden/fr70/glance_code.png b/test/golden/fr70/glance_code.png Binary files differnew file mode 100644 index 0000000..4a348fd --- /dev/null +++ b/test/golden/fr70/glance_code.png diff --git a/test/golden/fr965/about.png b/test/golden/fr965/about.png Binary files differindex f48bea7..4a2ffdc 100644 --- a/test/golden/fr965/about.png +++ b/test/golden/fr965/about.png diff --git a/test/golden/fr965/glance_code.png b/test/golden/fr965/glance_code.png Binary files differnew file mode 100644 index 0000000..1c180e9 --- /dev/null +++ b/test/golden/fr965/glance_code.png diff --git a/test/golden/instinct3amoled45mm/about.png b/test/golden/instinct3amoled45mm/about.png Binary files differindex bbad66e..d7f5087 100644 --- a/test/golden/instinct3amoled45mm/about.png +++ b/test/golden/instinct3amoled45mm/about.png diff --git a/test/golden/instinct3amoled45mm/glance_code.png b/test/golden/instinct3amoled45mm/glance_code.png Binary files differnew file mode 100644 index 0000000..5f848fe --- /dev/null +++ b/test/golden/instinct3amoled45mm/glance_code.png diff --git a/test/golden/venusq2/glance_code.png b/test/golden/venusq2/glance_code.png Binary files differnew file mode 100644 index 0000000..5082069 --- /dev/null +++ b/test/golden/venusq2/glance_code.png diff --git a/test/golden/venux1/glance_code.png b/test/golden/venux1/glance_code.png Binary files differnew file mode 100644 index 0000000..52980ae --- /dev/null +++ b/test/golden/venux1/glance_code.png diff --git a/tools/budgets.json b/tools/budgets.json index ba7432d..5e330c9 100644 --- a/tools/budgets.json +++ b/tools/budgets.json @@ -1,12 +1,13 @@ { "_author": "cirakg & claude ai", "_summary": "Simulator budgets checked by `make perf` and `make memtest` (README section 18).", - "_about": "Simulator timings depend on your PC. startup_ms (in-app, object creation to first full frame) measured 1-18 ms in M4; its budget leaves room for simulator jitter. first_frame_wall_ms includes the JVM start of monkeydo, loading the glance, the 2 s pause and the tap that opens the app (16-18 s in M4), so it only catches big regressions; set to about 2x. Memory is a hard limit: at most 70 % of the watch-app memory of the device.", + "_about": "Simulator timings depend on your PC. startup_ms (in-app, object creation to first full frame) measured 1-18 ms in M4; its budget leaves room for simulator jitter. first_frame_wall_ms includes the JVM start of monkeydo, loading the glance, the 2 s pause and the tap that opens the app (16-18 s in M4), so it only catches big regressions; set to about 2x. Memory is a hard limit: at most 70 % of the watch-app memory of the device, and 70 % of the glance's 64 KB.", "perf": { "startup_ms": 100, "first_frame_wall_ms": 36000 }, "mem": { - "mem_used_fraction": 0.7 + "mem_used_fraction": 0.7, + "glance_mem_used_fraction": 0.7 } } diff --git a/tools/check_budgets.py b/tools/check_budgets.py index 8b55a2b..8219a6d 100644 --- a/tools/check_budgets.py +++ b/tools/check_budgets.py @@ -7,6 +7,8 @@ Two kinds of lines are read from build/diag/<device>.log: DIAG first_frame startup_ms=1 mem_used=6984 mem_total=781888 printed by the debug build (source/util/Diag.mc) after its first frame; + DIAG glance mem_used=12552 mem_total=65536 + printed by the glance after its first draw (the simulator starts there); HARNESS first_frame wall_ms=2310 added by app-diag.sh: wall-clock time from launching the app with monkeydo until that DIAG line appeared, which includes loading the @@ -52,7 +54,11 @@ def measure(kind: str, events: dict[str, dict[str, float]]) -> dict[str, float]: values["first_frame_wall_ms"] = harness["wall_ms"] return values total = frame["mem_total"] - return {"mem_used_fraction": frame["mem_used"] / total if total else 1.0} + values = {"mem_used_fraction": frame["mem_used"] / total if total else 1.0} + glance = events.get("DIAG.glance") + if glance is not None: + values["glance_mem_used_fraction"] = glance["mem_used"] / glance["mem_total"] if glance["mem_total"] else 1.0 + return values def main(argv: list[str] | None = None) -> int: diff --git a/tools/tests/test_check_budgets.py b/tools/tests/test_check_budgets.py index 2262a0b..28ba710 100644 --- a/tools/tests/test_check_budgets.py +++ b/tools/tests/test_check_budgets.py @@ -29,6 +29,8 @@ def test_measure_perf_and_memory() -> None: assert cb.measure("perf", events) == {"startup_ms": 12.0, "first_frame_wall_ms": 2310.0} assert cb.measure("mem", events) == {"mem_used_fraction": 100000 / 524288} assert cb.measure("perf", {}) == {} + with_glance = cb.read_events(f"DIAG glance mem_used=16384 mem_total=65536\n{FRAME}") + assert cb.measure("mem", with_glance) == {"mem_used_fraction": 100000 / 524288, "glance_mem_used_fraction": 0.25} def run(tmp_path: Path, kind: str, log_text: str, budgets: Mapping[str, Mapping[str, float]]) -> int: diff --git a/tools/tests/test_ui_flow.py b/tools/tests/test_ui_flow.py index 7e0d336..d98b4c2 100644 --- a/tools/tests/test_ui_flow.py +++ b/tools/tests/test_ui_flow.py @@ -1,5 +1,5 @@ # Author: cirakg & claude ai -# Summary: Tests for the UI flow driver's pure parts: device key layout, JSON lookup and the app-log reader. +# Summary: Tests for the UI flow driver's pure parts: key layout, JSON lookup, the app-log reader, barcode reading. from __future__ import annotations import json @@ -8,7 +8,9 @@ import time from pathlib import Path import pytest +import segno import ui_flow +from PIL import Image def write_sim(tmp_path: Path, config: dict[str, object]) -> Path: @@ -71,3 +73,11 @@ def test_app_log_latest_returns_the_last_match(tmp_path: Path) -> None: match = ui_flow.AppLog(log_path).latest(r"DIAG list_focus y=(\d+)") assert match is not None and match.group(1) == "42" assert ui_flow.AppLog(tmp_path / "missing.log").latest("x") is None + + +def test_barcodes_on_reads_a_drawn_code(tmp_path: Path) -> None: + path = tmp_path / "qr.png" + segno.make("OTPW stress", error="m").save(str(path), scale=4, border=4) + screen = Image.open(path).convert("RGB") + assert ui_flow.barcodes_on(screen) == ["OTPW stress"] + assert ui_flow.barcodes_on(Image.new("RGB", (100, 100), "white")) == [] diff --git a/tools/ui_flow.py b/tools/ui_flow.py index edfc9a4..a17d435 100644 --- a/tools/ui_flow.py +++ b/tools/ui_flow.py @@ -28,8 +28,11 @@ waiting for the PIN and confirmed from the menu (watches with buttons), the list, a TOTP entry, a HOTP entry and its next code, the PIN screen, a wrong PIN, a correct PIN opening a locked entry, auto-lock closing it, the menu, the About screen (watches with buttons; touch-only -menus have no focus to move), the lockout after 5 wrong PINs, and an EAN-13 -code whose screenshot must decode with zxing-cpp (README 12.4). +menus have no focus to move), the lockout after 5 wrong PINs, the barcodes +whose screenshots must decode with zxing-cpp (README 12.4). The glance with +the favourite's code is a separate launch (--shot glance_code, from +screenshots.sh): the simulator doesn't return to the glance when the app +exits. Each screenshot is compared with test/golden/<device>/<screen>.png like sim_screenshot.py does (or saved there with --approve). @@ -50,6 +53,7 @@ from pathlib import Path import sim_screenshot as shots import zxingcpp +from PIL import Image EXIT_OK, EXIT_DIFFERENT, EXIT_FLOW = 0, 1, 2 SCREEN_TIMEOUT_S = 20.0 @@ -90,6 +94,11 @@ class Key: hold: bool +def barcodes_on(screen: Image.Image) -> list[str]: + """The texts of every barcode zxing-cpp finds on a screenshot.""" + return [r.text for r in zxingcpp.read_barcodes(screen)] + + def load_keys(device_dir: Path) -> dict[str, Key]: """Button centres in device-drawing coordinates, by button id (enter, up, down, esc, menu).""" sim = json.loads((device_dir / "simulator.json").read_text(encoding="utf-8")) @@ -326,7 +335,7 @@ class Flow: return if expect_code is not None: # README 12.4 check 3: the code on the screenshot must decode. - read = [r.text for r in zxingcpp.read_barcodes(screen)] + read = barcodes_on(screen) if expect_code not in read: print(f"[{label}] the barcode didn't decode to {expect_code!r}: read {read}") self.results[name] = EXIT_DIFFERENT @@ -569,6 +578,50 @@ def open_from_glance(window: str, device_dir: Path, raw_dir: Path) -> None: sim.press("enter") +def decode_screen(window: str, device_dir: Path, out: Path, text: str) -> int: + """Screenshot the simulator; EXIT_OK when a barcode on it reads `text`.""" + name = device_dir.name + try: + sim = Simulator(window, device_dir, out / "raw") + screen = shots.extract_screen(sim.capture("decode"), device_dir, out / "decode.png") + except (FlowError, shots.ScreenshotError, subprocess.CalledProcessError, OSError) as err: + print(f"[{name}] no screenshot to decode: {err}") + return EXIT_FLOW + read = barcodes_on(screen) + if text not in read: + print(f"[{name}] the barcode on the screen didn't decode to {text!r}: read {read}") + return EXIT_DIFFERENT + print(f"[{name}] the barcode on the screen decodes to {text!r}") + return EXIT_OK + + +def open_app(window: str, device_dir: Path, out: Path) -> int: + """--open-from-glance: EXIT_OK once the input was sent.""" + try: + open_from_glance(window, device_dir, out / "raw") + except (FlowError, subprocess.CalledProcessError, OSError) as err: + print(f"[{device_dir.name}] could not open the app from its glance: {err}") + return EXIT_FLOW + return EXIT_OK + + +def single_shot(args: argparse.Namespace) -> int: + """Screenshot the simulator as args.shot and compare it with its golden image (or approve it).""" + window, device_dir, out, golden, name = args.window, args.device_dir, args.out, args.golden, args.shot + label = f"{device_dir.name}/{name}" + try: + sim = Simulator(window, device_dir, out / "raw") + screen = shots.extract_screen(sim.capture(name), device_dir, out / f"{name}.png") + except (FlowError, shots.ScreenshotError, subprocess.CalledProcessError, OSError) as err: + print(f"[{label}] {err}") + return EXIT_FLOW + if args.approve: + shots.save_png(screen, golden / f"{name}.png", f"Approved simulator screen '{name}' of {device_dir.name}") + print(f"[{label}] approved") + return EXIT_OK + return shots.check_golden(screen, label, out / f"{name}.png", golden / f"{name}.png") + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Scripted UI flow with screenshot checks.") parser.add_argument("--device-dir", type=Path, required=True) @@ -578,15 +631,18 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--golden", type=Path, help="directory of the approved images") parser.add_argument("--approve", action="store_true") parser.add_argument("--open-from-glance", action="store_true", help="only open the app from its glance") + parser.add_argument("--decode", metavar="TEXT", help="only check that the screen shows a barcode reading TEXT") + parser.add_argument("--shot", metavar="NAME", help="only screenshot the current screen as NAME (needs --golden)") args = parser.parse_args(argv) name = args.device_dir.name + if args.decode is not None: + return decode_screen(args.window, args.device_dir, args.out, args.decode) + if args.shot is not None: + if args.golden is None: + parser.error("--shot needs --golden") + return single_shot(args) if args.open_from_glance: - try: - open_from_glance(args.window, args.device_dir, args.out / "raw") - except (FlowError, subprocess.CalledProcessError, OSError) as err: - print(f"[{name}] could not open the app from its glance: {err}") - return EXIT_FLOW - return EXIT_OK + return open_app(args.window, args.device_dir, args.out) if args.log is None or args.golden is None: parser.error("--log and --golden are required for the flow") try: diff --git a/uitest.jungle b/uitest.jungle index 5639dea..eab75b8 100644 --- a/uitest.jungle +++ b/uitest.jungle @@ -5,4 +5,4 @@ # it keeps the (:uitest) code and drops the (:no_uitest) alternatives, so the # app starts with a reset store full of fake entries and a frozen clock # (source/util/TestMode.mc). Never shipped: release builds use monkey.jungle. -base.excludeAnnotations = no_uitest +base.excludeAnnotations = no_uitest;stress |
