diff options
Diffstat (limited to 'tools/ui_flow.py')
| -rw-r--r-- | tools/ui_flow.py | 417 |
1 files changed, 405 insertions, 12 deletions
diff --git a/tools/ui_flow.py b/tools/ui_flow.py index 3c7afb2..3796e85 100644 --- a/tools/ui_flow.py +++ b/tools/ui_flow.py @@ -6,7 +6,8 @@ Usage (inside the dev container, with the simulator running and the UI-test build launched by docker/scripts/screenshots.sh): ui_flow.py --device fr70 --device-dir DIR --log APP.log --window ID - --out build/screenshots/fr70 --golden test/golden/fr70 [--approve] + --out build/screenshots/fr70 --golden test/golden/fr70 + [--scenario main|extra] [--approve] Input goes to the simulator window with xdotool: clicks on the device drawing's buttons (positions from DIR/simulator.json) and taps and swipes on @@ -29,10 +30,31 @@ list, a TOTP entry, a HOTP entry and its next code, the PIN screen, a wrong PIN, a correct PIN opening a locked entry, auto-lock closing it, the menu, the About screen (watches with buttons; touch-only menus have no focus to move), the lockout after 5 wrong PINs, the barcodes -whose screenshots must decode with zxing-cpp (README 12.4). The glance with -the favourite's code is a separate launch (--shot glance_code, from -screenshots.sh): the simulator doesn't return to the glance when the app -exits. +whose screenshots must decode with zxing-cpp (README 12.4), the Steam and +Library entries, and the text screens the menu opens (settings results, +entry IDs, clock status, About on touch-only watches, the benchmark), +scrolled with the buttons or a swipe. The glance with the favourite's code +is a separate launch (--shot glance_code, from screenshots.sh): the +simulator doesn't return to the glance when the app exits. + +--scenario extra drives the second UI-test build (uitest-extra.jungle, +Fixtures.loadExtra): a "Require PIN" from the Manage form waiting while the +watch is locked; UPC-A and Code 39 cards that must decode; an EAN-13 with a +wrong check digit (the code error screen); a QR whose encoding progress must +start low and only go up; a 6-digit PIN from before the 4-digit rule, +unlocked with OK and replaced (new PIN, a repeat that differs, the repeat); +the waiting lock applied at that PIN; auto-lock closing a locked code; +Change PIN; Reset PIN and creating a PIN. Its glance (no favourite code) is +checked as glance.png. + +Menus are reached by label: the debug build logs the menu's items ("DIAG +menu items=..."). Watches with buttons move the focus; touch-only watches +(Venu Sq 2, Venu X1) have none, so the item is tapped where that watch's +menu draws it (TOUCH_MENU_ROWS), after slow drags whose distance is +measured on the screen. + +Screens whose content changes from run to run (the encoding progress, the +benchmark's timings) are saved for review only, never compared. Each screenshot is compared with test/golden/<device>/<screen>.png like sim_screenshot.py does (or saved there with --approve). @@ -43,6 +65,7 @@ Exit: 0 all screens match, 1 a screen differs or is missing a golden image, from __future__ import annotations import argparse +import itertools import json import re import subprocess @@ -53,7 +76,7 @@ from pathlib import Path import sim_screenshot as shots import zxingcpp -from PIL import Image +from PIL import Image, ImageChops, ImageStat EXIT_OK, EXIT_DIFFERENT, EXIT_FLOW = 0, 1, 2 SCREEN_TIMEOUT_S = 20.0 @@ -80,6 +103,27 @@ LOCKER_CODE128 = "FAKE-0042" # source/app/Fixtures.mc: the Add code form's Code # The Add OTP form's TOTP ("Hello!" demo key) at TestMode.NOW is 309848, next # 489290 (RFC 6238 computed in Python); the golden image otp_form shows them. QR_TIMEOUT_S = 30.0 # the QR is encoded in timer steps the first time +LEGACY_PIN = "123456" # source/app/Fixtures.mc FIXTURE_LEGACY_PIN: a PIN from before the 4-digit rule +NEW_PIN, OTHER_PIN, CHANGED_PIN, CREATED_PIN = "5678", "5679", "2468", "1357" +SELECTOR_LEGACY = 11 # the selector adds "OK" (position 10) for a legacy PIN +KEY_OK = -2 # source/ui/PinView.mc: the keypad's OK key (legacy PINs only) +MODE_UNLOCK, MODE_CREATE, MODE_CHANGE = 0, 1, 2 # source/ui/PinView.mc +# source/app/Fixtures.mc loadExtra: the UPC-A 036000291452, which zxing-cpp +# reports as the EAN-13 it is a subset of, and the Code 39 "FAKE-39" with the +# mod 43 check character the app adds ("L"), which zxing-cpp reports as text. +SHOP_UPCA = "0036000291452" +BADGE_CODE39 = "FAKE-39L" +BIG_QR_TEXT = "Fake event ticket 0042 for the UI test." + " Row 12, seat 34, gate b, valid all day." * 4 +PROGRESS_FIRST_MAX = 30 # the encoding progress must start below this (percent) +PROGRESS_MIN_STEPS = 3 # and show at least this many different values +BENCH_TIMEOUT_S = 30.0 +# Touch-only menus (no focus): the centre of the first item and the item +# pitch, in screen pixels, at the top of a Menu2 with a title, as these +# watches draw it (measured from the simulator). +TOUCH_MENU_ROWS = {"venusq2": (100, 100), "venux1": (184, 140)} +DRAG_STEPS = 30 # a slow drag: the list follows the finger without a fling +DRAG_HOLD_S = 0.6 +MAX_MENU_DRAGS = 16 class FlowError(Exception): @@ -99,6 +143,22 @@ def barcodes_on(screen: Image.Image) -> list[str]: return [r.text for r in zxingcpp.read_barcodes(screen)] +def vertical_shift(before: Image.Image, after: Image.Image) -> int: + """How many pixels the content moved up from `before` to `after` (a + scrolled list; negative: down): the shift whose overlapping rows differ + least, compared in the middle half of the screen.""" + a, b = before.convert("L"), after.convert("L") + width, height = a.size + best, best_score = 0, float("inf") + top, bottom = height // 4, height * 3 // 4 + for shift in range(-(height // 4), height // 4 + 1): + moved = ImageChops.difference(a.crop((0, top + shift, width, bottom + shift)), b.crop((0, top, width, bottom))) + score = ImageStat.Stat(moved).mean[0] + if score < best_score: + best, best_score = shift, score + return best + + def load_keys(device_dir: Path) -> dict[str, Key]: """Button centres in device-drawing coordinates, by button id (enter, up, down, esc, menu).""" sim = json.loads((device_dir / "simulator.json").read_text(encoding="utf-8")) @@ -215,6 +275,18 @@ class Simulator: self._xdotool("mouseup", "1") time.sleep(CLICK_PAUSE_S) + def drag(self, sx: int, sy0: int, sy1: int) -> None: + """A slow vertical drag that stops before lifting the finger, so a + list moves with it and doesn't fling on.""" + self._move(*self._to_window(self.display.x + sx, self.display.y + sy0)) + self._xdotool("mousedown", "1") + for i in range(1, DRAG_STEPS + 1): + self._move(*self._to_window(self.display.x + sx, self.display.y + sy0 + (sy1 - sy0) * i // DRAG_STEPS)) + time.sleep(0.03) + time.sleep(DRAG_HOLD_S) + self._xdotool("mouseup", "1") + time.sleep(CLICK_PAUSE_S) + def swipe(self, sx0: int, sy0: int, sx1: int, sy1: int) -> None: self._move(*self._to_window(self.display.x + sx0, self.display.y + sy0)) self._xdotool("mousedown", "1") @@ -288,7 +360,11 @@ class Flow: else: self.sim.tap(self._w() // 2, self._h() // 2) - def enter_pin(self, pin: str) -> None: + def enter_pin(self, pin: str, legacy: bool = False, ok: bool | None = None) -> None: + """Type a PIN; the 4th digit submits it. A legacy PIN (any length) + is submitted with OK (unless ok is False: more digits follow), after + which the screen's selector is back at 0.""" + submit = legacy if ok is None else ok if self.touch: keys = self.log.latest(r"DIAG pin_keys (\S+)") if keys is None: @@ -298,15 +374,101 @@ class Flow: value, xy = item.split("=") x, y = xy.split(",") centres[int(value)] = (int(x), int(y)) - for digit in pin: # the 4th digit submits + for digit in pin: self.sim.tap(*centres[int(digit)]) + if submit: + if KEY_OK not in centres: + raise FlowError("the legacy PIN keypad has no OK key") + self.sim.tap(*centres[KEY_OK]) return - for position in [int(d) for d in pin]: - steps = (position - self.selector) % SELECTOR_DIGITS + positions = SELECTOR_LEGACY if legacy else SELECTOR_DIGITS + for position in [int(d) for d in pin] + ([SELECTOR_DIGITS] if submit else []): + steps = (position - self.selector) % positions for _ in range(steps): self.sim.press("down") self.selector = position self.sim.press("enter") + if submit: + self.selector = 0 # the new-PIN screen has no OK position + + def expect_pin(self, mode: int, digits: int = 0, repeat: bool = False, busy: bool = False) -> None: + """Wait until the PIN screen shows this state (PinView MODE_*: 0 + unlock, 1 create, 2 change): every key press redraws the screen, so + "shown pin" alone can come before the last digit arrived.""" + state = f"mode={mode} digits={digits} repeat={int(repeat)} busy={int(busy)}" + self.log.wait_for(rf"DIAG pin_state {state}\s*$", SCREEN_TIMEOUT_S) + time.sleep(SETTLE_S) + + def menu_items(self) -> list[str]: + """The open menu's item labels, from its debug log line.""" + items = self.log.latest(r"DIAG menu items=(.*?)\s*$") + if items is None: + raise FlowError("the menu didn't report its items") + return items.group(1).split("|") + + def open_menu_item(self, label: str) -> None: + """Open the menu and select the item `label`.""" + self.act() + self.menu() + self.log.wait_for(r"DIAG menu items=", SCREEN_TIMEOUT_S) + time.sleep(MENU_SETTLE_S) + items = self.menu_items() + if label not in items: + raise FlowError(f"the menu has no item {label!r}: {items}") + index = items.index(label) + self.act() + if self.sim.has("down"): + for _ in range(index): + self.next() + time.sleep(MENU_SETTLE_S) + self.select() + return + self.tap_touch_menu_item(index) + + def tap_touch_menu_item(self, index: int) -> None: + """Tap item `index` of a menu without focus, dragging it into view + first; each drag's real distance is measured on the screen.""" + name = self.sim.device_dir.name + if name not in TOUCH_MENU_ROWS: + raise FlowError(f"no menu layout for the touch-only watch {name} in TOUCH_MENU_ROWS") + first, pitch = TOUCH_MENU_ROWS[name] + x, h = self._w() // 2, self._h() + # A new menu doesn't always start at the top: drag down to it first. + for _ in range(MAX_MENU_DRAGS): + before = self.screen("menu-before") + self.sim.drag(x, h // 4, h // 4 + h // 5) + time.sleep(MENU_SETTLE_S) + if vertical_shift(before, self.screen("menu-after")) == 0: + break + y = first + index * pitch + for _ in range(MAX_MENU_DRAGS): + if y <= h - pitch // 2: + break + before = self.screen("menu-before") + distance = min(y - h // 2, h // 5) + self.sim.drag(x, h * 3 // 4, h * 3 // 4 - distance) + time.sleep(MENU_SETTLE_S) + shift = vertical_shift(before, self.screen("menu-after")) + if shift <= 0: + break # the end of the menu + y -= shift + if not 0 < y < h: + raise FlowError(f"menu item {index} can't be brought into view") + self.sim.tap(x, y) + + def screen(self, name: str) -> Image.Image: + """The current screen, uncompared (saved under raw/).""" + return shots.extract_screen(self.sim.capture(name), self.sim.device_dir, self.out / "raw" / f"{name}.png") + + def mark_after(self, pattern: str, timeout: float = SCREEN_TIMEOUT_S) -> None: + """Wait for a line matching `pattern`; later waits look only after it.""" + self.log.wait_for(pattern, timeout) + regex = re.compile(pattern) + lines = self.log.lines() + for index in range(len(lines) - 1, self.log.mark - 1, -1): + if regex.search(lines[index]): + self.log.mark = index + 1 + return # -- checking ----------------------------------------------------------- @@ -322,7 +484,9 @@ class Flow: """Call before an input whose result `expect` should wait for.""" self.log.set_mark() - def shot(self, name: str, expect_code: str | None = None) -> None: + def shot(self, name: str, expect_code: str | None = None, review: bool = False) -> None: + """Screenshot as `name` and compare it with its golden image; with + review, only save it (screens that differ from run to run).""" raw = self.sim.capture(name) out = self.out / f"{name}.png" label = f"{self.sim.device_dir.name}/{name}" @@ -332,6 +496,9 @@ class Flow: print(f"[{label}] {err}") self.results[name] = EXIT_DIFFERENT return + if review: + print(f"[{label}] saved for review only: {out}") + return if expect_code is not None: # README 12.4 check 3: the code on the screenshot must decode. read = barcodes_on(screen) @@ -562,6 +729,60 @@ def flow_code(f: Flow) -> None: f.shot(name, expect_code=expected) +def flow_details(f: Flow) -> None: + """From the Aztec card (index 10): the Steam (4) and Library (5) entries.""" + f.act() + f.back() + f.expect("list") + for _ in range(6): + f.prev() + for name in ("otp_steam", "otp_library"): + f.act() + f.open_focused_row() + f.expect("otp") + f.shot(name) + f.act() + f.back() + f.expect("list") + f.next() + + +def flow_info_screens(f: Flow) -> None: + """The text screens the menu opens, each scrolled with a swipe on touch + screens; the entry IDs (110 entries) also one page down. Touch-only + watches get About here (the others in flow_menu). The benchmark's + timings vary, so its screens are kept for review only.""" + screens = [ + ("Settings results", "results", "menu_results"), + ("Entry IDs", "ids", "entry_ids"), + ("Clock status", "clock", "clock_status"), + ] + if not f.sim.has("down"): + screens.append(("About", "about", "about")) + for label, screen, name in screens: + f.open_menu_item(label) + f.expect(screen) + f.shot(name) + flow_swipes(f, screen) + if screen == "ids": + f.act() + f.next() + f.log.wait_for(r"DIAG info_scroll first=[1-9]", SCREEN_TIMEOUT_S) + time.sleep(SETTLE_S) + f.shot("entry_ids_next") + f.act() + f.back() + f.expect("list") + f.open_menu_item("Debug: benchmark") + f.expect("bench_running") + f.shot("bench_running", review=True) + f.expect("bench", BENCH_TIMEOUT_S) + f.shot("bench", review=True) + f.act() + f.back() + f.expect("list") + + def run_flow(f: Flow) -> None: """The scripted session; see the module docstring.""" flow_import(f) @@ -572,6 +793,177 @@ def run_flow(f: Flow) -> None: flow_queue(f) flow_lockout(f) flow_code(f) + flow_details(f) + flow_info_screens(f) + + +def open_row(f: Flow, moves: int, screen: str, timeout: float = SCREEN_TIMEOUT_S) -> None: + """Move the list focus by `moves` rows (negative: up) and open that row.""" + for _ in range(abs(moves)): + if moves > 0: + f.next() + else: + f.prev() + f.act() + f.open_focused_row() + f.expect(screen, timeout) + + +def close_to_list(f: Flow) -> None: + f.act() + f.back() + f.expect("list") + + +def extra_codes(f: Flow) -> None: + """From GitHub (0): UPC-A (2) and Code 39 (3) must decode, the EAN-13 + with a wrong check digit (4) shows the code error, and the large QR (5) + shows its encoding progress, which must start low and only go up.""" + open_row(f, 2, "code") + f.shot("code_upca", expect_code=SHOP_UPCA) + close_to_list(f) + open_row(f, 1, "code") + f.shot("code_39", expect_code=BADGE_CODE39) + close_to_list(f) + open_row(f, 1, "code_error") + f.shot("code_error") + close_to_list(f) + f.next() + f.act() + f.open_focused_row() + f.log.wait_for(r"DIAG code_progress percent=", SCREEN_TIMEOUT_S) + f.shot("code_progress", review=True) + f.expect("code", QR_TIMEOUT_S) + check_progress(f) + f.shot("code_qr_large", expect_code=BIG_QR_TEXT) + close_to_list(f) + + +def check_progress(f: Flow) -> None: + """The encoding progress since the last mark starts low, never goes + back, and moves through several values.""" + regex = re.compile(r"DIAG code_progress percent=(\d+)") + seen = [int(m.group(1)) for line in f.log.lines()[f.log.mark :] if (m := regex.search(line))] + steps = sorted(set(seen)) + print(f"[{f.sim.device_dir.name}] encoding progress: {' '.join(str(p) for p in steps)} %") + if not seen or seen[0] >= PROGRESS_FIRST_MAX: + raise FlowError(f"the encoding progress started at {seen[:1]}, not below {PROGRESS_FIRST_MAX} %") + if any(b < a for a, b in itertools.pairwise(seen)): + raise FlowError(f"the encoding progress went back: {seen}") + if len(steps) < PROGRESS_MIN_STEPS: + raise FlowError(f"the encoding progress showed only {steps}") + + +def extra_legacy_pin(f: Flow) -> None: + """From the large QR (5): Bank (1) asks for the legacy 6-digit PIN with + OK; it unlocks, applies the waiting "Require PIN", then asks for a new + 4-digit PIN: a repeat that differs, then the same PIN twice opens Bank.""" + open_row(f, -4, "pin") + f.opened_pin_screen() + f.shot("pin_legacy") + f.act() + f.enter_pin(LEGACY_PIN[:-1], legacy=True, ok=False) + f.expect_pin(MODE_UNLOCK, digits=len(LEGACY_PIN) - 1) + f.shot("pin_legacy_typed") + f.act() + f.enter_pin(LEGACY_PIN[-1], legacy=True) + f.log.wait_for(r"DIAG locks_applied count=1\b", SCREEN_TIMEOUT_S) + f.expect_pin(MODE_CHANGE) + f.shot("pin_new_forced") + f.act() + f.enter_pin(NEW_PIN) + f.expect_pin(MODE_CHANGE, repeat=True) + f.shot("pin_repeat") + f.act() + f.enter_pin(OTHER_PIN) + f.expect("pin_error") + f.shot("pin_differ") + f.act() + f.enter_pin(NEW_PIN) + f.expect_pin(MODE_CHANGE, repeat=True) + f.act() + f.enter_pin(NEW_PIN) + f.expect("otp") + close_to_list(f) + f.shot("list_locked") + + +def extra_auto_lock_code(f: Flow) -> None: + """The Shop card (2) now requires the PIN; the session is unlocked, so + it opens directly, and auto-lock closes it back to the list.""" + open_row(f, 1, "code") + f.act() + f.expect("list", AUTO_LOCK_TIMEOUT_S) + + +def extra_change_pin(f: Flow) -> None: + """Locked: Change PIN asks for the current PIN, then a new one twice. + After "Lock now", the new PIN opens Bank (1).""" + f.open_menu_item("Change PIN") + f.expect_pin(MODE_UNLOCK) + f.opened_pin_screen() + f.act() + f.enter_pin(NEW_PIN) + f.expect_pin(MODE_CHANGE) + f.opened_pin_screen() + f.shot("pin_change") + f.act() + f.enter_pin(CHANGED_PIN) + f.expect_pin(MODE_CHANGE, repeat=True) + f.act() + f.enter_pin(CHANGED_PIN) + f.expect("list") + f.open_menu_item("Lock now") # changing the PIN left the watch unlocked + f.expect("list") + open_row(f, -1, "pin") + f.opened_pin_screen() + f.act() + f.enter_pin(CHANGED_PIN) + f.expect("otp") + close_to_list(f) + + +def extra_reset_pin(f: Flow) -> None: + """Reset PIN: the confirmation, then creating a new PIN (entered twice).""" + f.open_menu_item("Reset PIN") + time.sleep(MENU_SETTLE_S) + f.shot("reset_confirm") + f.act() + confirm(f) + f.expect_pin(MODE_CREATE) + f.opened_pin_screen() + f.shot("pin_create") + f.act() + f.enter_pin(CREATED_PIN) + f.expect_pin(MODE_CREATE, repeat=True) + f.act() + f.enter_pin(CREATED_PIN) + f.expect("list") + f.shot("list_reset") + + +def confirm(f: Flow) -> None: + """Answer yes in a native confirmation: UP to "Yes" and START with + buttons, a tap on the right half ("Confirm") on touch-only watches.""" + if f.sim.has("up"): + f.sim.press("up") + time.sleep(MENU_SETTLE_S) + f.sim.press("enter") + else: + f.sim.tap(f._w() * 3 // 4, f._h() * 5 // 8) + + +def run_extra_flow(f: Flow) -> None: + """The extra scenario's session; see the module docstring.""" + f.expect("results", IMPORT_TIMEOUT_S) # the Manage form's "Require PIN" waits for the PIN + f.shot("lock_queued") + close_to_list(f) + f.shot("list_extra") + extra_codes(f) + extra_legacy_pin(f) + extra_auto_lock_code(f) + extra_change_pin(f) + extra_reset_pin(f) def find_key(data: object, key: str) -> object: @@ -655,6 +1047,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--log", type=Path, help="the app's console log (monkeydo output)") parser.add_argument("--golden", type=Path, help="directory of the approved images") parser.add_argument("--approve", action="store_true") + parser.add_argument("--scenario", choices=["main", "extra"], default="main", help="which UI-test build runs") parser.add_argument("--open-from-glance", action="store_true", help="only open the app from its glance") parser.add_argument("--decode", metavar="TEXT", help="only check that the screen shows a barcode reading TEXT") parser.add_argument("--shot", metavar="NAME", help="only screenshot the current screen as NAME (needs --golden)") @@ -673,7 +1066,7 @@ def main(argv: list[str] | None = None) -> int: try: sim = Simulator(args.window, args.device_dir, args.out / "raw") flow = Flow(sim, AppLog(args.log), args.out, args.golden, args.approve) - run_flow(flow) + (run_extra_flow if args.scenario == "extra" else run_flow)(flow) except (FlowError, subprocess.CalledProcessError, OSError) as err: print(f"[{name}] UI flow failed: {err}") return EXIT_FLOW |