feat(import): parse otpauth URIs and unencrypted Aegis JSON in resumable steps

Streaming JSON tokenizer with depth and length limits and native skipping
of long strings; Aegis reader for database versions 1-3 with groups and
encrypted-export detection; otpauth parser with ';' separators and Steam;
import job with Replace mode, duplicate and replacement checks, a sealed
multi-part buffer and a report. Google Authenticator exports deferred.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

1 files changed, 486 insertions, 0 deletions

diff --git a/source/import/Json.mc b/source/import/Json.mc
new file mode 100644
index 0000000..c19150e
--- /dev/null
+++ b/source/import/Json.mc
@@ -0,0 +1,486 @@
+// Author:  cirakg & claude ai
+// Summary: A streaming JSON tokenizer over UTF-8 bytes: one token per call, strict grammar, depth and length limits.
+//
+// README 11.3: the Aegis reader must stream, never build large objects, and
+// pause between steps (watchdog). This tokenizer keeps only its position, a
+// small container stack and the current token, so a reader can stop after any
+// token and continue in the next event.
+//
+// Limits and guarantees:
+//   * Strict RFC 8259 grammar; anything else is T_ERROR with a message.
+//   * Nesting deeper than MAX_DEPTH is an error.
+//   * Strings longer than MAX_STRING_BYTES, or with more than STRING_WINDOW
+//     input bytes (icons in Aegis exports), are not decoded: they come back as T_LONG_STRING and are skipped with native
+//     searches, so a 50 KB icon costs a few calls, not 50,000 loop rounds.
+//   * Escapes, including \u with surrogate pairs, are decoded; the result
+//     must be valid UTF-8 without NUL (Bytes.toUtf8).
+//   * Input that ends inside the document gives T_INCOMPLETE (not an error),
+//     so a multi-part import can wait for the rest.
+//   * Numbers are returned as Long when they are integers that fit;
+//     numberIsInteger is false otherwise (Aegis only needs integers).
+//   * Work per call is bounded: at most STRING_WINDOW bytes are looked at
+//     in Monkey C per token; searches over longer stretches are native.
+
+import Toybox.Lang;
+
+module Json {
+
+    const T_END = 0;          // the document is complete
+    const T_BEGIN_OBJECT = 1;
+    const T_END_OBJECT = 2;
+    const T_BEGIN_ARRAY = 3;
+    const T_END_ARRAY = 4;
+    const T_KEY = 5;          // text holds the key
+    const T_STRING = 6;       // text holds the value
+    const T_NUMBER = 7;       // number / numberIsInteger
+    const T_TRUE = 8;
+    const T_FALSE = 9;
+    const T_NULL = 10;
+    const T_LONG_STRING = 11; // a string value over MAX_STRING_BYTES, skipped
+    const T_INCOMPLETE = 12;  // the input ended inside the document
+    const T_ERROR = 13;       // error holds the reason
+
+    const MAX_DEPTH = 16;
+    // Longest decoded string; values the importer needs (names, secrets,
+    // group IDs) are far shorter.
+    const MAX_STRING_BYTES = 512;
+    // Input bytes examined for one string: escapes may double its size.
+    const STRING_WINDOW = 2 * MAX_STRING_BYTES + 2;
+    // Chunk size for native searches when skipping long strings.
+    const SKIP_CHUNK = 4096;
+
+    // Parser states.
+    const S_VALUE = 0;
+    const S_VALUE_OR_END = 1; // after '['
+    const S_KEY_OR_END = 2;   // after '{'
+    const S_KEY = 3;          // after ',' in an object
+    const S_COLON = 4;
+    const S_COMMA_OR_END = 5;
+    const S_DONE = 6;
+    const S_FAILED = 7;
+
+    const QUOTE = 0x22;
+    const BACKSLASH = 0x5C;
+
+    class Tokenizer {
+
+        var pos as Number = 0;
+        var depth as Number = 0;
+        var text as String = "";
+        var number as Long = 0l;
+        var numberIsInteger as Boolean = false;
+        var error as String = "";
+
+        private var _data as ByteArray;
+        private var _stack as ByteArray = []b; // 1 = object, 2 = array
+        private var _state as Number = S_VALUE;
+
+        function initialize(data as ByteArray) {
+            _data = data;
+        }
+
+        // Bytes of input.
+        function size() as Number {
+            return _data.size();
+        }
+
+        // The next token (T_*). After T_END, T_INCOMPLETE or T_ERROR every
+        // further call returns the same token.
+        function next() as Number {
+            while (true) {
+                if (_state == S_FAILED) {
+                    return T_ERROR;
+                }
+                skipWhitespace();
+                if (pos >= _data.size()) {
+                    return _state == S_DONE ? T_END : T_INCOMPLETE;
+                }
+                var b = _data[pos];
+                if (_state == S_DONE) {
+                    return fail("unexpected data after the end of the JSON document");
+                }
+                if (_state == S_COLON) {
+                    if (b != 0x3A) {
+                        return fail("':' expected");
+                    }
+                    pos++;
+                    _state = S_VALUE;
+                    continue;
+                }
+                if (_state == S_COMMA_OR_END) {
+                    var top = _stack[_stack.size() - 1];
+                    if (b == 0x2C) {
+                        pos++;
+                        _state = top == 1 ? S_KEY : S_VALUE;
+                        continue;
+                    }
+                    if ((top == 1 && b == 0x7D) || (top == 2 && b == 0x5D)) {
+                        return close(top);
+                    }
+                    return fail(top == 1 ? "',' or '}' expected" : "',' or ']' expected");
+                }
+                if (_state == S_KEY_OR_END && b == 0x7D) {
+                    return close(1);
+                }
+                if (_state == S_KEY_OR_END || _state == S_KEY) {
+                    if (b != QUOTE) {
+                        return fail("a quoted key expected");
+                    }
+                    var key = readString(false);
+                    if (key != T_STRING) {
+                        return key;
+                    }
+                    _state = S_COLON;
+                    return T_KEY;
+                }
+                if (_state == S_VALUE_OR_END && b == 0x5D) {
+                    return close(2);
+                }
+                return value(b);
+            }
+            return T_ERROR; // not reached
+        }
+
+        private function value(b as Number) as Number {
+            if (b == 0x7B || b == 0x5B) {
+                if (depth >= MAX_DEPTH) {
+                    return fail("nested deeper than " + MAX_DEPTH + " levels");
+                }
+                pos++;
+                depth++;
+                _stack.add(b == 0x7B ? 1 : 2);
+                _state = b == 0x7B ? S_KEY_OR_END : S_VALUE_OR_END;
+                return b == 0x7B ? T_BEGIN_OBJECT : T_BEGIN_ARRAY;
+            }
+            var token = T_ERROR;
+            if (b == QUOTE) {
+                token = readString(true);
+            } else if (b == 0x74) {
+                token = literal("true", T_TRUE);
+            } else if (b == 0x66) {
+                token = literal("false", T_FALSE);
+            } else if (b == 0x6E) {
+                token = literal("null", T_NULL);
+            } else if (b == 0x2D || (b >= 0x30 && b <= 0x39)) {
+                token = readNumber();
+            } else {
+                return fail("a value expected");
+            }
+            if (token != T_INCOMPLETE && token != T_ERROR) {
+                _state = depth == 0 ? S_DONE : S_COMMA_OR_END;
+            }
+            return token;
+        }
+
+        private function close(kind as Number) as Number {
+            pos++;
+            depth--;
+            _stack = _stack.slice(0, _stack.size() - 1);
+            _state = depth == 0 ? S_DONE : S_COMMA_OR_END;
+            return kind == 1 ? T_END_OBJECT : T_END_ARRAY;
+        }
+
+        private function fail(message as String) as Number {
+            error = message + " at byte " + pos;
+            _state = S_FAILED;
+            return T_ERROR;
+        }
+
+        private function skipWhitespace() as Void {
+            var data = _data;
+            var n = data.size();
+            while (pos < n) {
+                var b = data[pos];
+                if (b != 0x20 && b != 0x0A && b != 0x0D && b != 0x09) {
+                    return;
+                }
+                pos++;
+            }
+        }
+
+        private function literal(word as String, token as Number) as Number {
+            var bytes = Bytes.fromUtf8(word);
+            for (var i = 0; i < bytes.size(); i++) {
+                if (pos + i >= _data.size()) {
+                    return T_INCOMPLETE;
+                }
+                if (_data[pos + i] != bytes[i]) {
+                    return fail("invalid literal");
+                }
+            }
+            pos += bytes.size();
+            return token;
+        }
+
+        // -?(0|[1-9][0-9]*)(.[0-9]+)?([eE][+-]?[0-9]+)?
+        private function readNumber() as Number {
+            var data = _data;
+            var n = data.size();
+            var start = pos;
+            var i = pos;
+            var negative = false;
+            if (data[i] == 0x2D) {
+                negative = true;
+                i++;
+            }
+            var intStart = i;
+            var value = 0l;
+            var overflow = false;
+            while (i < n && data[i] >= 0x30 && data[i] <= 0x39) {
+                if (i - intStart >= 18) {
+                    overflow = true;
+                } else {
+                    value = value * 10 + (data[i] - 0x30);
+                }
+                i++;
+            }
+            if (i == intStart) {
+                return i >= n ? T_INCOMPLETE : fail("invalid number");
+            }
+            if (i - intStart > 1 && data[intStart] == 0x30) {
+                return fail("leading zero in a number");
+            }
+            var integer = !overflow;
+            if (i < n && data[i] == 0x2E) {
+                integer = false;
+                i++;
+                var fracStart = i;
+                while (i < n && data[i] >= 0x30 && data[i] <= 0x39) {
+                    i++;
+                }
+                if (i == fracStart) {
+                    return i >= n ? T_INCOMPLETE : fail("invalid number");
+                }
+            }
+            if (i < n && (data[i] == 0x65 || data[i] == 0x45)) {
+                integer = false;
+                i++;
+                if (i < n && (data[i] == 0x2B || data[i] == 0x2D)) {
+                    i++;
+                }
+                var expStart = i;
+                while (i < n && data[i] >= 0x30 && data[i] <= 0x39) {
+                    i++;
+                }
+                if (i == expStart) {
+                    return i >= n ? T_INCOMPLETE : fail("invalid number");
+                }
+            }
+            if (i >= n && depth > 0) {
+                // The number may continue in the next part.
+                pos = start;
+                return T_INCOMPLETE;
+            }
+            pos = i;
+            numberIsInteger = integer;
+            number = negative ? -value : value;
+            return T_NUMBER;
+        }
+
+        // Reads a string starting at the opening quote. Values over
+        // MAX_STRING_BYTES are skipped (T_LONG_STRING); keys that long are an
+        // error. Returns T_STRING with `text` set, or another token.
+        private function readString(isValue as Boolean) as Number {
+            var data = _data;
+            var n = data.size();
+            var start = pos + 1;
+            // Find the closing quote within the window, with native searches.
+            var window = data.slice(start, start + STRING_WINDOW < n ? start + STRING_WINDOW : n);
+            var end = closingQuote(window);
+            if (end < 0) {
+                if (window.size() < STRING_WINDOW) {
+                    return T_INCOMPLETE; // the input ends inside the string
+                }
+                return isValue ? skipLongString(start) : fail("key too long");
+            }
+            if (end > MAX_STRING_BYTES && window.slice(0, end).indexOf(BACKSLASH) < 0) {
+                // Without escapes the decoded text is as long as the input.
+                pos = start + end + 1;
+                return isValue ? T_LONG_STRING : fail("key too long");
+            }
+            var out = []b;
+            var i = 0;
+            while (i < end) {
+                var b = window[i];
+                if (b < 0x20) {
+                    pos = start + i;
+                    return fail("control character in a string");
+                }
+                if (b != BACKSLASH) {
+                    out.add(b);
+                    i++;
+                    continue;
+                }
+                var e = window[i + 1];
+                if (e == 0x75) {
+                    var consumed = unicodeEscape(window, i, end, out);
+                    if (consumed < 0) {
+                        pos = start + i;
+                        return fail("invalid \\u escape");
+                    }
+                    i += consumed;
+                    continue;
+                }
+                var mapped = escapeByte(e);
+                if (mapped < 0) {
+                    pos = start + i;
+                    return fail("invalid escape");
+                }
+                out.add(mapped);
+                i += 2;
+            }
+            if (out.size() > MAX_STRING_BYTES) {
+                pos = start + end + 1;
+                if (!isValue) {
+                    return fail("key too long");
+                }
+                return T_LONG_STRING;
+            }
+            var decoded = Bytes.toUtf8(out);
+            if (decoded == null) {
+                pos = start;
+                return fail("invalid UTF-8 or NUL in a string");
+            }
+            text = decoded;
+            pos = start + end + 1;
+            return T_STRING;
+        }
+
+        // Index of the first unescaped quote in bytes, or -1. Quotes are
+        // found natively; one preceded by an odd number of backslashes is
+        // escaped.
+        private function closingQuote(bytes as ByteArray) as Number {
+            var offset = 0;
+            while (offset < bytes.size()) {
+                var rel = (offset == 0 ? bytes : bytes.slice(offset, null)).indexOf(QUOTE);
+                if (rel < 0) {
+                    return -1;
+                }
+                var q = offset + rel;
+                var backslashes = 0;
+                while (q - 1 - backslashes >= 0 && bytes[q - 1 - backslashes] == BACKSLASH) {
+                    backslashes++;
+                }
+                if (backslashes % 2 == 0) {
+                    return q;
+                }
+                offset = q + 1;
+            }
+            return -1;
+        }
+
+        // Skips a long string value from `start` (just after its opening
+        // quote) using native searches; escaped quotes are recognised by the
+        // number of backslashes before them.
+        private function skipLongString(start as Number) as Number {
+            var data = _data;
+            var n = data.size();
+            var p = start;
+            while (p < n) {
+                var chunkEnd = p + SKIP_CHUNK < n ? p + SKIP_CHUNK : n;
+                var found = data.slice(p, chunkEnd).indexOf(QUOTE);
+                if (found < 0) {
+                    p = chunkEnd;
+                    continue;
+                }
+                var q = p + found;
+                var backslashes = 0;
+                while (q - 1 - backslashes >= start && data[q - 1 - backslashes] == BACKSLASH) {
+                    backslashes++;
+                }
+                if (backslashes % 2 == 0) {
+                    pos = q + 1;
+                    return T_LONG_STRING;
+                }
+                p = q + 1;
+            }
+            return T_INCOMPLETE;
+        }
+
+        // The byte for a one-character escape, or -1.
+        private function escapeByte(e as Number) as Number {
+            if (e == QUOTE || e == BACKSLASH || e == 0x2F) {
+                return e;
+            }
+            if (e == 0x62) {
+                return 0x08;
+            }
+            if (e == 0x66) {
+                return 0x0C;
+            }
+            if (e == 0x6E) {
+                return 0x0A;
+            }
+            if (e == 0x72) {
+                return 0x0D;
+            }
+            if (e == 0x74) {
+                return 0x09;
+            }
+            return -1;
+        }
+
+        // Decodes \uXXXX (and a following low surrogate) at bytes[i] into
+        // UTF-8. Returns the number of input bytes used, or -1.
+        private function unicodeEscape(bytes as ByteArray, i as Number, end as Number, out as ByteArray) as Number {
+            var cp = hex4(bytes, i + 2, end);
+            if (cp < 0) {
+                return -1;
+            }
+            var used = 6;
+            if (cp >= 0xD800 && cp <= 0xDBFF) {
+                if (i + 11 >= end + 1 || bytes[i + 6] != BACKSLASH || bytes[i + 7] != 0x75) {
+                    return -1;
+                }
+                var low = hex4(bytes, i + 8, end);
+                if (low < 0xDC00 || low > 0xDFFF) {
+                    return -1;
+                }
+                cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00);
+                used = 12;
+            } else if (cp >= 0xDC00 && cp <= 0xDFFF) {
+                return -1;
+            }
+            if (cp < 0x80) {
+                out.add(cp);
+            } else if (cp < 0x800) {
+                out.add(0xC0 | (cp >> 6));
+                out.add(0x80 | (cp & 0x3F));
+            } else if (cp < 0x10000) {
+                out.add(0xE0 | (cp >> 12));
+                out.add(0x80 | ((cp >> 6) & 0x3F));
+                out.add(0x80 | (cp & 0x3F));
+            } else {
+                out.add(0xF0 | (cp >> 18));
+                out.add(0x80 | ((cp >> 12) & 0x3F));
+                out.add(0x80 | ((cp >> 6) & 0x3F));
+                out.add(0x80 | (cp & 0x3F));
+            }
+            return used;
+        }
+
+        // Four hex digits at bytes[at..at+3] (all before `end`), or -1.
+        private function hex4(bytes as ByteArray, at as Number, end as Number) as Number {
+            if (at + 4 > end) {
+                return -1;
+            }
+            var v = 0;
+            for (var k = 0; k < 4; k++) {
+                var c = bytes[at + k];
+                var d = -1;
+                if (c >= 0x30 && c <= 0x39) {
+                    d = c - 0x30;
+                } else if (c >= 0x41 && c <= 0x46) {
+                    d = c - 0x37;
+                } else if (c >= 0x61 && c <= 0x66) {
+                    d = c - 0x57;
+                }
+                if (d < 0) {
+                    return -1;
+                }
+                v = (v << 4) | d;
+            }
+            return v;
+        }
+    }
+}