path: root/source/import/Json.mc

// Author:  cirakg & claude ai
// Summary: A streaming JSON tokenizer over UTF-8 bytes: one token per call, strict grammar, depth and length limits.
//
// README 11.3: the Aegis reader must stream, never build large objects, and
// pause between steps (watchdog). This tokenizer keeps only its position, a
// small container stack and the current token, so a reader can stop after any
// token and continue in the next event.
//
// Limits and guarantees:
//   * Strict RFC 8259 grammar; anything else is T_ERROR with a message.
//   * Nesting deeper than MAX_DEPTH is an error.
//   * Strings longer than MAX_STRING_BYTES, or with more than STRING_WINDOW
//     input bytes (icons in Aegis exports), are not decoded: they come back as T_LONG_STRING and are skipped with native
//     searches, so a 50 KB icon costs a few calls, not 50,000 loop rounds.
//   * Escapes, including \u with surrogate pairs, are decoded; the result
//     must be valid UTF-8 without NUL (Bytes.toUtf8).
//   * Input that ends inside the document gives T_INCOMPLETE (not an error),
//     so a multi-part import can wait for the rest.
//   * Numbers are returned as Long when they are integers that fit;
//     numberIsInteger is false otherwise (Aegis only needs integers).
//   * Work per call is bounded: at most STRING_WINDOW bytes are looked at
//     in Monkey C per token; searches over longer stretches are native.

import Toybox.Lang;

module Json {

    const T_END = 0;          // the document is complete
    const T_BEGIN_OBJECT = 1;
    const T_END_OBJECT = 2;
    const T_BEGIN_ARRAY = 3;
    const T_END_ARRAY = 4;
    const T_KEY = 5;          // text holds the key
    const T_STRING = 6;       // text holds the value
    const T_NUMBER = 7;       // number / numberIsInteger
    const T_TRUE = 8;
    const T_FALSE = 9;
    const T_NULL = 10;
    const T_LONG_STRING = 11; // a string value over MAX_STRING_BYTES, skipped
    const T_INCOMPLETE = 12;  // the input ended inside the document
    const T_ERROR = 13;       // error holds the reason

    const MAX_DEPTH = 16;
    // Longest decoded string; values the importer needs (names, secrets,
    // group IDs) are far shorter.
    const MAX_STRING_BYTES = 512;
    // Input bytes examined for one string: escapes may double its size.
    const STRING_WINDOW = 2 * MAX_STRING_BYTES + 2;
    // Chunk size for native searches when skipping long strings.
    const SKIP_CHUNK = 4096;

    // Parser states.
    const S_VALUE = 0;
    const S_VALUE_OR_END = 1; // after '['
    const S_KEY_OR_END = 2;   // after '{'
    const S_KEY = 3;          // after ',' in an object
    const S_COLON = 4;
    const S_COMMA_OR_END = 5;
    const S_DONE = 6;
    const S_FAILED = 7;

    const QUOTE = 0x22;
    const BACKSLASH = 0x5C;

    class Tokenizer {

        var pos as Number = 0;
        var depth as Number = 0;
        var text as String = "";
        var number as Long = 0l;
        var numberIsInteger as Boolean = false;
        var error as String = "";

        private var _data as ByteArray;
        private var _stack as ByteArray = []b; // 1 = object, 2 = array
        private var _state as Number = S_VALUE;

        function initialize(data as ByteArray) {
            _data = data;
        }

        // Bytes of input.
        function size() as Number {
            return _data.size();
        }

        // The next token (T_*). After T_END, T_INCOMPLETE or T_ERROR every
        // further call returns the same token.
        function next() as Number {
            while (true) {
                if (_state == S_FAILED) {
                    return T_ERROR;
                }
                skipWhitespace();
                if (pos >= _data.size()) {
                    return _state == S_DONE ? T_END : T_INCOMPLETE;
                }
                var b = _data[pos];
                if (_state == S_DONE) {
                    return fail("unexpected data after the end of the JSON document");
                }
                if (_state == S_COLON) {
                    if (b != 0x3A) {
                        return fail("':' expected");
                    }
                    pos++;
                    _state = S_VALUE;
                    continue;
                }
                if (_state == S_COMMA_OR_END) {
                    var top = _stack[_stack.size() - 1];
                    if (b == 0x2C) {
                        pos++;
                        _state = top == 1 ? S_KEY : S_VALUE;
                        continue;
                    }
                    if ((top == 1 && b == 0x7D) || (top == 2 && b == 0x5D)) {
                        return close(top);
                    }
                    return fail(top == 1 ? "',' or '}' expected" : "',' or ']' expected");
                }
                if (_state == S_KEY_OR_END && b == 0x7D) {
                    return close(1);
                }
                if (_state == S_KEY_OR_END || _state == S_KEY) {
                    if (b != QUOTE) {
                        return fail("a quoted key expected");
                    }
                    var key = readString(false);
                    if (key != T_STRING) {
                        return key;
                    }
                    _state = S_COLON;
                    return T_KEY;
                }
                if (_state == S_VALUE_OR_END && b == 0x5D) {
                    return close(2);
                }
                return value(b);
            }
            return T_ERROR; // not reached
        }

        private function value(b as Number) as Number {
            if (b == 0x7B || b == 0x5B) {
                if (depth >= MAX_DEPTH) {
                    return fail("nested deeper than " + MAX_DEPTH + " levels");
                }
                pos++;
                depth++;
                _stack.add(b == 0x7B ? 1 : 2);
                _state = b == 0x7B ? S_KEY_OR_END : S_VALUE_OR_END;
                return b == 0x7B ? T_BEGIN_OBJECT : T_BEGIN_ARRAY;
            }
            var token = T_ERROR;
            if (b == QUOTE) {
                token = readString(true);
            } else if (b == 0x74) {
                token = literal("true", T_TRUE);
            } else if (b == 0x66) {
                token = literal("false", T_FALSE);
            } else if (b == 0x6E) {
                token = literal("null", T_NULL);
            } else if (b == 0x2D || (b >= 0x30 && b <= 0x39)) {
                token = readNumber();
            } else {
                return fail("a value expected");
            }
            if (token != T_INCOMPLETE && token != T_ERROR) {
                _state = depth == 0 ? S_DONE : S_COMMA_OR_END;
            }
            return token;
        }

        private function close(kind as Number) as Number {
            pos++;
            depth--;
            _stack = _stack.slice(0, _stack.size() - 1);
            _state = depth == 0 ? S_DONE : S_COMMA_OR_END;
            return kind == 1 ? T_END_OBJECT : T_END_ARRAY;
        }

        private function fail(message as String) as Number {
            error = message + " at byte " + pos;
            _state = S_FAILED;
            return T_ERROR;
        }

        private function skipWhitespace() as Void {
            var data = _data;
            var n = data.size();
            while (pos < n) {
                var b = data[pos];
                if (b != 0x20 && b != 0x0A && b != 0x0D && b != 0x09) {
                    return;
                }
                pos++;
            }
        }

        private function literal(word as String, token as Number) as Number {
            var bytes = Bytes.fromUtf8(word);
            for (var i = 0; i < bytes.size(); i++) {
                if (pos + i >= _data.size()) {
                    return T_INCOMPLETE;
                }
                if (_data[pos + i] != bytes[i]) {
                    return fail("invalid literal");
                }
            }
            pos += bytes.size();
            return token;
        }

        // -?(0|[1-9][0-9]*)(.[0-9]+)?([eE][+-]?[0-9]+)?
        private function readNumber() as Number {
            var data = _data;
            var n = data.size();
            var start = pos;
            var i = pos;
            var negative = false;
            if (data[i] == 0x2D) {
                negative = true;
                i++;
            }
            var intStart = i;
            var value = 0l;
            var overflow = false;
            while (i < n && data[i] >= 0x30 && data[i] <= 0x39) {
                if (i - intStart >= 18) {
                    overflow = true;
                } else {
                    value = value * 10 + (data[i] - 0x30);
                }
                i++;
            }
            if (i == intStart) {
                return i >= n ? T_INCOMPLETE : fail("invalid number");
            }
            if (i - intStart > 1 && data[intStart] == 0x30) {
                return fail("leading zero in a number");
            }
            var integer = !overflow;
            if (i < n && data[i] == 0x2E) {
                integer = false;
                i++;
                var fracStart = i;
                while (i < n && data[i] >= 0x30 && data[i] <= 0x39) {
                    i++;
                }
                if (i == fracStart) {
                    return i >= n ? T_INCOMPLETE : fail("invalid number");
                }
            }
            if (i < n && (data[i] == 0x65 || data[i] == 0x45)) {
                integer = false;
                i++;
                if (i < n && (data[i] == 0x2B || data[i] == 0x2D)) {
                    i++;
                }
                var expStart = i;
                while (i < n && data[i] >= 0x30 && data[i] <= 0x39) {
                    i++;
                }
                if (i == expStart) {
                    return i >= n ? T_INCOMPLETE : fail("invalid number");
                }
            }
            if (i >= n && depth > 0) {
                // The number may continue in the next part.
                pos = start;
                return T_INCOMPLETE;
            }
            pos = i;
            numberIsInteger = integer;
            number = negative ? -value : value;
            return T_NUMBER;
        }

        // Reads a string starting at the opening quote. Values over
        // MAX_STRING_BYTES are skipped (T_LONG_STRING); keys that long are an
        // error. Returns T_STRING with `text` set, or another token.
        private function readString(isValue as Boolean) as Number {
            var data = _data;
            var n = data.size();
            var start = pos + 1;
            // Find the closing quote within the window, with native searches.
            var window = data.slice(start, start + STRING_WINDOW < n ? start + STRING_WINDOW : n);
            var end = closingQuote(window);
            if (end < 0) {
                if (window.size() < STRING_WINDOW) {
                    return T_INCOMPLETE; // the input ends inside the string
                }
                return isValue ? skipLongString(start) : fail("key too long");
            }
            if (end > MAX_STRING_BYTES && window.slice(0, end).indexOf(BACKSLASH) < 0) {
                // Without escapes the decoded text is as long as the input.
                pos = start + end + 1;
                return isValue ? T_LONG_STRING : fail("key too long");
            }
            var out = []b;
            var i = 0;
            while (i < end) {
                var b = window[i];
                if (b < 0x20) {
                    pos = start + i;
                    return fail("control character in a string");
                }
                if (b != BACKSLASH) {
                    out.add(b);
                    i++;
                    continue;
                }
                var e = window[i + 1];
                if (e == 0x75) {
                    var consumed = unicodeEscape(window, i, end, out);
                    if (consumed < 0) {
                        pos = start + i;
                        return fail("invalid \\u escape");
                    }
                    i += consumed;
                    continue;
                }
                var mapped = escapeByte(e);
                if (mapped < 0) {
                    pos = start + i;
                    return fail("invalid escape");
                }
                out.add(mapped);
                i += 2;
            }
            if (out.size() > MAX_STRING_BYTES) {
                pos = start + end + 1;
                if (!isValue) {
                    return fail("key too long");
                }
                return T_LONG_STRING;
            }
            var decoded = Bytes.toUtf8(out);
            if (decoded == null) {
                pos = start;
                return fail("invalid UTF-8 or NUL in a string");
            }
            text = decoded;
            pos = start + end + 1;
            return T_STRING;
        }

        // Index of the first unescaped quote in bytes, or -1. Quotes are
        // found natively; one preceded by an odd number of backslashes is
        // escaped.
        private function closingQuote(bytes as ByteArray) as Number {
            var offset = 0;
            while (offset < bytes.size()) {
                var rel = (offset == 0 ? bytes : bytes.slice(offset, null)).indexOf(QUOTE);
                if (rel < 0) {
                    return -1;
                }
                var q = offset + rel;
                var backslashes = 0;
                while (q - 1 - backslashes >= 0 && bytes[q - 1 - backslashes] == BACKSLASH) {
                    backslashes++;
                }
                if (backslashes % 2 == 0) {
                    return q;
                }
                offset = q + 1;
            }
            return -1;
        }

        // Skips a long string value from `start` (just after its opening
        // quote) using native searches; escaped quotes are recognised by the
        // number of backslashes before them.
        private function skipLongString(start as Number) as Number {
            var data = _data;
            var n = data.size();
            var p = start;
            while (p < n) {
                var chunkEnd = p + SKIP_CHUNK < n ? p + SKIP_CHUNK : n;
                var found = data.slice(p, chunkEnd).indexOf(QUOTE);
                if (found < 0) {
                    p = chunkEnd;
                    continue;
                }
                var q = p + found;
                var backslashes = 0;
                while (q - 1 - backslashes >= start && data[q - 1 - backslashes] == BACKSLASH) {
                    backslashes++;
                }
                if (backslashes % 2 == 0) {
                    pos = q + 1;
                    return T_LONG_STRING;
                }
                p = q + 1;
            }
            return T_INCOMPLETE;
        }

        // The byte for a one-character escape, or -1.
        private function escapeByte(e as Number) as Number {
            if (e == QUOTE || e == BACKSLASH || e == 0x2F) {
                return e;
            }
            if (e == 0x62) {
                return 0x08;
            }
            if (e == 0x66) {
                return 0x0C;
            }
            if (e == 0x6E) {
                return 0x0A;
            }
            if (e == 0x72) {
                return 0x0D;
            }
            if (e == 0x74) {
                return 0x09;
            }
            return -1;
        }

        // Decodes \uXXXX (and a following low surrogate) at bytes[i] into
        // UTF-8. Returns the number of input bytes used, or -1.
        private function unicodeEscape(bytes as ByteArray, i as Number, end as Number, out as ByteArray) as Number {
            var cp = hex4(bytes, i + 2, end);
            if (cp < 0) {
                return -1;
            }
            var used = 6;
            if (cp >= 0xD800 && cp <= 0xDBFF) {
                if (i + 11 >= end + 1 || bytes[i + 6] != BACKSLASH || bytes[i + 7] != 0x75) {
                    return -1;
                }
                var low = hex4(bytes, i + 8, end);
                if (low < 0xDC00 || low > 0xDFFF) {
                    return -1;
                }
                cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00);
                used = 12;
            } else if (cp >= 0xDC00 && cp <= 0xDFFF) {
                return -1;
            }
            if (cp < 0x80) {
                out.add(cp);
            } else if (cp < 0x800) {
                out.add(0xC0 | (cp >> 6));
                out.add(0x80 | (cp & 0x3F));
            } else if (cp < 0x10000) {
                out.add(0xE0 | (cp >> 12));
                out.add(0x80 | ((cp >> 6) & 0x3F));
                out.add(0x80 | (cp & 0x3F));
            } else {
                out.add(0xF0 | (cp >> 18));
                out.add(0x80 | ((cp >> 12) & 0x3F));
                out.add(0x80 | ((cp >> 6) & 0x3F));
                out.add(0x80 | (cp & 0x3F));
            }
            return used;
        }

        // Four hex digits at bytes[at..at+3] (all before `end`), or -1.
        private function hex4(bytes as ByteArray, at as Number, end as Number) as Number {
            if (at + 4 > end) {
                return -1;
            }
            var v = 0;
            for (var k = 0; k < 4; k++) {
                var c = bytes[at + k];
                var d = -1;
                if (c >= 0x30 && c <= 0x39) {
                    d = c - 0x30;
                } else if (c >= 0x41 && c <= 0x46) {
                    d = c - 0x37;
                } else if (c >= 0x61 && c <= 0x66) {
                    d = c - 0x57;
                }
                if (d < 0) {
                    return -1;
                }
                v = (v << 4) | d;
            }
            return v;
        }
    }
}