From a69e23fb5266dd942d125239e7dcbe8a7c8ab956 Mon Sep 17 00:00:00 2001 From: cirakg Date: Wed, 21 Oct 2026 03:55:01 +0200 Subject: perf(import): keep parsed items packed and the pasted text once A parsed import item with its Entry took about 760 bytes as objects on the Forerunner 255S; packed into one Codec ByteArray it takes about 110, and the importer unpacks one item per store step (wiping the packed secret). Aegis group names are resolved when unpacking, so no step repacks all items. The JSON is parsed from the untrimmed text's bytes and the string is dropped; an incomplete part is saved from those bytes. The form hash is computed before the importer copies the text. The peak of a 100-entry import on the 255S falls from 56 % to 45 %. Co-Authored-By: Claude Opus 5.5 --- source/import/Aegis.mc | 44 ++++++++++-------- source/import/ImportBuffer.mc | 10 ++-- source/import/ImportItem.mc | 102 ++++++++++++++++++++++++++++++++++++++++- source/import/Importer.mc | 40 ++++++++++------ source/import/Json.mc | 5 ++ source/settings/SettingsJob.mc | 27 +++++++---- 6 files changed, 177 insertions(+), 51 deletions(-) (limited to 'source') diff --git a/source/import/Aegis.mc b/source/import/Aegis.mc index 05905e9..0d515cd 100644 --- a/source/import/Aegis.mc +++ b/source/import/Aegis.mc @@ -4,7 +4,8 @@ // README 11.3. The reader consumes the token stream of Json.Tokenizer and // tracks the path of the current value ("/db/entries/[]/info/secret"), so it // can stop after any token and continue in the next event. It keeps only -// the parsed items and the group names, never the document. +// the parsed items (packed, see ImportItem) and the group names; the +// document's bytes stay with the tokenizer. // // * Encrypted exports are refused: header.slots is not null, or db is a // string. @@ -34,7 +35,10 @@ class AegisReader { static const ENCRYPTED_MESSAGE = "This Aegis export is encrypted. In Aegis choose Export, then turn encryption off, and paste that file."; - var items as Array = [] as Array; + // Parsed items, packed (ImportItem.pack); unpack with item(i). + var items as Array = [] as Array; + // Group UUID -> name, from db.groups. + var groups as Dictionary = {} as Dictionary; var error as String? = null; private var _tok as Json.Tokenizer; @@ -44,7 +48,6 @@ class AegisReader { private var _fields as ImportFields? = null; private var _seen as Dictionary = {} as Dictionary; private var _group as Dictionary? = null; - private var _groups as Dictionary = {} as Dictionary; private var _encrypted as Boolean = false; private var _vaultVersion as Long = -1l; private var _sawEntries as Boolean = false; @@ -53,11 +56,25 @@ class AegisReader { _tok = new Json.Tokenizer(data); } - // Bytes read so far (for progress). + // Bytes read so far, and in all (for progress). function position() as Number { return _tok.pos; } + function size() as Number { + return _tok.size(); + } + + // The document's bytes (saved when it is incomplete). + function data() as ByteArray { + return _tok.data(); + } + + // Parsed item i, unpacked, with its group name resolved. + function item(i as Number) as ImportItem { + return ImportItem.unpack(items[i], groups); + } + // Process up to TOKENS_PER_STEP tokens. Returns RUNNING, DONE, // INCOMPLETE (the text ends early: wait for more) or FAILED (error set). function step() as Number { @@ -139,14 +156,14 @@ class AegisReader { item.label = fields.issuer.length() > 0 ? fields.issuer : fields.account; } fields.secret = null; - items.add(item); + items.add(item.pack()); } _item = null; _fields = null; } else if (p.equals("/db/groups/[]")) { var g = _group; if (g != null && g.hasKey("uuid") && g.hasKey("name")) { - _groups[g["uuid"] as String] = g["name"] as String; + groups[g["uuid"] as String] = g["name"] as String; } _group = null; } @@ -244,19 +261,8 @@ class AegisReader { return fail(_vaultVersion > 1 ? "Aegis export version " + _vaultVersion + " is not supported" : "not an Aegis export (no version 1 vault with db.entries)"); } - // Resolve group references now that db.groups has been read. - for (var i = 0; i < items.size(); i++) { - var item = items[i]; - var ref = item.groupRef; - var e = item.entry; - if (ref == null || e == null) { - continue; - } - var name = ref.find("uuid:") == 0 ? _groups[ref.substring(5, ref.length()) as String] : ref.substring(5, ref.length()); - if (name != null) { - e.group = fitBytes(Text.trim(name as String), Entry.MAX_GROUP_BYTES); - } - } + // Group references are resolved when items are unpacked (db.groups + // may come after the entries). return DONE; } diff --git a/source/import/ImportBuffer.mc b/source/import/ImportBuffer.mc index 0aca548..dee0012 100644 --- a/source/import/ImportBuffer.mc +++ b/source/import/ImportBuffer.mc @@ -54,17 +54,15 @@ module ImportBuffer { return text; } - // Save text as the waiting import. Returns false (and saves nothing) - // when it is over MAX_TEXT_BYTES. - function save(store as Persist.Backend, keys as SealedBox.Keys, text as String, now as Long) as Boolean { - var plain = Bytes.fromUtf8(text); + // Save the text's UTF-8 bytes as the waiting import. Returns false (and + // saves nothing) when they are over MAX_TEXT_BYTES. The caller's bytes + // are left as they are. + function save(store as Persist.Backend, keys as SealedBox.Keys, plain as ByteArray, now as Long) as Boolean { if (plain.size() > MAX_TEXT_BYTES) { - Bytes.wipe(plain); return false; } clear(store); var box = SealedBox.seal(keys, Bytes.fromUtf8(AAD), plain); - Bytes.wipe(plain); var n = 0; for (var at = 0; at < box.size(); at += CHUNK_BYTES) { store.setValue(META_KEY + "." + n, box.slice(at, at + CHUNK_BYTES < box.size() ? at + CHUNK_BYTES : box.size())); diff --git a/source/import/ImportItem.mc b/source/import/ImportItem.mc index a242aa0..cc772af 100644 --- a/source/import/ImportItem.mc +++ b/source/import/ImportItem.mc @@ -1,10 +1,17 @@ // Author: cirakg & claude ai -// Summary: One parsed import candidate (an entry, an error or a skip) and the shared field validation of both formats. +// Summary: One parsed import candidate (an entry, an error or a skip), its packed form, and the shared field validation of both formats. // // Both parsers (otpauth URIs and Aegis JSON) turn text fields into an Entry // through ImportFields.toEntry, so the same rules (README 10.4) and the same // messages apply to both. Messages name the field and the problem, never the // secret itself. +// +// Memory: an item with its Entry takes about 760 bytes as objects (measured +// on the Forerunner 255S in M8), so 150 parsed items would take 110 KB. The +// parsers keep each finished item packed into one ByteArray (Codec TLV, +// about 110 bytes) instead, and the importer unpacks one item per step. +// The packed form holds the secret in clear, like the Entry it replaces; +// the importer wipes it as soon as the item is unpacked. import Toybox.Lang; @@ -25,10 +32,103 @@ class ImportItem { // the reader once the whole document is read. var groupRef as String? = null; + // Packed field tags. + private static const P_INDEX = 1; + private static const P_LABEL = 2; + private static const P_PROBLEM = 3; + private static const P_SKIPPED = 4; + private static const P_NOTE = 5; + private static const P_GROUP_REF = 6; + private static const P_TYPE = 10; + private static const P_NAME = 11; + private static const P_ACCOUNT = 12; + private static const P_FAVORITE = 13; + private static const P_ALGORITHM = 14; + private static const P_DIGITS = 15; + private static const P_PERIOD = 16; + private static const P_COUNTER = 17; + private static const P_SECRET = 18; + function initialize(position as Number) { index = position; } + // This item as one ByteArray. Wipes the entry's secret: the packed copy + // is the only one left. + function pack() as ByteArray { + var w = new Codec.Writer(); + w.addInt(P_INDEX, index); + w.addText(P_LABEL, label); + addText(w, P_PROBLEM, problem); + addText(w, P_SKIPPED, skipped); + addText(w, P_NOTE, note); + addText(w, P_GROUP_REF, groupRef); + var e = entry; + if (e != null && e.secret != null) { + w.addInt(P_TYPE, e.type); + w.addText(P_NAME, e.name); + w.addText(P_ACCOUNT, e.account); + w.addBool(P_FAVORITE, e.favorite); + w.addInt(P_ALGORITHM, e.algorithm); + w.addInt(P_DIGITS, e.digits); + w.addInt(P_PERIOD, e.period); + w.addInt(P_COUNTER, e.counter); + w.addBytes(P_SECRET, e.secret as ByteArray); + e.wipeSecrets(); + } + return w.bytes; + } + + private static function addText(w as Codec.Writer, tag as Number, value as String?) as Void { + if (value != null) { + w.addText(tag, value); + } + } + + // The item packed by pack(), with its group resolved: groupRef is + // "name:" or "uuid:" looked up in groups (Aegis db.groups). + // The caller wipes `packed`. + static function unpack(packed as ByteArray, groups as Dictionary) as ImportItem { + var map = Codec.decode(packed); + var item = new ImportItem(0); + if (map == null) { + item.problem = "invalid"; // only if memory was corrupted + return item; + } + item.index = (map[P_INDEX] as Long).toNumber(); + item.label = map[P_LABEL] as String; + item.problem = map[P_PROBLEM] as String?; + item.skipped = map[P_SKIPPED] as String?; + item.note = map[P_NOTE] as String?; + var secret = map[P_SECRET]; + if (secret instanceof ByteArray) { + var e = new Entry(); + e.kind = Entry.KIND_OTP; + e.source = Entry.SOURCE_IMPORT; + e.type = (map[P_TYPE] as Long).toNumber(); + e.name = map[P_NAME] as String; + e.account = map[P_ACCOUNT] as String; + e.favorite = map[P_FAVORITE] as Boolean; + e.algorithm = (map[P_ALGORITHM] as Long).toNumber() as Hashes.Algorithm; + e.digits = (map[P_DIGITS] as Long).toNumber(); + e.period = (map[P_PERIOD] as Long).toNumber(); + e.counter = map[P_COUNTER] as Long; + e.secret = secret; + e.group = groupName(map[P_GROUP_REF] as String?, groups); + item.entry = e; + } + return item; + } + + private static function groupName(ref as String?, groups as Dictionary) as String { + if (ref == null) { + return ""; + } + var key = ref.substring(5, ref.length()) as String; + var name = ref.find("uuid:") == 0 ? groups[key] : key; + return name != null ? AegisReader.fitBytes(Text.trim(name), Entry.MAX_GROUP_BYTES) : ""; + } + // "entry 7 'GitHub'" or "URI 3", for messages. function describe(kind as String) as String { return label.length() > 0 ? kind + " " + index + " '" + label + "'" : kind + " " + index; diff --git a/source/import/Importer.mc b/source/import/Importer.mc index 9513358..825221b 100644 --- a/source/import/Importer.mc +++ b/source/import/Importer.mc @@ -12,6 +12,10 @@ // STORE one item per step: skip, report, or add to the vault. A crash // leaves only whole entries. // The report is complete when step() returns true. +// +// Memory: while JSON is parsed the pasted text is held once, as the +// tokenizer's bytes, and parsed items are kept packed (ImportItem.pack); +// the M8 numbers are in docs/PERFORMANCE.md. import Toybox.Lang; @@ -40,7 +44,8 @@ class Importer extends Jobs.Job { private var _text as String = ""; private var _uris as Array? = null; private var _aegis as AegisReader? = null; - private var _items as Array = [] as Array; + private var _items as Array = [] as Array; + private var _groups as Dictionary = {} as Dictionary; private var _cursor as Number = 0; private var _ids as Array = [] as Array; private var _fingerprints as Dictionary = {} as Dictionary; @@ -57,7 +62,7 @@ class Importer extends Jobs.Job { _now = now; var keys = vault.keyManager.deviceKeys(); var earlier = ImportBuffer.load(vault.storage(), keys, now); - _text = earlier + text; + _text = earlier.length() > 0 ? earlier + text : text; detect(); } @@ -65,9 +70,8 @@ class Importer extends Jobs.Job { function percent() as Number { if (phase == P_PARSE) { var a = _aegis; - var size = _text.length() > 0 ? _text.length() : 1; if (a != null) { - return 40 * a.position() / (size > 0 ? size : 1); + return 40 * a.position() / (a.size() > 0 ? a.size() : 1); } var u = _uris; return u != null && u.size() > 0 ? 40 * _cursor / u.size() : 0; @@ -85,7 +89,10 @@ class Importer extends Jobs.Job { return; } if (trimmed.find("{") == 0) { - _aegis = new AegisReader(Bytes.fromUtf8(trimmed)); + // The untrimmed text: an incomplete part is saved exactly as + // received (a cut can fall on a space inside a string). + _aegis = new AegisReader(Bytes.fromUtf8(_text)); + _text = ""; return; } if (Text.findIgnoreCase(trimmed, OtpAuth.MIGRATION_PREFIX, 0) >= 0) { @@ -133,31 +140,32 @@ class Importer extends Jobs.Job { return; } if (status == AegisReader.INCOMPLETE) { - var size = Bytes.fromUtf8(_text).size(); - if (ImportBuffer.save(_vault.storage(), _vault.keyManager.deviceKeys(), _text, _now)) { - report.waitingChars = _text.length(); + var data = aegis.data(); + _aegis = null; + if (ImportBuffer.save(_vault.storage(), _vault.keyManager.deviceKeys(), data, _now)) { + var text = Bytes.toUtf8(data); + report.waitingChars = text != null ? text.length() : data.size(); phase = P_DONE; } else { - finishWith("the JSON is incomplete and too long to wait for more (" + size + " bytes, at most " + finishWith("the JSON is incomplete and too long to wait for more (" + data.size() + " bytes, at most " + ImportBuffer.MAX_TEXT_BYTES + ")"); } - _text = ""; return; } if (status == AegisReader.FAILED) { + _aegis = null; finishWith(aegis.error as String); - _text = ""; return; } _items = aegis.items; + _groups = aegis.groups; _aegis = null; - _text = ""; startReplace(); return; } var uris = _uris as Array; for (var n = 0; n < URIS_PER_STEP && _cursor < uris.size(); n++) { - _items.add(OtpAuth.parse(uris[_cursor], _cursor + 1)); + _items.add(OtpAuth.parse(uris[_cursor], _cursor + 1).pack()); _cursor++; } if (_cursor >= uris.size()) { @@ -239,8 +247,10 @@ class Importer extends Jobs.Job { phase = P_DONE; return; } - var item = _items[_cursor]; - _items[_cursor] = new ImportItem(item.index); // drop the parsed secret + var packed = _items[_cursor]; + _items[_cursor] = []b; + var item = ImportItem.unpack(packed, _groups); + Bytes.wipe(packed); // the parsed secret; about 110 bytes _cursor++; var kind = _kind; var note = item.note; diff --git a/source/import/Json.mc b/source/import/Json.mc index c19150e..e3e9fa8 100644 --- a/source/import/Json.mc +++ b/source/import/Json.mc @@ -84,6 +84,11 @@ module Json { return _data.size(); } + // The input itself (kept when the document is incomplete). + function data() as ByteArray { + return _data; + } + // The next token (T_*). After T_END, T_INCOMPLETE or T_ERROR every // further call returns the same token. function next() as Number { diff --git a/source/settings/SettingsJob.mc b/source/settings/SettingsJob.mc index e6e7bcf..2fc4a3d 100644 --- a/source/settings/SettingsJob.mc +++ b/source/settings/SettingsJob.mc @@ -105,24 +105,31 @@ class SettingsJob extends Jobs.Job { endImport(["Import: waiting import cancelled."] as Array, "Waiting import cancelled", false); return; } - var content = new Forms.Content("import"); - content.add(importMode); - content.add(locked); - var text = ""; - for (var i = 1; i <= IMPORT_TEXT_FIELDS; i++) { - var part = Forms.rawText(_store, "impText" + i); - content.add(part); - text += part; - } - _hash = Forms.hash(_vault, content); + _hash = importHash(importMode, locked); if (Forms.alreadyApplied(_vault, _hash)) { endImport(["Import: already applied. Change the text to import again."] as Array, "Already imported", true); return; } + var text = ""; + for (var i = 1; i <= IMPORT_TEXT_FIELDS; i++) { + text += Forms.rawText(_store, "impText" + i); + } _importStart = System.getTimer(); _importer = new Importer(_vault, text, importMode, locked, _now); } + // The import form's hash. Its own function, so the field bytes it hashes + // are freed before the importer makes its copy of the text. + private function importHash(importMode as Number, locked as Boolean) as String { + var content = new Forms.Content("import"); + content.add(importMode); + content.add(locked); + for (var i = 1; i <= IMPORT_TEXT_FIELDS; i++) { + content.add(Forms.rawText(_store, "impText" + i)); + } + return Forms.hash(_vault, content); + } + private function finishImport(importer as Importer) as Void { _importer = null; var report = importer.report; -- cgit v1.2.3