diff options
| -rw-r--r-- | source/import/Aegis.mc | 44 | ||||
| -rw-r--r-- | source/import/ImportBuffer.mc | 10 | ||||
| -rw-r--r-- | source/import/ImportItem.mc | 102 | ||||
| -rw-r--r-- | source/import/Importer.mc | 40 | ||||
| -rw-r--r-- | source/import/Json.mc | 5 | ||||
| -rw-r--r-- | source/settings/SettingsJob.mc | 27 | ||||
| -rw-r--r-- | test/import/AegisTest.mc | 6 | ||||
| -rw-r--r-- | test/import/FuzzTest.mc | 2 | ||||
| -rw-r--r-- | test/import/ImportSupport.mc | 9 | ||||
| -rw-r--r-- | test/import/ImporterTest.mc | 59 |
10 files changed, 248 insertions, 56 deletions
diff --git a/source/import/Aegis.mc b/source/import/Aegis.mc index 05905e9..0d515cd 100644 --- a/source/import/Aegis.mc +++ b/source/import/Aegis.mc @@ -4,7 +4,8 @@ // README 11.3. The reader consumes the token stream of Json.Tokenizer and // tracks the path of the current value ("/db/entries/[]/info/secret"), so it // can stop after any token and continue in the next event. It keeps only -// the parsed items and the group names, never the document. +// the parsed items (packed, see ImportItem) and the group names; the +// document's bytes stay with the tokenizer. // // * Encrypted exports are refused: header.slots is not null, or db is a // string. @@ -34,7 +35,10 @@ class AegisReader { static const ENCRYPTED_MESSAGE = "This Aegis export is encrypted. In Aegis choose Export, then turn encryption off, and paste that file."; - var items as Array<ImportItem> = [] as Array<ImportItem>; + // Parsed items, packed (ImportItem.pack); unpack with item(i). + var items as Array<ByteArray> = [] as Array<ByteArray>; + // Group UUID -> name, from db.groups. + var groups as Dictionary<String, String> = {} as Dictionary<String, String>; var error as String? = null; private var _tok as Json.Tokenizer; @@ -44,7 +48,6 @@ class AegisReader { private var _fields as ImportFields? = null; private var _seen as Dictionary<String, Boolean> = {} as Dictionary<String, Boolean>; private var _group as Dictionary<String, String>? = null; - private var _groups as Dictionary<String, String> = {} as Dictionary<String, String>; private var _encrypted as Boolean = false; private var _vaultVersion as Long = -1l; private var _sawEntries as Boolean = false; @@ -53,11 +56,25 @@ class AegisReader { _tok = new Json.Tokenizer(data); } - // Bytes read so far (for progress). + // Bytes read so far, and in all (for progress). function position() as Number { return _tok.pos; } + function size() as Number { + return _tok.size(); + } + + // The document's bytes (saved when it is incomplete). + function data() as ByteArray { + return _tok.data(); + } + + // Parsed item i, unpacked, with its group name resolved. + function item(i as Number) as ImportItem { + return ImportItem.unpack(items[i], groups); + } + // Process up to TOKENS_PER_STEP tokens. Returns RUNNING, DONE, // INCOMPLETE (the text ends early: wait for more) or FAILED (error set). function step() as Number { @@ -139,14 +156,14 @@ class AegisReader { item.label = fields.issuer.length() > 0 ? fields.issuer : fields.account; } fields.secret = null; - items.add(item); + items.add(item.pack()); } _item = null; _fields = null; } else if (p.equals("/db/groups/[]")) { var g = _group; if (g != null && g.hasKey("uuid") && g.hasKey("name")) { - _groups[g["uuid"] as String] = g["name"] as String; + groups[g["uuid"] as String] = g["name"] as String; } _group = null; } @@ -244,19 +261,8 @@ class AegisReader { return fail(_vaultVersion > 1 ? "Aegis export version " + _vaultVersion + " is not supported" : "not an Aegis export (no version 1 vault with db.entries)"); } - // Resolve group references now that db.groups has been read. - for (var i = 0; i < items.size(); i++) { - var item = items[i]; - var ref = item.groupRef; - var e = item.entry; - if (ref == null || e == null) { - continue; - } - var name = ref.find("uuid:") == 0 ? _groups[ref.substring(5, ref.length()) as String] : ref.substring(5, ref.length()); - if (name != null) { - e.group = fitBytes(Text.trim(name as String), Entry.MAX_GROUP_BYTES); - } - } + // Group references are resolved when items are unpacked (db.groups + // may come after the entries). return DONE; } diff --git a/source/import/ImportBuffer.mc b/source/import/ImportBuffer.mc index 0aca548..dee0012 100644 --- a/source/import/ImportBuffer.mc +++ b/source/import/ImportBuffer.mc @@ -54,17 +54,15 @@ module ImportBuffer { return text; } - // Save text as the waiting import. Returns false (and saves nothing) - // when it is over MAX_TEXT_BYTES. - function save(store as Persist.Backend, keys as SealedBox.Keys, text as String, now as Long) as Boolean { - var plain = Bytes.fromUtf8(text); + // Save the text's UTF-8 bytes as the waiting import. Returns false (and + // saves nothing) when they are over MAX_TEXT_BYTES. The caller's bytes + // are left as they are. + function save(store as Persist.Backend, keys as SealedBox.Keys, plain as ByteArray, now as Long) as Boolean { if (plain.size() > MAX_TEXT_BYTES) { - Bytes.wipe(plain); return false; } clear(store); var box = SealedBox.seal(keys, Bytes.fromUtf8(AAD), plain); - Bytes.wipe(plain); var n = 0; for (var at = 0; at < box.size(); at += CHUNK_BYTES) { store.setValue(META_KEY + "." + n, box.slice(at, at + CHUNK_BYTES < box.size() ? at + CHUNK_BYTES : box.size())); diff --git a/source/import/ImportItem.mc b/source/import/ImportItem.mc index a242aa0..cc772af 100644 --- a/source/import/ImportItem.mc +++ b/source/import/ImportItem.mc @@ -1,10 +1,17 @@ // Author: cirakg & claude ai -// Summary: One parsed import candidate (an entry, an error or a skip) and the shared field validation of both formats. +// Summary: One parsed import candidate (an entry, an error or a skip), its packed form, and the shared field validation of both formats. // // Both parsers (otpauth URIs and Aegis JSON) turn text fields into an Entry // through ImportFields.toEntry, so the same rules (README 10.4) and the same // messages apply to both. Messages name the field and the problem, never the // secret itself. +// +// Memory: an item with its Entry takes about 760 bytes as objects (measured +// on the Forerunner 255S in M8), so 150 parsed items would take 110 KB. The +// parsers keep each finished item packed into one ByteArray (Codec TLV, +// about 110 bytes) instead, and the importer unpacks one item per step. +// The packed form holds the secret in clear, like the Entry it replaces; +// the importer wipes it as soon as the item is unpacked. import Toybox.Lang; @@ -25,10 +32,103 @@ class ImportItem { // the reader once the whole document is read. var groupRef as String? = null; + // Packed field tags. + private static const P_INDEX = 1; + private static const P_LABEL = 2; + private static const P_PROBLEM = 3; + private static const P_SKIPPED = 4; + private static const P_NOTE = 5; + private static const P_GROUP_REF = 6; + private static const P_TYPE = 10; + private static const P_NAME = 11; + private static const P_ACCOUNT = 12; + private static const P_FAVORITE = 13; + private static const P_ALGORITHM = 14; + private static const P_DIGITS = 15; + private static const P_PERIOD = 16; + private static const P_COUNTER = 17; + private static const P_SECRET = 18; + function initialize(position as Number) { index = position; } + // This item as one ByteArray. Wipes the entry's secret: the packed copy + // is the only one left. + function pack() as ByteArray { + var w = new Codec.Writer(); + w.addInt(P_INDEX, index); + w.addText(P_LABEL, label); + addText(w, P_PROBLEM, problem); + addText(w, P_SKIPPED, skipped); + addText(w, P_NOTE, note); + addText(w, P_GROUP_REF, groupRef); + var e = entry; + if (e != null && e.secret != null) { + w.addInt(P_TYPE, e.type); + w.addText(P_NAME, e.name); + w.addText(P_ACCOUNT, e.account); + w.addBool(P_FAVORITE, e.favorite); + w.addInt(P_ALGORITHM, e.algorithm); + w.addInt(P_DIGITS, e.digits); + w.addInt(P_PERIOD, e.period); + w.addInt(P_COUNTER, e.counter); + w.addBytes(P_SECRET, e.secret as ByteArray); + e.wipeSecrets(); + } + return w.bytes; + } + + private static function addText(w as Codec.Writer, tag as Number, value as String?) as Void { + if (value != null) { + w.addText(tag, value); + } + } + + // The item packed by pack(), with its group resolved: groupRef is + // "name:<group>" or "uuid:<id>" looked up in groups (Aegis db.groups). + // The caller wipes `packed`. + static function unpack(packed as ByteArray, groups as Dictionary<String, String>) as ImportItem { + var map = Codec.decode(packed); + var item = new ImportItem(0); + if (map == null) { + item.problem = "invalid"; // only if memory was corrupted + return item; + } + item.index = (map[P_INDEX] as Long).toNumber(); + item.label = map[P_LABEL] as String; + item.problem = map[P_PROBLEM] as String?; + item.skipped = map[P_SKIPPED] as String?; + item.note = map[P_NOTE] as String?; + var secret = map[P_SECRET]; + if (secret instanceof ByteArray) { + var e = new Entry(); + e.kind = Entry.KIND_OTP; + e.source = Entry.SOURCE_IMPORT; + e.type = (map[P_TYPE] as Long).toNumber(); + e.name = map[P_NAME] as String; + e.account = map[P_ACCOUNT] as String; + e.favorite = map[P_FAVORITE] as Boolean; + e.algorithm = (map[P_ALGORITHM] as Long).toNumber() as Hashes.Algorithm; + e.digits = (map[P_DIGITS] as Long).toNumber(); + e.period = (map[P_PERIOD] as Long).toNumber(); + e.counter = map[P_COUNTER] as Long; + e.secret = secret; + e.group = groupName(map[P_GROUP_REF] as String?, groups); + item.entry = e; + } + return item; + } + + private static function groupName(ref as String?, groups as Dictionary<String, String>) as String { + if (ref == null) { + return ""; + } + var key = ref.substring(5, ref.length()) as String; + var name = ref.find("uuid:") == 0 ? groups[key] : key; + return name != null ? AegisReader.fitBytes(Text.trim(name), Entry.MAX_GROUP_BYTES) : ""; + } + // "entry 7 'GitHub'" or "URI 3", for messages. function describe(kind as String) as String { return label.length() > 0 ? kind + " " + index + " '" + label + "'" : kind + " " + index; diff --git a/source/import/Importer.mc b/source/import/Importer.mc index 9513358..825221b 100644 --- a/source/import/Importer.mc +++ b/source/import/Importer.mc @@ -12,6 +12,10 @@ // STORE one item per step: skip, report, or add to the vault. A crash // leaves only whole entries. // The report is complete when step() returns true. +// +// Memory: while JSON is parsed the pasted text is held once, as the +// tokenizer's bytes, and parsed items are kept packed (ImportItem.pack); +// the M8 numbers are in docs/PERFORMANCE.md. import Toybox.Lang; @@ -40,7 +44,8 @@ class Importer extends Jobs.Job { private var _text as String = ""; private var _uris as Array<String>? = null; private var _aegis as AegisReader? = null; - private var _items as Array<ImportItem> = [] as Array<ImportItem>; + private var _items as Array<ByteArray> = [] as Array<ByteArray>; + private var _groups as Dictionary<String, String> = {} as Dictionary<String, String>; private var _cursor as Number = 0; private var _ids as Array<String> = [] as Array<String>; private var _fingerprints as Dictionary<String, String> = {} as Dictionary<String, String>; @@ -57,7 +62,7 @@ class Importer extends Jobs.Job { _now = now; var keys = vault.keyManager.deviceKeys(); var earlier = ImportBuffer.load(vault.storage(), keys, now); - _text = earlier + text; + _text = earlier.length() > 0 ? earlier + text : text; detect(); } @@ -65,9 +70,8 @@ class Importer extends Jobs.Job { function percent() as Number { if (phase == P_PARSE) { var a = _aegis; - var size = _text.length() > 0 ? _text.length() : 1; if (a != null) { - return 40 * a.position() / (size > 0 ? size : 1); + return 40 * a.position() / (a.size() > 0 ? a.size() : 1); } var u = _uris; return u != null && u.size() > 0 ? 40 * _cursor / u.size() : 0; @@ -85,7 +89,10 @@ class Importer extends Jobs.Job { return; } if (trimmed.find("{") == 0) { - _aegis = new AegisReader(Bytes.fromUtf8(trimmed)); + // The untrimmed text: an incomplete part is saved exactly as + // received (a cut can fall on a space inside a string). + _aegis = new AegisReader(Bytes.fromUtf8(_text)); + _text = ""; return; } if (Text.findIgnoreCase(trimmed, OtpAuth.MIGRATION_PREFIX, 0) >= 0) { @@ -133,31 +140,32 @@ class Importer extends Jobs.Job { return; } if (status == AegisReader.INCOMPLETE) { - var size = Bytes.fromUtf8(_text).size(); - if (ImportBuffer.save(_vault.storage(), _vault.keyManager.deviceKeys(), _text, _now)) { - report.waitingChars = _text.length(); + var data = aegis.data(); + _aegis = null; + if (ImportBuffer.save(_vault.storage(), _vault.keyManager.deviceKeys(), data, _now)) { + var text = Bytes.toUtf8(data); + report.waitingChars = text != null ? text.length() : data.size(); phase = P_DONE; } else { - finishWith("the JSON is incomplete and too long to wait for more (" + size + " bytes, at most " + finishWith("the JSON is incomplete and too long to wait for more (" + data.size() + " bytes, at most " + ImportBuffer.MAX_TEXT_BYTES + ")"); } - _text = ""; return; } if (status == AegisReader.FAILED) { + _aegis = null; finishWith(aegis.error as String); - _text = ""; return; } _items = aegis.items; + _groups = aegis.groups; _aegis = null; - _text = ""; startReplace(); return; } var uris = _uris as Array<String>; for (var n = 0; n < URIS_PER_STEP && _cursor < uris.size(); n++) { - _items.add(OtpAuth.parse(uris[_cursor], _cursor + 1)); + _items.add(OtpAuth.parse(uris[_cursor], _cursor + 1).pack()); _cursor++; } if (_cursor >= uris.size()) { @@ -239,8 +247,10 @@ class Importer extends Jobs.Job { phase = P_DONE; return; } - var item = _items[_cursor]; - _items[_cursor] = new ImportItem(item.index); // drop the parsed secret + var packed = _items[_cursor]; + _items[_cursor] = []b; + var item = ImportItem.unpack(packed, _groups); + Bytes.wipe(packed); // the parsed secret; about 110 bytes _cursor++; var kind = _kind; var note = item.note; diff --git a/source/import/Json.mc b/source/import/Json.mc index c19150e..e3e9fa8 100644 --- a/source/import/Json.mc +++ b/source/import/Json.mc @@ -84,6 +84,11 @@ module Json { return _data.size(); } + // The input itself (kept when the document is incomplete). + function data() as ByteArray { + return _data; + } + // The next token (T_*). After T_END, T_INCOMPLETE or T_ERROR every // further call returns the same token. function next() as Number { diff --git a/source/settings/SettingsJob.mc b/source/settings/SettingsJob.mc index e6e7bcf..2fc4a3d 100644 --- a/source/settings/SettingsJob.mc +++ b/source/settings/SettingsJob.mc @@ -105,24 +105,31 @@ class SettingsJob extends Jobs.Job { endImport(["Import: waiting import cancelled."] as Array<String>, "Waiting import cancelled", false); return; } - var content = new Forms.Content("import"); - content.add(importMode); - content.add(locked); - var text = ""; - for (var i = 1; i <= IMPORT_TEXT_FIELDS; i++) { - var part = Forms.rawText(_store, "impText" + i); - content.add(part); - text += part; - } - _hash = Forms.hash(_vault, content); + _hash = importHash(importMode, locked); if (Forms.alreadyApplied(_vault, _hash)) { endImport(["Import: already applied. Change the text to import again."] as Array<String>, "Already imported", true); return; } + var text = ""; + for (var i = 1; i <= IMPORT_TEXT_FIELDS; i++) { + text += Forms.rawText(_store, "impText" + i); + } _importStart = System.getTimer(); _importer = new Importer(_vault, text, importMode, locked, _now); } + // The import form's hash. Its own function, so the field bytes it hashes + // are freed before the importer makes its copy of the text. + private function importHash(importMode as Number, locked as Boolean) as String { + var content = new Forms.Content("import"); + content.add(importMode); + content.add(locked); + for (var i = 1; i <= IMPORT_TEXT_FIELDS; i++) { + content.add(Forms.rawText(_store, "impText" + i)); + } + return Forms.hash(_vault, content); + } + private function finishImport(importer as Importer) as Void { _importer = null; var report = importer.report; diff --git a/test/import/AegisTest.mc b/test/import/AegisTest.mc index eaad039..1644e5d 100644 --- a/test/import/AegisTest.mc +++ b/test/import/AegisTest.mc @@ -30,7 +30,7 @@ function aegisReadsDatabaseVersion3(logger as Test.Logger) as Boolean { logger.error("error " + reader.error + ", items " + reader.items.size()); return false; } - var it = reader.items; + var it = AegisFixtures.items(reader); var github = it[0].entry; var bank = it[1].entry; var vpn = it[2].entry; @@ -69,7 +69,7 @@ function aegisReadsOlderDatabaseVersions(logger as Test.Logger) as Boolean { for (var v = 1; v <= 2; v++) { var reader = AegisFixtures.read(AegisFixtures.export(v, [AegisFixtures.entry("totp", "Mail", "ann", AegisFixtures.SECRET_1, "SHA1", 6, 30, "\"group\": \"Private\", ")], "")); - var e = reader.items.size() == 1 ? reader.items[0].entry : null; + var e = reader.items.size() == 1 ? reader.item(0).entry : null; if (e == null || !e.group.equals("Private")) { logger.error("database version " + v + " failed: " + reader.error); return false; @@ -98,7 +98,7 @@ function aegisReportsBadEntriesAndDocuments(logger as Test.Logger) as Boolean { "{\"type\": \"totp\", \"name\": \"z\", \"info\": {\"secret\": \"" + AegisFixtures.SECRET_1 + "\", \"digits\": 6.5}}" ]; var reader = AegisFixtures.read(AegisFixtures.export(3, entries, "")); - var it = reader.items; + var it = AegisFixtures.items(reader); if (reader.error != null || it.size() != 4) { logger.error("error " + reader.error); return false; diff --git a/test/import/FuzzTest.mc b/test/import/FuzzTest.mc index 7573ffa..fcad345 100644 --- a/test/import/FuzzTest.mc +++ b/test/import/FuzzTest.mc @@ -50,7 +50,7 @@ function fuzzAegisReaderGivesDefiniteResults(logger as Test.Logger) as Boolean { if (status == AegisReader.DONE) { done++; for (var k = 0; k < reader.items.size(); k++) { - var item = reader.items[k]; + var item = reader.item(k); var e = item.entry; if (e == null ? item.problem == null && item.skipped == null : e.validate() != null) { logger.error("input " + i + " item " + k + " is neither valid nor explained"); diff --git a/test/import/ImportSupport.mc b/test/import/ImportSupport.mc index ba891a7..1c84ec3 100644 --- a/test/import/ImportSupport.mc +++ b/test/import/ImportSupport.mc @@ -36,6 +36,15 @@ module AegisFixtures { } // Runs the reader to the end (or a non-running status). + // Every parsed item, unpacked. + function items(reader as AegisReader) as Array<ImportItem> { + var out = [] as Array<ImportItem>; + for (var i = 0; i < reader.items.size(); i++) { + out.add(reader.item(i)); + } + return out; + } + function read(text as String) as AegisReader { var reader = new AegisReader(Bytes.fromUtf8(text)); for (var i = 0; i < 10000; i++) { diff --git a/test/import/ImporterTest.mc b/test/import/ImporterTest.mc index 411e528..3443f4e 100644 --- a/test/import/ImporterTest.mc +++ b/test/import/ImporterTest.mc @@ -1,5 +1,5 @@ // Author: cirakg & claude ai -// Summary: Tests for the import job: reports, duplicates, replacements, locking, Replace mode, multi-part buffer, failures, a full store, 100 entries. +// Summary: Tests for the import job: reports, duplicates, replacements, locking, Replace mode, multi-part buffer, failures, a full store, 100 entries, packed items. import Toybox.Lang; import Toybox.System; @@ -206,3 +206,60 @@ function importerHandlesOneHundredEntries(logger as Test.Logger) as Boolean { logger.debug("BENCH import_100_aegis ms=" + elapsed + " steps=" + steps + " json_bytes=" + text.length()); return job.report.imported == 100 && vault.ids().size() == 100; } + +// Packed items (M8 memory) keep every field, resolve the group when +// unpacked, and leave no second copy of the secret. +(:test) +function importItemsPackAndUnpack(logger as Test.Logger) as Boolean { + var e = new Entry(); + e.type = Entry.OTP_HOTP; + e.name = "Bänk €"; + e.account = "bob"; + e.favorite = true; + e.algorithm = Hashes.SHA512; + e.digits = 8; + e.period = 60; + e.counter = 5000000000l; + var secret = [1, 2, 3, 4, 5]b; + e.secret = secret; + var item = new ImportItem(7); + item.label = e.name; + item.note = "issuer differs"; + item.groupRef = "uuid:g-1"; + item.entry = e; + var packed = item.pack(); + var wiped = secret[0] == 0 && secret[4] == 0 && e.secret == null; + var back = ImportItem.unpack(packed, {"g-1" => "Work"} as Dictionary<String, String>); + var b = back.entry; + if (!wiped || b == null) { + logger.error("wiped " + wiped + ", entry " + b); + return false; + } + var fieldsOk = back.index == 7 && back.label.equals("Bänk €") && "issuer differs".equals(back.note) && back.problem == null + && b.kind == Entry.KIND_OTP && b.source == Entry.SOURCE_IMPORT && b.type == Entry.OTP_HOTP && b.name.equals("Bänk €") + && b.account.equals("bob") && b.favorite && b.algorithm == Hashes.SHA512 && b.digits == 8 && b.period == 60 + && b.counter == 5000000000l && Bytes.equal(b.secret as ByteArray, [1, 2, 3, 4, 5]b) && b.group.equals("Work") + && b.validate() == null; + + var bad = new ImportItem(3); + bad.label = "x"; + bad.problem = "secret missing"; + bad.groupRef = "name:Private"; + var badBack = ImportItem.unpack(bad.pack(), {} as Dictionary<String, String>); + var named = new ImportItem(4); + named.groupRef = "name: Private "; + var namedEntry = new Entry(); + namedEntry.name = "Mail"; + namedEntry.secret = [9]b; + named.entry = namedEntry; + var namedBack = ImportItem.unpack(named.pack(), {} as Dictionary<String, String>).entry; + var missing = new ImportItem(5); + missing.groupRef = "uuid:nope"; + var missingEntry = new Entry(); + missingEntry.name = "Mail"; + missingEntry.secret = [9]b; + missing.entry = missingEntry; + var missingBack = ImportItem.unpack(missing.pack(), {} as Dictionary<String, String>).entry; + return fieldsOk && badBack.entry == null && "secret missing".equals(badBack.problem) && badBack.index == 3 + && namedBack != null && namedBack.group.equals("Private") && missingBack != null && missingBack.group.equals(""); +} |
