1 files changed, 25 insertions, 8 deletions

diff --git a/source/util/Bytes.mc b/source/util/Bytes.mc
index aa5a28b..5fd7a96 100644
--- a/source/util/Bytes.mc
+++ b/source/util/Bytes.mc
@@ -60,6 +60,14 @@ module Bytes {
         if (!isValidUtf8(data)) {
             return null;
         }
+        return toUtf8Checked(data);
+    }
+
+    // Bytes that utf8Prefix() found well-formed to the end, as a string.
+    function toUtf8Checked(data as ByteArray) as String {
+        if (data.size() == 0) {
+            return "";
+        }
         return StringUtil.convertEncodedString(data, {
             :fromRepresentation => StringUtil.REPRESENTATION_BYTE_ARRAY,
             :toRepresentation => StringUtil.REPRESENTATION_STRING_PLAIN_TEXT,
@@ -70,15 +78,24 @@ module Bytes {
     // True for well-formed UTF-8 (RFC 3629) without NUL bytes: no overlong
     // forms, no surrogates (U+D800-DFFF), nothing above U+10FFFF.
     function isValidUtf8(data as ByteArray) as Boolean {
-        var i = 0;
+        return utf8Prefix(data, 0, data.size()) >= 0;
+    }
+
+    // Checks the sequences that start in [from, to) (to <= size) as
+    // isValidUtf8() does:
+    // the index where the next sequence starts (`to` or up to 3 past it),
+    // or -1 when one is malformed. A large array is checked in steps this
+    // way (the check is a byte loop in the VM).
+    function utf8Prefix(data as ByteArray, from as Number, to as Number) as Number {
+        var i = from;
         var n = data.size();
-        while (i < n) {
+        while (i < to) {
             var b = data[i];
             var more = 0;
             var min = 0;
             var cp = 0;
             if (b == 0) {
-                return false;
+                return -1;
             } else if (b < 0x80) {
                 i++;
                 continue;
@@ -89,24 +106,24 @@ module Bytes {
             } else if (b >= 0xF0 && b <= 0xF4) {
                 more = 3; min = 0x10000; cp = b & 0x07;
             } else {
-                return false;
+                return -1;
             }
             if (i + more >= n) {
-                return false;
+                return -1;
             }
             for (var k = 1; k <= more; k++) {
                 var c = data[i + k];
                 if ((c & 0xC0) != 0x80) {
-                    return false;
+                    return -1;
                 }
                 cp = (cp << 6) | (c & 0x3F);
             }
             if (cp < min || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) {
-                return false;
+                return -1;
             }
             i += more + 1;
         }
-        return true;
+        return i;
     }
 
     // A new array holding a followed by b; neither input is changed.