From 46f388b9d643e289ce8d70719a97fca17e27410a Mon Sep 17 00:00:00 2001 From: Andreas Rossberg Date: Wed, 21 Oct 2026 15:31:39 +0200 Subject: [PATCH 1/2] [interpreter/spec] Fix a couple of UTF-8 issues --- interpreter/binary/utf8.ml | 4 ++++ interpreter/text/lexer.mll | 17 ++++++++++------- specification/wasm-3.0/6.1-text.values.spectec | 2 +- .../wasm-latest/6.1-text.values.spectec | 2 +- spectec/test-frontend/TEST.md | 2 +- spectec/test-latex/TEST.md | 2 +- spectec/test-middlend/TEST.md | 6 +++--- test/core/utf8-invalid-encoding.wast | 9 +++++++++ 8 files changed, 30 insertions(+), 14 deletions(-) diff --git a/interpreter/binary/utf8.ml b/interpreter/binary/utf8.ml index b922954882..142436303a 100644 --- a/interpreter/binary/utf8.ml +++ b/interpreter/binary/utf8.ml @@ -14,6 +14,10 @@ and encode' = function n :: encode' ns | n::ns when n < 0x800 -> 0xc0 lor (n lsr 6) :: con n :: encode' ns + | n::ns when n >= 0xd800 && n < 0xe000 -> + raise Utf8 + | n::ns when n < 0x10000 -> + 0xe0 lor (n lsr 12) :: con (n lsr 6) :: con n :: encode' ns | n::ns when n < 0x10000 -> 0xe0 lor (n lsr 12) :: con (n lsr 6) :: con n :: encode' ns | n::ns when n < 0x110000 -> diff --git a/interpreter/text/lexer.mll b/interpreter/text/lexer.mll index 653c93a5b3..ec9b6bb08c 100644 --- a/interpreter/text/lexer.mll +++ b/interpreter/text/lexer.mll @@ -21,7 +21,7 @@ let error_nest start lexbuf msg = let unknown lexbuf = error lexbuf ("unknown operator " ^ Lexing.lexeme lexbuf) -let string s = +let string lexbuf s = let b = Buffer.create (String.length s) in let i = ref 1 in while !i < String.length s - 1 do @@ -37,7 +37,10 @@ let string s = let j = !i + 2 in i := String.index_from s j '}'; let n = int_of_string ("0x" ^ String.sub s j (!i - j)) in - let bs = Utf8.encode [n] in + let bs = + try Utf8.encode [n] + with Utf8.Utf8 -> error lexbuf "malformed UTF-8 codepoint" + in Buffer.add_substring b bs 0 (String.length bs - 1); bs.[String.length bs - 1] | h -> @@ -49,7 +52,7 @@ let string s = Buffer.contents b let annot_id lexbuf s = - let s' = string s in + let s' = string lexbuf s in if s' = "" then error lexbuf "empty annotation id"; try Utf8.decode s' with Utf8.Utf8 -> error lexbuf "malformed UTF-8 encoding" @@ -133,7 +136,7 @@ rule token = parse | int as s { INT s } | float as s { FLOAT s } - | string as s { STRING (string s) } + | string as s { STRING (string lexbuf s) } | '"'character*(newline|eof) { error lexbuf "unclosed string literal" } | '"'character*(control#ascii_newline) { error lexbuf "illegal control character in string literal" } @@ -814,7 +817,7 @@ rule token = parse | '$'(id as s) { VAR s } | '$'(string as s) - { let s' = string s in + { let s' = string lexbuf s in if s' = "" then error lexbuf "empty identifier"; VAR s' } | '$' { error lexbuf "empty identifier" } @@ -872,13 +875,13 @@ and annot start = parse (Annot.Var s @@ r) :: annot start lexbuf } | '$'(string as s) { let r = region lexbuf in - let s' = string s in + let s' = string lexbuf s in if s' = "" then error lexbuf "empty identifier"; (Annot.Var s' @@ r) :: annot start lexbuf } | '$' { error lexbuf "empty identifier" } | string as s { let r = region lexbuf in - (Annot.String (string s) @@ r) :: annot start lexbuf } + (Annot.String (string lexbuf s) @@ r) :: annot start lexbuf } | reserved as s { let r = region lexbuf in (Annot.Atom s @@ r) :: annot start lexbuf } diff --git a/specification/wasm-3.0/6.1-text.values.spectec b/specification/wasm-3.0/6.1-text.values.spectec index 09b6bbd2ea..81b86a1110 100644 --- a/specification/wasm-3.0/6.1-text.values.spectec +++ b/specification/wasm-3.0/6.1-text.values.spectec @@ -95,7 +95,7 @@ grammar Tstringchar : char = | "\\\"" => U+22 | "\\'" => U+27 | "\\\\" => U+5C - | "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE800 <= n < 0x110000 + | "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE000 <= n < 0x110000 grammar Tstringelem : byte* = | c:Tstringchar => $utf8(c) diff --git a/specification/wasm-latest/6.1-text.values.spectec b/specification/wasm-latest/6.1-text.values.spectec index 09b6bbd2ea..81b86a1110 100644 --- a/specification/wasm-latest/6.1-text.values.spectec +++ b/specification/wasm-latest/6.1-text.values.spectec @@ -95,7 +95,7 @@ grammar Tstringchar : char = | "\\\"" => U+22 | "\\'" => U+27 | "\\\\" => U+5C - | "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE800 <= n < 0x110000 + | "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE000 <= n < 0x110000 grammar Tstringelem : byte* = | c:Tstringchar => $utf8(c) diff --git a/spectec/test-frontend/TEST.md b/spectec/test-frontend/TEST.md index 463de773fc..5636866077 100644 --- a/spectec/test-frontend/TEST.md +++ b/spectec/test-frontend/TEST.md @@ -10457,7 +10457,7 @@ grammar Tstringchar : char prod "\\\\" => `%`_char(92,) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,) - -- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112))) + -- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112))) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec grammar Tstringelem : byte* diff --git a/spectec/test-latex/TEST.md b/spectec/test-latex/TEST.md index 30daae604e..5cee409a45 100644 --- a/spectec/test-latex/TEST.md +++ b/spectec/test-latex/TEST.md @@ -12701,7 +12701,7 @@ $$ & & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)\kern-0.02em{'}\kern-0.05em{'}\kern-0.02em}’} & \quad\Rightarrow\quad{} & \mathrm{U{+}22} \\ & & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)\kern0.03em{'}\kern0.03em}’} & \quad\Rightarrow\quad{} & \mathrm{U{+}27} \\ & & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)\(\mathtt{\backslash}\)}’} & \quad\Rightarrow\quad{} & \mathrm{U{+}5C} \\ -& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)u{\{}}’}~~n{:}{\mathtt{hexnum}}~~\mbox{‘\texttt{{\}}}’} & \quad\Rightarrow\quad{} & n & \quad \mbox{if}~ n < \mathtt{0xD800} \lor \mathtt{0xE800} \leq n < \mathtt{0x110000} \\ +& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)u{\{}}’}~~n{:}{\mathtt{hexnum}}~~\mbox{‘\texttt{{\}}}’} & \quad\Rightarrow\quad{} & n & \quad \mbox{if}~ n < \mathtt{0xD800} \lor \mathtt{0xE000} \leq n < \mathtt{0x110000} \\ & {\mathtt{stringelem}} & ::= & c{:}{\mathtt{stringchar}} & \quad\Rightarrow\quad{} & {\mathrm{utf{\kern-0.1em\scriptstyle 8}}}(c) \\ & & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)}’}~~h_1{:}{\mathtt{hexdigit}}~~h_2{:}{\mathtt{hexdigit}} & \quad\Rightarrow\quad{} & 16 \, h_1 + h_2 \\ & {\mathtt{string}} & ::= & \mbox{‘\texttt{\kern-0.02em{'}\kern-0.05em{'}\kern-0.02em}’}~~{({b^\ast}{:}{\mathtt{stringelem}})^\ast}~~\mbox{‘\texttt{\kern-0.02em{'}\kern-0.05em{'}\kern-0.02em}’} & \quad\Rightarrow\quad{} & {\bigoplus}\, {{b^\ast}^\ast} & \quad \mbox{if}~ {|{\bigoplus}\, {{b^\ast}^\ast}|} < {2^{32}} \\ diff --git a/spectec/test-middlend/TEST.md b/spectec/test-middlend/TEST.md index a636e157c2..f8ea59c197 100644 --- a/spectec/test-middlend/TEST.md +++ b/spectec/test-middlend/TEST.md @@ -9980,7 +9980,7 @@ grammar Tstringchar : char prod "\\\\" => `%`_char(92,) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,) - -- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112))) + -- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112))) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec grammar Tstringelem : byte* @@ -21896,7 +21896,7 @@ grammar Tstringchar : char prod "\\\\" => `%`_char(92,) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,) - -- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112))) + -- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112))) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec grammar Tstringelem : byte* @@ -34054,7 +34054,7 @@ grammar Tstringchar : char prod "\\\\" => `%`_char(92,) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,) - -- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112))) + -- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112))) ;; ../../../../specification/wasm-latest/6.1-text.values.spectec grammar Tstringelem : byte* diff --git a/test/core/utf8-invalid-encoding.wast b/test/core/utf8-invalid-encoding.wast index cfd596930c..9906e0f8fb 100644 --- a/test/core/utf8-invalid-encoding.wast +++ b/test/core/utf8-invalid-encoding.wast @@ -174,3 +174,12 @@ (assert_malformed (module quote "(func (export \"\\ff\"))") "malformed UTF-8 encoding") (assert_malformed (module quote "(func (export \"\\ff\\fe\\00\\00\"))") "malformed UTF-8 encoding") (assert_malformed (module quote "(func (export \"\\ff\\fe\"))") "malformed UTF-8 encoding") + +(assert_malformed + (module quote "(func (export \"\\u{110000}\"))") + "malformed UTF-8 codepoint" +) +(assert_malformed + (module quote "(func (export \"\\u{D800}\"))") + "malformed UTF-8 codepoint" +) From 514355d846fc60658a0ef93160d26223e680ec37 Mon Sep 17 00:00:00 2001 From: Andreas Rossberg Date: Wed, 21 Oct 2026 23:31:58 +0200 Subject: [PATCH 2/2] Comments --- interpreter/binary/utf8.ml | 2 -- interpreter/text/lexer.mll | 7 ++++--- test/core/utf8-invalid-encoding.wast | 13 +++++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/interpreter/binary/utf8.ml b/interpreter/binary/utf8.ml index 142436303a..2d69bd3f56 100644 --- a/interpreter/binary/utf8.ml +++ b/interpreter/binary/utf8.ml @@ -18,8 +18,6 @@ and encode' = function raise Utf8 | n::ns when n < 0x10000 -> 0xe0 lor (n lsr 12) :: con (n lsr 6) :: con n :: encode' ns - | n::ns when n < 0x10000 -> - 0xe0 lor (n lsr 12) :: con (n lsr 6) :: con n :: encode' ns | n::ns when n < 0x110000 -> 0xf0 lor (n lsr 18) :: con (n lsr 12) :: con (n lsr 6) :: con n :: encode' ns diff --git a/interpreter/text/lexer.mll b/interpreter/text/lexer.mll index ec9b6bb08c..618ee2cff3 100644 --- a/interpreter/text/lexer.mll +++ b/interpreter/text/lexer.mll @@ -36,10 +36,11 @@ let string lexbuf s = | 'u' -> let j = !i + 2 in i := String.index_from s j '}'; - let n = int_of_string ("0x" ^ String.sub s j (!i - j)) in let bs = - try Utf8.encode [n] - with Utf8.Utf8 -> error lexbuf "malformed UTF-8 codepoint" + try + let n = int_of_string ("0x" ^ String.sub s j (!i - j)) in + Utf8.encode [n] + with Failure _ | Utf8.Utf8 -> error lexbuf "malformed UTF-8 codepoint" in Buffer.add_substring b bs 0 (String.length bs - 1); bs.[String.length bs - 1] diff --git a/test/core/utf8-invalid-encoding.wast b/test/core/utf8-invalid-encoding.wast index 9906e0f8fb..b7d95b7ec8 100644 --- a/test/core/utf8-invalid-encoding.wast +++ b/test/core/utf8-invalid-encoding.wast @@ -175,6 +175,11 @@ (assert_malformed (module quote "(func (export \"\\ff\\fe\\00\\00\"))") "malformed UTF-8 encoding") (assert_malformed (module quote "(func (export \"\\ff\\fe\"))") "malformed UTF-8 encoding") + +(module quote "(func (export \"\\u{D7FF}\"))") +(module quote "(func (export \"\\u{E000}\"))") +(module quote "(func (export \"\\u{00D7FF}\"))") + (assert_malformed (module quote "(func (export \"\\u{110000}\"))") "malformed UTF-8 codepoint" @@ -183,3 +188,11 @@ (module quote "(func (export \"\\u{D800}\"))") "malformed UTF-8 codepoint" ) +(assert_malformed + (module quote "(func (export \"\\u{DFFF}\"))") + "malformed UTF-8 codepoint" +) +(assert_malformed + (module quote "(func (export \"\\u{00D800}\"))") + "malformed UTF-8 codepoint" +)