Skip to content

[interpreter/spec] Fix a couple of UTF-8 issues #2295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rossberg merged 2 commits into from
Oct 21, 2026
Merged

[interpreter/spec] Fix a couple of UTF-8 issues #2295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
46f388b
[interpreter/spec] Fix a couple of UTF-8 issues
rossberg Oct 21, 2026
514355d
Comments
rossberg Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions interpreter/binary/utf8.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ and encode' = function
n :: encode' ns
| n::ns when n < 0x800 ->
0xc0 lor (n lsr 6) :: con n :: encode' ns
| n::ns when n >= 0xd800 && n < 0xe000 ->
raise Utf8
| n::ns when n < 0x10000 ->

@tlively tlively Oct 21, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like a duplicate of the next case.

0xe0 lor (n lsr 12) :: con (n lsr 6) :: con n :: encode' ns
| n::ns when n < 0x110000 ->
Expand Down
20 changes: 12 additions & 8 deletions interpreter/text/lexer.mll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ let error_nest start lexbuf msg =

let unknown lexbuf = error lexbuf ("unknown operator " ^ Lexing.lexeme lexbuf)

let string s =
let string lexbuf s =
let b = Buffer.create (String.length s) in
let i = ref 1 in
while !i < String.length s - 1 do
Expand All @@ -36,8 +36,12 @@ let string s =
| 'u' ->
let j = !i + 2 in
i := String.index_from s j '}';
let n = int_of_string ("0x" ^ String.sub s j (!i - j)) in
let bs = Utf8.encode [n] in
let bs =
try
let n = int_of_string ("0x" ^ String.sub s j (!i - j)) in
Utf8.encode [n]
with Failure _ | Utf8.Utf8 -> error lexbuf "malformed UTF-8 codepoint"
in
Buffer.add_substring b bs 0 (String.length bs - 1);
bs.[String.length bs - 1]
| h ->
Expand All @@ -49,7 +53,7 @@ let string s =
Buffer.contents b

let annot_id lexbuf s =
let s' = string s in
let s' = string lexbuf s in
if s' = "" then error lexbuf "empty annotation id";
try Utf8.decode s' with Utf8.Utf8 -> error lexbuf "malformed UTF-8 encoding"

Expand Down Expand Up @@ -133,7 +137,7 @@ rule token = parse
| int as s { INT s }
| float as s { FLOAT s }

| string as s { STRING (string s) }
| string as s { STRING (string lexbuf s) }
| '"'character*(newline|eof) { error lexbuf "unclosed string literal" }
| '"'character*(control#ascii_newline)
{ error lexbuf "illegal control character in string literal" }
Expand Down Expand Up @@ -814,7 +818,7 @@ rule token = parse

| '$'(id as s) { VAR s }
| '$'(string as s)
{ let s' = string s in
{ let s' = string lexbuf s in
if s' = "" then error lexbuf "empty identifier"; VAR s' }
| '$' { error lexbuf "empty identifier" }

Expand Down Expand Up @@ -872,13 +876,13 @@ and annot start = parse
(Annot.Var s @@ r) :: annot start lexbuf }
| '$'(string as s)
{ let r = region lexbuf in
let s' = string s in
let s' = string lexbuf s in
if s' = "" then error lexbuf "empty identifier";
(Annot.Var s' @@ r) :: annot start lexbuf }
| '$' { error lexbuf "empty identifier" }
| string as s
{ let r = region lexbuf in
(Annot.String (string s) @@ r) :: annot start lexbuf }
(Annot.String (string lexbuf s) @@ r) :: annot start lexbuf }
| reserved as s
{ let r = region lexbuf in
(Annot.Atom s @@ r) :: annot start lexbuf }
Expand Down
2 changes: 1 addition & 1 deletion specification/wasm-3.0/6.1-text.values.spectec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ grammar Tstringchar : char =
| "\\\"" => U+22
| "\\'" => U+27
| "\\\\" => U+5C
| "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE800 <= n < 0x110000
| "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE000 <= n < 0x110000

grammar Tstringelem : byte* =
| c:Tstringchar => $utf8(c)
Expand Down
2 changes: 1 addition & 1 deletion specification/wasm-latest/6.1-text.values.spectec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ grammar Tstringchar : char =
| "\\\"" => U+22
| "\\'" => U+27
| "\\\\" => U+5C
| "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE800 <= n < 0x110000
| "\\u{" n:Thexnum "}" => n -- if n < 0xD800 \/ 0xE000 <= n < 0x110000

grammar Tstringelem : byte* =
| c:Tstringchar => $utf8(c)
Expand Down
2 changes: 1 addition & 1 deletion spectec/test-frontend/TEST.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10457,7 +10457,7 @@ grammar Tstringchar : char
prod "\\\\" => `%`_char(92,)
;; ../../../../specification/wasm-latest/6.1-text.values.spectec
prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,)
-- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112)))
-- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112)))

;; ../../../../specification/wasm-latest/6.1-text.values.spectec
grammar Tstringelem : byte*
Expand Down
2 changes: 1 addition & 1 deletion spectec/test-latex/TEST.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12701,7 +12701,7 @@ $$
& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)\kern-0.02em{'}\kern-0.05em{'}\kern-0.02em}’} & \quad\Rightarrow\quad{} & \mathrm{U{+}22} \\
& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)\kern0.03em{'}\kern0.03em}’} & \quad\Rightarrow\quad{} & \mathrm{U{+}27} \\
& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)\(\mathtt{\backslash}\)}’} & \quad\Rightarrow\quad{} & \mathrm{U{+}5C} \\
& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)u{\{}}’}~~n{:}{\mathtt{hexnum}}~~\mbox{‘\texttt{{\}}}’} & \quad\Rightarrow\quad{} & n & \quad \mbox{if}~ n < \mathtt{0xD800} \lor \mathtt{0xE800} \leq n < \mathtt{0x110000} \\
& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)u{\{}}’}~~n{:}{\mathtt{hexnum}}~~\mbox{‘\texttt{{\}}}’} & \quad\Rightarrow\quad{} & n & \quad \mbox{if}~ n < \mathtt{0xD800} \lor \mathtt{0xE000} \leq n < \mathtt{0x110000} \\
& {\mathtt{stringelem}} & ::= & c{:}{\mathtt{stringchar}} & \quad\Rightarrow\quad{} & {\mathrm{utf{\kern-0.1em\scriptstyle 8}}}(c) \\
& & | & \mbox{‘\texttt{\(\mathtt{\backslash}\)}’}~~h_1{:}{\mathtt{hexdigit}}~~h_2{:}{\mathtt{hexdigit}} & \quad\Rightarrow\quad{} & 16 \, h_1 + h_2 \\
& {\mathtt{string}} & ::= & \mbox{‘\texttt{\kern-0.02em{'}\kern-0.05em{'}\kern-0.02em}’}~~{({b^\ast}{:}{\mathtt{stringelem}})^\ast}~~\mbox{‘\texttt{\kern-0.02em{'}\kern-0.05em{'}\kern-0.02em}’} & \quad\Rightarrow\quad{} & {\bigoplus}\, {{b^\ast}^\ast} & \quad \mbox{if}~ {|{\bigoplus}\, {{b^\ast}^\ast}|} < {2^{32}} \\
Expand Down
6 changes: 3 additions & 3 deletions spectec/test-middlend/TEST.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9980,7 +9980,7 @@ grammar Tstringchar : char
prod "\\\\" => `%`_char(92,)
;; ../../../../specification/wasm-latest/6.1-text.values.spectec
prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,)
-- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112)))
-- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112)))

;; ../../../../specification/wasm-latest/6.1-text.values.spectec
grammar Tstringelem : byte*
Expand Down Expand Up @@ -21896,7 +21896,7 @@ grammar Tstringchar : char
prod "\\\\" => `%`_char(92,)
;; ../../../../specification/wasm-latest/6.1-text.values.spectec
prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,)
-- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112)))
-- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112)))

;; ../../../../specification/wasm-latest/6.1-text.values.spectec
grammar Tstringelem : byte*
Expand Down Expand Up @@ -34054,7 +34054,7 @@ grammar Tstringchar : char
prod "\\\\" => `%`_char(92,)
;; ../../../../specification/wasm-latest/6.1-text.values.spectec
prod{n : n} {{"\\u{"} {n:Thexnum} {"}"}} => `%`_char(n,)
-- if ((n < 55296) \/ ((59392 <= n) /\ (n < 1114112)))
-- if ((n < 55296) \/ ((57344 <= n) /\ (n < 1114112)))

;; ../../../../specification/wasm-latest/6.1-text.values.spectec
grammar Tstringelem : byte*
Expand Down
22 changes: 22 additions & 0 deletions test/core/utf8-invalid-encoding.wast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,3 +174,25 @@
(assert_malformed (module quote "(func (export \"\\ff\"))") "malformed UTF-8 encoding")
(assert_malformed (module quote "(func (export \"\\ff\\fe\\00\\00\"))") "malformed UTF-8 encoding")
(assert_malformed (module quote "(func (export \"\\ff\\fe\"))") "malformed UTF-8 encoding")


(module quote "(func (export \"\\u{D7FF}\"))")
(module quote "(func (export \"\\u{E000}\"))")
(module quote "(func (export \"\\u{00D7FF}\"))")

(assert_malformed
(module quote "(func (export \"\\u{110000}\"))")
"malformed UTF-8 codepoint"
)
(assert_malformed
(module quote "(func (export \"\\u{D800}\"))")
"malformed UTF-8 codepoint"
)

@tlively tlively Oct 21, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be good to add a negative test for \u{DFFF} and a positive test for \u{E000} as well. Might as well do \u{D7FF} as well.

(assert_malformed
(module quote "(func (export \"\\u{DFFF}\"))")
"malformed UTF-8 codepoint"
)
(assert_malformed
(module quote "(func (export \"\\u{00D800}\"))")
"malformed UTF-8 codepoint"
)
Loading