diff --git a/http-core/src/main/scala/org/apache/pekko/http/impl/engine/http2/RequestParsing.scala b/http-core/src/main/scala/org/apache/pekko/http/impl/engine/http2/RequestParsing.scala index b03fd520b8..476050a133 100644 --- a/http-core/src/main/scala/org/apache/pekko/http/impl/engine/http2/RequestParsing.scala +++ b/http-core/src/main/scala/org/apache/pekko/http/impl/engine/http2/RequestParsing.scala @@ -13,6 +13,7 @@ package org.apache.pekko.http.impl.engine.http2 +import java.nio.charset.StandardCharsets import javax.net.ssl.SSLSession import scala.annotation.tailrec @@ -218,9 +219,13 @@ private[http2] object RequestParsing { // or use a separate, simpler, parser for Http2 // The odd-looking 'x' below is a by-product of how current parser and HTTP/1.1 work. // Without '\r\n\x' (x being any additional byte) parsing will fail. See HttpHeaderParserSpec for examples. + // HPACK decodes string literals as ISO-8859-1, one char per octet, so encoding the line as ISO-8859-1 again hands + // the parser the octets that were on the wire. It then decodes UTF-8 in a value exactly as it does for HTTP/1.1. + // Encoding as UTF-8 here would encode each octet above 0x7F as a char of its own, so a UTF-8 value such as "café" + // came out as "café". val concHeaderLine = name + ": " + value + "\r\nx" try { - httpHeaderParser.parseHeaderLine(ByteString(concHeaderLine))() + httpHeaderParser.parseHeaderLine(ByteString(concHeaderLine, StandardCharsets.ISO_8859_1))() httpHeaderParser.resultHeader } catch { // the HTTP/1.1 parser reports a malformed field with its own, internal exception type, which nothing on the diff --git a/http-core/src/main/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParser.scala b/http-core/src/main/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParser.scala index cd1ee89157..80a6c34bab 100644 --- a/http-core/src/main/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParser.scala +++ b/http-core/src/main/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParser.scala @@ -609,42 +609,37 @@ private[http] object HttpHeaderParser { case '\n' => if (WSP(byteChar(input, ix + 1))) scanHeaderValue(hhp, input, start, limit, log, mode)(appended(' '), ix + 2) else (if (sb != null) sb.toString else asciiString(input, start, ix), ix + 1) + case c if c >= 0xC0 && c <= 0xF7 => // lead byte of a 2-, 3- or 4-byte UTF-8 sequence + val length = if (c < 0xE0) 2 else if (c < 0xF0) 3 else 4 + var i = 0 + while (i < length) { + hhp.byteBuffer.put(byteAt(input, ix + i)) + i += 1 + } + val cc = hhp.decodeByteBuffer() + if (cc != -1) scanHeaderValue(hhp, input, start, limit, log, mode)(appended2(cc), ix + length) + else { + // Not valid UTF-8: copy the bytes as they are, one char per byte. Only the lead byte and the continuation + // bytes (10xxxxxx) right after it are taken as the sequence. Anything else - CR and LF in particular - is + // left for the next step, so that a stray lead byte at the end of a value cannot consume the line end + // and pull the next header line into this value. + var end = ix + 1 + while (end < ix + length && (byteAt(input, end) & 0xC0) == 0x80) end += 1 + val nsb = + if (sb != null) { + var j = ix + while (j < end) { + sb.append(byteChar(input, j)) + j += 1 + } + sb + } else null + scanHeaderValue(hhp, input, start, limit, log, mode)(nsb, end) + } case c => - var nix = ix + 1 val nsb = if (' ' <= c && c <= '\u007F') if (sb != null) sb.append(c) else null // legal 7-Bit ASCII - else if ((c & 0xE0) == 0xC0) { // 2-byte UTF-8 sequence? - hhp.byteBuffer.put(c.toByte) - hhp.byteBuffer.put(byteAt(input, ix + 1)) - nix = ix + 2 - hhp.decodeByteBuffer() match { // if we cannot decode as UTF8 we don't decode but simply copy - case -1 => if (sb != null) sb.append(c).append(byteChar(input, ix + 1)) else null - case cc => appended2(cc) - } - } else if ((c & 0xF0) == 0xE0) { // 3-byte UTF-8 sequence? - hhp.byteBuffer.put(c.toByte) - hhp.byteBuffer.put(byteAt(input, ix + 1)) - hhp.byteBuffer.put(byteAt(input, ix + 2)) - nix = ix + 3 - hhp.decodeByteBuffer() match { // if we cannot decode as UTF8 we don't decode but simply copy - case -1 => - if (sb != null) sb.append(c).append(byteChar(input, ix + 1)).append(byteChar(input, ix + 2)) else null - case cc => appended2(cc) - } - } else if ((c & 0xF8) == 0xF0) { // 4-byte UTF-8 sequence? - hhp.byteBuffer.put(c.toByte) - hhp.byteBuffer.put(byteAt(input, ix + 1)) - hhp.byteBuffer.put(byteAt(input, ix + 2)) - hhp.byteBuffer.put(byteAt(input, ix + 3)) - nix = ix + 4 - hhp.decodeByteBuffer() match { // if we cannot decode as UTF8 we don't decode but simply copy - case -1 => - if (sb != null) sb.append(c).append(byteChar(input, ix + 1)).append(byteChar(input, ix + 2)).append( - byteChar(input, ix + 3)) - else null - case cc => appended2(cc) - } - } else { + else { mode match { case IllegalResponseHeaderValueProcessingMode.Error => fail(s"Illegal character '${escape(c)}' in header value") @@ -656,9 +651,8 @@ private[http] object HttpHeaderParser { // just ignore the illegal character sb } - } - scanHeaderValue(hhp, input, start, limit, log, mode)(nsb, nix) + scanHeaderValue(hhp, input, start, limit, log, mode)(nsb, ix + 1) } else fail(s"HTTP header value exceeds the configured limit of ${limit - start - 2} characters", StatusCodes.RequestHeaderFieldsTooLarge) diff --git a/http-core/src/test/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParserSpec.scala b/http-core/src/test/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParserSpec.scala index 32b18dba72..ba7aa5e2b7 100644 --- a/http-core/src/test/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParserSpec.scala +++ b/http-core/src/test/scala/org/apache/pekko/http/impl/engine/parsing/HttpHeaderParserSpec.scala @@ -195,6 +195,27 @@ abstract class HttpHeaderParserSpec(mode: String, newLine: String) extends Pekko RawHeader("4-UTF8-Bytes", "Surrogate pairs: \uD801\uDC1B\uD801\uDC04\uD801\uDC1B!") } + "not let an invalid UTF-8 sequence at the end of a value consume the line end" in new TestSetup() { + // lead bytes of a 2-, 3- and 4-byte sequence, followed directly by the line end + for (lead <- Seq(0xC3, 0xE9, 0xF0)) { + val bytes = ByteString("X-A: caf") ++ ByteString(lead.toByte) ++ ByteString(s"${newLine}X-B: b${newLine}x") + val (nextLine, header) = parseLineFromBytes(bytes) + header shouldEqual RawHeader("X-A", "caf" + lead.toChar) + nextLine shouldEqual 9 + newLine.length + parser.parseHeaderLine(bytes, nextLine)() + parser.resultHeader shouldEqual RawHeader("X-B", "b") + } + } + + "copy an invalid UTF-8 sequence into the value one char per byte" in new TestSetup() { + // 0xE9 0x80 is cut short by 'a', which is then read as an ordinary char + parseLineFromBytes(ByteString("X-A: caf") ++ ByteString(0xE9.toByte, 0x80.toByte) ++ + ByteString(s"abc${newLine}x"))._2 shouldEqual RawHeader("X-A", "café\u0080abc") + // the same with a tab earlier in the value, which makes the parser build the value char by char + parseLineFromBytes(ByteString("X-B: x\tcaf") ++ ByteString(0xE9.toByte) ++ + ByteString(s"${newLine}x"))._2 shouldEqual RawHeader("X-B", "x café") + } + "parse multiple header lines subsequently with UTF-8 characters one after another without crashing" in new TestSetup { parseLine(s"""Content-Disposition: form-data; name="test"; filename="λ"${newLine}x""") diff --git a/http2-tests/src/test/scala/org/apache/pekko/http/impl/engine/http2/RequestParsingSpec.scala b/http2-tests/src/test/scala/org/apache/pekko/http/impl/engine/http2/RequestParsingSpec.scala index 63fc77c6d0..8b1d242027 100644 --- a/http2-tests/src/test/scala/org/apache/pekko/http/impl/engine/http2/RequestParsingSpec.scala +++ b/http2-tests/src/test/scala/org/apache/pekko/http/impl/engine/http2/RequestParsingSpec.scala @@ -15,6 +15,7 @@ package org.apache.pekko.http.impl.engine.http2 import java.net.InetAddress import java.net.InetSocketAddress +import java.nio.charset.StandardCharsets import FrameEvent._ @@ -115,6 +116,27 @@ class RequestParsingSpec extends PekkoSpecWithMaterializer with Inside with Insp (request.headers.map(h => h.lowercaseName -> h.value) should contain).allOf("x-a" -> "a b", "x-b" -> "") } + "decode non-ASCII octets in a header value as the HTTP/1.1 parser does" should { + // HPACK hands out string literals as ISO-8859-1, one char per octet, and `encodeHeaderPairs` writes each + // char back as that octet: a pair value here is what was on the wire + def wire(s: String): String = new String(s.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1) + def request(extra: (String, String)*): Vector[(String, String)] = + Vector(":method" -> "GET", ":scheme" -> "https", ":path" -> "/") ++ extra + def headerValues(request: HttpRequest): Seq[(String, String)] = + request.headers.map(h => h.lowercaseName -> h.value) + + "a UTF-8 value" in { + val req = parseExpectOk(request("x-a" -> wire("café €"), "x-b" -> "b")) + (headerValues(req) should contain).allOf("x-a" -> "café €", "x-b" -> "b") + } + "an octet that is not valid UTF-8, copied as one char" in { + // 0xE9 alone is the lead byte of a 3-byte sequence: it must not take the line end with it, or the next + // field would be lost + val req = parseExpectOk(request("x-a" -> "café", "x-b" -> "b")) + (headerValues(req) should contain).allOf("x-a" -> "café", "x-b" -> "b") + } + } + "reject a malformed header field with a bad request rather than accepting or failing the connection" should { // RFC 9113 8.2.1: a field name or value carrying NUL, CR or LF makes the message malformed def request(extra: (String, String)*): Vector[(String, String)] =