PostgreSQL: Parse hexadecimal, octal and binary integer literals (#2758)

Author: LucaCappelletti94

fuzz/fuzz_seeds/postgres_non_decimal_integer_literals.sql

@@ -0,0 +1 @@
+SELECT 0x1F, 0X_1F, 0o17, 0b1_01, X'1F'

src/dialect/mod.rs

@@ -608,6 +608,13 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if the dialect supports integer literals in hexadecimal,
+    /// octal and binary, e.g. `0x1F`, `0o17` and `0b101`, which then parse as
+    /// numbers instead of `0x` hexadecimal strings.
+    fn supports_non_decimal_integer_literals(&self) -> bool {
+        false
+    }
+
     /// Returns true if the dialects supports specifying null treatment
     /// as part of a window function's parameter list as opposed
     /// to after the parameter list.

src/dialect/postgresql.rs

@@ -490,6 +490,11 @@ impl Dialect for PostgreSqlDialect {
         true
     }
 
+    /// See: <https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS-NUMERIC>
+    fn supports_non_decimal_integer_literals(&self) -> bool {
+        true
+    }
+
     /// See: <https://www.postgresql.org/docs/current/arrays.html#ARRAYS-DECLARATION>
     fn supports_array_typedef_with_brackets(&self) -> bool {
         true

src/parser/mod.rs

@@ -5135,6 +5135,30 @@ impl<'a> Parser<'a> {
         })
     }
 
+    #[cfg(not(feature = "bigdecimal"))]
+    fn parse_number_literal(s: String, _loc: Location) -> Result<String, ParserError> {
+        Ok(s)
+    }
+
+    #[cfg(feature = "bigdecimal")]
+    fn parse_number_literal(
+        s: String,
+        loc: Location,
+    ) -> Result<bigdecimal::BigDecimal, ParserError> {
+        let radix = match s.get(..2) {
+            Some("0x" | "0X") => 16,
+            Some("0o" | "0O") => 8,
+            Some("0b" | "0B") => 2,
+            _ => return Self::parse(s, loc),
+        };
+        let digits = s[2..].strip_prefix('_').unwrap_or(&s[2..]);
+        bigdecimal::num_bigint::BigInt::parse_bytes(digits.as_bytes(), radix)
+            .map(bigdecimal::BigDecimal::from)
+            .ok_or_else(|| {
+                ParserError::ParserError(format!("Could not parse '{s}' as an integer{loc}"))
+            })
+    }
+
     /// Parse a comma-separated list of 1+ SelectItem
     pub fn parse_projection(&mut self) -> Result<Vec<SelectItem>, ParserError> {
         // BigQuery and Snowflake allow trailing commas, but only in project lists
@@ -12990,10 +13014,10 @@ impl<'a> Parser<'a> {
                     },
                 ),
             },
-            // The call to n.parse() returns a bigdecimal when the
-            // bigdecimal feature is enabled, and is otherwise a no-op
-            // (i.e., it returns the input string).
-            Token::Number(n, l) => ok_value(Value::Number(Self::parse(n, span.start)?, l)),
+            // A BigDecimal with the bigdecimal feature, the literal as written otherwise.
+            Token::Number(n, l) => {
+                ok_value(Value::Number(Self::parse_number_literal(n, span.start)?, l))
+            }
             Token::SingleQuotedString(ref s) => ok_value(Value::SingleQuotedString(
                 self.maybe_concat_string_literal(s.to_string()),
             )),

src/tokenizer.rs

@@ -1606,6 +1606,20 @@ impl<'a> Tokenizer<'a> {
                         return Ok(Some(Token::Number(s, false)));
                     }
 
+                    if s == "0" && self.dialect.supports_non_decimal_integer_literals() {
+                        let radix = match chars.peek() {
+                            Some('x' | 'X') => Some((16, "a hexadecimal")),
+                            Some('o' | 'O') => Some((8, "an octal")),
+                            Some('b' | 'B') => Some((2, "a binary")),
+                            _ => None,
+                        };
+                        if let Some((radix, name)) = radix {
+                            return self
+                                .tokenize_non_decimal_integer(chars, radix, name)
+                                .map(Some);
+                        }
+                    }
+
                     // match binary literal that starts with 0x
                     if s == "0" && chars.peek() == Some(&'x') {
                         chars.next();
@@ -2302,6 +2316,32 @@ impl<'a> Tokenizer<'a> {
         Ok(s)
     }
 
+    /// Tokenizes an integer literal such as `0x1F` after its leading `0`,
+    /// keeping the spelling as written.
+    fn tokenize_non_decimal_integer(
+        &self,
+        chars: &mut State,
+        radix: u32,
+        name: &str,
+    ) -> Result<Token, TokenizerError> {
+        let mut s = String::from('0');
+        s.extend(chars.next());
+        // An underscore may directly follow the prefix, as in `0x_1F`.
+        if chars.peek() == Some(&'_') && self.dialect.supports_numeric_literal_underscores() {
+            chars.next();
+            s.push('_');
+        }
+        let digits = self.tokenize_number_part(chars, |ch| ch.is_digit(radix))?;
+        if digits.is_empty() {
+            return self.tokenizer_error(
+                chars.location(),
+                format!("Expected {name} digit after '{}'", &s[..2]),
+            );
+        }
+        s.push_str(&digits);
+        Ok(Token::Number(s, false))
+    }
+
     // Consume characters until newline
     fn tokenize_single_line_comment(&self, chars: &mut State) -> String {
         peeking_take_while(chars, |ch| match ch {
@@ -3035,10 +3075,20 @@ mod tests {
             ],
         );
 
-        numeric_underscore_dialects.tokenizes_to(
+        let hex_string_dialects = all_dialects_where(|dialect| {
+            dialect.supports_numeric_literal_underscores()
+                && !dialect.supports_non_decimal_integer_literals()
+        });
+        hex_string_dialects.tokenizes_to(
             "0xFF_FF",
             vec![Token::HexStringLiteral("FF_FF".to_string())],
         );
+        for dialect in &hex_string_dialects.dialects {
+            let err = Tokenizer::new(&**dialect, "SELECT 0x_1")
+                .tokenize()
+                .unwrap_err();
+            assert_eq!("Unexpected character '_'", err.message);
+        }
 
         for dialect in &numeric_underscore_dialects.dialects {
             for sql in [
@@ -3050,7 +3100,6 @@ mod tests {
                 "SELECT 1e_1",
                 "SELECT 1e1_",
                 "SELECT 1e1__0",
-                "SELECT 0x_1",
                 "SELECT 0x1_",
             ] {
                 let err = Tokenizer::new(&**dialect, sql).tokenize().unwrap_err();

tests/sqlparser_postgres.rs

@@ -11351,3 +11351,57 @@ fn parse_drop_foreign_table_rejects_modifiers() {
         );
     }
 }
+
+#[test]
+fn parse_non_decimal_integer_literals() {
+    let sql = "SELECT 0x1F, 0X1F, 0o17, 0O17, 0b101, 0B101, 0x_1F, 0xFF_FF, 0xFFFFFFFFFFFFFFFFFFFF";
+    let canonical = if cfg!(feature = "bigdecimal") {
+        "SELECT 31, 31, 15, 15, 5, 5, 31, 65535, 1208925819614629174706175"
+    } else {
+        sql
+    };
+    let select = pg().verified_only_select_with_canonical(sql, canonical);
+    for item in &select.projection {
+        assert!(
+            matches!(
+                item,
+                SelectItem::UnnamedExpr(Expr::Value(ValueWithSpan {
+                    value: Value::Number(_, false),
+                    ..
+                }))
+            ),
+            "{item:?}"
+        );
+    }
+
+    assert_eq!(
+        expr_from_projection(&pg().verified_only_select("SELECT X'1F'").projection[0]),
+        &Expr::Value(Value::HexStringLiteral("1F".to_string()).with_empty_span())
+    );
+
+    for (sql, message) in [
+        ("SELECT 0x", "Expected a hexadecimal digit after '0x'"),
+        ("SELECT 0Xg", "Expected a hexadecimal digit after '0X'"),
+        ("SELECT 0o8", "Expected an octal digit after '0o'"),
+        ("SELECT 0b2", "Expected a binary digit after '0b'"),
+        ("SELECT 0x_", "Expected a hexadecimal digit after '0x'"),
+        ("SELECT 0x__1", "Unexpected character '_'"),
+        ("SELECT 0x1_", "Unexpected character '_'"),
+    ] {
+        let err = pg().parse_sql_statements(sql).unwrap_err().to_string();
+        assert!(err.contains(message), "{sql}: {err}");
+    }
+}
+
+#[cfg(feature = "bigdecimal")]
+#[test]
+fn parse_non_decimal_integer_token_with_invalid_digits() {
+    let err = Parser::new(&PostgreSqlDialect {})
+        .with_tokens(vec![Token::Number("0xZZ".to_string(), false)])
+        .parse_value()
+        .unwrap_err();
+    assert_eq!(
+        err,
+        ParserError::ParserError("Could not parse '0xZZ' as an integer".to_string())
+    );
+}