Merge pull request #17 from JLDLaughlin/binding_wip

Parse positional parameters

Author: benesch

src/ast/mod.rs

@@ -177,6 +177,8 @@ pub enum Expr {
     QualifiedWildcard(Vec<Ident>),
     /// Multi-part identifier, e.g. `table_alias.column` or `schema.table.col`
     CompoundIdentifier(Vec<Ident>),
+    /// A positional parameter, e.g., `$1` or `$42`
+    Parameter(usize),
     /// `IS NULL` expression
     IsNull(Box<Expr>),
     /// `IS NOT NULL` expression
@@ -255,9 +257,9 @@ pub enum Expr {
     /// `<expr> <op> ALL (<query>)`
     All {
         left: Box<Expr>,
-        op:  BinaryOperator,
+        op: BinaryOperator,
         right: Box<Query>,
-    }
+    },
 }
 
 impl fmt::Display for Expr {
@@ -267,6 +269,7 @@ impl fmt::Display for Expr {
             Expr::Wildcard => f.write_str("*"),
             Expr::QualifiedWildcard(q) => write!(f, "{}.*", display_separated(q, ".")),
             Expr::CompoundIdentifier(s) => write!(f, "{}", display_separated(s, ".")),
+            Expr::Parameter(n) => write!(f, "${}", n),
             Expr::IsNull(ast) => write!(f, "{} IS NULL", ast),
             Expr::IsNotNull(ast) => write!(f, "{} IS NOT NULL", ast),
             Expr::InList {
@@ -333,8 +336,20 @@ impl fmt::Display for Expr {
             }
             Expr::Exists(s) => write!(f, "EXISTS ({})", s),
             Expr::Subquery(s) => write!(f, "({})", s),
-            Expr::Any{left, op, right, some} => write!(f, "{} {} {} ({})", left, op, if *some { "SOME" } else { "ANY" }, right),
-            Expr::All{left, op, right} => write!(f, "{} {} ALL ({})", left, op, right),
+            Expr::Any {
+                left,
+                op,
+                right,
+                some,
+            } => write!(
+                f,
+                "{} {} {} ({})",
+                left,
+                op,
+                if *some { "SOME" } else { "ANY" },
+                right
+            ),
+            Expr::All { left, op, right } => write!(f, "{} {} ALL ({})", left, op, right),
         }
     }
 }
@@ -532,9 +547,7 @@ pub enum Statement {
         with_options: Vec<SqlOption>,
     },
     /// `FLUSH SOURCE`
-    FlushSource {
-        name: ObjectName
-    },
+    FlushSource { name: ObjectName },
     /// `FLUSH ALL SOURCES`
     FlushAllSources,
     /// `CREATE VIEW`
@@ -618,9 +631,7 @@ pub enum Statement {
         filter: Option<ShowStatementFilter>,
     },
     /// `SHOW CREATE VIEW <view>`
-    ShowCreateView {
-        view_name: ObjectName,
-    },
+    ShowCreateView { view_name: ObjectName },
     /// `{ BEGIN [ TRANSACTION | WORK ] | START TRANSACTION } ...`
     StartTransaction { modes: Vec<TransactionMode> },
     /// `SET TRANSACTION ...`
@@ -902,9 +913,7 @@ impl fmt::Display for Statement {
                 }
                 Ok(())
             }
-            Statement::ShowCreateView {
-                view_name,
-            } => {
+            Statement::ShowCreateView { view_name } => {
                 f.write_str("SHOW CREATE VIEW ")?;
                 write!(f, "{}", view_name)
             }

src/ast/visit_macro.rs

@@ -186,6 +186,8 @@ macro_rules! make_visitor {
                 visit_qualified_wildcard(self, idents)
             }
 
+            fn visit_parameter(&mut self, _n: usize) {}
+
             fn visit_is_null(&mut self, expr: &'ast $($mut)* Expr) {
                 visit_is_null(self, expr)
             }
@@ -887,6 +889,7 @@ macro_rules! make_visitor {
                 Expr::Wildcard => visitor.visit_wildcard(),
                 Expr::QualifiedWildcard(idents) => visitor.visit_qualified_wildcard(idents),
                 Expr::CompoundIdentifier(idents) => visitor.visit_compound_identifier(idents),
+                Expr::Parameter(n) => visitor.visit_parameter(*n),
                 Expr::IsNull(expr) => visitor.visit_is_null(expr),
                 Expr::IsNotNull(expr) => visitor.visit_is_not_null(expr),
                 Expr::InList {
@@ -971,6 +974,13 @@ macro_rules! make_visitor {
             }
         }
 
+        pub fn visit_parameter<'ast, V: $name<'ast> + ?Sized>(
+            visitor: &mut V,
+            n: usize,
+        ) {
+            visitor.visit_parameter(n)
+        }
+
         pub fn visit_is_null<'ast, V: $name<'ast> + ?Sized>(visitor: &mut V, expr: &'ast $($mut)* Expr) {
             visitor.visit_expr(expr);
         }

src/parser.rs

@@ -56,7 +56,7 @@ use IsLateral::*;
 
 impl From<TokenizerError> for ParserError {
     fn from(e: TokenizerError) -> Self {
-        ParserError::TokenizerError(format!("{:?}", e))
+        ParserError::TokenizerError(format!("{}", e))
     }
 }
 
@@ -263,6 +263,10 @@ impl Parser {
                 self.prev_token();
                 Ok(Expr::Value(self.parse_value()?))
             }
+            Token::Parameter(s) => Ok(Expr::Parameter(match s.parse() {
+                Ok(n) => n,
+                Err(err) => return parser_err!("unable to parse parameter: {}", err),
+            })),
             Token::LParen => {
                 let expr = if self.parse_keyword("SELECT") || self.parse_keyword("WITH") {
                     self.prev_token();
@@ -684,18 +688,19 @@ impl Parser {
                 let query = self.parse_query()?;
                 self.expect_token(&Token::RParen)?;
                 if any || some {
-                    Ok(Expr::Any{
+                    Ok(Expr::Any {
                         left: Box::new(expr),
                         op,
                         right: Box::new(query),
                         some,
                     })
                 } else {
-                    Ok(Expr::All{
+                    Ok(Expr::All {
                         left: Box::new(expr),
                         op,
                         right: Box::new(query),
-                    })}
+                    })
+                }
             } else {
                 Ok(Expr::BinaryOp {
                     left: Box::new(expr),
@@ -2399,7 +2404,7 @@ impl Parser {
             Ok(Statement::FlushAllSources)
         } else if self.parse_keyword("SOURCE") {
             Ok(Statement::FlushSource {
-                name: self.parse_object_name()?
+                name: self.parse_object_name()?,
             })
         } else {
             self.expected("ALL SOURCES or SOURCE", self.peek_token())?

src/tokenizer.rs

@@ -21,6 +21,7 @@ use std::str::Chars;
 
 use super::dialect::keywords::ALL_KEYWORDS;
 use super::dialect::Dialect;
+use std::error::Error;
 use std::fmt;
 
 /// SQL Token enumeration
@@ -38,6 +39,10 @@ pub enum Token {
     NationalStringLiteral(String),
     /// Hexadecimal string literal: i.e.: X'deadbeef'
     HexStringLiteral(String),
+    /// An unsigned numeric literal representing positional
+    /// parameters like $1, $2, etc. in prepared statements and
+    /// function definitions
+    Parameter(String),
     /// Comma
     Comma,
     /// Whitespace (space, tab, etc)
@@ -99,6 +104,7 @@ impl fmt::Display for Token {
             Token::SingleQuotedString(ref s) => write!(f, "'{}'", s),
             Token::NationalStringLiteral(ref s) => write!(f, "N'{}'", s),
             Token::HexStringLiteral(ref s) => write!(f, "X'{}'", s),
+            Token::Parameter(n) => write!(f, "${}", n),
             Token::Comma => f.write_str(","),
             Token::Whitespace(ws) => write!(f, "{}", ws),
             Token::Eq => f.write_str("="),
@@ -212,6 +218,14 @@ impl fmt::Display for Whitespace {
 #[derive(Debug, PartialEq)]
 pub struct TokenizerError(String);
 
+impl fmt::Display for TokenizerError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+impl Error for TokenizerError {}
+
 /// SQL Tokenizer
 pub struct Tokenizer<'a> {
     dialect: &'a dyn Dialect,
@@ -249,6 +263,7 @@ impl<'a> Tokenizer<'a> {
                 Token::Word(w) if w.quote_style != None => self.col += w.value.len() as u64 + 2,
                 Token::Number(s) => self.col += s.len() as u64,
                 Token::SingleQuotedString(s) => self.col += s.len() as u64,
+                Token::Parameter(s) => self.col += s.len() as u64,
                 _ => self.col += 1,
             }
 
@@ -415,6 +430,7 @@ impl<'a> Tokenizer<'a> {
                 '&' => self.consume_and_return(chars, Token::Ampersand),
                 '{' => self.consume_and_return(chars, Token::LBrace),
                 '}' => self.consume_and_return(chars, Token::RBrace),
+                '$' => self.tokenize_parameter(chars),
                 other => self.consume_and_return(chars, Token::Char(other)),
             },
             None => Ok(None),
@@ -490,6 +506,31 @@ impl<'a> Tokenizer<'a> {
         }
     }
 
+    /// PostgreSQL supports positional parameters (like $1, $2, etc.) for
+    /// prepared statements and function definitions.
+    /// Grab the positional argument following a $ to parse it.
+    fn tokenize_parameter(
+        &self,
+        chars: &mut Peekable<Chars<'_>>,
+    ) -> Result<Option<Token>, TokenizerError> {
+        assert_eq!(Some('$'), chars.next());
+
+        let n = peeking_take_while(chars, |ch| match ch {
+            '0'..='9' => true,
+            _ => false,
+        });
+
+        if n.is_empty() {
+            return Err(TokenizerError(
+                "parameter marker ($) was not followed by \
+                 at least one digit"
+                    .into(),
+            ));
+        }
+
+        Ok(Some(Token::Parameter(n)))
+    }
+
     fn consume_and_return(
         &self,
         chars: &mut Peekable<Chars<'_>>,

tests/sqlparser_common.rs

@@ -364,6 +364,46 @@ fn parse_select_count_distinct() {
     );
 }
 
+#[test]
+fn parse_parameters() {
+    let select = verified_only_select("SELECT $1");
+    assert_eq!(
+        &Expr::Parameter(1),
+        expr_from_projection(only(&select.projection)),
+    );
+
+    assert_eq!(
+        Expr::BinaryOp {
+            left: Box::new(Expr::Parameter(91)),
+            op: BinaryOperator::Plus,
+            right: Box::new(Expr::Parameter(42)),
+        },
+        verified_expr("$91 + $42"),
+    );
+
+    let res = parse_sql_statements("SELECT $q");
+    assert_eq!(
+        ParserError::TokenizerError(
+            "parameter marker ($) was not followed by at least one digit".into()
+        ),
+        res.unwrap_err()
+    );
+
+    let res = parse_sql_statements("SELECT $1$2");
+    assert_eq!(
+        ParserError::ParserError("Expected end of statement, found: $2".into()),
+        res.unwrap_err()
+    );
+
+    let res = parse_sql_statements(&format!("SELECT $18446744073709551616"));
+    assert_eq!(
+        ParserError::ParserError(
+            "unable to parse parameter: number too large to fit in target type".into(),
+        ),
+        res.unwrap_err()
+    );
+}
+
 #[test]
 fn parse_not() {
     let sql = "SELECT id FROM customer WHERE NOT salary = ''";
@@ -3323,10 +3363,7 @@ fn parse_explain() {
 #[test]
 fn parse_flush() {
     let ast = verified_stmt("FLUSH ALL SOURCES");
-    assert_eq!(
-        ast,
-        Statement::FlushAllSources,
-    );
+    assert_eq!(ast, Statement::FlushAllSources,);
 
     let ast = verified_stmt("FLUSH SOURCE foo");
     assert_eq!(

tests/sqlparser_postgres.rs

@@ -232,6 +232,7 @@ fn parse_create_table_with_inherit() {
     pg().verified_stmt(sql);
 }
 
+#[ignore] // NOTE(benesch): this test is doomed. COPY data should not be tokenized/parsed.
 #[test]
 fn parse_copy_example() {
     let sql = r#"COPY public.actor (actor_id, first_name, last_name, last_update, value) FROM stdin;