From 3f6a369e11d494b4d78de4928605896fb8dbdb66 Mon Sep 17 00:00:00 2001 From: shenli Date: Wed, 9 Sep 2015 14:33:48 +0800 Subject: [PATCH 1/5] parser: Add more keywords into UnReservedKeyword list See: https://dev.mysql.com/doc/refman/5.7/en/keywords.html --- parser/parser.y | 14 ++++++++----- parser/parser_test.go | 20 +++++++++++++++--- parser/scanner.l | 48 ++++++++++++++++++++++++++++--------------- 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/parser/parser.y b/parser/parser.y index cd80e9ac09..8404064f03 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -422,6 +422,7 @@ import ( Identifier "identifier or unreserved keyword" UnReservedKeyword "MySQL unreserved keywords" + NotKeywordToken "Tokes not mysql keyword but treate as keyword" WhenClause "When clause" WhenClauseList "When clause list" @@ -1521,15 +1522,18 @@ IndexType: /**********************************Identifier********************************************/ Identifier: - identifier | UnReservedKeyword + identifier | UnReservedKeyword | NotKeywordToken // TODO: Add Data Type UnReserved Keywords UnReservedKeyword: - "AUTO_INCREMENT" | "BEGIN" | "BIT" | "BOOL" | "BOOLEAN" | "CHARSET" | "COLUMN" | "COLUMNS" | "DATE" | "DATETIME" -| "ENGINE" | "FULL" | "LOCAL" | "NAMES" | "OFFSET" | "PASSWORD" | "QUICK" | "ROLLBACK" | "SESSION" | "GLOBAL" -| "TABLES"| "TEXT" | "TIME" | "TIMESTAMP" | "TRANSACTION" | "TRUNCATE" | "VALUE" | "WARNINGS" | "YEAR" | "NOW" -| "SUBSTRING" | "MODE" + "AUTO_INCREMENT" | "AFTER" | "BEGIN" | "BIT" | "BOOL" | "BOOLEAN" | "CHARSET" | "COLUMNS" | "COMMIT" +| "DATE" | "DATETIME" | "DEALLOCATE" | "DO" | "END" | "ENGINE" | "ENGINES" | "EXECUTE" | "FIRST" | "FULL" +| "LOCAL" | "NAMES" | "OFFSET" | "PASSWORD" | "PREPARE" | "QUICK" | "ROLLBACK" | "SESSION" | "SIGNED" +| "START" | "GLOBAL" | "TABLES"| "TEXT" | "TIME" | "TIMESTAMP" | "TRANSACTION" | "TRUNCATE" | "UNKNOWN" +| "VALUE" | "WARNINGS" | "YEAR" | "NOW" | "MODE" +NotKeywordToken: + "SQL_CALC_FOUND_ROWS" | "SUBSTRING" /************************************************************************************ * diff --git a/parser/parser_test.go b/parser/parser_test.go index a5adfbde04..6296aa0dc3 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -268,9 +268,6 @@ func (s *testParserSuite) TestParser0(c *C) { // For time fsp {"CREATE TABLE t( c1 TIME(2), c2 DATETIME(2), c3 TIMESTAMP(2) );", true}, - - // For unreserved keywords - {"SELECT id, user_id, repo_id, mode FROM access WHERE repo_id=1 AND mode>=1;", true}, } for _, t := range table { @@ -287,6 +284,23 @@ func (s *testParserSuite) TestParser0(c *C) { } } + // Testcase for unreserved keywords + unreservedKws := []string{ + "auto_increment", "after", "begin", "bit", "bool", "boolean", "charset", "columns", "commit", + "date", "datetime", "deallocate", "do", "end", "engine", "engines", "execute", "first", "full", + "local", "names", "offset", "password", "prepare", "quick", "rollback", "session", "signed", + "start", "global", "tables", "text", "time", "timestamp", "transaction", "truncate", "unknown", + "value", "warnings", "year", "now", "substring", "mode", + } + for _, kw := range unreservedKws { + src := fmt.Sprintf("SELECT %s FROM tbl;", kw) + fmt.Printf("%s\n", src) + l := NewLexer(src) + ok := yyParse(l) == 0 + c.Assert(ok, IsTrue) + c.Assert(l.errs, HasLen, 0) + } + // Testcase for prepared statement src := "SELECT id+?, id+? from t;" l := NewLexer(src) diff --git a/parser/scanner.l b/parser/scanner.l index c77d3d97a1..75a4299acf 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -448,7 +448,8 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} "?" return placeholder {add} return add -{after} return after +{after} lval.item = string(l.val) + return after {all} return all {alter} return alter {and} return and @@ -463,20 +464,22 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {case} return caseKwd {cast} return cast {character} return character -{charset} return charsetKwd +{charset} lval.item = string(l.val) + return charsetKwd {collate} return collation -{column} lval.item = string(l.val) - return column +{column} return column {columns} lval.item = string(l.val) return columns -{commit} return commit +{commit} lval.item = string(l.val) + return commit {constraint} return constraint {convert} return convert {create} return create {cross} return cross {database} return database {databases} return databases -{deallocate} return deallocate +{deallocate} lval.item = string(l.val) + return deallocate {default} return defaultKwd {delayed} return delayed {delete} return deleteKwd @@ -485,17 +488,23 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {drop} return drop {distinct} return distinct {div} return div -{do} return do -{duplicate} return duplicate +{do} lval.item = string(l.val) + return do +{duplicate} lval.item = string(l.val) + return duplicate {else} return elseKwd -{end} return end +{end} lval.item = string(l.val) + return end {engine} lval.item = string(l.val) return engine -{engines} return engines -{execute} return execute +{engines} lval.item = string(l.val) + return engines +{execute} lval.item = string(l.val) + return execute {exists} return exists {explain} return explain -{first} return first +{first} lval.item = string(l.val) + return first {for} return forKwd {foreign} return foreign {from} return from @@ -536,7 +545,8 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {outer} return outer {password} lval.item = string(l.val) return password -{prepare} return prepare +{prepare} lval.item = string(l.val) + return prepare {primary} return primary {quick} lval.item = string(l.val) return quick @@ -547,7 +557,8 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {schemas} return schemas {session} lval.item = string(l.val) return session -{start} return start +{start} lval.item = string(l.val) + return start {global} lval.item = string(l.val) return global {regexp} return regexp @@ -578,7 +589,8 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {update} return update {union} return union {unique} return unique -{unknown} return unknown +{unknown} lval.item = string(l.val) + return unknown {use} return use {using} return using {value} lval.item = string(l.val) @@ -590,7 +602,8 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {where} return where {xor} return xor -{signed} return signed +{signed} lval.item = string(l.val) + return signed {unsigned} return unsigned {zerofill} return zerofill @@ -601,7 +614,8 @@ sys_var "@@"(({global}".")|({session}".")|{local}".")?{ident} {true} return trueKwd -{calc_found_rows} return calcFoundRows +{calc_found_rows} lval.item = string(l.val) + return calcFoundRows {current_ts} return currentTs {localtime} return localTime From f5a46df71e4a2caeecd84bf0c2f04ee22477b376 Mon Sep 17 00:00:00 2001 From: Shen Li Date: Wed, 9 Sep 2015 22:41:54 +0800 Subject: [PATCH 2/5] parser: Change lower_than_eq to lowerThanEq Fix golint error --- parser/parser.y | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parser/parser.y b/parser/parser.y index 14717607d4..f3552e8ce1 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -471,7 +471,7 @@ import ( %left xor %left andand and %left between -%precedence lower_than_eq +%precedence lowerThanEq %left eq ge le neq neqSynonym '>' '<' is like in %left '|' %left '&' @@ -1532,7 +1532,7 @@ Identifier: UnReservedKeyword: "AUTO_INCREMENT" | "AFTER" | "BEGIN" | "BIT" | "BOOL" | "BOOLEAN" | "CHARSET" | "COLUMNS" | "COMMIT" | "DATE" | "DATETIME" | "DEALLOCATE" | "DO" | "END" | "ENGINE" | "ENGINES" | "EXECUTE" | "FIRST" | "FULL" -| "LOCAL" | "NAMES" | "OFFSET" | "PASSWORD" %prec lower_than_eq | "PREPARE" | "QUICK" | "ROLLBACK" | "SESSION" | "SIGNED" +| "LOCAL" | "NAMES" | "OFFSET" | "PASSWORD" %prec lowerThanEq | "PREPARE" | "QUICK" | "ROLLBACK" | "SESSION" | "SIGNED" | "START" | "GLOBAL" | "TABLES"| "TEXT" | "TIME" | "TIMESTAMP" | "TRANSACTION" | "TRUNCATE" | "UNKNOWN" | "VALUE" | "WARNINGS" | "YEAR" | "NOW" | "MODE" From 211eadbb098fd7201c6cdcd5833a1d1e8f4f520d Mon Sep 17 00:00:00 2001 From: Shen Li Date: Wed, 9 Sep 2015 23:08:27 +0800 Subject: [PATCH 3/5] parser: Fix typo Address comment --- parser/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/parser.y b/parser/parser.y index f3552e8ce1..f3557e88d7 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -425,7 +425,7 @@ import ( Identifier "identifier or unreserved keyword" UnReservedKeyword "MySQL unreserved keywords" - NotKeywordToken "Tokes not mysql keyword but treate as keyword" + NotKeywordToken "Tokens not mysql keyword but treated specially" WhenClause "When clause" WhenClauseList "When clause list" From bd7817ea38e018e37a892fde6f4b43109f89ae7b Mon Sep 17 00:00:00 2001 From: Shen Li Date: Wed, 9 Sep 2015 23:28:20 +0800 Subject: [PATCH 4/5] parser: Address comment --- parser/parser_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 8b91689b3a..94a2895756 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -296,8 +296,7 @@ func (s *testParserSuite) TestParser0(c *C) { src := fmt.Sprintf("SELECT %s FROM tbl;", kw) fmt.Printf("%s\n", src) l := NewLexer(src) - ok := yyParse(l) == 0 - c.Assert(ok, IsTrue) + c.Assert(yyParse(l), Equals, 0) c.Assert(l.errs, HasLen, 0) } From 25eac139ad4e0e57c752a823381232de760c7cae Mon Sep 17 00:00:00 2001 From: Shen Li Date: Thu, 10 Sep 2015 07:41:43 +0800 Subject: [PATCH 5/5] parser: Address comment --- parser/parser_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 94a2895756..12b46a28d9 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -294,10 +294,9 @@ func (s *testParserSuite) TestParser0(c *C) { } for _, kw := range unreservedKws { src := fmt.Sprintf("SELECT %s FROM tbl;", kw) - fmt.Printf("%s\n", src) l := NewLexer(src) c.Assert(yyParse(l), Equals, 0) - c.Assert(l.errs, HasLen, 0) + c.Assert(l.errs, HasLen, 0, Commentf("source %s", src)) } // Testcase for prepared statement