From fce58065d77eb779b2f8da53d3db9d90cf569059 Mon Sep 17 00:00:00 2001 From: winkyao <82091309@qq.com> Date: Mon, 21 Aug 2017 12:37:27 +0800 Subject: [PATCH] parser,plan,ast: fix #4239, concatenates string literals which placed each other, and use first string as projection name (#4252) --- ast/expressions.go | 12 ++++++ executor/executor_test.go | 71 ++++++++++++++++++++++++++++++++++++ parser/lexer.go | 19 +--------- parser/lexer_test.go | 11 ++++-- parser/parser.y | 36 +++++++++++++++++- plan/logical_plan_builder.go | 7 +++- 6 files changed, 133 insertions(+), 23 deletions(-) diff --git a/ast/expressions.go b/ast/expressions.go index 21142e75b7..9298b83581 100644 --- a/ast/expressions.go +++ b/ast/expressions.go @@ -52,6 +52,7 @@ var ( // ValueExpr is the simple value expression. type ValueExpr struct { exprNode + projectionOffset int } // NewValueExpr creates a ValueExpr with value, and sets default field type. @@ -62,9 +63,20 @@ func NewValueExpr(value interface{}) *ValueExpr { ve := &ValueExpr{} ve.SetValue(value) types.DefaultTypeForValue(value, &ve.Type) + ve.projectionOffset = -1 return ve } +// SetProjectionOffset sets ValueExpr.projectionOffset for logical plan builder. +func (n *ValueExpr) SetProjectionOffset(offset int) { + n.projectionOffset = offset +} + +// GetProjectionOffset returns ValueExpr.projectionOffset. +func (n *ValueExpr) GetProjectionOffset() int { + return n.projectionOffset +} + // Accept implements Node interface. func (n *ValueExpr) Accept(v Visitor) (Node, bool) { newNode, skipChildren := v.Enter(n) diff --git a/executor/executor_test.go b/executor/executor_test.go index 90cced5a89..b53e90a009 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -462,6 +462,77 @@ func (s *testSuite) TestSelectStringLiteral(c *C) { c.Check(err, IsNil) c.Check(len(fields), Equals, 1) c.Check(fields[0].Column.Name.O, Equals, "abc 123 ") + + // Issue #4239. + sql = `select 'a' ' ' 'string';` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("a string")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "a") + + sql = `select 'a' " " "string";` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("a string")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "a") + + sql = `select 'string' 'string';` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("stringstring")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "string") + + sql = `select "ss" "a";` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("ssa")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "ss") + + sql = `select "ss" "a" "b";` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("ssab")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "ss") + + sql = `select "ss" "a" ' ' "b";` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("ssa b")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "ss") + + sql = `select "ss" "a" ' ' "b" ' ' "d";` + r = tk.MustQuery(sql) + r.Check(testkit.Rows("ssa b d")) + rs, err = tk.Exec(sql) + c.Check(err, IsNil) + fields, err = rs.Fields() + c.Check(err, IsNil) + c.Check(len(fields), Equals, 1) + c.Check(fields[0].Column.Name.O, Equals, "ss") } func (s *testSuite) TestSelectLimit(c *C) { diff --git a/parser/lexer.go b/parser/lexer.go index 3abb4bb627..7ad0fdc57e 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -451,24 +451,7 @@ func scanQuotedIdent(s *Scanner) (tok int, pos Pos, lit string) { } func startString(s *Scanner) (tok int, pos Pos, lit string) { - tok, pos, lit = s.scanString() - if tok == unicode.ReplacementChar { - return - } - - // Quoted strings placed next to each other are concatenated to a single string. - // See http://dev.mysql.com/doc/refman/5.7/en/string-literals.html - ch := s.skipWhitespace() - if s.sqlMode&mysql.ModeANSIQuotes > 0 && - ch == '"' || s.r.s[pos.Offset] == '"' { - return - } - for ch == '\'' || ch == '"' { - _, _, lit1 := s.scanString() - lit = lit + lit1 - ch = s.skipWhitespace() - } - return + return s.scanString() } // lazyBuf is used to avoid allocation if possible. diff --git a/parser/lexer_test.go b/parser/lexer_test.go index c28e928b49..39fb3fe82c 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -184,8 +184,6 @@ func (s *testLexerSuite) TestscanString(c *C) { expect string }{ {`' \n\tTest String'`, " \n\tTest String"}, - {`'a' ' ' 'string'`, "a string"}, - {`'a' " " "string"`, "a string"}, {`'\x\B'`, "xB"}, {`'\0\'\"\b\n\r\t\\'`, "\000'\"\b\n\r\t\\"}, {`'\Z'`, string(26)}, @@ -315,7 +313,6 @@ func (s *testLexerSuite) TestSQLModeANSIQuotes(c *C) { {"`identifier`", identifier, "identifier"}, {`"identifier""and"`, identifier, `identifier"and`}, {`'string''string'`, stringLit, "string'string"}, - {`'string' 'string'`, stringLit, "stringstring"}, {`"identifier"'and'`, identifier, "identifier"}, {`'string'"identifier"`, stringLit, "string"}, } @@ -328,6 +325,14 @@ func (s *testLexerSuite) TestSQLModeANSIQuotes(c *C) { c.Assert(tok, Equals, t.tok) c.Assert(v.ident, Equals, t.ident) } + scanner.reset(`'string' 'string'`) + var v yySymType + tok := scanner.Lex(&v) + c.Assert(tok, Equals, stringLit) + c.Assert(v.ident, Equals, "string") + tok = scanner.Lex(&v) + c.Assert(tok, Equals, stringLit) + c.Assert(v.ident, Equals, "string") } func (s *testLexerSuite) TestIllegal(c *C) { diff --git a/parser/parser.y b/parser/parser.y index 29d9365321..fa3d640813 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -752,6 +752,7 @@ import ( StatsPersistentVal "stats_persistent value" StringName "string literal or identifier" StringList "string list" + StringLiteral "text literal" ExplainableStmt "explainable statement" SubSelect "Sub Select" Symbol "Constraint Symbol" @@ -891,6 +892,9 @@ import ( %precedence lowerThanIntervalKeyword %precedence interval +%precedence lowerThanStringLitToken +%precedence stringLit + %precedence lowerThanSetKeyword %precedence set @@ -2622,7 +2626,6 @@ Literal: | floatLit | decLit | intLit -| stringLit { tp := types.NewFieldType(mysql.TypeString) tp.Charset, tp.Collate = parser.charset, parser.collation @@ -2630,6 +2633,10 @@ Literal: expr.SetType(tp) $$ = expr } +| StringLiteral %prec lowerThanStringLitToken + { + $$ = $1 + } | "UNDERSCORE_CHARSET" stringLit { // See https://dev.mysql.com/doc/refman/5.7/en/charset-literal.html @@ -2648,6 +2655,33 @@ Literal: | hexLit | bitLit +StringLiteral: + stringLit + { + tp := types.NewFieldType(mysql.TypeString) + tp.Charset, tp.Collate = parser.charset, parser.collation + expr := ast.NewValueExpr($1) + expr.SetType(tp) + $$ = expr + } +| StringLiteral stringLit + { + valExpr := $1.(*ast.ValueExpr) + strLit := valExpr.GetString() + tp := types.NewFieldType(mysql.TypeString) + tp.Charset, tp.Collate = parser.charset, parser.collation + expr := ast.NewValueExpr(strLit+$2) + // Fix #4239, use first string literal as projection name. + if valExpr.GetProjectionOffset() >= 0 { + expr.SetProjectionOffset(valExpr.GetProjectionOffset()) + } else { + expr.SetProjectionOffset(len(strLit)) + } + expr.SetType(tp) + $$ = expr + } + + Operand: Literal { diff --git a/plan/logical_plan_builder.go b/plan/logical_plan_builder.go index da495e79ab..3fdaaca044 100644 --- a/plan/logical_plan_builder.go +++ b/plan/logical_plan_builder.go @@ -437,9 +437,14 @@ func (b *planBuilder) buildProjectionFieldNameFromExpressions(field *ast.SelectF // Literal: Need special processing switch valueExpr.Kind() { case types.KindString: + projName := valueExpr.GetString() + projOffset := valueExpr.GetProjectionOffset() + if projOffset >= 0 { + projName = projName[:projOffset] + } // See #3686, #3994: // For string literals, string content is used as column name. Non-graph initial characters are trimmed. - fieldName := strings.TrimLeftFunc(valueExpr.GetString(), func(r rune) bool { + fieldName := strings.TrimLeftFunc(projName, func(r rune) bool { return !unicode.IsOneOf(mysql.RangeGraph, r) }) return model.NewCIStr(fieldName)