diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 974da5d2aa..2dfb542708 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -56,6 +56,7 @@ import org.apache.logging.log4j.Logger; import java.lang.reflect.Method; import java.util.BitSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -228,7 +229,7 @@ public class NereidsParser { } private List parseSQLWithDialect(String sql, - SessionVariable sessionVariable) { + SessionVariable sessionVariable) { @Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect()); if (sqlDialect == null) { return parseSQL(sql); @@ -244,7 +245,7 @@ public class NereidsParser { } } catch (Throwable throwable) { LOG.warn("Parse sql with dialect {} failed, plugin: {}, sql: {}.", - sqlDialect, plugin.getClass().getSimpleName(), sql, throwable); + sqlDialect, plugin.getClass().getSimpleName(), sql, throwable); } } @@ -280,7 +281,7 @@ public class NereidsParser { } public List> parseMultiple(String sql, - @Nullable LogicalPlanBuilder logicalPlanBuilder) { + @Nullable LogicalPlanBuilder logicalPlanBuilder) { return parse(sql, logicalPlanBuilder, DorisParser::multiStatements); } @@ -325,30 +326,31 @@ public class NereidsParser { } private T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, - Function parseFunction) { - ParserRuleContext tree = toAst(sql, parseFunction); + Function parseFunction) { + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, parseFunction); LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null - ? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder; + ? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint)) + : logicalPlanBuilder; return (T) realLogicalPlanBuilder.visit(tree); } public LogicalPlan parseForCreateView(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView( - getHintMap(sql, DorisParser::selectHint)); + getHintMap(sql, tokenStream, DorisParser::selectHint)); return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } /** get hint map */ - public static Map getHintMap(String sql, - Function parseFunction) { + public static Map getHintMap(String sql, CommonTokenStream hintTokenStream, + Function parseFunction) { // parse hint first round - DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer); - Map selectHintMap = Maps.newHashMap(); - Token hintToken = hintTokenStream.getTokenSource().nextToken(); + Iterator tokenIterator = hintTokenStream.getTokens().iterator(); + Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; while (hintToken != null && hintToken.getType() != DorisLexer.EOF) { if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') { String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1); @@ -358,15 +360,19 @@ public class NereidsParser { ParserRuleContext hintContext = parseFunction.apply(hintParser); selectHintMap.put(hintToken.getStartIndex(), hintContext); } - hintToken = hintTokenStream.getTokenSource().nextToken(); + hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; } return selectHintMap; } + public static ParserRuleContext toAst( + String sql, Function parseFunction) { + return toAst(parseAllTokens(sql), parseFunction); + } + /** toAst */ - public static ParserRuleContext toAst(String sql, Function parseFunction) { - DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); + public static ParserRuleContext toAst( + CommonTokenStream tokenStream, Function parseFunction) { DorisParser parser = new DorisParser(tokenStream); parser.addParseListener(POST_PROCESSOR); @@ -397,9 +403,7 @@ public class NereidsParser { * will be normalized to: select \/*+SET_VAR(key=value)*\/ * , a, b from table */ public static String removeCommentAndTrimBlank(String sql) { - DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); - tokenStream.fill(); + CommonTokenStream tokenStream = parseAllTokens(sql); // maybe add more space char StringBuilder newSql = new StringBuilder((int) (sql.length() * 1.2)); @@ -426,4 +430,11 @@ public class NereidsParser { } return newSql.toString().trim(); } + + private static CommonTokenStream parseAllTokens(String sql) { + DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); + CommonTokenStream tokenStream = new CommonTokenStream(lexer); + tokenStream.fill(); + return tokenStream; + } }