database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Nereids) support reuse sql cache between different comment (#40065)

Browse Source 
cherry pick from #40049
This commit is contained in:
924060929
2024-08-28 22:15:55 +08:00
committed by GitHub
parent c9aca7d852
commit 688b97e183
5 changed files with 89 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.nereids.SqlCacheContext.FullTableName;
import org.apache.doris.nereids.SqlCacheContext.ScanTable;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundVariable;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
import org.apache.doris.nereids.rules.analysis.UserAuthentication;
@ -127,7 +128,7 @@ public class NereidsSqlCacheManager {
SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get();
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL
? currentUserIdentity.toString() + ":" + sql.trim()
? currentUserIdentity.toString() + ":" + normalizeSql(sql.trim())
: currentUserIdentity.toString() + ":" + DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5());
if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5() != null
&& sqlCacheContext.getResultSetInFe().isPresent()) {
@ -147,7 +148,7 @@ public class NereidsSqlCacheManager {
SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get();
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL
? currentUserIdentity.toString() + ":" + sql.trim()
? currentUserIdentity.toString() + ":" + normalizeSql(sql.trim())
: currentUserIdentity.toString() + ":" + DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5());
if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5() != null) {
SqlCache cache = (SqlCache) analyzer.getCache();
@ -168,7 +169,7 @@ public class NereidsSqlCacheManager {
/** tryParseSql */
public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, String sql) {
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
String key = currentUserIdentity + ":" + sql.trim();
String key = currentUserIdentity + ":" + normalizeSql(sql.trim());
SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key);
if (sqlCacheContext == null) {
return Optional.empty();
@ -201,6 +202,10 @@ public class NereidsSqlCacheManager {
}
}
private String normalizeSql(String sql) {
return NereidsParser.removeCommentAndTrimBlank(sql);
}
private Optional<LogicalSqlCache> tryParseSqlWithoutCheckVariable(
ConnectContext connectContext, String key,
SqlCacheContext sqlCacheContext, UserIdentity currentUserIdentity) {

39
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View File

@ -40,6 +40,7 @@ import com.google.common.collect.Maps;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.atn.PredictionMode;
@ -334,4 +335,42 @@ public class NereidsParser {
}
return tree;
}
/**
* removeCommentAndTrimBlank
*
* for example: select \/*+SET_VAR(key=value)*\/ \/* trace_id: 1234 *\/ *, a, \n b from table
*
* will be normalized to: select \/*+SET_VAR(key=value)*\/ * , a, b from table
*/
public static String removeCommentAndTrimBlank(String sql) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
// maybe add more space char
StringBuilder newSql = new StringBuilder((int) (sql.length() * 1.2));
for (Token token : tokenStream.getTokens()) {
int tokenType = token.getType();
switch (tokenType) {
case DorisLexer.SIMPLE_COMMENT:
case DorisLexer.WS:
case Recognizer.EOF:
break;
case DorisLexer.BRACKETED_COMMENT:
String bracketedComment = token.getText();
// append hint
if (bracketedComment.startsWith("/*+")) {
newSql.append(bracketedComment);
newSql.append(" ");
}
break;
default:
newSql.append(token.getText());
newSql.append(" ");
}
}
return newSql.toString().trim();
}
}