database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature] (sql-digest) support sql digest (#8919)

Browse Source
This commit is contained in:
Henry2SS
2022-05-08 17:25:41 +08:00
committed by GitHub
parent 52a2db18c0
commit c633402ce3
37 changed files with 735 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
View File

@ -838,6 +838,36 @@ public class AnalyticExpr extends Expr {
return sb.toString();
}
@Override
public String toDigestImpl() {
StringBuilder sb = new StringBuilder();
sb.append(fnCall.toDigest()).append(" OVER (");
boolean needsSpace = false;
if (!partitionExprs.isEmpty()) {
sb.append("PARTITION BY ").append(exprListToDigest(partitionExprs));
needsSpace = true;
}
if (!orderByElements.isEmpty()) {
List<String> orderByStrings = Lists.newArrayList();
for (OrderByElement e : orderByElements) {
orderByStrings.add(e.toDigest());
}
if (needsSpace) {
sb.append(" ");
}
sb.append("ORDER BY ").append(Joiner.on(", ").join(orderByStrings));
needsSpace = true;
}
if (window != null) {
if (needsSpace) {
sb.append(" ");
}
sb.append(window.toDigest());
}
sb.append(")");
return sb.toString();
}
private String exprListToSql(List<? extends Expr> exprs) {
if (exprs == null || exprs.isEmpty())
return "";
@ -847,4 +877,15 @@ public class AnalyticExpr extends Expr {
}
return Joiner.on(", ").join(strings);
}
private String exprListToDigest(List<? extends Expr> exprs) {
if (exprs == null || exprs.isEmpty()) {
return "";
}
List<String> strings = Lists.newArrayList();
for (Expr expr : exprs) {
strings.add(expr.toDigest());
}
return Joiner.on(", ").join(strings);
}
}

26
fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java
View File

@ -167,6 +167,17 @@ public class AnalyticWindow {
return sb.toString();
}
public String toDigest() {
StringBuilder sb = new StringBuilder();
if (expr != null) {
sb.append(expr.toDigest()).append(" ");
}
sb.append(type.toString());
return sb.toString();
}
public TAnalyticWindowBoundary toThrift(Type windowType) {
TAnalyticWindowBoundary result = new TAnalyticWindowBoundary(type.toThrift());
@ -296,6 +307,21 @@ public class AnalyticWindow {
return sb.toString();
}
public String toDigest() {
StringBuilder sb = new StringBuilder();
sb.append(type_.toString()).append(" ");
if (rightBoundary_ == null) {
sb.append(leftBoundary_.toDigest());
} else {
sb.append("BETWEEN ").append(leftBoundary_.toDigest()).append(" AND ");
sb.append(rightBoundary_.toDigest());
}
return sb.toString();
}
public TAnalyticWindow toThrift() {
TAnalyticWindow result = new TAnalyticWindow(type_.toThrift());

9
fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
View File

@ -221,6 +221,15 @@ public class ArithmeticExpr extends Expr {
}
}
@Override
public String toDigestImpl() {
if (children.size() == 1) {
return op.toString() + " " + getChild(0).toDigest();
} else {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}
}
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.ARITHMETIC_EXPR;

8
fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
View File

@ -72,6 +72,14 @@ public class ArrayLiteral extends LiteralExpr {
return "ARRAY(" + StringUtils.join(list, ", ") + ")";
}
@Override
public String toDigestImpl() {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(v.toDigestImpl()));
return "ARRAY(" + StringUtils.join(list, ", ") + ")";
}
@Override
public String getStringValue() {
List<String> list = new ArrayList<>(children.size());

9
fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java
View File

@ -96,7 +96,14 @@ public class BetweenPredicate extends Predicate {
public String toSqlImpl() {
String notStr = (isNotBetween) ? "NOT " : "";
return children.get(0).toSql() + " " + notStr + "BETWEEN " +
children.get(1).toSql() + " AND " + children.get(2).toSql();
children.get(1).toSql() + " AND " + children.get(2).toSql();
}
@Override
public String toDigestImpl() {
String notStr = (isNotBetween) ? "NOT " : "";
return children.get(0).toDigest() + " " + notStr + "BETWEEN " +
children.get(1).toDigest() + " AND " + children.get(2).toDigest();
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
View File

@ -242,6 +242,11 @@ public class BinaryPredicate extends Predicate implements Writable {
return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
}
@Override
public String toDigestImpl() {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.BINARY_PRED;

18
fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java
View File

@ -129,6 +129,24 @@ public class CaseExpr extends Expr {
return output.toString();
}
@Override
public String toDigestImpl() {
StringBuilder sb = new StringBuilder("CASE");
int childIdx = 0;
if (hasCaseExpr) {
sb.append(" ").append(children.get(childIdx++).toDigest());
}
while (childIdx + 2 <= children.size()) {
sb.append(" WHEN ").append(children.get(childIdx++).toDigest());
sb.append(" THEN ").append(children.get(childIdx++).toDigest());
}
if (hasElseExpr) {
sb.append(" ELSE ").append(children.get(children.size() - 1).toDigest());
}
sb.append(" END");
return sb.toString();
}
@Override
public boolean isVectorized() {
return false;

21
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
View File

@ -206,6 +206,27 @@ public class CastExpr extends Expr {
}
}
@Override
public String toDigestImpl() {
boolean isVerbose = ConnectContext.get() != null &&
ConnectContext.get().getExecutor() != null &&
ConnectContext.get().getExecutor().getParsedStmt() != null &&
ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions() != null &&
ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions().isVerbose();
if (isImplicit && !isVerbose) {
return getChild(0).toDigest();
}
if (isAnalyzed) {
if (type.isStringType()) {
return "CAST(" + getChild(0).toDigest() + " AS " + "CHARACTER" + ")";
} else {
return "CAST(" + getChild(0).toDigest() + " AS " + type.toString() + ")";
}
} else {
return "CAST(" + getChild(0).toDigest() + " AS " + targetTypeDef.toString() + ")";
}
}
@Override
protected void treeToThriftHelper(TExpr container) {
if (noOp) {

9
fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
View File

@ -94,6 +94,15 @@ public class CompoundPredicate extends Predicate {
}
}
@Override
public String toDigestImpl() {
if (children.size() == 1) {
return "NOT " + getChild(0).toDigest();
} else {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}
}
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.COMPOUND_PRED;

13
fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java
View File

@ -58,6 +58,7 @@ public class ExistsPredicate extends Predicate {
@Override
public Expr clone() { return new ExistsPredicate(this); }
@Override
public String toSqlImpl() {
StringBuilder strBuilder = new StringBuilder();
if (notExists) {
@ -69,6 +70,18 @@ public class ExistsPredicate extends Predicate {
return strBuilder.toString();
}
@Override
public String toDigestImpl() {
StringBuilder strBuilder = new StringBuilder();
if (notExists) {
strBuilder.append("NOT ");
}
strBuilder.append("EXISTS ");
strBuilder.append(getChild(0).toDigest());
return strBuilder.toString();
}
@Override
public int hashCode() {
return 31 * super.hashCode() + Boolean.hashCode(notExists);

21
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -874,12 +874,25 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl();
}
public String toDigest() {
return (printSqlInParens) ? "(" + toDigestImpl() + ")" : toDigestImpl();
}
/**
* Returns a SQL string representing this expr. Subclasses should override this method
* instead of toSql() to ensure that parenthesis are properly added around the toSql().
*/
protected abstract String toSqlImpl();
/**
* !!!!!! Important !!!!!!
* Subclasses should override this method if
* sql digest should be represented different from tosqlImpl().
*/
protected String toDigestImpl() {
return toSqlImpl();
}
public String toMySql() {
return toSql();
}
@ -952,6 +965,14 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
// }
// }
public List<String> childrenToDigest() {
List<String> childrenDigestList = Lists.newArrayList();
for (Expr child : children) {
childrenDigestList.add(child.toDigest());
}
return childrenDigestList;
}
public static com.google.common.base.Predicate<Expr> isAggregatePredicate() {
return IS_AGGREGATE_PREDICATE;
}

11
fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
View File

@ -193,6 +193,17 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
return builder.toString();
}
public String toDigest() {
StringBuilder builder = new StringBuilder();
if (!tableRefs_.isEmpty()) {
builder.append(" FROM");
for (int i = 0; i < tableRefs_.size(); ++i) {
builder.append(" " + tableRefs_.get(i).toDigest());
}
}
return builder.toString();
}
public boolean isEmpty() { return tableRefs_.isEmpty(); }
@Override

55
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -292,6 +292,61 @@ public class FunctionCallExpr extends Expr {
return sb.toString();
}
private String paramsToDigest() {
StringBuilder sb = new StringBuilder();
sb.append("(");
if (fnParams.isStar()) {
sb.append("*");
}
if (fnParams.isDistinct()) {
sb.append("DISTINCT ");
}
int len = children.size();
List<String> result = Lists.newArrayList();
if (fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
len = len - 1;
}
if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) {
len = len - 1;
}
for (int i = 0; i < len; ++i) {
if (i == 1 && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) {
result.add("\'***\'");
} else {
result.add(children.get(i).toDigest());
}
}
sb.append(Joiner.on(", ").join(result)).append(")");
return sb.toString();
}
@Override
public String toDigestImpl() {
Expr expr;
if (originStmtFnExpr != null) {
expr = originStmtFnExpr;
} else {
expr = this;
}
StringBuilder sb = new StringBuilder();
sb.append(((FunctionCallExpr) expr).fnName);
sb.append(paramsToDigest());
if (fnName.getFunction().equalsIgnoreCase("json_quote") ||
fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
return forJSON(sb.toString());
}
return sb.toString();
}
@Override
public String debugString() {
return MoreObjects.toStringHelper(this)/*.add("op", aggOp)*/.add("name", fnName).add("isStar",

13
fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
View File

@ -269,6 +269,19 @@ public class InPredicate extends Predicate {
return strBuilder.toString();
}
@Override
public String toDigestImpl() {
StringBuilder strBuilder = new StringBuilder();
String notStr = (isNotIn) ? "NOT " : "";
strBuilder.append(getChild(0).toDigest() + " " + notStr + "IN (");
for (int i = 1; i < children.size(); ++i) {
strBuilder.append(getChild(i).toDigest());
strBuilder.append((i + 1 != children.size()) ? ", " : "");
}
strBuilder.append(")");
return strBuilder.toString();
}
@Override
public String toString() {
return toSql();

20
fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View File

@ -463,4 +463,24 @@ public class InlineViewRef extends TableRef {
return sb.toString();
}
@Override
public String tableRefToDigest() {
String aliasSql = null;
String alias = getExplicitAlias();
if (alias != null) {
aliasSql = ToSqlUtils.getIdentSql(alias);
}
if (view != null) {
return name.toSql() + (aliasSql == null ? "" : " " + aliasSql);
}
StringBuilder sb = new StringBuilder()
.append("(")
.append(queryStmt.toDigest())
.append(") ")
.append(aliasSql);
return sb.toString();
}
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
View File

@ -97,6 +97,11 @@ public class IsNullPredicate extends Predicate {
return getChild(0).toSql() + (isNotNull ? " IS NOT NULL" : " IS NULL");
}
@Override
public String toDigestImpl() {
return getChild(0).toDigest() + (isNotNull ? " IS NOT NULL" : " IS NULL");
}
public boolean isSlotRefChildren() {
return (children.get(0) instanceof SlotRef);
}

6
fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
View File

@ -108,6 +108,12 @@ public class LikePredicate extends Predicate {
return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
}
@Override
public String toDigestImpl() {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.FUNCTION_CALL;

12
fe/fe-core/src/main/java/org/apache/doris/analysis/LimitElement.java
View File

@ -96,6 +96,18 @@ public class LimitElement {
return sb.toString();
}
public String toDigest() {
if (limit == -1) {
return "";
}
StringBuilder sb = new StringBuilder(" LIMIT ");
if (offset != 0) {
sb.append(offset + "?, ");
}
sb.append("" + " ? ");
return sb.toString();
}
public void analyze(Analyzer analyzer) {
if (limit == 0) analyzer.setHasEmptyResultSet();
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View File

@ -198,6 +198,11 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
return buffer;
}
@Override
public String toDigestImpl() {
return " ? ";
}
// Swaps the sign of numeric literals.
// Throws for non-numeric literals.
public void swapSign() throws NotImplementedException {

16
fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java
View File

@ -133,6 +133,22 @@ public class OrderByElement {
return strBuilder.toString();
}
public String toDigest() {
StringBuilder strBuilder = new StringBuilder();
strBuilder.append(expr.toDigest());
strBuilder.append(isAsc ? " ASC" : " DESC");
if (nullsFirstParam != null) {
if (isAsc && nullsFirstParam) {
// If ascending, nulls are last by default, so only add if nulls first.
strBuilder.append(" NULLS FIRST");
} else if (!isAsc && !nullsFirstParam) {
// If descending, nulls are first by default, so only add if nulls last.
strBuilder.append(" NULLS LAST");
}
}
return strBuilder.toString();
}
@Override
public String toString() {
return toSql();

9
fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
View File

@ -529,6 +529,15 @@ public class OutFileClause {
return sb.toString();
}
public String toDigest() {
StringBuilder sb = new StringBuilder();
sb.append(" INTO OUTFILE '").append(" ? ").append(" FORMAT AS ").append(" ? ");
if (properties != null && !properties.isEmpty()) {
sb.append(" PROPERTIES(").append(" ? ").append(")");
}
return sb.toString();
}
public TResultFileSinkOptions toSinkOptions() {
TResultFileSinkOptions sinkOptions = new TResultFileSinkOptions(filePath, fileFormatType);
if (isCsvFormat()) {

4
fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
View File

@ -706,6 +706,10 @@ public abstract class QueryStmt extends StatementBase {
return outFileClause != null ? outFileClause.clone() : null;
}
public String toDigest() {
return "";
}
/**
* C'tor for cloning.
*/

14
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java
View File

@ -97,6 +97,20 @@ public class SelectListItem {
}
}
public String toDigest() {
if (!isStar) {
Preconditions.checkNotNull(expr);
String aliasSql = null;
if (alias != null) {
aliasSql = "`" + alias + "`";
}
return expr.toDigest() + ((aliasSql == null) ? "" : " " + aliasSql);
} else if (tblName != null) {
return tblName.toString() + ".*";
} else {
return "*";
}
}
/**
* Return a column label for the select list item.
*/

77
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -1710,6 +1710,83 @@ public class SelectStmt extends QueryStmt {
return strBuilder.toString();
}
@Override
public String toDigest() {
StringBuilder strBuilder = new StringBuilder();
if (withClause_ != null) {
strBuilder.append(withClause_.toDigest());
strBuilder.append(" ");
}
// Select list
strBuilder.append("SELECT ");
if (selectList.isDistinct()) {
strBuilder.append("DISTINCT ");
}
if (originalExpr == null) {
originalExpr = Expr.cloneList(resultExprs);
}
if (resultExprs.isEmpty()) {
for (int i = 0; i < selectList.getItems().size(); ++i) {
if (i != 0) {
strBuilder.append(", ");
}
strBuilder.append(selectList.getItems().get(i).toDigest());
}
} else {
for (int i = 0; i < originalExpr.size(); ++i) {
if (i != 0) {
strBuilder.append(", ");
}
strBuilder.append(originalExpr.get(i).toDigest());
strBuilder.append(" AS ").append(SqlUtils.getIdentSql(colLabels.get(i)));
}
}
// From clause
if (!fromClause_.isEmpty()) {
strBuilder.append(fromClause_.toDigest());
}
// Where clause
if (whereClause != null) {
strBuilder.append(" WHERE ");
strBuilder.append(whereClause.toDigest());
}
// Group By clause
if (groupByClause != null) {
strBuilder.append(" GROUP BY ");
strBuilder.append(groupByClause.toSql());
}
// Having clause
if (havingClause != null) {
strBuilder.append(" HAVING ");
strBuilder.append(havingClause.toDigest());
}
// Order By clause
if (orderByElements != null) {
strBuilder.append(" ORDER BY ");
for (int i = 0; i < orderByElements.size(); ++i) {
strBuilder.append(orderByElements.get(i).getExpr().toDigest());
if (sortInfo != null) {
strBuilder.append((sortInfo.getIsAscOrder().get(i)) ? " ASC" : " DESC");
}
strBuilder.append((i + 1 != orderByElements.size()) ? ", " : "");
}
}
// Limit clause.
if (hasLimitClause()) {
strBuilder.append(limitElement.toDigest());
}
if (hasOutFileClause()) {
strBuilder.append(outFileClause.toDigest());
}
return strBuilder.toString();
}
/**
* If the select statement has a sort/top that is evaluated, then the sort tuple
* is materialized. Else, if there is aggregation then the aggregate tuple id is

51
fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
View File

@ -659,6 +659,57 @@ public class SetOperationStmt extends QueryStmt {
return strBuilder.toString();
}
@Override
public String toDigest() {
StringBuilder strBuilder = new StringBuilder();
if (withClause_ != null) {
strBuilder.append(withClause_.toDigest());
strBuilder.append(" ");
}
strBuilder.append(operands.get(0).getQueryStmt().toDigest());
for (int i = 1; i < operands.size() - 1; ++i) {
strBuilder.append(
" " + operands.get(i).getOperation().toString() + " "
+ ((operands.get(i).getQualifier() == Qualifier.ALL) ? "ALL " : ""));
if (operands.get(i).getQueryStmt() instanceof SetOperationStmt) {
strBuilder.append("(");
}
strBuilder.append(operands.get(i).getQueryStmt().toDigest());
if (operands.get(i).getQueryStmt() instanceof SetOperationStmt) {
strBuilder.append(")");
}
}
// Determine whether we need parenthesis around the last Set operand.
SetOperand lastOperand = operands.get(operands.size() - 1);
QueryStmt lastQueryStmt = lastOperand.getQueryStmt();
strBuilder.append(" " + lastOperand.getOperation().toString() + " "
+ ((lastOperand.getQualifier() == Qualifier.ALL) ? "ALL " : ""));
if (lastQueryStmt instanceof SetOperationStmt || ((hasOrderByClause() || hasLimitClause()) &&
!lastQueryStmt.hasLimitClause() &&
!lastQueryStmt.hasOrderByClause())) {
strBuilder.append("(");
strBuilder.append(lastQueryStmt.toDigest());
strBuilder.append(")");
} else {
strBuilder.append(lastQueryStmt.toDigest());
}
// Order By clause
if (hasOrderByClause()) {
strBuilder.append(" ORDER BY ");
for (int i = 0; i < orderByElements.size(); ++i) {
strBuilder.append(orderByElements.get(i).getExpr().toDigest());
strBuilder.append(orderByElements.get(i).getIsAsc() ? " ASC" : " DESC");
strBuilder.append((i + 1 != orderByElements.size()) ? ", " : "");
}
}
// Limit clause.
if (hasLimitClause()) {
strBuilder.append(limitElement.toDigest());
}
return strBuilder.toString();
}
@Override
public ArrayList<String> getColLabels() {
Preconditions.checkState(operands.size() > 0);

5
fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
View File

@ -56,6 +56,11 @@ public class Subquery extends Expr {
@Override
public String toSqlImpl() { return "(" + stmt.toSql() + ")"; }
@Override
public String toDigestImpl() {
return "(" + stmt.toDigest() + ")";
}
/**
* C'tor that initializes a Subquery from a QueryStmt.
*/

24
fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
View File

@ -649,6 +649,10 @@ public class TableRef implements ParseNode, Writable {
return tblName;
}
public String tableRefToDigest() {
return tableRefToSql();
}
@Override
public String toSql() {
if (joinOp == null) {
@ -670,6 +674,26 @@ public class TableRef implements ParseNode, Writable {
return output.toString();
}
public String toDigest() {
if (joinOp == null) {
// prepend "," if we're part of a sequence of table refs w/o an
// explicit JOIN clause
return (leftTblRef != null ? ", " : "") + tableRefToDigest();
}
StringBuilder output = new StringBuilder(" " + joinOpToSql() + " ");
if (joinHints != null && !joinHints.isEmpty()) {
output.append("[").append(Joiner.on(", ").join(joinHints)).append("] ");
}
output.append(tableRefToDigest()).append(" ");
if (usingColNames != null) {
output.append("USING (").append(Joiner.on(", ").join(usingColNames)).append(")");
} else if (onClause != null) {
output.append("ON ").append(onClause.toDigest());
}
return output.toString();
}
public String getAlias() {
if (!hasExplicitAlias()) {
return name.toString();

38
fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
View File

@ -325,6 +325,44 @@ public class TimestampArithmeticExpr extends Expr {
return strBuilder.toString();
}
@Override
public String toDigestImpl() {
StringBuilder strBuilder = new StringBuilder();
if (funcName != null) {
if (funcName.equalsIgnoreCase("TIMESTAMPDIFF") || funcName.equalsIgnoreCase("TIMESTAMPADD")) {
strBuilder.append(funcName).append("(");
strBuilder.append(timeUnitIdent).append(", ");
strBuilder.append(getChild(1).toDigest()).append(", ");
strBuilder.append(getChild(0).toDigest()).append(")");
return strBuilder.toString();
}
// Function-call like version.
strBuilder.append(funcName).append("(");
strBuilder.append(getChild(0).toDigest()).append(", ");
strBuilder.append("INTERVAL ");
strBuilder.append(getChild(1).toDigest());
strBuilder.append(" ").append(timeUnitIdent);
strBuilder.append(")");
return strBuilder.toString();
}
if (intervalFirst) {
// Non-function-call like version with interval as first operand.
strBuilder.append("INTERVAL ");
strBuilder.append(getChild(1).toDigest() + " ");
strBuilder.append(timeUnitIdent);
strBuilder.append(" ").append(op.toString()).append(" ");
strBuilder.append(getChild(0).toDigest());
} else {
// Non-function-call like version with interval as second operand.
strBuilder.append(getChild(0).toDigest());
strBuilder.append(" " + op.toString() + " ");
strBuilder.append("INTERVAL ");
strBuilder.append(getChild(1).toDigest() + " ");
strBuilder.append(timeUnitIdent);
}
return strBuilder.toString();
}
// Time units supported in timestamp arithmetic.
public enum TimeUnit {
YEAR("YEAR"), // YEARS

15
fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java
View File

@ -143,5 +143,20 @@ public class WithClause implements ParseNode {
return "WITH " + Joiner.on(",").join(viewStrings);
}
public String toDigest() {
List<String> viewStrings = Lists.newArrayList();
for (View view : views_) {
// Enclose the view alias and explicit labels in quotes if Hive cannot parse it
// without quotes. This is needed for view compatibility between Impala and Hive.
String aliasSql = ToSqlUtils.getIdentSql(view.getName());
if (view.hasColLabels()) {
aliasSql += "(" + Joiner.on(", ").join(
ToSqlUtils.getIdentSqlList(view.getOriginalColLabels())) + ")";
}
viewStrings.add(aliasSql + " AS (" + view.getQueryStmt().toDigest() + ")");
}
return "WITH " + Joiner.on(",").join(viewStrings);
}
public List<View> getViews() { return views_; }
}

7
fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
View File

@ -84,6 +84,8 @@ public class AuditEvent {
public String sqlHash = "";
@AuditField(value = "peakMemoryBytes")
public long peakMemoryBytes = -1;
@AuditField(value = "SqlDigest")
public String sqlDigest = "";
public static class AuditEventBuilder {
@ -186,6 +188,11 @@ public class AuditEvent {
return this;
}
public AuditEventBuilder setSqlDigest(String sqlDigest) {
auditEvent.sqlDigest = sqlDigest;
return this;
}
public AuditEvent build() {
return this.auditEvent;
}

7
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.qe;
import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@ -29,6 +30,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Table;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@ -133,6 +135,10 @@ public class ConnectProcessor {
} else {
// ok query
MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
if (elapseMs > Config.qe_slow_log_ms) {
String sqlDigest = DigestUtils.md5Hex(((QueryStmt) parsedStmt).toDigest());
ctx.getAuditEventBuilder().setSqlDigest(sqlDigest);
}
}
ctx.getAuditEventBuilder().setIsQuery(true);
ctx.getQueryDetail().setEventTime(endTime);
@ -183,7 +189,6 @@ public class ConnectProcessor {
}
String sqlHash = DigestUtils.md5Hex(originStmt);
ctx.setSqlHash(sqlHash);
ctx.getAuditEventBuilder().reset();
ctx.getAuditEventBuilder()
.setTimestamp(System.currentTimeMillis())

142
fe/fe-core/src/test/java/org/apache/doris/planner/SqlDigestTest.java Normal file
View File

@ -0,0 +1,142 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.Assert;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.util.UUID;
public class SqlDigestTest {
private static String runningDir = "fe/mocked/SqlDigestTest/" + UUID.randomUUID().toString() + "/";
private static ConnectContext connectContext;
@BeforeClass
public static void setUp() throws Exception {
UtFrameUtils.createDorisCluster(runningDir);
connectContext = UtFrameUtils.createDefaultCtx();
String createDbStmtStr = "create database db1;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
Catalog.getCurrentCatalog().createDb(createDbStmt);
// 3. create table tbl1
String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2 varchar(32), k3 varchar(32), k4 int) "
+ "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, connectContext);
Catalog.getCurrentCatalog().createTable(createTableStmt);
}
@AfterClass
public static void tearDown() {
File file = new File(runningDir);
file.delete();
}
@Test
public void testWhere() throws Exception {
String sql1 = "select k4 from tbl1 where k1 > 1";
String sql2 = "select k4 from tbl1 where k1 > 5";
String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
sql1 = "select k4 from tbl1 where k1 like 'xxx' ";
sql2 = "select k4 from tbl1 where k1 like 'kkskkkkkkkkk' ";
digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
sql1 = "select k4 from tb1 where k1 < 2 and k2 > -1 ";
sql2 = "select k4 from tb1 where k1 < 1000 and k2 > 100000 ";
digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
sql1 = "select k4 from tb1 where k1 < 2 or k2 > -1 ";
sql2 = "select k4 from tb1 where k1 < 3 or k2 > 100000 ";
digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
sql1 = "select k4 from tb1 where not k1 < 2 ";
sql2 = "select k4 from tb1 where not k1 < 3";
digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
sql1 = "select k4 from tb1 where not k1 < 2 ";
sql2 = "select k4 from tb1 where not k1 > 3";
digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertNotEquals(digest1, digest2);
}
@Test
public void testLimit() throws Exception {
String sql1 = "select k4 from tb1 where k1 > 1 limit 1";
String sql2 = "select k4 from tb1 where k1 > 5 limit 20";
String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
sql1 = "select k4 from tb1 where k1 > 1 order by k1 limit 1";
sql2 = "select k4 from tb1 where k1 > 5 order by k1 limit 20";
digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
}
@Test
public void testFunction() throws Exception {
String sql1 = "select substr(k4, 1, 2) from tb1 where k1 > 1 limit 1";
String sql2 = "select substr(k4, 1, 5) from tb1 where k1 > 1 limit 1";
String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
}
@Test
public void testArithmetic() throws Exception {
String sql1 = "select k1 + 1 from tb1";
String sql2 = "select k1 + 2 from tb1";
String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
}
@Test
public void testCaseWhen() throws Exception {
String sql1 = "select k1+20, case k2 when k3 then 1 else 0 end from tbl1 where k4 is null";
String sql2 = "select k1+20, case k2 when k3 then 1000 else 9999999 end from tbl1 where k4 is null";
String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
Assert.assertEquals(digest1, digest2);
}
}

15
fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.utframe;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@ -45,9 +46,12 @@ import org.apache.doris.utframe.MockedFrontend.EnvVarNotSetException;
import org.apache.doris.utframe.MockedFrontend.FeStartException;
import org.apache.doris.utframe.MockedFrontend.NotInitException;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import java.io.File;
@ -297,5 +301,16 @@ public class UtFrameUtils {
return null;
}
}
public static String getStmtDigest(ConnectContext connectContext, String originStmt) throws Exception {
SqlScanner input =
new SqlScanner(new StringReader(originStmt), connectContext.getSessionVariable().getSqlMode());
SqlParser parser = new SqlParser(input);
StatementBase statementBase = SqlParserUtils.getFirstStmt(parser);
Preconditions.checkState(statementBase instanceof QueryStmt);
QueryStmt queryStmt = (QueryStmt) statementBase;
String digest = queryStmt.toDigest();
return DigestUtils.md5Hex(digest);
}
}