database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add PreAgg Hint (#1617)

Browse Source 
eg:
SELECT xxx FROM tbl /*+ PREAGGOPEN */ 
This will open pre-aggregation forcibly for the specified table
This commit is contained in:
worker24h
2019-09-06 06:47:18 -05:00
committed by Mingyu Chen
parent 981e0feb99
commit 2f52ae7988
7 changed files with 143 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
fe/src/main/cup/sql_parser.cup
View File

@ -231,6 +231,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A
terminal COMMA, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
terminal BITAND, BITOR, BITXOR, BITNOT;
terminal EQUAL, NOT, LESSTHAN, GREATERTHAN, SET_VAR;
terminal COMMENTED_PLAN_HINT_START, COMMENTED_PLAN_HINT_END;
terminal String IDENT;
terminal String NUMERIC_OVERFLOW;
terminal Long INTEGER_LITERAL;
@ -338,6 +339,7 @@ nonterminal ArrayList<String> opt_plan_hints;
nonterminal ArrayList<String> opt_sort_hints;
nonterminal Expr sign_chain_expr;
nonterminal Qualifier union_op;
nonterminal ArrayList<String> opt_common_hints;
nonterminal ArrayList<PartitionName> opt_partition_name_list, partition_name_list;
nonterminal PartitionName partition_name;
@ -2985,9 +2987,24 @@ base_table_ref_list ::=
;
base_table_ref ::=
table_name:name opt_using_partition:parts opt_table_alias:alias
table_name:name opt_using_partition:parts opt_table_alias:alias opt_common_hints:common_hints
{:
RESULT = new TableRef(name, alias, parts);
RESULT = new TableRef(name, alias, parts, common_hints);
:}
;
opt_common_hints ::=
COMMENTED_PLAN_HINT_START ident_list:l COMMENTED_PLAN_HINT_END
{:
RESULT = l;
:}
| LBRACKET ident_list:l RBRACKET
{:
RESULT = l;
:}
|
{:
RESULT = null;
:}
;
@ -3067,6 +3084,10 @@ opt_plan_hints ::=
}
RESULT = hints;
:}
| COMMENTED_PLAN_HINT_START ident_list:l COMMENTED_PLAN_HINT_END
{:
RESULT = l;
:}
| LBRACKET ident_list:l RBRACKET
{:
RESULT = l;

1
fe/src/main/java/org/apache/doris/analysis/BaseTableRef.java
View File

@ -70,6 +70,7 @@ public class BaseTableRef extends TableRef {
isAnalyzed = true; // true that we have assigned desc
analyzeJoin(analyzer);
analyzeSortHints();
analyzeHints();
}
}

16
fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View File

@ -17,6 +17,9 @@
package org.apache.doris.analysis;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.InlineView;
import org.apache.doris.catalog.View;
@ -25,12 +28,8 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.rewrite.ExprRewriter;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
@ -265,7 +264,12 @@ public class InlineViewRef extends TableRef {
}
columnSet.add(colAlias);
columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType()));
if (selectItemExpr instanceof SlotRef && ((SlotRef)selectItemExpr).getDesc().getColumn() != null) {
SlotRef slotRef = (SlotRef) selectItemExpr;
columnList.add(new Column(slotRef.getDesc().getColumn()));
} else {
columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType()));
}
}
InlineView inlineView = (view != null) ? new InlineView(view, columnList) : new InlineView(getExplicitAlias(), columnList);

40
fe/src/main/java/org/apache/doris/analysis/TableRef.java
View File

@ -17,6 +17,10 @@
package org.apache.doris.analysis;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
@ -25,12 +29,6 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.rewrite.ExprRewriter;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -90,7 +88,8 @@ public class TableRef implements ParseNode, Writable {
protected List<String> usingColNames;
private ArrayList<String> joinHints;
private ArrayList<String> sortHints;
private ArrayList<String> commonHints; //The Hints is set by user
private boolean isForcePreAggOpened;
// ///////////////////////////////////////
// BEGIN: Members that need to be reset()
@ -137,6 +136,10 @@ public class TableRef implements ParseNode, Writable {
}
public TableRef(TableName name, String alias, List<String> partitions) {
this(name, alias, partitions, null);
}
public TableRef(TableName name, String alias, List<String> partitions, ArrayList<String> commonHints) {
this.name = name;
if (alias != null) {
aliases_ = new String[] { alias };
@ -145,9 +148,9 @@ public class TableRef implements ParseNode, Writable {
hasExplicitAlias_ = false;
}
this.partitions = partitions;
this.commonHints = commonHints;
isAnalyzed = false;
}
// Only used to clone
// this will reset all the 'analyzed' stuff
protected TableRef(TableRef other) {
@ -163,6 +166,7 @@ public class TableRef implements ParseNode, Writable {
onClause = (other.onClause != null) ? other.onClause.clone().reset() : null;
partitions =
(other.partitions != null) ? Lists.newArrayList(other.partitions) : null;
commonHints = other.commonHints;
usingColNames =
(other.usingColNames != null) ? Lists.newArrayList(other.usingColNames) : null;
@ -301,6 +305,10 @@ public class TableRef implements ParseNode, Writable {
return isPartitionJoin;
}
public boolean isForcePreAggOpened() {
return isForcePreAggOpened;
}
public void setSortHints(ArrayList<String> hints) {
this.sortHints = hints;
}
@ -342,6 +350,22 @@ public class TableRef implements ParseNode, Writable {
}
}
/**
* Parse PreAgg hints.
*/
protected void analyzeHints() throws AnalysisException {
if (commonHints == null || commonHints.isEmpty()) {
return;
}
// Currently only 'PREAGGOPEN' is supported
for (String hint : commonHints) {
if (hint.toUpperCase().equals("PREAGGOPEN")) {
isForcePreAggOpened = true;
break;
}
}
}
/**
* Analyze the join clause.
* The join clause can only be analyzed after the left table has been analyzed

26
fe/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -17,6 +17,13 @@
package org.apache.doris.planner;
import com.google.common.base.Objects;
import com.google.common.base.Objects.ToStringHelper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.BaseTableRef;
import org.apache.doris.analysis.TupleDescriptor;
@ -50,15 +57,6 @@ import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TScanRange;
import org.apache.doris.thrift.TScanRangeLocation;
import org.apache.doris.thrift.TScanRangeLocations;
import com.google.common.base.Objects;
import com.google.common.base.Objects.ToStringHelper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -80,6 +78,7 @@ public class OlapScanNode extends ScanNode {
private boolean isPreAggregation = false;
private String reasonOfPreAggregation = null;
private boolean canTurnOnPreAggr = true;
private boolean forceOpenPreAgg = false;
private ArrayList<String> tupleColumns = new ArrayList<String>();
private HashSet<String> predicateColumns = new HashSet<String>();
private OlapTable olapTable = null;
@ -109,7 +108,6 @@ public class OlapScanNode extends ScanNode {
this.reasonOfPreAggregation = reason;
}
public boolean isPreAggregation() {
return isPreAggregation;
}
@ -122,6 +120,14 @@ public class OlapScanNode extends ScanNode {
this.canTurnOnPreAggr = canChangePreAggr;
}
public boolean getForceOpenPreAgg() {
return forceOpenPreAgg;
}
public void setForceOpenPreAgg(boolean forceOpenPreAgg) {
this.forceOpenPreAgg = forceOpenPreAgg;
}
public OlapTable getOlapTable() {
return olapTable;
}

35
fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -17,6 +17,11 @@
package org.apache.doris.planner;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.doris.analysis.AggregateInfo;
import org.apache.doris.analysis.AnalyticInfo;
import org.apache.doris.analysis.Analyzer;
@ -53,13 +58,6 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.Reference;
import org.apache.doris.common.UserException;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -318,13 +316,18 @@ public class SingleNodePlanner {
private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyzer analyzer, PlanNode root) {
String turnOffReason = null;
do {
if (null == aggInfo) {
turnOffReason = "No AggregateInfo";
if (!(root instanceof OlapScanNode)) {
turnOffReason = "left-deep Node is not OlapScanNode";
break;
}
if (!(root instanceof OlapScanNode)) {
turnOffReason = "left-deep Node is not OlapScanNode";
if (((OlapScanNode)root).getForceOpenPreAgg()) {
((OlapScanNode)root).setIsPreAggregation(true, "");
return;
}
if (null == aggInfo) {
turnOffReason = "No AggregateInfo";
break;
}
@ -395,8 +398,8 @@ public class SingleNodePlanner {
for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) {
if (!slot.getColumn().isKey()) {
if (conjunctSlotIds.contains(slot.getId())) {
turnOffReason = "conjunct on " + slot.getColumn().getName() +
" which is StorageEngine value column";
turnOffReason = "conjunct on `" + slot.getColumn().getName() +
"` which is StorageEngine value column";
valueColumnValidate = false;
break;
}
@ -643,7 +646,7 @@ public class SingleNodePlanner {
rowTuples.addAll(tblRef.getMaterializedTupleIds());
}
if (analyzer.hasEmptySpjResultSet()) {
if (analyzer.hasEmptySpjResultSet()) {
final PlanNode emptySetNode = new EmptySetNode(ctx_.getNextNodeId(), rowTuples);
emptySetNode.init(analyzer);
emptySetNode.setOutputSmap(selectStmt.getBaseTblSmap());
@ -1156,7 +1159,9 @@ public class SingleNodePlanner {
switch (tblRef.getTable().getType()) {
case OLAP:
scanNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened());
scanNode = olapNode;
break;
case MYSQL:
scanNode = new MysqlScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), (MysqlTable) tblRef.getTable());

62
fe/src/main/jflex/sql_scanner.flex
View File

@ -441,20 +441,36 @@ QuotedIdentifier = \`(\`\`|[^\`])*\`
SingleQuoteStringLiteral = \'(\\.|[^\\\'])*\'
DoubleQuoteStringLiteral = \"(\\.|[^\\\"])*\"
// Both types of plan hints must appear within a single line.
TraditionalCommentedPlanHints = "/*" [ ]* "+" [^\r\n*]* "*/"
// Must end with a line terminator.
EndOfLineCommentedPlanHints = "--" [ ]* "+" {NonTerminator}* {LineTerminator}
FLit1 = [0-9]+ \. [0-9]*
FLit2 = \. [0-9]+
FLit3 = [0-9]+
Exponent = [eE] [+-]? [0-9]+
DoubleLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}?
EolHintBegin = "--" " "* "+"
CommentedHintBegin = "/*" " "* "+"
CommentedHintEnd = "*/"
// Both types of plan hints must appear within a single line.
HintContent = " "* "+" [^\r\n]*
Comment = {TraditionalComment} | {EndOfLineComment}
TraditionalComment = "/*" [^*] ~"*/" | "/*" "*"+ "/"
EndOfLineComment = "--" {NonTerminator}* {LineTerminator}?
// Match anything that has a comment end (*/) in it.
ContainsCommentEnd = [^]* "*/" [^]*
// Match anything that has a line terminator in it.
ContainsLineTerminator = [^]* {LineTerminator} [^]*
// A traditional comment is anything that starts and ends like a comment and has neither a
// plan hint inside nor a CommentEnd (*/).
TraditionalComment = "/*" !({HintContent}|{ContainsCommentEnd}) "*/"
// Similar for a end-of-line comment.
EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminator}?
// This additional state is needed because newlines signal the end of a end-of-line hint
// if one has been started earlier. Hence we need to discern between newlines within and
// outside of end-of-line hints.
%state EOLHINT
%%
@ -514,6 +530,24 @@ EndOfLineComment = "--" {NonTerminator}* {LineTerminator}?
escapeBackSlash(yytext().substring(1, yytext().length()-1)));
}
{CommentedHintBegin} {
return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_START, null);
}
{CommentedHintEnd} {
return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_END, null);
}
{EolHintBegin} {
yybegin(EOLHINT);
return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_START, null);
}
<EOLHINT> {LineTerminator} {
yybegin(YYINITIAL);
return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_END, null);
}
{IntegerLiteral} {
BigInteger val = null;
try {
@ -544,19 +578,5 @@ EndOfLineComment = "--" {NonTerminator}* {LineTerminator}?
return newToken(SqlParserSymbols.DECIMAL_LITERAL, decimal_val);
}
{TraditionalCommentedPlanHints} {
String text = yytext();
// Remove everything before the first '+' as well as the trailing "*/"
String hintStr = text.substring(text.indexOf('+') + 1, text.length() - 2);
return newToken(SqlParserSymbols.COMMENTED_PLAN_HINTS, hintStr.trim());
}
{EndOfLineCommentedPlanHints} {
String text = yytext();
// Remove everything before the first '+'
String hintStr = text.substring(text.indexOf('+') + 1);
return newToken(SqlParserSymbols.COMMENTED_PLAN_HINTS, hintStr.trim());
}
{Comment} { /* ignore */ }
{Whitespace} { /* ignore */ }