database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](inverted index) fix bug when match condition in hash join (#23105)

Browse Source 
* [Fix](inverted index) fix bug when match condition in hash join
This commit is contained in:
airborne12
2023-08-23 17:48:31 +08:00
committed by GitHub
parent 6aeacf252e
commit 8140fc737e
7 changed files with 78 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
View File

@ -480,6 +480,10 @@ public class SlotRef extends Expr {
this.table = table;
}
public TableIf getTableDirect() {
return this.table;
}
public TableIf getTable() {
if (desc == null && table != null) {
return table;

54
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.FunctionParams;
import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.MatchPredicate;
import org.apache.doris.analysis.OrderByElement;
@ -169,23 +170,39 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
NullableMode.DEPEND_ON_ARGUMENT);
}
@Override
public Expr visitMatch(Match match, PlanTranslatorContext context) {
String invertedIndexParser = null;
String invertedIndexParserMode = null;
SlotRef left = (SlotRef) match.left().accept(this, context);
SlotDescriptor slotDesc = left.getDesc();
private OlapTable getOlapTableFromSlotDesc(SlotDescriptor slotDesc) {
if (slotDesc != null && slotDesc.isScanSlot()) {
TupleDescriptor slotParent = slotDesc.getParent();
OlapTable olapTbl = (OlapTable) slotParent.getTable();
if (olapTbl == null) {
throw new AnalysisException("slotRef in matchExpression failed to get OlapTable");
}
List<Index> indexes = olapTbl.getIndexes();
return (OlapTable) slotParent.getTable();
}
return null;
}
private OlapTable getOlapTableDirectly(SlotRef left) {
if (left.getTableDirect() instanceof OlapTable) {
return (OlapTable) left.getTableDirect();
}
return null;
}
@Override
public Expr visitMatch(Match match, PlanTranslatorContext context) {
String invertedIndexParser = InvertedIndexUtil.INVERTED_INDEX_PARSER_UNKNOWN;
String invertedIndexParserMode = InvertedIndexUtil.INVERTED_INDEX_PARSER_FINE_GRANULARITY;
SlotRef left = (SlotRef) match.left().accept(this, context);
OlapTable olapTbl = Optional.ofNullable(getOlapTableFromSlotDesc(left.getDesc()))
.orElse(getOlapTableDirectly(left));
if (olapTbl == null) {
throw new AnalysisException("slotRef in matchExpression failed to get OlapTable");
}
List<Index> indexes = olapTbl.getIndexes();
if (indexes != null) {
for (Index index : indexes) {
if (index.getIndexType() == IndexDef.IndexType.INVERTED) {
List<String> columns = index.getColumns();
if (left.getColumnName().equals(columns.get(0))) {
if (columns != null && !columns.isEmpty() && left.getColumnName().equals(columns.get(0))) {
invertedIndexParser = index.getInvertedIndexParser();
invertedIndexParserMode = index.getInvertedIndexParserMode();
break;
@ -193,14 +210,15 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
}
}
}
MatchPredicate.Operator op = match.op();
return new MatchPredicate(op,
match.left().accept(this, context),
match.right().accept(this, context),
match.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT,
invertedIndexParser,
invertedIndexParserMode);
match.left().accept(this, context),
match.right().accept(this, context),
match.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT,
invertedIndexParser,
invertedIndexParserMode);
}
@Override

21
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -1130,7 +1130,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove it after refactor
sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, leftSlotDescriptor);
} else {
sd = context.createSlotDesc(intermediateDescriptor, sf);
sd = context.createSlotDesc(intermediateDescriptor, sf, leftSlotDescriptor.getParent().getTable());
//sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(leftSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), leftSlotDescriptor.getId());
@ -1150,7 +1151,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove it after refactor
sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, rightSlotDescriptor);
} else {
sd = context.createSlotDesc(intermediateDescriptor, sf);
sd = context.createSlotDesc(intermediateDescriptor, sf, rightSlotDescriptor.getParent().getTable());
//sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(rightSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), rightSlotDescriptor.getId());
@ -1169,7 +1171,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove it after refactor
sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, leftSlotDescriptor);
} else {
sd = context.createSlotDesc(intermediateDescriptor, sf);
sd = context.createSlotDesc(intermediateDescriptor, sf, leftSlotDescriptor.getParent().getTable());
//sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(leftSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), leftSlotDescriptor.getId());
@ -1187,7 +1190,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove it after refactor
sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, rightSlotDescriptor);
} else {
sd = context.createSlotDesc(intermediateDescriptor, sf);
sd = context.createSlotDesc(intermediateDescriptor, sf, rightSlotDescriptor.getParent().getTable());
//sd = context.createSlotDesc(intermediateDescriptor, sf);
if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) {
hashJoinNode.addSlotIdToHashOutputSlotIds(rightSlotDescriptor.getId());
hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), rightSlotDescriptor.getId());
@ -1355,7 +1359,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove it after refactor
sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, leftSlotDescriptor);
} else {
sd = context.createSlotDesc(intermediateDescriptor, sf);
sd = context.createSlotDesc(intermediateDescriptor, sf, leftSlotDescriptor.getParent().getTable());
//sd = context.createSlotDesc(intermediateDescriptor, sf);
}
leftIntermediateSlotDescriptor.add(sd);
}
@ -1369,7 +1374,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: temporary code for two phase read, should remove it after refactor
sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, rightSlotDescriptor);
} else {
sd = context.createSlotDesc(intermediateDescriptor, sf);
sd = context.createSlotDesc(intermediateDescriptor, sf, rightSlotDescriptor.getParent().getTable());
//sd = context.createSlotDesc(intermediateDescriptor, sf);
}
rightIntermediateSlotDescriptor.add(sd);
}
@ -1533,7 +1539,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (requiredByProjectSlotIdSet.size() != requiredSlotIdSet.size()
|| new HashSet<>(projectionExprs).size() != projectionExprs.size()
|| projectionExprs.stream().anyMatch(expr -> !(expr instanceof SlotRef))) {
projectionTuple = generateTupleDesc(slots, null, context);
projectionTuple = generateTupleDesc(slots,
((ScanNode) inputPlanNode).getTupleDesc().getTable(), context);
inputPlanNode.setProjectList(projectionExprs);
inputPlanNode.setOutputTupleDesc(projectionTuple);
} else {

1
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
View File

@ -210,6 +210,7 @@ public class PlanTranslatorContext {
} else {
slotRef = new SlotRef(slotDescriptor);
}
slotRef.setTable(table);
slotRef.setLabel(slotReference.getName());
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
slotDescriptor.setIsNullable(slotReference.nullable());

8
regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out Normal file
View File

@ -0,0 +1,8 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !q100 --
site_0 144200
site_1 519
site_2 539
site_3 504
site_4 563

2
regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql
View File

@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS web_site (
INDEX web_site_sk_idx(web_site_sk) USING INVERTED COMMENT "web_site_sk index",
INDEX web_site_id_idx(web_site_id) USING INVERTED COMMENT "web_site_id index",
INDEX web_rec_start_date_idx(web_rec_start_date) USING INVERTED COMMENT "web_rec_start_date index",
INDEX web_name_idx(web_name) USING INVERTED COMMENT "web_name index",
INDEX web_name_idx(web_name) USING INVERTED PROPERTIES("parser"="unicode") COMMENT "web_name index",
INDEX web_class_idx(web_class) USING INVERTED COMMENT "web_class index",
INDEX web_manager_idx(web_manager) USING INVERTED PROPERTIES("parser"="standard") COMMENT "web_manager index",
INDEX web_market_manager_idx(web_market_manager) USING INVERTED PROPERTIES("parser"="none") COMMENT "web_market_manager index",

14
regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql Normal file
View File

@ -0,0 +1,14 @@
SELECT
t1.web_name,
COUNT(t1.web_name) AS count
FROM
web_site t1
LEFT JOIN
web_sales t2 ON t1.web_site_sk = t2.ws_web_site_sk
WHERE
t1.web_name MATCH 'site_0'
OR t2.ws_item_sk > 17934
GROUP BY
t1.web_name
ORDER BY
t1.web_name;