From 8140fc737eef7bfb76045ffd20904d8701d3b8ef Mon Sep 17 00:00:00 2001 From: airborne12 Date: Wed, 23 Aug 2023 17:48:31 +0800 Subject: [PATCH] [Fix](inverted index) fix bug when match condition in hash join (#23105) * [Fix](inverted index) fix bug when match condition in hash join --- .../org/apache/doris/analysis/SlotRef.java | 4 ++ .../glue/translator/ExpressionTranslator.java | 54 ++++++++++++------- .../translator/PhysicalPlanTranslator.java | 21 +++++--- .../translator/PlanTranslatorContext.java | 1 + .../tpcds_sf1_index/sql/q100.out | 8 +++ .../tpcds_sf1_index/ddl/web_site.sql | 2 +- .../tpcds_sf1_index/sql/q100.sql | 14 +++++ 7 files changed, 78 insertions(+), 26 deletions(-) create mode 100644 regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out create mode 100644 regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index de0cb93b60..0bb2a37cdb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -480,6 +480,10 @@ public class SlotRef extends Expr { this.table = table; } + public TableIf getTableDirect() { + return this.table; + } + public TableIf getTable() { if (desc == null && table != null) { return table; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index a67513d0f0..7b5a13e73e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -31,6 +31,7 @@ import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.FunctionName; import org.apache.doris.analysis.FunctionParams; import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.OrderByElement; @@ -169,23 +170,39 @@ public class ExpressionTranslator extends DefaultExpressionVisitor indexes = olapTbl.getIndexes(); + return (OlapTable) slotParent.getTable(); + } + return null; + } + + private OlapTable getOlapTableDirectly(SlotRef left) { + if (left.getTableDirect() instanceof OlapTable) { + return (OlapTable) left.getTableDirect(); + } + return null; + } + + @Override + public Expr visitMatch(Match match, PlanTranslatorContext context) { + String invertedIndexParser = InvertedIndexUtil.INVERTED_INDEX_PARSER_UNKNOWN; + String invertedIndexParserMode = InvertedIndexUtil.INVERTED_INDEX_PARSER_FINE_GRANULARITY; + SlotRef left = (SlotRef) match.left().accept(this, context); + OlapTable olapTbl = Optional.ofNullable(getOlapTableFromSlotDesc(left.getDesc())) + .orElse(getOlapTableDirectly(left)); + + if (olapTbl == null) { + throw new AnalysisException("slotRef in matchExpression failed to get OlapTable"); + } + + List indexes = olapTbl.getIndexes(); + if (indexes != null) { for (Index index : indexes) { if (index.getIndexType() == IndexDef.IndexType.INVERTED) { List columns = index.getColumns(); - if (left.getColumnName().equals(columns.get(0))) { + if (columns != null && !columns.isEmpty() && left.getColumnName().equals(columns.get(0))) { invertedIndexParser = index.getInvertedIndexParser(); invertedIndexParserMode = index.getInvertedIndexParserMode(); break; @@ -193,14 +210,15 @@ public class ExpressionTranslator extends DefaultExpressionVisitor(projectionExprs).size() != projectionExprs.size() || projectionExprs.stream().anyMatch(expr -> !(expr instanceof SlotRef))) { - projectionTuple = generateTupleDesc(slots, null, context); + projectionTuple = generateTupleDesc(slots, + ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); inputPlanNode.setProjectList(projectionExprs); inputPlanNode.setOutputTupleDesc(projectionTuple); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index a9c9dae012..1c176da409 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -210,6 +210,7 @@ public class PlanTranslatorContext { } else { slotRef = new SlotRef(slotDescriptor); } + slotRef.setTable(table); slotRef.setLabel(slotReference.getName()); this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef); slotDescriptor.setIsNullable(slotReference.nullable()); diff --git a/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out new file mode 100644 index 0000000000..fbb2285d06 --- /dev/null +++ b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q100.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q100 -- +site_0 144200 +site_1 519 +site_2 539 +site_3 504 +site_4 563 + diff --git a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql index b6f930a728..640926c35a 100644 --- a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql +++ b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/ddl/web_site.sql @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS web_site ( INDEX web_site_sk_idx(web_site_sk) USING INVERTED COMMENT "web_site_sk index", INDEX web_site_id_idx(web_site_id) USING INVERTED COMMENT "web_site_id index", INDEX web_rec_start_date_idx(web_rec_start_date) USING INVERTED COMMENT "web_rec_start_date index", - INDEX web_name_idx(web_name) USING INVERTED COMMENT "web_name index", + INDEX web_name_idx(web_name) USING INVERTED PROPERTIES("parser"="unicode") COMMENT "web_name index", INDEX web_class_idx(web_class) USING INVERTED COMMENT "web_class index", INDEX web_manager_idx(web_manager) USING INVERTED PROPERTIES("parser"="standard") COMMENT "web_manager index", INDEX web_market_manager_idx(web_market_manager) USING INVERTED PROPERTIES("parser"="none") COMMENT "web_market_manager index", diff --git a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql new file mode 100644 index 0000000000..8653806f9e --- /dev/null +++ b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q100.sql @@ -0,0 +1,14 @@ +SELECT + t1.web_name, + COUNT(t1.web_name) AS count +FROM + web_site t1 +LEFT JOIN + web_sales t2 ON t1.web_site_sk = t2.ws_web_site_sk +WHERE + t1.web_name MATCH 'site_0' + OR t2.ws_item_sk > 17934 +GROUP BY + t1.web_name +ORDER BY + t1.web_name;