diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h index 66979a7b6a..381323c049 100644 --- a/be/src/vec/core/block.h +++ b/be/src/vec/core/block.h @@ -121,7 +121,11 @@ public: void initialize_index_by_name(); /// References are invalidated after calling functions above. - ColumnWithTypeAndName& get_by_position(size_t position) { return data[position]; } + ColumnWithTypeAndName& get_by_position(size_t position) { + DCHECK(data.size() > position) + << ", data.size()=" << data.size() << ", position=" << position; + return data[position]; + } const ColumnWithTypeAndName& get_by_position(size_t position) const { return data[position]; } // need exception safety diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h index f6a9504fc3..0d4509012e 100644 --- a/be/src/vec/exec/scan/pip_scanner_context.h +++ b/be/src/vec/exec/scan/pip_scanner_context.h @@ -17,6 +17,7 @@ #pragma once +#include "runtime/descriptors.h" #include "scanner_context.h" namespace doris { @@ -156,7 +157,7 @@ public: return _current_used_bytes < _max_bytes_in_queue / 2 * _max_queue_size; } - virtual void _dispose_coloate_blocks_not_in_queue() override { + void _dispose_coloate_blocks_not_in_queue() override { if (_need_colocate_distribute) { for (int i = 0; i < _max_queue_size; ++i) { std::scoped_lock s(*_colocate_block_mutexs[i], *_queue_mutexs[i]); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 8ca8386bae..ea4be00ba0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -200,18 +200,18 @@ public class OlapScanNode extends ScanNode { olapTable = (OlapTable) desc.getTable(); distributionColumnIds = Sets.newTreeSet(); - Set distColumnName = olapTable != null - ? olapTable.getDistributionColumnNames() : Sets.newTreeSet(); - int columnId = 0; + Set distColumnName = getDistributionColumnNames(); // use for Nereids to generate uniqueId set for inverted index to avoid scan unnecessary big size column + + int columnId = 0; for (SlotDescriptor slotDescriptor : desc.getSlots()) { if (slotDescriptor.getColumn() != null) { outputColumnUniqueIds.add(slotDescriptor.getColumn().getUniqueId()); if (distColumnName.contains(slotDescriptor.getColumn().getName().toLowerCase())) { distributionColumnIds.add(columnId); } + columnId++; } - columnId++; } } @@ -1506,13 +1506,28 @@ public class OlapScanNode extends ScanNode { cardinality = cardinality == -1 ? 0 : cardinality; } + Set getDistributionColumnNames() { + return olapTable != null + ? olapTable.getDistributionColumnNames() + : Sets.newTreeSet(); + } + @Override public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) { outputColumnUniqueIds.clear(); + distributionColumnIds.clear(); + + Set distColumnName = getDistributionColumnNames(); + + int columnId = 0; for (SlotDescriptor slot : context.getTupleDesc(this.getTupleId()).getSlots()) { if (requiredByProjectSlotIdSet.contains(slot.getId()) && slot.getColumn() != null) { outputColumnUniqueIds.add(slot.getColumn().getUniqueId()); + if (distColumnName.contains(slot.getColumn().getName().toLowerCase())) { + distributionColumnIds.add(columnId); + } + columnId++; } } } diff --git a/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out b/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out index d71b056b27..740959669c 100644 --- a/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out +++ b/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out @@ -110,3 +110,15 @@ 13 1 14 1 +-- !sql15 -- +1 1 +3 1 +5 1 +7 1 +9 1 +10 1 +11 1 +12 1 +13 1 +14 1 + diff --git a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy index 82528a44b0..992c2df831 100644 --- a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy +++ b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy @@ -78,6 +78,10 @@ suite("test_bitmap_filter_nereids") { qt_sql15 "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in (select k2 from ${tbl2} b2) order by k1;" + sql "set enable_pipeline_engine=true;" + qt_sql15 "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in (select k2 from ${tbl2} b2) order by k1;" + sql "set enable_pipeline_engine=false;" + explain{ sql "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) order by k1;" contains "RF000[bitmap]"