[fix](nereids)runtime filter translator failed on set operator (#28102)

* runtime filter translator failed on set operator
This commit is contained in:
minghong
2023-12-08 12:58:42 +08:00
committed by GitHub
parent 341822ec05
commit 61d556c718
21 changed files with 42 additions and 35 deletions

View File

@ -52,6 +52,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* runtime filter context used at post process and translation.
@ -273,6 +274,7 @@ public class RuntimeFilterContext {
l.addAll(r);
return l;
});
filters = filters.stream().distinct().collect(Collectors.toList());
filters.sort((a, b) -> a.getId().compareTo(b.getId()));
return filters;
}

View File

@ -122,8 +122,12 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi
ctx.getRuntimeFilterBySrcAndType(src, type, builderNode);
Preconditions.checkState(scanSlot != null, "scan slot is null");
if (filter != null) {
this.addAppliedRuntimeFilter(filter);
filter.addTargetSlot(scanSlot);
filter.addTargetExpression(scanSlot);
ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId());
ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
ctx.setTargetsOnScanNode(aliasTransferMap.get(probeExpr).first, scanSlot);
} else {
filter = new RuntimeFilter(generator.getNextId(),
src, ImmutableList.of(scanSlot), type, exprOrder, builderNode, buildSideNdv);

View File

@ -275,6 +275,7 @@ public class RuntimeFilterTest extends SSBTestBase {
checkRuntimeFilterExprs(filters, ImmutableList.of(
Pair.of("s_suppkey", "lo_partkey"),
Pair.of("p_partkey", "lo_partkey")));
connectContext.getSessionVariable().enableRuntimeFilterPrune = false;
connectContext.getSessionVariable().expandRuntimeFilterByInnerJoin = true;
filters = getRuntimeFilters(sql).get();
Assertions.assertEquals(2, filters.size());

View File

@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 )
--------------------PhysicalOlapScan[web_sales] apply RFs: RF0
----------------PhysicalDistribute
------------------PhysicalProject
--------------------PhysicalOlapScan[catalog_sales]
--------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
--------------PhysicalDistribute
----------------PhysicalProject
------------------PhysicalOlapScan[date_dim]

View File

@ -24,7 +24,7 @@ PhysicalResultSink
------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[store_returns]
------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))
@ -46,7 +46,7 @@ PhysicalResultSink
------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[catalog_returns]
------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF1
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))
@ -72,7 +72,7 @@ PhysicalResultSink
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[web_returns]
----------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF4
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))

View File

@ -68,7 +68,7 @@ PhysicalResultSink
------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
--------------------------------------------------------------------PhysicalDistribute
----------------------------------------------------------------------PhysicalProject
------------------------------------------------------------------------PhysicalOlapScan[web_sales]
------------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
------------------------------------------------------------------PhysicalDistribute
--------------------------------------------------------------------PhysicalProject
----------------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country'))

View File

@ -26,7 +26,7 @@ PhysicalResultSink
----------------------------PhysicalProject
------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
--------------------------------PhysicalProject
----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF3
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2002))
@ -35,7 +35,7 @@ PhysicalResultSink
----------------------------PhysicalProject
------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
--------------------------------PhysicalProject
----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
--------------------------------PhysicalDistribute
----------------------------------PhysicalProject
------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2002))

View File

@ -23,7 +23,7 @@ PhysicalResultSink
----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------PhysicalOlapScan[store_returns]
----------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF1
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[store]
@ -44,7 +44,7 @@ PhysicalResultSink
----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------PhysicalOlapScan[catalog_returns]
----------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF2 RF3
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[catalog_page]
@ -67,9 +67,9 @@ PhysicalResultSink
--------------------------------------PhysicalProject
----------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_item_sk = web_sales.ws_item_sk) and (web_returns.wr_order_number = web_sales.ws_order_number)) otherCondition=() build RFs:RF4 wr_item_sk->[ws_item_sk];RF5 wr_order_number->[ws_order_number]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5
--------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5 RF6
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[web_returns]
--------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF7
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[web_site]

View File

@ -34,7 +34,7 @@ PhysicalResultSink
--------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
----------------------------------------------------PhysicalDistribute
------------------------------------------------------PhysicalProject
--------------------------------------------------------PhysicalOlapScan[web_sales]
--------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
--------------------------------------------------PhysicalDistribute
----------------------------------------------------PhysicalProject
------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998))

View File

@ -25,7 +25,7 @@ PhysicalResultSink
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF3 RF4
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))
@ -34,7 +34,7 @@ PhysicalResultSink
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF4
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))

View File

@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 )
--------------------PhysicalOlapScan[web_sales] apply RFs: RF0
----------------PhysicalDistribute
------------------PhysicalProject
--------------------PhysicalOlapScan[catalog_sales]
--------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
--------------PhysicalDistribute
----------------PhysicalProject
------------------PhysicalOlapScan[date_dim]

View File

@ -23,7 +23,7 @@ PhysicalResultSink
----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------PhysicalOlapScan[store_returns]
----------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF1
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[store]
@ -44,7 +44,7 @@ PhysicalResultSink
----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------PhysicalOlapScan[catalog_returns]
----------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF2 RF3
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[catalog_page]
@ -67,9 +67,9 @@ PhysicalResultSink
--------------------------------------PhysicalProject
----------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_item_sk = web_sales.ws_item_sk) and (web_returns.wr_order_number = web_sales.ws_order_number)) otherCondition=() build RFs:RF4 wr_item_sk->[ws_item_sk];RF5 wr_order_number->[ws_order_number]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5
--------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5 RF6
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[web_returns]
--------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF7
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[web_site]

View File

@ -34,7 +34,7 @@ PhysicalResultSink
--------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
----------------------------------------------------PhysicalDistribute
------------------------------------------------------PhysicalProject
--------------------------------------------------------PhysicalOlapScan[web_sales]
--------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
--------------------------------------------------PhysicalDistribute
----------------------------------------------------PhysicalProject
------------------------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 1998))

View File

@ -25,7 +25,7 @@ PhysicalResultSink
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF3 RF4
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))
@ -34,7 +34,7 @@ PhysicalResultSink
--------------------------PhysicalProject
----------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
------------------------------PhysicalProject
--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF4
------------------------------PhysicalDistribute
--------------------------------PhysicalProject
----------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))

View File

@ -24,7 +24,7 @@ PhysicalResultSink
------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[store_returns]
------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))
@ -46,7 +46,7 @@ PhysicalResultSink
------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[catalog_returns]
------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF1
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))
@ -72,7 +72,7 @@ PhysicalResultSink
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[web_returns]
----------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF4
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))

View File

@ -48,7 +48,7 @@ PhysicalResultSink
------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
--------------------------------------------------------------------------PhysicalDistribute
----------------------------------------------------------------------------PhysicalProject
------------------------------------------------------------------------------PhysicalOlapScan[web_sales]
------------------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
------------------------------------------------------------------------PhysicalDistribute
--------------------------------------------------------------------------PhysicalProject
----------------------------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity'))

View File

@ -27,7 +27,7 @@ PhysicalResultSink
------------------------------PhysicalProject
--------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
----------------------------------PhysicalProject
------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF3
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))
@ -36,7 +36,7 @@ PhysicalResultSink
------------------------------PhysicalProject
--------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
----------------------------------PhysicalProject
------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))

View File

@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 )
--------------------PhysicalOlapScan[web_sales] apply RFs: RF0
----------------PhysicalDistribute
------------------PhysicalProject
--------------------PhysicalOlapScan[catalog_sales]
--------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
--------------PhysicalDistribute
----------------PhysicalProject
------------------PhysicalOlapScan[date_dim]

View File

@ -24,7 +24,7 @@ PhysicalResultSink
------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[store_returns]
------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))
@ -46,7 +46,7 @@ PhysicalResultSink
------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
--------------------------------------PhysicalDistribute
----------------------------------------PhysicalProject
------------------------------------------PhysicalOlapScan[catalog_returns]
------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF1
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))
@ -72,7 +72,7 @@ PhysicalResultSink
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3
--------------------------------------------PhysicalProject
----------------------------------------------PhysicalOlapScan[web_returns]
----------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF4
------------------------------------PhysicalDistribute
--------------------------------------PhysicalProject
----------------------------------------filter((date_dim.d_date <= '2000-09-02') and (date_dim.d_date >= '2000-08-19'))

View File

@ -48,7 +48,7 @@ PhysicalResultSink
------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
--------------------------------------------------------------------------PhysicalDistribute
----------------------------------------------------------------------------PhysicalProject
------------------------------------------------------------------------------PhysicalOlapScan[web_sales]
------------------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
------------------------------------------------------------------------PhysicalDistribute
--------------------------------------------------------------------------PhysicalProject
----------------------------------------------------------------------------filter((item.i_category = 'Women') and (item.i_class = 'maternity'))

View File

@ -27,7 +27,7 @@ PhysicalResultSink
------------------------------PhysicalProject
--------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
----------------------------------PhysicalProject
------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1
------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF3
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))
@ -36,7 +36,7 @@ PhysicalResultSink
------------------------------PhysicalProject
--------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
----------------------------------PhysicalProject
------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
----------------------------------PhysicalDistribute
------------------------------------PhysicalProject
--------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998))