From abd9000368a431478ced1af7737c09c641b2a311 Mon Sep 17 00:00:00 2001
From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com>
Date: Thu, 4 Jan 2024 17:51:06 +0800
Subject: [PATCH] [Feat](Nereids) add distribute hint to leading hint (#28562)
add distribute hint to leading hint, we can use leading like:
/*+ leading(t1 broadcase{t2 t3}) */ after this commit
---
.../docs/query-acceleration/hint/joinHint.md | 64 +
.../org/apache/doris/nereids/DorisLexer.g4 | 2 +
.../org/apache/doris/nereids/DorisParser.g4 | 8 +-
.../apache/doris/nereids/NereidsPlanner.java | 15 +-
.../doris/nereids/hint/DistributeHint.java | 76 +
.../org/apache/doris/nereids/hint/Hint.java | 4 +
.../doris/nereids/hint/LeadingHint.java | 76 +-
.../jobs/joinorder/hypergraph/HyperGraph.java | 6 +-
.../hypergraph/receiver/PlanReceiver.java | 12 +-
.../nereids/parser/LogicalPlanBuilder.java | 36 +-
.../nereids/properties/DistributionSpec.java | 4 +
.../properties/RequestPropertyDeriver.java | 12 +-
.../rules/analysis/BindExpression.java | 4 +-
.../analysis/EliminateLogicalSelectHint.java | 4 +-
.../exploration/join/InnerJoinLAsscom.java | 2 +-
.../join/InnerJoinLAsscomProject.java | 2 +-
.../join/InnerJoinLeftAssociate.java | 2 +-
.../join/InnerJoinLeftAssociateProject.java | 2 +-
.../join/InnerJoinRightAssociate.java | 2 +-
.../join/InnerJoinRightAssociateProject.java | 2 +-
.../rules/exploration/join/JoinCommute.java | 2 +-
.../rules/exploration/join/JoinExchange.java | 15 +-
.../join/JoinExchangeBothProject.java | 16 +-
.../join/JoinExchangeLeftProject.java | 16 +-
.../join/JoinExchangeRightProject.java | 16 +-
.../join/LogicalJoinSemiJoinTranspose.java | 4 +-
.../LogicalJoinSemiJoinTransposeProject.java | 8 +-
.../join/OuterJoinAssocProject.java | 2 +-
.../exploration/join/OuterJoinLAsscom.java | 2 +-
.../join/OuterJoinLAsscomProject.java | 2 +-
.../PushDownProjectThroughInnerOuterJoin.java | 4 +-
.../join/PushDownProjectThroughSemiJoin.java | 4 +-
.../join/SemiJoinSemiJoinTranspose.java | 2 +-
.../SemiJoinSemiJoinTransposeProject.java | 2 +-
.../rules/expression/ExpressionRewrite.java | 2 +-
.../implementation/LogicalJoinToHashJoin.java | 2 +-
.../rules/rewrite/ExistsApplyToJoin.java | 13 +-
.../rewrite/ExtractFilterFromCrossJoin.java | 2 +-
.../rewrite/FindHashConditionForJoin.java | 2 +-
.../nereids/rules/rewrite/InApplyToJoin.java | 9 +-
.../nereids/rules/rewrite/OrExpansion.java | 6 +-
.../rules/rewrite/PullUpJoinFromUnionAll.java | 7 +-
.../rewrite/PushDownFilterThroughJoin.java | 2 +-
.../rewrite/PushDownJoinOtherCondition.java | 2 +-
.../rules/rewrite/PushFilterInsideJoin.java | 2 +-
.../nereids/rules/rewrite/ReorderJoin.java | 22 +-
.../rules/rewrite/ScalarApplyToJoin.java | 7 +-
.../rules/rewrite/SemiJoinCommute.java | 2 +-
.../rewrite/TransposeSemiJoinLogicalJoin.java | 2 +-
.../TransposeSemiJoinLogicalJoinProject.java | 2 +-
.../trees/copier/LogicalPlanDeepCopier.java | 2 +-
.../{JoinHint.java => DistributeType.java} | 22 +-
.../nereids/trees/plans/algebra/Join.java | 25 +-
.../trees/plans/logical/LogicalJoin.java | 37 +-
.../trees/plans/logical/UsingJoin.java | 12 +-
.../plans/physical/AbstractPhysicalJoin.java | 15 +-
.../plans/physical/PhysicalDistribute.java | 7 +
.../plans/physical/PhysicalHashJoin.java | 8 +-
.../physical/PhysicalNestedLoopJoin.java | 7 +-
...nHintTest.java => DistributeHintTest.java} | 6 +-
...nHintTest.java => DistributeHintTest.java} | 2 +-
.../nereids/parser/NereidsParserTest.java | 12 +-
.../ChildOutputPropertyDeriverTest.java | 27 +-
.../RequestPropertyDeriverTest.java | 7 +-
.../rewrite/FindHashConditionForJoinTest.java | 7 +-
.../nereids/trees/plans/PlanEqualsTest.java | 7 +-
.../nereids/trees/plans/PlanToStringTest.java | 1 +
.../doris/nereids/util/HyperGraphBuilder.java | 9 +-
.../nereids/util/LogicalPlanBuilder.java | 5 +-
.../data/empty_relation/eliminate_empty.out | 10 +-
.../cte/test_cte_filter_pushdown.out | 14 +-
.../eliminate_outer_join.out | 224 ++-
.../data/nereids_p0/hint/fix_leading.out | 10 +-
.../data/nereids_p0/hint/test_leading.out | 1494 ++++++++++++++++-
.../eager_aggregate/basic.out | 66 +-
.../eager_aggregate/basic_one_side.out | 60 +-
.../push_down_count_through_join.out | 740 ++++----
.../push_down_count_through_join_one_side.out | 744 ++++----
.../push_down_max_through_join.out | 419 +++--
.../push_down_min_through_join.out | 419 +++--
.../push_down_sum_through_join.out | 439 ++---
.../push_down_sum_through_join_one_side.out | 419 +++--
.../eliminate_join_condition.out | 43 +-
.../eliminate_not_null/eliminate_not_null.out | 59 +-
.../eliminate_outer_join.out | 70 +-
.../push_down_alias_through_join.out | 52 +-
.../push_down_expression_in_hash_join.out | 64 +-
.../push_down_filter_other_condition.out | 410 +++--
.../push_filter_inside_join.out | 139 +-
.../infer_set_operator_distinct.out | 200 +--
.../limit_push_down/limit_push_down.out | 1002 ++++++-----
.../limit_push_down/order_push_down.out | 830 +++++----
.../predicate_infer/infer_predicate.out | 625 ++++---
.../push_down_limit_distinct.out | 24 +-
...push_down_top_n_distinct_through_union.out | 182 +-
.../push_down_top_n_through_union.out | 245 +--
.../transposeJoin/transposeSemiJoinAgg.out | 132 +-
.../data/nereids_shape_check/load.out | 16 +-
.../nereids_ssb_shape_sf100_p0/shape/flat.out | 8 +-
.../nereids_ssb_shape_sf100_p0/shape/q1.1.out | 4 +-
.../nereids_ssb_shape_sf100_p0/shape/q1.2.out | 4 +-
.../nereids_ssb_shape_sf100_p0/shape/q1.3.out | 4 +-
.../nereids_ssb_shape_sf100_p0/shape/q2.1.out | 10 +-
.../nereids_ssb_shape_sf100_p0/shape/q2.2.out | 10 +-
.../nereids_ssb_shape_sf100_p0/shape/q2.3.out | 10 +-
.../nereids_ssb_shape_sf100_p0/shape/q3.1.out | 12 +-
.../nereids_ssb_shape_sf100_p0/shape/q3.2.out | 12 +-
.../nereids_ssb_shape_sf100_p0/shape/q3.3.out | 10 +-
.../nereids_ssb_shape_sf100_p0/shape/q3.4.out | 10 +-
.../nereids_ssb_shape_sf100_p0/shape/q4.1.out | 16 +-
.../nereids_ssb_shape_sf100_p0/shape/q4.2.out | 16 +-
.../nereids_ssb_shape_sf100_p0/shape/q4.3.out | 12 +-
.../push_filter_through_ptopn.out | 10 +-
.../push_filter_through_window.out | 10 +-
.../shape/query1.out | 20 +-
.../shape/query10.out | 20 +-
.../shape/query11.out | 22 +-
.../shape/query12.out | 12 +-
.../shape/query13.out | 16 +-
.../shape/query14.out | 80 +-
.../shape/query15.out | 14 +-
.../shape/query16.out | 14 +-
.../shape/query17.out | 20 +-
.../shape/query18.out | 24 +-
.../shape/query19.out | 16 +-
.../shape/query2.out | 22 +-
.../shape/query20.out | 10 +-
.../shape/query21.out | 10 +-
.../shape/query22.out | 8 +-
.../shape/query23.out | 38 +-
.../shape/query24.out | 30 +-
.../shape/query25.out | 20 +-
.../shape/query26.out | 14 +-
.../shape/query27.out | 14 +-
.../shape/query28.out | 22 +-
.../shape/query29.out | 18 +-
.../shape/query3.out | 8 +-
.../shape/query30.out | 22 +-
.../shape/query31.out | 30 +-
.../shape/query32.out | 8 +-
.../shape/query33.out | 46 +-
.../shape/query34.out | 12 +-
.../shape/query35.out | 26 +-
.../shape/query36.out | 14 +-
.../shape/query37.out | 10 +-
.../shape/query38.out | 26 +-
.../shape/query39.out | 14 +-
.../shape/query4.out | 30 +-
.../shape/query40.out | 10 +-
.../shape/query41.out | 8 +-
.../shape/query42.out | 8 +-
.../shape/query43.out | 8 +-
.../shape/query44.out | 30 +-
.../shape/query45.out | 20 +-
.../shape/query46.out | 20 +-
.../shape/query47.out | 18 +-
.../shape/query48.out | 12 +-
.../shape/query49.out | 28 +-
.../shape/query5.out | 34 +-
.../shape/query50.out | 10 +-
.../shape/query51.out | 20 +-
.../shape/query52.out | 8 +-
.../shape/query53.out | 12 +-
.../shape/query54.out | 38 +-
.../shape/query55.out | 8 +-
.../shape/query56.out | 38 +-
.../shape/query57.out | 18 +-
.../shape/query58.out | 44 +-
.../shape/query59.out | 22 +-
.../shape/query6.out | 26 +-
.../shape/query60.out | 46 +-
.../shape/query61.out | 34 +-
.../shape/query62.out | 12 +-
.../shape/query63.out | 12 +-
.../shape/query64.out | 54 +-
.../shape/query65.out | 20 +-
.../shape/query66.out | 24 +-
.../shape/query67.out | 12 +-
.../shape/query68.out | 16 +-
.../shape/query69.out | 28 +-
.../shape/query7.out | 14 +-
.../shape/query70.out | 20 +-
.../shape/query71.out | 20 +-
.../shape/query72.out | 34 +-
.../shape/query73.out | 12 +-
.../shape/query74.out | 22 +-
.../shape/query75.out | 28 +-
.../shape/query76.out | 26 +-
.../shape/query77.out | 38 +-
.../shape/query78.out | 14 +-
.../shape/query79.out | 14 +-
.../shape/query8.out | 18 +-
.../shape/query80.out | 52 +-
.../shape/query81.out | 24 +-
.../shape/query82.out | 10 +-
.../shape/query83.out | 38 +-
.../shape/query84.out | 12 +-
.../shape/query85.out | 18 +-
.../shape/query86.out | 10 +-
.../shape/query87.out | 26 +-
.../shape/query88.out | 78 +-
.../shape/query89.out | 12 +-
.../shape/query9.out | 62 +-
.../shape/query90.out | 18 +-
.../shape/query91.out | 18 +-
.../shape/query92.out | 8 +-
.../shape/query93.out | 6 +-
.../shape/query94.out | 14 +-
.../shape/query95.out | 20 +-
.../shape/query96.out | 8 +-
.../shape/query97.out | 10 +-
.../shape/query98.out | 10 +-
.../shape/query99.out | 12 +-
.../noStatsRfPrune/query1.out | 18 +-
.../noStatsRfPrune/query10.out | 24 +-
.../noStatsRfPrune/query11.out | 22 +-
.../noStatsRfPrune/query12.out | 10 +-
.../noStatsRfPrune/query13.out | 14 +-
.../noStatsRfPrune/query14.out | 80 +-
.../noStatsRfPrune/query15.out | 14 +-
.../noStatsRfPrune/query16.out | 14 +-
.../noStatsRfPrune/query17.out | 18 +-
.../noStatsRfPrune/query18.out | 20 +-
.../noStatsRfPrune/query19.out | 16 +-
.../noStatsRfPrune/query2.out | 20 +-
.../noStatsRfPrune/query20.out | 10 +-
.../noStatsRfPrune/query21.out | 10 +-
.../noStatsRfPrune/query22.out | 8 +-
.../noStatsRfPrune/query23.out | 38 +-
.../noStatsRfPrune/query24.out | 24 +-
.../noStatsRfPrune/query25.out | 18 +-
.../noStatsRfPrune/query26.out | 12 +-
.../noStatsRfPrune/query27.out | 12 +-
.../noStatsRfPrune/query28.out | 22 +-
.../noStatsRfPrune/query29.out | 18 +-
.../noStatsRfPrune/query3.out | 8 +-
.../noStatsRfPrune/query30.out | 22 +-
.../noStatsRfPrune/query31.out | 28 +-
.../noStatsRfPrune/query32.out | 8 +-
.../noStatsRfPrune/query33.out | 40 +-
.../noStatsRfPrune/query34.out | 12 +-
.../noStatsRfPrune/query35.out | 26 +-
.../noStatsRfPrune/query36.out | 12 +-
.../noStatsRfPrune/query37.out | 12 +-
.../noStatsRfPrune/query38.out | 26 +-
.../noStatsRfPrune/query39.out | 14 +-
.../noStatsRfPrune/query4.out | 30 +-
.../noStatsRfPrune/query40.out | 10 +-
.../noStatsRfPrune/query41.out | 8 +-
.../noStatsRfPrune/query42.out | 8 +-
.../noStatsRfPrune/query43.out | 8 +-
.../noStatsRfPrune/query44.out | 34 +-
.../noStatsRfPrune/query45.out | 18 +-
.../noStatsRfPrune/query46.out | 20 +-
.../noStatsRfPrune/query47.out | 18 +-
.../noStatsRfPrune/query48.out | 12 +-
.../noStatsRfPrune/query49.out | 28 +-
.../noStatsRfPrune/query5.out | 34 +-
.../noStatsRfPrune/query50.out | 10 +-
.../noStatsRfPrune/query51.out | 20 +-
.../noStatsRfPrune/query52.out | 8 +-
.../noStatsRfPrune/query53.out | 12 +-
.../noStatsRfPrune/query54.out | 36 +-
.../noStatsRfPrune/query55.out | 8 +-
.../noStatsRfPrune/query56.out | 34 +-
.../noStatsRfPrune/query57.out | 18 +-
.../noStatsRfPrune/query58.out | 50 +-
.../noStatsRfPrune/query59.out | 18 +-
.../noStatsRfPrune/query6.out | 26 +-
.../noStatsRfPrune/query60.out | 40 +-
.../noStatsRfPrune/query61.out | 32 +-
.../noStatsRfPrune/query62.out | 12 +-
.../noStatsRfPrune/query63.out | 12 +-
.../noStatsRfPrune/query64.out | 48 +-
.../noStatsRfPrune/query65.out | 18 +-
.../noStatsRfPrune/query66.out | 24 +-
.../noStatsRfPrune/query67.out | 12 +-
.../noStatsRfPrune/query68.out | 20 +-
.../noStatsRfPrune/query69.out | 24 +-
.../noStatsRfPrune/query7.out | 12 +-
.../noStatsRfPrune/query70.out | 22 +-
.../noStatsRfPrune/query71.out | 20 +-
.../noStatsRfPrune/query72.out | 24 +-
.../noStatsRfPrune/query73.out | 12 +-
.../noStatsRfPrune/query74.out | 22 +-
.../noStatsRfPrune/query75.out | 28 +-
.../noStatsRfPrune/query76.out | 22 +-
.../noStatsRfPrune/query77.out | 42 +-
.../noStatsRfPrune/query78.out | 14 +-
.../noStatsRfPrune/query79.out | 14 +-
.../noStatsRfPrune/query8.out | 18 +-
.../noStatsRfPrune/query80.out | 34 +-
.../noStatsRfPrune/query81.out | 24 +-
.../noStatsRfPrune/query82.out | 12 +-
.../noStatsRfPrune/query83.out | 38 +-
.../noStatsRfPrune/query84.out | 14 +-
.../noStatsRfPrune/query85.out | 20 +-
.../noStatsRfPrune/query86.out | 10 +-
.../noStatsRfPrune/query87.out | 26 +-
.../noStatsRfPrune/query88.out | 78 +-
.../noStatsRfPrune/query89.out | 12 +-
.../noStatsRfPrune/query9.out | 62 +-
.../noStatsRfPrune/query90.out | 18 +-
.../noStatsRfPrune/query91.out | 20 +-
.../noStatsRfPrune/query92.out | 8 +-
.../noStatsRfPrune/query93.out | 6 +-
.../noStatsRfPrune/query94.out | 14 +-
.../noStatsRfPrune/query95.out | 20 +-
.../noStatsRfPrune/query96.out | 8 +-
.../noStatsRfPrune/query97.out | 10 +-
.../noStatsRfPrune/query98.out | 10 +-
.../noStatsRfPrune/query99.out | 12 +-
.../no_stats_shape/query1.out | 18 +-
.../no_stats_shape/query10.out | 24 +-
.../no_stats_shape/query11.out | 22 +-
.../no_stats_shape/query12.out | 10 +-
.../no_stats_shape/query13.out | 14 +-
.../no_stats_shape/query14.out | 80 +-
.../no_stats_shape/query15.out | 14 +-
.../no_stats_shape/query16.out | 14 +-
.../no_stats_shape/query17.out | 18 +-
.../no_stats_shape/query18.out | 20 +-
.../no_stats_shape/query19.out | 16 +-
.../no_stats_shape/query2.out | 20 +-
.../no_stats_shape/query20.out | 10 +-
.../no_stats_shape/query21.out | 10 +-
.../no_stats_shape/query22.out | 8 +-
.../no_stats_shape/query23.out | 38 +-
.../no_stats_shape/query24.out | 24 +-
.../no_stats_shape/query25.out | 18 +-
.../no_stats_shape/query26.out | 12 +-
.../no_stats_shape/query27.out | 12 +-
.../no_stats_shape/query28.out | 22 +-
.../no_stats_shape/query29.out | 18 +-
.../no_stats_shape/query3.out | 8 +-
.../no_stats_shape/query30.out | 22 +-
.../no_stats_shape/query31.out | 28 +-
.../no_stats_shape/query32.out | 8 +-
.../no_stats_shape/query33.out | 40 +-
.../no_stats_shape/query34.out | 12 +-
.../no_stats_shape/query35.out | 26 +-
.../no_stats_shape/query36.out | 12 +-
.../no_stats_shape/query37.out | 12 +-
.../no_stats_shape/query38.out | 26 +-
.../no_stats_shape/query39.out | 14 +-
.../no_stats_shape/query4.out | 30 +-
.../no_stats_shape/query40.out | 10 +-
.../no_stats_shape/query41.out | 8 +-
.../no_stats_shape/query42.out | 8 +-
.../no_stats_shape/query43.out | 8 +-
.../no_stats_shape/query44.out | 34 +-
.../no_stats_shape/query45.out | 18 +-
.../no_stats_shape/query46.out | 20 +-
.../no_stats_shape/query47.out | 18 +-
.../no_stats_shape/query48.out | 12 +-
.../no_stats_shape/query49.out | 28 +-
.../no_stats_shape/query5.out | 34 +-
.../no_stats_shape/query50.out | 10 +-
.../no_stats_shape/query51.out | 20 +-
.../no_stats_shape/query52.out | 8 +-
.../no_stats_shape/query53.out | 12 +-
.../no_stats_shape/query54.out | 36 +-
.../no_stats_shape/query55.out | 8 +-
.../no_stats_shape/query56.out | 34 +-
.../no_stats_shape/query57.out | 18 +-
.../no_stats_shape/query58.out | 50 +-
.../no_stats_shape/query59.out | 18 +-
.../no_stats_shape/query6.out | 26 +-
.../no_stats_shape/query60.out | 40 +-
.../no_stats_shape/query61.out | 32 +-
.../no_stats_shape/query62.out | 12 +-
.../no_stats_shape/query63.out | 12 +-
.../no_stats_shape/query64.out | 48 +-
.../no_stats_shape/query65.out | 18 +-
.../no_stats_shape/query66.out | 24 +-
.../no_stats_shape/query67.out | 12 +-
.../no_stats_shape/query68.out | 20 +-
.../no_stats_shape/query69.out | 24 +-
.../no_stats_shape/query7.out | 12 +-
.../no_stats_shape/query70.out | 22 +-
.../no_stats_shape/query71.out | 20 +-
.../no_stats_shape/query72.out | 24 +-
.../no_stats_shape/query73.out | 12 +-
.../no_stats_shape/query74.out | 22 +-
.../no_stats_shape/query75.out | 28 +-
.../no_stats_shape/query76.out | 22 +-
.../no_stats_shape/query77.out | 42 +-
.../no_stats_shape/query78.out | 14 +-
.../no_stats_shape/query79.out | 14 +-
.../no_stats_shape/query8.out | 18 +-
.../no_stats_shape/query80.out | 34 +-
.../no_stats_shape/query81.out | 24 +-
.../no_stats_shape/query82.out | 12 +-
.../no_stats_shape/query83.out | 38 +-
.../no_stats_shape/query84.out | 14 +-
.../no_stats_shape/query85.out | 20 +-
.../no_stats_shape/query86.out | 10 +-
.../no_stats_shape/query87.out | 26 +-
.../no_stats_shape/query88.out | 78 +-
.../no_stats_shape/query89.out | 12 +-
.../no_stats_shape/query9.out | 62 +-
.../no_stats_shape/query90.out | 18 +-
.../no_stats_shape/query91.out | 20 +-
.../no_stats_shape/query92.out | 8 +-
.../no_stats_shape/query93.out | 6 +-
.../no_stats_shape/query94.out | 14 +-
.../no_stats_shape/query95.out | 20 +-
.../no_stats_shape/query96.out | 8 +-
.../no_stats_shape/query97.out | 10 +-
.../no_stats_shape/query98.out | 10 +-
.../no_stats_shape/query99.out | 12 +-
.../rf_prune/query1.out | 20 +-
.../rf_prune/query10.out | 20 +-
.../rf_prune/query11.out | 22 +-
.../rf_prune/query12.out | 12 +-
.../rf_prune/query13.out | 14 +-
.../rf_prune/query14.out | 82 +-
.../rf_prune/query15.out | 14 +-
.../rf_prune/query16.out | 14 +-
.../rf_prune/query17.out | 20 +-
.../rf_prune/query18.out | 22 +-
.../rf_prune/query19.out | 16 +-
.../rf_prune/query2.out | 22 +-
.../rf_prune/query20.out | 12 +-
.../rf_prune/query21.out | 10 +-
.../rf_prune/query22.out | 8 +-
.../rf_prune/query23.out | 38 +-
.../rf_prune/query24.out | 30 +-
.../rf_prune/query25.out | 20 +-
.../rf_prune/query26.out | 14 +-
.../rf_prune/query27.out | 14 +-
.../rf_prune/query28.out | 22 +-
.../rf_prune/query29.out | 18 +-
.../rf_prune/query3.out | 8 +-
.../rf_prune/query30.out | 22 +-
.../rf_prune/query31.out | 30 +-
.../rf_prune/query32.out | 8 +-
.../rf_prune/query33.out | 44 +-
.../rf_prune/query34.out | 12 +-
.../rf_prune/query35.out | 26 +-
.../rf_prune/query36.out | 12 +-
.../rf_prune/query37.out | 10 +-
.../rf_prune/query38.out | 26 +-
.../rf_prune/query39.out | 14 +-
.../rf_prune/query4.out | 30 +-
.../rf_prune/query40.out | 10 +-
.../rf_prune/query41.out | 8 +-
.../rf_prune/query42.out | 8 +-
.../rf_prune/query43.out | 8 +-
.../rf_prune/query44.out | 28 +-
.../rf_prune/query45.out | 20 +-
.../rf_prune/query46.out | 20 +-
.../rf_prune/query47.out | 18 +-
.../rf_prune/query48.out | 12 +-
.../rf_prune/query49.out | 28 +-
.../rf_prune/query5.out | 34 +-
.../rf_prune/query50.out | 10 +-
.../rf_prune/query51.out | 20 +-
.../rf_prune/query52.out | 8 +-
.../rf_prune/query53.out | 12 +-
.../rf_prune/query54.out | 40 +-
.../rf_prune/query55.out | 8 +-
.../rf_prune/query56.out | 36 +-
.../rf_prune/query57.out | 20 +-
.../rf_prune/query58.out | 44 +-
.../rf_prune/query59.out | 22 +-
.../rf_prune/query6.out | 28 +-
.../rf_prune/query60.out | 40 +-
.../rf_prune/query61.out | 28 +-
.../rf_prune/query62.out | 12 +-
.../rf_prune/query63.out | 12 +-
.../rf_prune/query64.out | 52 +-
.../rf_prune/query65.out | 20 +-
.../rf_prune/query66.out | 24 +-
.../rf_prune/query67.out | 12 +-
.../rf_prune/query68.out | 16 +-
.../rf_prune/query69.out | 26 +-
.../rf_prune/query7.out | 14 +-
.../rf_prune/query70.out | 20 +-
.../rf_prune/query71.out | 22 +-
.../rf_prune/query72.out | 32 +-
.../rf_prune/query73.out | 12 +-
.../rf_prune/query74.out | 22 +-
.../rf_prune/query75.out | 28 +-
.../rf_prune/query76.out | 22 +-
.../rf_prune/query77.out | 38 +-
.../rf_prune/query78.out | 14 +-
.../rf_prune/query79.out | 14 +-
.../rf_prune/query8.out | 18 +-
.../rf_prune/query80.out | 34 +-
.../rf_prune/query81.out | 24 +-
.../rf_prune/query82.out | 10 +-
.../rf_prune/query83.out | 34 +-
.../rf_prune/query84.out | 12 +-
.../rf_prune/query85.out | 16 +-
.../rf_prune/query86.out | 12 +-
.../rf_prune/query87.out | 26 +-
.../rf_prune/query88.out | 78 +-
.../rf_prune/query89.out | 12 +-
.../rf_prune/query9.out | 62 +-
.../rf_prune/query90.out | 18 +-
.../rf_prune/query91.out | 20 +-
.../rf_prune/query92.out | 8 +-
.../rf_prune/query93.out | 6 +-
.../rf_prune/query94.out | 14 +-
.../rf_prune/query95.out | 20 +-
.../rf_prune/query96.out | 8 +-
.../rf_prune/query97.out | 10 +-
.../rf_prune/query98.out | 12 +-
.../rf_prune/query99.out | 12 +-
.../shape/query1.out | 20 +-
.../shape/query10.out | 20 +-
.../shape/query11.out | 22 +-
.../shape/query12.out | 12 +-
.../shape/query13.out | 14 +-
.../shape/query14.out | 82 +-
.../shape/query15.out | 14 +-
.../shape/query16.out | 14 +-
.../shape/query17.out | 20 +-
.../shape/query18.out | 22 +-
.../shape/query19.out | 16 +-
.../shape/query2.out | 22 +-
.../shape/query20.out | 12 +-
.../shape/query21.out | 10 +-
.../shape/query22.out | 8 +-
.../shape/query23.out | 38 +-
.../shape/query24.out | 30 +-
.../shape/query25.out | 20 +-
.../shape/query26.out | 14 +-
.../shape/query27.out | 14 +-
.../shape/query28.out | 22 +-
.../shape/query29.out | 18 +-
.../shape/query3.out | 8 +-
.../shape/query30.out | 22 +-
.../shape/query31.out | 30 +-
.../shape/query32.out | 8 +-
.../shape/query33.out | 44 +-
.../shape/query34.out | 12 +-
.../shape/query35.out | 26 +-
.../shape/query36.out | 12 +-
.../shape/query37.out | 10 +-
.../shape/query38.out | 26 +-
.../shape/query39.out | 14 +-
.../shape/query4.out | 30 +-
.../shape/query40.out | 10 +-
.../shape/query41.out | 8 +-
.../shape/query42.out | 8 +-
.../shape/query43.out | 8 +-
.../shape/query44.out | 28 +-
.../shape/query45.out | 20 +-
.../shape/query46.out | 20 +-
.../shape/query47.out | 18 +-
.../shape/query48.out | 12 +-
.../shape/query49.out | 28 +-
.../shape/query5.out | 34 +-
.../shape/query50.out | 10 +-
.../shape/query51.out | 20 +-
.../shape/query52.out | 8 +-
.../shape/query53.out | 12 +-
.../shape/query54.out | 40 +-
.../shape/query55.out | 8 +-
.../shape/query56.out | 36 +-
.../shape/query57.out | 20 +-
.../shape/query58.out | 44 +-
.../shape/query59.out | 22 +-
.../shape/query6.out | 28 +-
.../shape/query60.out | 40 +-
.../shape/query61.out | 28 +-
.../shape/query62.out | 12 +-
.../shape/query63.out | 12 +-
.../shape/query64.out | 52 +-
.../shape/query65.out | 20 +-
.../shape/query66.out | 24 +-
.../shape/query67.out | 12 +-
.../shape/query68.out | 16 +-
.../shape/query69.out | 26 +-
.../shape/query7.out | 14 +-
.../shape/query70.out | 20 +-
.../shape/query71.out | 22 +-
.../shape/query72.out | 32 +-
.../shape/query73.out | 12 +-
.../shape/query74.out | 22 +-
.../shape/query75.out | 28 +-
.../shape/query76.out | 22 +-
.../shape/query77.out | 38 +-
.../shape/query78.out | 14 +-
.../shape/query79.out | 14 +-
.../shape/query8.out | 18 +-
.../shape/query80.out | 34 +-
.../shape/query81.out | 24 +-
.../shape/query82.out | 10 +-
.../shape/query83.out | 34 +-
.../shape/query84.out | 12 +-
.../shape/query85.out | 16 +-
.../shape/query86.out | 12 +-
.../shape/query87.out | 26 +-
.../shape/query88.out | 78 +-
.../shape/query89.out | 12 +-
.../shape/query9.out | 62 +-
.../shape/query90.out | 18 +-
.../shape/query91.out | 20 +-
.../shape/query92.out | 8 +-
.../shape/query93.out | 6 +-
.../shape/query94.out | 14 +-
.../shape/query95.out | 20 +-
.../shape/query96.out | 8 +-
.../shape/query97.out | 10 +-
.../shape/query98.out | 12 +-
.../shape/query99.out | 12 +-
.../nostats_rf_prune/q1.out | 4 +-
.../nostats_rf_prune/q10.out | 12 +-
.../nostats_rf_prune/q11.out | 20 +-
.../nostats_rf_prune/q12.out | 4 +-
.../nostats_rf_prune/q13.out | 8 +-
.../nostats_rf_prune/q14.out | 6 +-
.../nostats_rf_prune/q15.out | 12 +-
.../nostats_rf_prune/q16.out | 6 +-
.../nostats_rf_prune/q17.out | 6 +-
.../nostats_rf_prune/q18.out | 8 +-
.../nostats_rf_prune/q19.out | 6 +-
.../nostats_rf_prune/q2.out | 12 +-
.../nostats_rf_prune/q20-rewrite.out | 12 +-
.../nostats_rf_prune/q20.out | 12 +-
.../nostats_rf_prune/q21.out | 10 +-
.../nostats_rf_prune/q22.out | 12 +-
.../nostats_rf_prune/q3.out | 8 +-
.../nostats_rf_prune/q4.out | 4 +-
.../nostats_rf_prune/q5.out | 16 +-
.../nostats_rf_prune/q6.out | 2 +-
.../nostats_rf_prune/q7.out | 16 +-
.../nostats_rf_prune/q8.out | 20 +-
.../nostats_rf_prune/q9.out | 14 +-
.../rf_prune/q1.out | 4 +-
.../rf_prune/q10.out | 10 +-
.../rf_prune/q11.out | 14 +-
.../rf_prune/q12.out | 4 +-
.../rf_prune/q13.out | 8 +-
.../rf_prune/q14.out | 4 +-
.../rf_prune/q15.out | 12 +-
.../rf_prune/q16.out | 8 +-
.../rf_prune/q17.out | 6 +-
.../rf_prune/q18.out | 6 +-
.../rf_prune/q19.out | 4 +-
.../rf_prune/q2.out | 12 +-
.../rf_prune/q20-rewrite.out | 12 +-
.../rf_prune/q20.out | 12 +-
.../rf_prune/q21.out | 8 +-
.../rf_prune/q22.out | 12 +-
.../rf_prune/q3.out | 8 +-
.../rf_prune/q4.out | 4 +-
.../rf_prune/q5.out | 14 +-
.../rf_prune/q6.out | 2 +-
.../rf_prune/q7.out | 16 +-
.../rf_prune/q8.out | 18 +-
.../rf_prune/q9.out | 18 +-
.../nereids_tpch_shape_sf1000_p0/shape/q1.out | 4 +-
.../shape/q10.out | 10 +-
.../shape/q11.out | 14 +-
.../shape/q12.out | 4 +-
.../shape/q13.out | 8 +-
.../shape/q14.out | 4 +-
.../shape/q15.out | 12 +-
.../shape/q16.out | 8 +-
.../shape/q17.out | 6 +-
.../shape/q18.out | 6 +-
.../shape/q19.out | 4 +-
.../nereids_tpch_shape_sf1000_p0/shape/q2.out | 12 +-
.../shape/q20-rewrite.out | 12 +-
.../shape/q20.out | 12 +-
.../shape/q21.out | 8 +-
.../shape/q22.out | 12 +-
.../nereids_tpch_shape_sf1000_p0/shape/q3.out | 8 +-
.../nereids_tpch_shape_sf1000_p0/shape/q4.out | 4 +-
.../nereids_tpch_shape_sf1000_p0/shape/q5.out | 14 +-
.../nereids_tpch_shape_sf1000_p0/shape/q6.out | 2 +-
.../nereids_tpch_shape_sf1000_p0/shape/q7.out | 16 +-
.../nereids_tpch_shape_sf1000_p0/shape/q8.out | 18 +-
.../nereids_tpch_shape_sf1000_p0/shape/q9.out | 18 +-
.../shape_no_stats/q1.out | 4 +-
.../shape_no_stats/q10.out | 12 +-
.../shape_no_stats/q11.out | 20 +-
.../shape_no_stats/q12.out | 4 +-
.../shape_no_stats/q13.out | 8 +-
.../shape_no_stats/q14.out | 6 +-
.../shape_no_stats/q15.out | 12 +-
.../shape_no_stats/q16.out | 6 +-
.../shape_no_stats/q17.out | 6 +-
.../shape_no_stats/q18.out | 8 +-
.../shape_no_stats/q19.out | 6 +-
.../shape_no_stats/q2.out | 12 +-
.../shape_no_stats/q20-rewrite.out | 12 +-
.../shape_no_stats/q20.out | 12 +-
.../shape_no_stats/q21.out | 10 +-
.../shape_no_stats/q22.out | 12 +-
.../shape_no_stats/q3.out | 8 +-
.../shape_no_stats/q4.out | 4 +-
.../shape_no_stats/q5.out | 16 +-
.../shape_no_stats/q6.out | 2 +-
.../shape_no_stats/q7.out | 16 +-
.../shape_no_stats/q8.out | 20 +-
.../shape_no_stats/q9.out | 14 +-
.../nereids_p0/hint/test_leading.groovy | 153 ++
702 files changed, 12310 insertions(+), 9230 deletions(-)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/hint/DistributeHint.java
rename fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/{JoinHint.java => DistributeType.java} (72%)
rename fe/fe-core/src/test/java/org/apache/doris/nereids/{JoinHintTest.java => DistributeHintTest.java} (96%)
rename fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/{JoinHintTest.java => DistributeHintTest.java} (99%)
diff --git a/docs/en/docs/query-acceleration/hint/joinHint.md b/docs/en/docs/query-acceleration/hint/joinHint.md
index 5ce63cfe70..af120425cc 100644
--- a/docs/en/docs/query-acceleration/hint/joinHint.md
+++ b/docs/en/docs/query-acceleration/hint/joinHint.md
@@ -614,6 +614,70 @@ mysql> explain shape plan select /*+ ordered */ count(*) from t2 join[broadcast]
16 rows in set (0.01 sec)
```
- the Explain shape plan will display inside distribute operator related information, including DistributionSpecReplicated said the operator will be corresponding data into a copy of all be node, DistributionSpecGather indicates that data is gathered to fe nodes, and DistributionSpecHash indicates that data is dispersed to different be nodes according to a specific hashKey and algorithm.
+# Leading Hint mixed with Distribute Hint
+When we want more flexible using of leading which we want to control distribute type when useing hint and distribute hint together.
+For example: we want to use hint to make plan like:
+```sql
+mysql> explain shape plan select /*+ leading(t3 broadcast {t2 shuffle t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;
++----------------------------------------------------------------------------------------+
+| Explain String(Nereids Planner) |
++----------------------------------------------------------------------------------------+
+| PhysicalResultSink |
+| --hashAgg[GLOBAL] |
+| ----PhysicalDistribute[DistributionSpecGather] |
+| ------hashAgg[LOCAL] |
+| --------PhysicalProject |
+| ----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() |
+| ------------PhysicalProject |
+| --------------PhysicalOlapScan[t3] |
+| ------------PhysicalDistribute[DistributionSpecHash] |
+| --------------PhysicalProject |
+| ----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() |
+| ------------------PhysicalProject |
+| --------------------PhysicalOlapScan[t2] |
+| ------------------PhysicalDistribute[DistributionSpecHash] |
+| --------------------PhysicalProject |
+| ----------------------PhysicalOlapScan[t1] |
+| |
+| Hint log: |
+| Used: leading(t3 { t2 shuffle t1 } ) |
+| UnUsed: [broadcast]_2 |
+| SyntaxError: |
++----------------------------------------------------------------------------------------+
+21 rows in set (0.06 sec)
+```
+which means we can make plan like: leading(t3 { t2 shuffle t1 } ) but [broadcast]_2 can not be used because of changing the meaning of original query
+When we have Distribute hint and Leading Hint + Distribute Hint together, we would try leading hint first. When leading hint is used all separate distribute hints turn to unused:
+```sql
+mysql> explain shape plan select /*+ leading(t3 {t2 t1}) */ count(*) from t1 left outer join [shuffle] t2 on c1 = c2 join [broadcast] t3 on c2 = c3;
++----------------------------------------------------------------------------------------+
+| Explain String(Nereids Planner) |
++----------------------------------------------------------------------------------------+
+| PhysicalResultSink |
+| --hashAgg[GLOBAL] |
+| ----PhysicalDistribute[DistributionSpecGather] |
+| ------hashAgg[LOCAL] |
+| --------PhysicalProject |
+| ----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() |
+| ------------PhysicalProject |
+| --------------PhysicalOlapScan[t3] |
+| ------------PhysicalDistribute[DistributionSpecHash] |
+| --------------PhysicalProject |
+| ----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() |
+| ------------------PhysicalProject |
+| --------------------PhysicalOlapScan[t2] |
+| ------------------PhysicalDistribute[DistributionSpecHash] |
+| --------------------PhysicalProject |
+| ----------------------PhysicalOlapScan[t1] |
+| |
+| Hint log: |
+| Used: leading(t3 { t2 t1 } ) |
+| UnUsed: [shuffle]_2 [broadcast]_3 |
+| SyntaxError: |
++----------------------------------------------------------------------------------------+
+21 rows in set (0.02 sec)
+```
+
# To be supported
- leadingHint indicates that the current query cannot be mixed with the subquery after it is promoted. A hint is required to control whether the subquery can be unnested
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index a1eea97161..488c8b82a0 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -590,6 +590,8 @@ STRING_LITERAL
LEADING_STRING
: LEFT_BRACE
| RIGHT_BRACE
+ | LEFT_BRACKET
+ | RIGHT_BRACKET
;
BIGINT_LITERAL
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a91f207e47..7159350556 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -334,13 +334,13 @@ relation
;
joinRelation
- : (joinType) JOIN joinHint? right=relationPrimary joinCriteria?
+ : (joinType) JOIN distributeType? right=relationPrimary joinCriteria?
;
// Just like `opt_plan_hints` in legacy CUP parser.
-joinHint
- : LEFT_BRACKET identifier RIGHT_BRACKET #bracketJoinHint
- | HINT_START identifier HINT_END #commentJoinHint
+distributeType
+ : LEFT_BRACKET identifier RIGHT_BRACKET #bracketDistributeType
+ | HINT_START identifier HINT_END #commentDistributeType
;
relationHint
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 4ac8223cc3..220a9024db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.jobs.executor.Optimizer;
import org.apache.doris.nereids.jobs.executor.Rewriter;
@@ -357,17 +358,25 @@ public class NereidsPlanner extends Planner {
String used = "";
String unUsed = "";
String syntaxError = "";
+ int distributeHintIndex = 1;
for (Hint hint : hints) {
+ String distributeIndex = "";
+ if (hint instanceof DistributeHint) {
+ distributeHintIndex++;
+ if (!hint.getExplainString().equals("")) {
+ distributeIndex = "_" + String.valueOf(distributeHintIndex);
+ }
+ }
switch (hint.getStatus()) {
case UNUSED:
- unUsed = unUsed + " " + hint.getExplainString();
+ unUsed = unUsed + " " + hint.getExplainString() + distributeIndex;
break;
case SYNTAX_ERROR:
- syntaxError = syntaxError + " " + hint.getExplainString()
+ syntaxError = syntaxError + " " + hint.getExplainString() + distributeIndex
+ " Msg:" + hint.getErrorMessage();
break;
case SUCCESS:
- used = used + " " + hint.getExplainString();
+ used = used + " " + hint.getExplainString() + distributeIndex;
break;
default:
break;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/DistributeHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/DistributeHint.java
new file mode 100644
index 0000000000..8c41ebfce8
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/DistributeHint.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.hint;
+
+import org.apache.doris.nereids.trees.plans.DistributeType;
+
+/**
+ * Hints for join.
+ *
+ * Hints for the right child of join are supported currently.
+ * Left input and right input of join could have different hints for further extension.
+ */
+public class DistributeHint extends Hint {
+ public DistributeType distributeType;
+
+ private boolean isSuccessInLeading = false;
+
+ public DistributeHint(DistributeType distributeType) {
+ super("Distribute");
+ this.distributeType = distributeType;
+ }
+
+ public void setSuccessInLeading(boolean successInLeading) {
+ isSuccessInLeading = successInLeading;
+ }
+
+ /**
+ * get explain string of distribute hint, when distribute hint success in leading, it would not show
+ * @return explain string of distribute hint
+ */
+ public String getExplainString() {
+ if (this.isSuccessInLeading) {
+ return "";
+ }
+ StringBuilder out = new StringBuilder();
+ switch (this.distributeType) {
+ case NONE:
+ break;
+ case SHUFFLE_RIGHT:
+ out.append("[shuffle]");
+ break;
+ case BROADCAST_RIGHT:
+ out.append("[broadcast]");
+ break;
+ default:
+ break;
+ }
+ return out.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return this.distributeType == ((DistributeHint) o).distributeType;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/Hint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/Hint.java
index 8640ab3f5b..b144e71830 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/Hint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/Hint.java
@@ -65,6 +65,10 @@ public class Hint {
return getStatus() == HintStatus.SYNTAX_ERROR;
}
+ public boolean isUnUsed() {
+ return getStatus() == HintStatus.UNUSED;
+ }
+
public String getHintName() {
return hintName;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
index 4d1abbbf95..00a1c01b0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
@@ -22,7 +22,7 @@ import org.apache.doris.common.Pair;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.plans.JoinHint;
+import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
@@ -33,11 +33,13 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.util.JoinUtils;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -51,9 +53,13 @@ import java.util.Stack;
*/
public class LeadingHint extends Hint {
private String originalString = "";
+
+ private List parameters;
private final List tablelist = new ArrayList<>();
private final List levellist = new ArrayList<>();
+ private final Map distributeHints = new HashMap<>();
+
private final Map relationIdToScanMap = Maps.newLinkedHashMap();
private final List> relationIdAndTableName = new ArrayList<>();
@@ -82,6 +88,7 @@ public class LeadingHint extends Hint {
public LeadingHint(String hintName, List parameters, String originalString) {
super(hintName);
this.originalString = originalString;
+ this.parameters = parameters;
int level = 0;
Stack brace = new Stack<>();
String lastParameter = "";
@@ -100,6 +107,18 @@ public class LeadingHint extends Hint {
} else {
level--;
}
+ } else if (parameter.equals("shuffle")) {
+ DistributeHint distributeHint = new DistributeHint(DistributeType.SHUFFLE_RIGHT);
+ distributeHints.put(tablelist.size(), distributeHint);
+ if (!ConnectContext.get().getStatementContext().getHints().contains(distributeHint)) {
+ ConnectContext.get().getStatementContext().addHint(distributeHint);
+ }
+ } else if (parameter.equals("broadcast")) {
+ DistributeHint distributeHint = new DistributeHint(DistributeType.BROADCAST_RIGHT);
+ distributeHints.put(tablelist.size(), distributeHint);
+ if (!ConnectContext.get().getStatementContext().getHints().contains(distributeHint)) {
+ ConnectContext.get().getStatementContext().addHint(distributeHint);
+ }
} else {
tablelist.add(parameter);
levellist.add(level);
@@ -122,9 +141,25 @@ public class LeadingHint extends Hint {
@Override
public String getExplainString() {
+ if (!this.isSuccess()) {
+ return originalString;
+ }
StringBuilder out = new StringBuilder();
- out.append(originalString);
- return out.toString();
+ int tableIndex = 0;
+ for (String parameter : parameters) {
+ if (parameter.equals("{") || parameter.equals("}") || parameter.equals("[") || parameter.equals("]")) {
+ out.append(parameter + " ");
+ } else if (parameter.equals("shuffle") || parameter.equals("broadcast")) {
+ DistributeHint distributeHint = distributeHints.get(tableIndex);
+ if (distributeHint.isSuccess()) {
+ out.append(parameter + " ");
+ }
+ } else {
+ out.append(parameter + " ");
+ tableIndex++;
+ }
+ }
+ return "leading(" + out.toString() + ")";
}
/**
@@ -420,7 +455,7 @@ public class LeadingHint extends Hint {
* @return plan
*/
public Plan generateLeadingJoinPlan() {
- Stack> stack = new Stack<>();
+ Stack>> stack = new Stack<>();
int index = 0;
LogicalPlan logicalPlan = getLogicalPlanByName(getTablelist().get(index));
if (logicalPlan == null) {
@@ -428,27 +463,28 @@ public class LeadingHint extends Hint {
}
logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan);
assert (logicalPlan != null);
- stack.push(Pair.of(getLevellist().get(index), logicalPlan));
+ stack.push(Pair.of(getLevellist().get(index), Pair.of(logicalPlan, index)));
int stackTopLevel = getLevellist().get(index++);
while (index < getTablelist().size()) {
int currentLevel = getLevellist().get(index);
if (currentLevel == stackTopLevel) {
// should return error if can not found table
logicalPlan = getLogicalPlanByName(getTablelist().get(index++));
+ int distributeIndex = index - 1;
if (logicalPlan == null) {
return null;
}
logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan);
- Pair newStackTop = stack.peek();
+ Pair> newStackTop = stack.peek();
while (!(stack.isEmpty() || stackTopLevel != newStackTop.first)) {
// check join is legal and get join type
newStackTop = stack.pop();
List conditions = getJoinConditions(
- getFilters(), newStackTop.second, logicalPlan);
+ getFilters(), newStackTop.second.first, logicalPlan);
Pair, List> pair = JoinUtils.extractExpressionForHashTable(
- newStackTop.second.getOutput(), logicalPlan.getOutput(), conditions);
+ newStackTop.second.first.getOutput(), logicalPlan.getOutput(), conditions);
// leading hint would set status inside if not success
- JoinType joinType = computeJoinType(getBitmap(newStackTop.second),
+ JoinType joinType = computeJoinType(getBitmap(newStackTop.second.first),
getBitmap(logicalPlan), conditions);
if (joinType == null) {
this.setStatus(HintStatus.SYNTAX_ERROR);
@@ -461,13 +497,15 @@ public class LeadingHint extends Hint {
return null;
}
// get joinType
+ DistributeHint distributeHint = getJoinHint(distributeIndex);
LogicalJoin logicalJoin = new LogicalJoin<>(joinType, pair.first,
pair.second,
- JoinHint.NONE,
+ distributeHint,
Optional.empty(),
- newStackTop.second,
+ newStackTop.second.first,
logicalPlan);
- logicalJoin.setBitmap(LongBitmap.or(getBitmap(newStackTop.second), getBitmap(logicalPlan)));
+ distributeIndex = newStackTop.second.second;
+ logicalJoin.setBitmap(LongBitmap.or(getBitmap(newStackTop.second.first), getBitmap(logicalPlan)));
if (stackTopLevel > 0) {
if (index < getTablelist().size()) {
if (stackTopLevel > getLevellist().get(index)) {
@@ -482,7 +520,7 @@ public class LeadingHint extends Hint {
}
logicalPlan = logicalJoin;
}
- stack.push(Pair.of(stackTopLevel, logicalPlan));
+ stack.push(Pair.of(stackTopLevel, Pair.of(logicalPlan, distributeIndex)));
} else {
// push
logicalPlan = getLogicalPlanByName(getTablelist().get(index++));
@@ -490,12 +528,12 @@ public class LeadingHint extends Hint {
return null;
}
logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan);
- stack.push(Pair.of(currentLevel, logicalPlan));
+ stack.push(Pair.of(currentLevel, Pair.of(logicalPlan, index)));
stackTopLevel = currentLevel;
}
}
- LogicalJoin finalJoin = (LogicalJoin) stack.pop().second;
+ LogicalJoin finalJoin = (LogicalJoin) stack.pop().second.first;
// we want all filters been remove
assert (filters.isEmpty());
if (finalJoin != null) {
@@ -504,6 +542,14 @@ public class LeadingHint extends Hint {
return finalJoin;
}
+ private DistributeHint getJoinHint(Integer index) {
+ if (distributeHints.get(index) == null) {
+ return new DistributeHint(DistributeType.NONE);
+ }
+ distributeHints.get(index).setSuccessInLeading(true);
+ return distributeHints.get(index);
+ }
+
private List getJoinConditions(List> filters,
LogicalPlan left, LogicalPlan right) {
List joinConditions = new ArrayList<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
index f094efe718..d78374d134 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.jobs.joinorder.hypergraph;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
@@ -32,8 +33,8 @@ import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.GroupPlan;
-import org.apache.doris.nereids.trees.plans.JoinHint;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -230,7 +231,8 @@ public class HyperGraph {
for (Map.Entry, Pair, List>> entry : conjuncts
.entrySet()) {
LogicalJoin, ?> singleJoin = new LogicalJoin<>(join.getJoinType(), entry.getValue().first,
- entry.getValue().second, JoinHint.NONE, join.getMarkJoinSlotReference(),
+ entry.getValue().second,
+ new DistributeHint(DistributeType.NONE), join.getMarkJoinSlotReference(),
Lists.newArrayList(join.left(), join.right()));
Pair ends = entry.getKey();
JoinEdge edge = new JoinEdge(singleJoin, joinEdges.size(), leftEdgeNodes.first, rightEdgeNodes.first,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
index e2fd4b2dee..eece2d8c3d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.jobs.joinorder.hypergraph.receiver;
+import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.cascades.CostAndEnforcerJob;
import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob;
@@ -34,8 +35,8 @@ import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.GroupPlan;
-import org.apache.doris.nereids.trees.plans.JoinHint;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
@@ -228,11 +229,13 @@ public class PlanReceiver implements AbstractReceiver {
right, left));
}
} else {
- plans.add(new PhysicalHashJoin<>(joinType, hashConjuncts, otherConjuncts, JoinHint.NONE, Optional.empty(),
+ plans.add(new PhysicalHashJoin<>(joinType, hashConjuncts, otherConjuncts,
+ new DistributeHint(DistributeType.NONE), Optional.empty(),
joinProperties,
left, right));
if (joinType.isSwapJoinType()) {
- plans.add(new PhysicalHashJoin<>(joinType.swap(), hashConjuncts, otherConjuncts, JoinHint.NONE,
+ plans.add(new PhysicalHashJoin<>(joinType.swap(), hashConjuncts, otherConjuncts,
+ new DistributeHint(DistributeType.NONE),
Optional.empty(),
joinProperties,
right, left));
@@ -310,7 +313,8 @@ public class PlanReceiver implements AbstractReceiver {
} else if (physicalPlan instanceof AbstractPhysicalJoin) {
AbstractPhysicalJoin physicalJoin = (AbstractPhysicalJoin) physicalPlan;
logicalPlan = new LogicalJoin<>(physicalJoin.getJoinType(), physicalJoin.getHashJoinConjuncts(),
- physicalJoin.getOtherJoinConjuncts(), JoinHint.NONE, physicalJoin.getMarkJoinSlotReference(),
+ physicalJoin.getOtherJoinConjuncts(),
+ new DistributeHint(DistributeType.NONE), physicalJoin.getMarkJoinSlotReference(),
groupExpression.children().stream().map(g -> new GroupPlan(g)).collect(Collectors.toList()));
} else {
throw new RuntimeException("DPhyp can only handle join and project operator");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 745ae17946..aaa01c78db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -51,7 +51,7 @@ import org.apache.doris.nereids.DorisParser.ArraySliceContext;
import org.apache.doris.nereids.DorisParser.BitOperationContext;
import org.apache.doris.nereids.DorisParser.BooleanExpressionContext;
import org.apache.doris.nereids.DorisParser.BooleanLiteralContext;
-import org.apache.doris.nereids.DorisParser.BracketJoinHintContext;
+import org.apache.doris.nereids.DorisParser.BracketDistributeTypeContext;
import org.apache.doris.nereids.DorisParser.BracketRelationHintContext;
import org.apache.doris.nereids.DorisParser.BuildModeContext;
import org.apache.doris.nereids.DorisParser.CallProcedureContext;
@@ -60,7 +60,7 @@ import org.apache.doris.nereids.DorisParser.CollateContext;
import org.apache.doris.nereids.DorisParser.ColumnDefContext;
import org.apache.doris.nereids.DorisParser.ColumnDefsContext;
import org.apache.doris.nereids.DorisParser.ColumnReferenceContext;
-import org.apache.doris.nereids.DorisParser.CommentJoinHintContext;
+import org.apache.doris.nereids.DorisParser.CommentDistributeTypeContext;
import org.apache.doris.nereids.DorisParser.CommentRelationHintContext;
import org.apache.doris.nereids.DorisParser.ComparisonContext;
import org.apache.doris.nereids.DorisParser.ComplexColTypeContext;
@@ -201,6 +201,7 @@ import org.apache.doris.nereids.analyzer.UnboundVariable.VariableType;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.NotSupportedException;
import org.apache.doris.nereids.exceptions.ParseException;
+import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.properties.SelectHint;
import org.apache.doris.nereids.properties.SelectHintLeading;
@@ -317,7 +318,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
-import org.apache.doris.nereids.trees.plans.JoinHint;
+import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.LimitPhase;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -2726,16 +2727,17 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor