[CP] add with_subquery, generate_with_clause, group by placement parameters

src/share/parameter/ob_parameter_seed.ipp

@@ -1457,3 +1457,15 @@ DEF_STR_WITH_CHECKER(rpc_server_authentication_method, OB_CLUSTER_PARAMETER, "AL
 DEF_BOOL(_enable_backtrace_function, OB_CLUSTER_PARAMETER, "True",
          "Decide whether to let the backtrace function take effect",
          ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
+DEF_INT(_with_subquery, OB_TENANT_PARAMETER, "0", "[0,2]",
+        "WITH subquery transformation，0: optimizer，1: materialize，2: inline",
+        ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
+DEF_BOOL(_xsolapi_generate_with_clause, OB_TENANT_PARAMETER, "True",
+        "OLAP API generates WITH clause",
+        ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+
+DEF_BOOL(_optimizer_group_by_placement, OB_TENANT_PARAMETER, "True",
+        "enable group by placement transform rule",
+        ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));

src/sql/rewrite/ob_transform_groupby_pullup.cpp

@@ -298,8 +298,16 @@ int ObTransformGroupByPullup::check_groupby_pullup_validity(ObDMLStmt *stmt,
     ObSelectStmt *sub_stmt = NULL;
     ObString dummy_str;
     const ObViewMergeHint *myhint = NULL;
+    ObSQLSessionInfo *session_info = NULL;
+    bool enable_group_by_placement_transform = false;
     OPT_TRACE("try", table);
-    if (OB_ISNULL(sub_stmt = table->ref_query_)) {
+    if (OB_ISNULL(ctx_) ||
+        OB_ISNULL(session_info = ctx_->session_info_)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("unexpect null param", K(ctx_), K(ret));
+    } else if (OB_FAIL(session_info->is_groupby_placement_transformation_enabled(enable_group_by_placement_transform))) {
+      LOG_WARN("failed to check group by placement transform enabled", K(ret));
+    } else if (OB_ISNULL(sub_stmt = table->ref_query_)) {
       ret = OB_ERR_UNEXPECTED;
       LOG_WARN("invalid generated table item", K(ret), K(*table));
     } else if (OB_FAIL(check_hint_valid(*stmt, *table->ref_query_, hint_valid))) {
@@ -308,6 +316,8 @@ int ObTransformGroupByPullup::check_groupby_pullup_validity(ObDMLStmt *stmt,
       // can not set is_valid as false, may pullup other table
       OPT_TRACE("hint reject transform");
     } else if (OB_FALSE_IT(myhint = static_cast<const ObViewMergeHint*>(sub_stmt->get_stmt_hint().get_normal_hint(T_MERGE_HINT)))) {
+    } else if (!enable_group_by_placement_transform && (NULL == myhint || myhint->enable_no_group_by_pull_up())) {
+      OPT_TRACE("system var disable group by placemebt");
     } else if (ignore_tables.has_member(stmt->get_table_bit_index(table->table_id_))) {
       // skip the generated table
       OPT_TRACE("ignore this table");

src/sql/rewrite/ob_transform_groupby_pushdown.cpp

@@ -655,8 +655,11 @@ int ObTransformGroupByPushdown::do_groupby_push_down(ObSelectStmt *stmt,
   int ret = OB_SUCCESS;
   ObSqlBitSet<> outer_table_set;
   trans_happend = false;
+  ObSQLSessionInfo *session_info = NULL;
+  bool enable_group_by_placement_transform = false;
   if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) ||
-      OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_)) {
+      OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_) ||
+      OB_ISNULL(session_info = ctx_->session_info_)) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("params are invalid", K(ret), K(ctx_), K(stmt));
   } else if (OB_FAIL(ctx_->stmt_factory_->create_stmt(trans_stmt))) {
@@ -717,10 +720,19 @@ int ObTransformGroupByPushdown::do_groupby_push_down(ObSelectStmt *stmt,
       }
     }
     LOG_TRACE("push down params", K(ret));
+    bool hint_force_pushdown = false;
     if (OB_FAIL(ret) || !is_valid) {
-    } else if (OB_FAIL(check_hint_valid(static_cast<ObDMLStmt &>(*stmt), params, is_valid))) {
+    } else if (OB_FAIL(session_info->is_groupby_placement_transformation_enabled(enable_group_by_placement_transform))) {
+      LOG_WARN("failed to check group by placement transform enabled", K(ret));
+    } else if (OB_FAIL(check_hint_valid(static_cast<ObDMLStmt &>(*stmt),
+                                        params,
+                                        hint_force_pushdown,
+                                        is_valid))) {
       LOG_WARN("check hint failed", K(ret));
     } else if (!is_valid) {
+      OPT_TRACE("hint disable group by pushdown");
+    } else if (!enable_group_by_placement_transform && !hint_force_pushdown) {
+      OPT_TRACE("system variable disable group by pushdown");
     } else if (OB_FAIL(transform_groupby_push_down(trans_stmt,
                                                    flattern_joined_tables,
                                                    outer_table_set,
@@ -1633,10 +1645,14 @@ int ObTransformGroupByPushdown::has_group_by_op(ObLogicalOperator *op, bool &bre
   return ret;
 }
 
-int ObTransformGroupByPushdown::check_hint_valid(ObDMLStmt &stmt, ObIArray<PushDownParam> &params, bool &is_valid)
+int ObTransformGroupByPushdown::check_hint_valid(ObDMLStmt &stmt,
+                                                 ObIArray<PushDownParam> &params,
+                                                 bool &hint_force_pushdown,
+                                                 bool &is_valid)
 {
   int ret = OB_SUCCESS;
   is_valid = false;
+  hint_force_pushdown = false;
   const ObQueryHint *query_hint = NULL; 
   const ObGroupByPlacementHint *hint = static_cast<const ObGroupByPlacementHint*>(get_hint(stmt.get_stmt_hint()));
   const ObHint *no_rewrite = stmt.get_stmt_hint().get_no_rewrite_hint();
@@ -1657,6 +1673,9 @@ int ObTransformGroupByPushdown::check_hint_valid(ObDMLStmt &stmt, ObIArray<PushD
         LOG_WARN("get unexpected null", K(ret), K(trans_tables));
       } else {
         is_valid = hint->enable_groupby_placement(query_hint->cs_type_, trans_tables.at(i));
+        if (is_valid) {
+          hint_force_pushdown = true;
+        }
         LOG_TRACE("succeed hint valid", K(is_valid), K(*trans_tables.at(i).at(0)), K(*hint));
         if (!is_valid) { break; }
       }

src/sql/rewrite/ob_transform_groupby_pushdown.h

@@ -179,7 +179,10 @@ private:
                              ObIArray<PushDownParam> &params, 
                              ObIArray<ObSEArray<TableItem *, 4>> &trans_tables,
                               bool disassemble_join = true);
-  int check_hint_valid(ObDMLStmt &stmt, ObIArray<PushDownParam> &params, bool &is_valid);
+  int check_hint_valid(ObDMLStmt &stmt,
+                       ObIArray<PushDownParam> &params,
+                       bool &hint_force_pushdown,
+                       bool &is_valid);
 
 private:
   // help functions

src/sql/rewrite/ob_transform_temp_table.cpp

@@ -53,16 +53,29 @@ int ObTransformTempTable::transform_one_stmt(common::ObIArray<ObParentDMLStmt> &
   hash::ObHashMap<uint64_t, uint64_t> param_level;
   uint64_t min_param_level = 0;
   trans_happened = false;
+  bool enable_temp_table_transform = false;
+  bool force_temp_table_inline = false;
+  ObSQLSessionInfo *session_info = NULL;
   //当前stmt是root stmt时才改写
   if (parent_stmts.empty()) {
     void *buf = NULL;
-    if (OB_ISNULL(buf = allocator_.alloc(sizeof(TempTableTransParam)))) {
+    if (OB_ISNULL(ctx_) ||
+        OB_ISNULL(session_info = ctx_->session_info_)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("unexpect null param", K(ctx_), K(ret));
+    } else if (OB_FAIL(session_info->is_temp_table_transformation_enabled(enable_temp_table_transform))) {
+      LOG_WARN("failed to check temp table transform enabled", K(ret));
+    } else if (OB_FAIL(session_info->is_force_temp_table_inline(force_temp_table_inline))) {
+      LOG_WARN("failed to check temp table force inline", K(ret));
+    } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(TempTableTransParam)))) {
       ret = OB_ERR_UNEXPECTED;
       LOG_WARN("failed to allocate memory", K(ret));
     } else {
       trans_param_ = new(buf)TempTableTransParam;
     }
     if (OB_FAIL(ret)) {
+    } else if (!enable_temp_table_transform || force_temp_table_inline) {
+      OPT_TRACE("session variable disable temp table transform");
     } else if (OB_FAIL(parent_map.create(128, "TempTable"))) {
       LOG_WARN("failed to init stmt map", K(ret));
     } else if (OB_FAIL(param_level.create(128, "TempTable"))) {
@@ -137,11 +150,25 @@ int ObTransformTempTable::expand_temp_table(ObIArray<TempTableInfo> &temp_table_
 {
   int ret = OB_SUCCESS;
   trans_happened = false;
+  bool system_force_inline_cte = false;
+  bool system_force_materialize_cte = false;
+  ObSQLSessionInfo *session_info = NULL;
+  if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) ||
+      OB_ISNULL(ctx_->allocator_) || OB_ISNULL(ctx_->expr_factory_) ||
+      OB_ISNULL(session_info = ctx_->session_info_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("unexpect null param", K(ctx_), K(ret));
+  } else if (OB_FAIL(session_info->is_force_temp_table_inline(system_force_inline_cte))) {
+    LOG_WARN("failed to check temp table force inline", K(ret));
+  } else if (OB_FAIL(session_info->is_force_temp_table_materialize(system_force_materialize_cte))) {
+    LOG_WARN("failed to check temp table force materialize", K(ret));
+  }
   for (int64_t i = 0; OB_SUCC(ret) && i < temp_table_info.count(); ++i) {
     TempTableInfo &helper = temp_table_info.at(i);
     bool can_materia = false;
     bool force_materia = false;
     bool force_inline = false;
+    bool need_expand = false;
     OPT_TRACE("try to expand temp table:", helper.temp_table_query_);
     if (OB_ISNULL(helper.temp_table_query_)) {
       ret = OB_ERR_UNEXPECTED;
@@ -150,20 +177,28 @@ int ObTransformTempTable::expand_temp_table(ObIArray<TempTableInfo> &temp_table_
                                                 force_inline,
                                                 force_materia))) {
       LOG_WARN("failed to check force materialize", K(ret));
+    } else if (force_inline) {
+      need_expand = true;
+      OPT_TRACE("hint force inline CTE");
     } else if (force_materia) {
-      OPT_TRACE("hint force materialize");
+      //do nothing
-    } else if (!force_inline &&
+      OPT_TRACE("hint force materialize CTE");
-               OB_FAIL(check_stmt_can_materialize(helper.temp_table_query_, can_materia))) {
+    } else if (system_force_materialize_cte) {
+      //do nothing
+      OPT_TRACE("system variable force materialize CTE");
+    } else if (system_force_inline_cte) {
+      need_expand = true;
+      OPT_TRACE("system variable force inline CTE");
+    } else if (1 == helper.table_infos_.count()) {
+      need_expand = true;
+      OPT_TRACE("CTE`s refer once, force inline");
+    } else if (OB_FAIL(check_stmt_can_materialize(helper.temp_table_query_, can_materia))) {
       LOG_WARN("failed to check extract cte valid", K(ret));
-    } else if (!can_materia ||
+    } else if (!can_materia) {
-               force_inline ||
+      need_expand = true;
-               1 == helper.table_infos_.count()) {
+      OPT_TRACE("transform rule force inline CTE");
-      if (force_inline) {
-        OPT_TRACE("hint force extend temp table");
-      }
-      if (1 == helper.table_infos_.count()) {
-        OPT_TRACE("temp table if used once, will extend");
     }
+    if (OB_SUCC(ret) && need_expand) {
       //深拷贝每一份查询，还原成generate table
       ObDMLStmt *orig_stmt = helper.temp_table_query_;
       if (OB_FAIL(inner_expand_temp_table(helper))) {

src/sql/session/ob_sql_session_info.cpp

@@ -358,6 +358,62 @@ bool ObSQLSessionInfo::is_encrypt_tenant()
   return ret;
 }
 
+int ObSQLSessionInfo::is_force_temp_table_inline(bool &force_inline) const
+{
+  int ret = OB_SUCCESS;
+  int64_t with_subquery_policy = 0;
+  force_inline = false;
+  int64_t tenant_id = get_effective_tenant_id();
+  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
+  if (tenant_config.is_valid()) {
+    int64_t with_subquery_policy = tenant_config->_with_subquery;
+    if (2 == with_subquery_policy) {
+      force_inline = true;
+    }
+  }
+  return ret;
+}
+
+int ObSQLSessionInfo::is_force_temp_table_materialize(bool &force_materialize) const
+{
+  int ret = OB_SUCCESS;
+  int64_t with_subquery_policy = 0;
+  force_materialize = false;
+  int64_t tenant_id = get_effective_tenant_id();
+  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
+  if (tenant_config.is_valid()) {
+    int64_t with_subquery_policy = tenant_config->_with_subquery;
+    if (1 == with_subquery_policy) {
+      force_materialize = true;
+    }
+  }
+  return ret;
+}
+
+int ObSQLSessionInfo::is_temp_table_transformation_enabled(bool &transformation_enabled) const
+{
+  int ret = OB_SUCCESS;
+  transformation_enabled = false;
+  int64_t tenant_id = get_effective_tenant_id();
+  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
+  if (tenant_config.is_valid()) {
+    transformation_enabled = tenant_config->_xsolapi_generate_with_clause;
+  }
+  return ret;
+}
+
+int ObSQLSessionInfo::is_groupby_placement_transformation_enabled(bool &transformation_enabled) const
+{
+  int ret = OB_SUCCESS;
+  transformation_enabled = false;
+  int64_t tenant_id = get_effective_tenant_id();
+  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
+  if (tenant_config.is_valid()) {
+    transformation_enabled = tenant_config->_optimizer_group_by_placement;
+  }
+  return ret;
+}
+
 void ObSQLSessionInfo::destroy(bool skip_sys_var)
 {
   if (is_inited_) {

src/sql/session/ob_sql_session_info.h

@@ -994,6 +994,10 @@ public:
   void set_expect_group_id(int64_t group_id) { expect_group_id_ = group_id; }
 	bool get_group_id_not_expected() const { return group_id_not_expected_; }
   void set_group_id_not_expected(bool value) { group_id_not_expected_ = value; }
+  int is_force_temp_table_inline(bool &force_inline) const;
+  int is_force_temp_table_materialize(bool &force_materialize) const;
+  int is_temp_table_transformation_enabled(bool &transformation_enabled) const;
+  int is_groupby_placement_transformation_enabled(bool &transformation_enabled) const;
 
   ObSessionDDLInfo &get_ddl_info() { return ddl_info_; }
   void set_ddl_info(const ObSessionDDLInfo &ddl_info) { ddl_info_ = ddl_info; }

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result

@@ -316,6 +316,7 @@ _ob_query_rate_limit
 _ob_ssl_invited_nodes
 _ob_trans_rpc_timeout
 _optimizer_ads_time_limit
+_optimizer_group_by_placement
 _parallel_max_active_sessions
 _parallel_min_message_pool
 _parallel_server_sleep_time
@@ -349,8 +350,10 @@ _temporary_file_io_area_size
 _trace_control_info
 _tx_result_retention
 _upgrade_stage
+_with_subquery
 _xa_gc_interval
 _xa_gc_timeout
+_xsolapi_generate_with_clause
 __balance_controller
 __easy_memory_limit
 __easy_memory_reserved_percentage