/**
 * Copyright (c) 2021 OceanBase
 * OceanBase CE is licensed under Mulan PubL v2.
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
 * You may obtain a copy of Mulan PubL v2 at:
 *          http://license.coscl.org.cn/MulanPubL-2.0
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PubL v2 for more details.
 */

#include "share/inner_table/ob_inner_table_schema_constants.h"
#define USING_LOG_PREFIX SQL_ENG
#include "ob_dbms_stats_utils.h"
#include "share/stat/ob_opt_column_stat.h"
#include "share/stat/ob_opt_osg_column_stat.h"
#include "share/object/ob_obj_cast.h"
#include "share/stat/ob_opt_stat_manager.h"
#include "share/stat/ob_opt_table_stat.h"
#include "share/schema/ob_schema_struct.h"
#include "sql/engine/ob_exec_context.h"
#include "share/stat/ob_stat_item.h"
#include "share/schema/ob_part_mgr_util.h"
#include "sql/engine/expr/ob_expr_lob_utils.h"
#include "sql/ob_result_set.h"
#include "sql/optimizer/ob_opt_selectivity.h"
#include "share/stat/ob_dbms_stats_preferences.h"

#ifdef OB_BUILD_ORACLE_PL
#include "pl/sys_package/ob_json_pl_utils.h"
#endif

namespace oceanbase
{
namespace common
{

int ObDbmsStatsUtils::init_col_stats(ObIAllocator &allocator,
                                     int64_t col_cnt,
                                     ObIArray<ObOptColumnStat*> &col_stats)
{
  int ret = OB_SUCCESS;
  if (col_cnt <= 0) {
    //do nothing
  } else if (OB_FAIL(col_stats.prepare_allocate(col_cnt))) {
    LOG_WARN("failed to prepare allocate column stat", K(ret));
  } else {
    for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) {
      ObOptColumnStat *&col_stat = col_stats.at(i);
      if (OB_ISNULL(col_stat = ObOptColumnStat::malloc_new_column_stat(allocator))) {
        ret = OB_ALLOCATE_MEMORY_FAILED;
        LOG_WARN("memory is not enough", K(ret), K(col_stat));
      }
    }
  }
  return ret;
}

/* @brief ObDbmsStatsUtils::check_range_skew, check data is skewed or not
 * Based on Oracle 12c:
 * 1.for frequency histogram:
 *  if the number of value in all buckets is the same, then it's even distributed, Otherwise, it's
 *  skewed.
 * 2.for hybrid histogram: ==> refine it, TODO@jiangxiu.wt
 *  if the repeat count of value in all buckets is less than total_not_null_row_count / bucket_num,
 *  then it's even distributed, Otherwise, it's skewed.
 */
int ObDbmsStatsUtils::check_range_skew(ObHistType hist_type,
                                       const ObHistogram::Buckets &bkts,
                                       int64_t standard_cnt,
                                       bool &is_even_distributed)
{
  int ret = OB_SUCCESS;
  is_even_distributed = false;
  if (hist_type == ObHistType::FREQUENCY) {
    is_even_distributed = true;
    for (int64_t i = 0; is_even_distributed && i < bkts.count(); ++i) {
      if (i == 0) {
        is_even_distributed = standard_cnt == bkts.at(i).endpoint_num_;
      } else {
        is_even_distributed = standard_cnt == bkts.at(i).endpoint_num_ -
                                                                       bkts.at(i - 1).endpoint_num_;
      }
    }
  } else if (hist_type == ObHistType::HYBIRD) {
    is_even_distributed = true;
    for (int64_t i = 0; is_even_distributed && i < bkts.count(); ++i) {
      is_even_distributed = bkts.at(i).endpoint_repeat_count_ <= standard_cnt;
    }
  } else {/*do nothing*/}
  return ret;
}

int ObDbmsStatsUtils::batch_write(share::schema::ObSchemaGetterGuard *schema_guard,
                                  const uint64_t tenant_id,
                                  sqlclient::ObISQLConnection *conn,
                                  ObIArray<ObOptTableStat *> &table_stats,
                                  ObIArray<ObOptColumnStat*> &column_stats,
                                  const int64_t current_time,
                                  const bool is_index_stat,
                                  const bool is_online_stat,
                                  const ObObjPrintParams &print_params)
{
  int ret = OB_SUCCESS;
  if (OB_FAIL(ObOptStatManager::get_instance().batch_write(schema_guard,
                                                           tenant_id,
                                                           conn,
                                                           table_stats,
                                                           column_stats,
                                                           current_time,
                                                           is_index_stat,
                                                           print_params))) {
    LOG_WARN("failed to batch write stats", K(ret));
  } else if (!is_online_stat) {
    for (int64_t i = 0; OB_SUCC(ret) && i < table_stats.count(); ++i) {
      if (NULL != table_stats.at(i)) {
        table_stats.at(i)->~ObOptTableStat();
        table_stats.at(i) = NULL;
      }
    }
    for (int64_t i = 0; OB_SUCC(ret) && i < column_stats.count(); ++i) {
      if (NULL != column_stats.at(i)) {
        column_stats.at(i)->~ObOptColumnStat();
        column_stats.at(i) = NULL;
      }
    }
  }
  return ret;
}

int ObDbmsStatsUtils::cast_number_to_double(const number::ObNumber &src_val, double &dst_val)
{
  int ret = OB_SUCCESS;
  ObObj src_obj;
  ObObj dest_obj;
  src_obj.set_number(src_val);
  ObArenaAllocator calc_buf("ObDbmsStatsUtil", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID());
  ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation());
  if (OB_FAIL(ObObjCaster::to_type(ObDoubleType, cast_ctx, src_obj, dest_obj))) {
    LOG_WARN("failed to cast number to double type", K(ret));
  } else if (OB_FAIL(dest_obj.get_double(dst_val))) {
    LOG_WARN("failed to get double", K(ret));
  } else {
    LOG_TRACE("succeed to cast number to double", K(src_val), K(dst_val));
  }
  return ret;
}

// gather statistic related inner table should not read or write during tenant restore or on 
// standby cluster.
int ObDbmsStatsUtils::check_table_read_write_valid(const uint64_t tenant_id, bool &is_valid)
{
  int ret = OB_SUCCESS;
  is_valid = true;
  bool in_restore = false;
  if (OB_ISNULL(GCTX.schema_service_)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null", K(ret));
  } else if (OB_FAIL(GCTX.schema_service_->check_tenant_is_restore(NULL, tenant_id, in_restore))) {
    LOG_WARN("failed to check tenant is restore", K(ret));
  } else if (OB_UNLIKELY(in_restore) || GCTX.is_standby_cluster()) {
    is_valid = false;
  }
  return ret;
}

//now we support user table、sys table and virtual table to gather and use optimizer stats.
int ObDbmsStatsUtils::check_is_stat_table(share::schema::ObSchemaGetterGuard &schema_guard,
                                          const uint64_t tenant_id,
                                          const int64_t table_id,
                                          bool &is_valid)
{
  bool ret = OB_SUCCESS;
  is_valid = false;
  const ObTableSchema *table_schema = NULL;
  if (is_sys_table(table_id)) {//check sys table
    if (OB_FAIL(check_is_sys_table(schema_guard, tenant_id, table_id, is_valid))) {
      LOG_WARN("failed to check is sys table", K(ret));
    }
  } else if (is_virtual_table(table_id)) {//check virtual table
    is_valid = !is_no_stat_virtual_table(table_id);
  } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) {
    LOG_WARN("failed to get table schema", K(ret), K(tenant_id), K(table_id));
  } else if (OB_ISNULL(table_schema) || OB_UNLIKELY(!table_schema->is_normal_schema())) {
    //do nothing
  } else {//check user table
    is_valid = table_schema->is_user_table()
               || table_schema->is_external_table()
               || table_schema->is_mlog_table();
  }
  return ret;
}

int ObDbmsStatsUtils::check_is_sys_table(share::schema::ObSchemaGetterGuard &schema_guard,
                                         const uint64_t tenant_id,
                                         const int64_t table_id,
                                         bool &is_valid)
{
  bool ret = OB_SUCCESS;
  const ObSimpleTenantSchema *tenant = NULL;
  is_valid = false;
  if (!is_sys_table(table_id) ||
      ObSysTableChecker::is_sys_table_index_tid(table_id) ||
      is_sys_lob_table(table_id) ||
      table_id == share::OB_ALL_CORE_TABLE_TID ||//circular dependency,
      table_id == share::OB_ALL_TABLE_STAT_TID ||
      table_id == share::OB_ALL_COLUMN_STAT_TID ||
      table_id == share::OB_ALL_HISTOGRAM_STAT_TID ||
      table_id == share::OB_ALL_TABLE_STAT_HISTORY_TID ||
      table_id == share::OB_ALL_COLUMN_STAT_HISTORY_TID ||
      table_id == share::OB_ALL_HISTOGRAM_STAT_HISTORY_TID ||
      table_id == share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TID ||//circular dependency
      table_id == share::OB_ALL_OPTSTAT_USER_PREFS_TID ||
//bug:
      table_id == share::OB_ALL_SYS_VARIABLE_TID ||//circular dependency
      table_id == share::OB_ALL_SYS_VARIABLE_HISTORY_TID ||//circular dependency
      table_id == share::OB_ALL_MONITOR_MODIFIED_TID) {
    is_valid = false;
  } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant))) {
    LOG_WARN("fail to get tenant info", KR(ret), K(tenant_id));
  //now sys table stat only use and gather in normal tenant
  } else if (OB_ISNULL(tenant) || !tenant->is_normal()) {
    is_valid = false;
  } else {
    is_valid = true;
  }
  return ret;
}

//following virtual table access error, we temporarily disable gather table stats.
bool ObDbmsStatsUtils::is_no_stat_virtual_table(const int64_t table_id)
{
  return is_virtual_index_table(table_id) ||
         table_id == share::OB_TENANT_VIRTUAL_ALL_TABLE_TID ||
         table_id == share::OB_TENANT_VIRTUAL_TABLE_COLUMN_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_DATABASE_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TABLE_TID ||
         table_id == share::OB_TENANT_VIRTUAL_CURRENT_TENANT_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_TABLES_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_PROCEDURE_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSTAT_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_SCHEMA_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_PARTITION_INFO_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_PARTITION_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_SUB_PARTITION_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TABLEGROUP_TID ||
         table_id == share::OB_TENANT_VIRTUAL_OBJECT_DEFINITION_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TRIGGER_TID ||
         table_id == share::OB_TENANT_VIRTUAL_ALL_TABLE_AGENT_TID ||
         table_id == share::OB_ALL_VIRTUAL_INFORMATION_COLUMNS_TID ||
         table_id == share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSION_EVENT_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_ROUTINE_TID ||
         table_id == share::OB_ALL_VIRTUAL_TX_DATA_TID ||
         table_id == share::OB_ALL_VIRTUAL_TRANS_LOCK_STAT_TID ||
         table_id == share::OB_ALL_VIRTUAL_TRANS_SCHEDULER_TID ||
         table_id == share::OB_ALL_VIRTUAL_SQL_AUDIT_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_RESTORE_PREVIEW_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSTAT_ORA_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TABLE_ORA_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_PROCEDURE_ORA_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TABLEGROUP_ORA_TID ||
         table_id == share::OB_TENANT_VIRTUAL_TABLE_COLUMN_ORA_TID ||
         table_id == share::OB_TENANT_VIRTUAL_OBJECT_DEFINITION_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_AUTO_INCREMENT_REAL_AGENT_ORA_TID ||
         table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TRIGGER_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_SCHEMA_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_PARTITION_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_PARTITION_INFO_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_PROXY_SUB_PARTITION_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_PLAN_STAT_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_TRACE_SPAN_INFO_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_LOCK_WAIT_STAT_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_TRANS_STAT_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_TRANS_LOCK_STAT_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_TRANS_SCHEDULER_ORA_TID ||
         table_id == share::OB_ALL_VIRTUAL_MDS_NODE_STAT_TID ||
         table_id == share::OB_ALL_VIRTUAL_CHECKPOINT_DIAGNOSE_MEMTABLE_INFO_TID ||
         table_id == share::OB_ALL_VIRTUAL_CHECKPOINT_DIAGNOSE_CHECKPOINT_UNIT_INFO_TID;
}

bool ObDbmsStatsUtils::is_virtual_index_table(const int64_t table_id)
{
  return table_id == share::OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSION_WAIT_HISTORY_ALL_VIRTUAL_SESSION_WAIT_HISTORY_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SYSTEM_EVENT_ALL_VIRTUAL_SYSTEM_EVENT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSTAT_ALL_VIRTUAL_SESSTAT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SYSSTAT_ALL_VIRTUAL_SYSSTAT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SQL_AUDIT_ALL_VIRTUAL_SQL_AUDIT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SQL_PLAN_MONITOR_ALL_VIRTUAL_SQL_PLAN_MONITOR_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SQL_AUDIT_ORA_ALL_VIRTUAL_SQL_AUDIT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ORA_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSION_WAIT_ORA_ALL_VIRTUAL_SESSION_WAIT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSION_WAIT_HISTORY_ORA_ALL_VIRTUAL_SESSION_WAIT_HISTORY_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SESSTAT_ORA_ALL_VIRTUAL_SESSTAT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SYSSTAT_ORA_ALL_VIRTUAL_SYSSTAT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SYSTEM_EVENT_ORA_ALL_VIRTUAL_SYSTEM_EVENT_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_SQL_PLAN_MONITOR_ORA_ALL_VIRTUAL_SQL_PLAN_MONITOR_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_ASH_ALL_VIRTUAL_ASH_I1_TID ||
         table_id == share::OB_ALL_VIRTUAL_ASH_ORA_ALL_VIRTUAL_ASH_I1_TID;
}

int ObDbmsStatsUtils::parse_granularity(const ObString &granularity, ObGranularityType &granu_type)
{
  int ret = OB_SUCCESS;
  if (0 == granularity.case_compare("all")) {
    granu_type = ObGranularityType::GRANULARITY_ALL;
  } else if (0 == granularity.case_compare("auto") ||
             0 == granularity.case_compare("Z")) {
    granu_type = ObGranularityType::GRANULARITY_AUTO;
  } else if (0 == granularity.case_compare("default") ||
             0 == granularity.case_compare("global and partition")) {
    granu_type = ObGranularityType::GRANULARITY_GLOBAL_AND_PARTITION;
  } else if (0 == granularity.case_compare("approx_global and partition")) {
    granu_type = ObGranularityType::GRANULARITY_APPROX_GLOBAL_AND_PARTITION;
  } else if (0 == granularity.case_compare("global")) {
    granu_type = ObGranularityType::GRANULARITY_GLOBAL;
  } else if (0 == granularity.case_compare("partition")) {
    granu_type = ObGranularityType::GRANULARITY_PARTITION;
  } else if (0 == granularity.case_compare("subpartition")) {
    granu_type = ObGranularityType::GRANULARITY_SUBPARTITION;
  } else {
    ret = OB_ERR_DBMS_STATS_PL;
    LOG_WARN("Illegal granularity : must be AUTO | ALL | GLOBAL | PARTITION | SUBPARTITION" \
             "| GLOBAL AND PARTITION | APPROX_GLOBAL AND PARTITION", K(ret));
    LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Illegal granularity : must be AUTO | ALL | GLOBAL |" \
             " PARTITION | SUBPARTITION | GLOBAL AND PARTITION | APPROX_GLOBAL AND PARTITION");
  }
  return ret;
}

int ObDbmsStatsUtils::split_batch_write(sql::ObExecContext &ctx,
                                        ObIArray<ObOptTableStat*> &table_stats,
                                        ObIArray<ObOptColumnStat*> &column_stats,
                                        const bool is_index_stat/*default false*/,
                                        const bool is_online_stat /*default false*/)
{
  int ret = OB_SUCCESS;
  ObMySQLTransaction trans;
  if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_sql_proxy())) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null", K(ret), K(ctx.get_my_session()), K(ctx.get_sql_proxy()));
  } else if (OB_FAIL(trans.start(ctx.get_sql_proxy(), ctx.get_my_session()->get_effective_tenant_id()))) {
    LOG_WARN("fail to start transaction", K(ret));
  } else if (OB_FAIL(split_batch_write(trans.get_connection(),
                                       ctx.get_virtual_table_ctx().schema_guard_,
                                       ctx.get_my_session(),
                                       table_stats,
                                       column_stats,
                                       is_index_stat,
                                       is_online_stat))) {
    LOG_WARN("failed to split batch write", K(ret));
  }
  //end trans after writing stats.
  if (OB_SUCC(ret)) {
    if (OB_FAIL(trans.end(true))) {
      LOG_WARN("fail to commit transaction", K(ret));
    }
  } else {
    int tmp_ret = OB_SUCCESS;
    if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
      LOG_WARN("fail to roll back transaction", K(tmp_ret));
    }
  }
  return ret;
}

int ObDbmsStatsUtils::split_batch_write(share::schema::ObSchemaGetterGuard *schema_guard,
                                        sql::ObSQLSessionInfo *session_info,
                                        common::ObMySQLProxy *sql_proxy,
                                        ObIArray<ObOptTableStat*> &table_stats,
                                        ObIArray<ObOptColumnStat*> &column_stats,
                                        const bool is_index_stat/*default false*/,
                                        const bool is_online_stat /*default false*/)
{
  int ret = OB_SUCCESS;
  ObMySQLTransaction trans;
  if (OB_ISNULL(session_info) || OB_ISNULL(sql_proxy)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null", K(ret), K(session_info), K(sql_proxy));
  } else if (OB_FAIL(trans.start(sql_proxy, session_info->get_effective_tenant_id()))) {
    LOG_WARN("fail to start transaction", K(ret));
  } else if (OB_FAIL(split_batch_write(trans.get_connection(),
                                       schema_guard,
                                       session_info,
                                       table_stats,
                                       column_stats,
                                       is_index_stat,
                                       is_online_stat))) {
    LOG_WARN("failed to split batch write", K(ret));
  }
  //end trans after writing stats.
  if (OB_SUCC(ret)) {
    if (OB_FAIL(trans.end(true))) {
      LOG_WARN("fail to commit transaction", K(ret));
    }
  } else {
    int tmp_ret = OB_SUCCESS;
    if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
      LOG_WARN("fail to roll back transaction", K(tmp_ret));
    }
  }
  return ret;
}

int ObDbmsStatsUtils::split_batch_write(sql::ObExecContext &ctx,
                                        sqlclient::ObISQLConnection *conn,
                                        ObIArray<ObOptTableStat*> &table_stats,
                                        ObIArray<ObOptColumnStat*> &column_stats,
                                        const bool is_index_stat/*default false*/,
                                        const bool is_online_stat /*default false*/)
{
  int ret = OB_SUCCESS;
  if (OB_FAIL(split_batch_write(conn,
                                ctx.get_virtual_table_ctx().schema_guard_,
                                ctx.get_my_session(),
                                table_stats,
                                column_stats,
                                is_index_stat,
                                is_online_stat))) {
    LOG_WARN("failed to split batch write", K(ret));
  }
  return ret;
}

int ObDbmsStatsUtils::split_batch_write(sqlclient::ObISQLConnection *conn,
                                        share::schema::ObSchemaGetterGuard *schema_guard,
                                        sql::ObSQLSessionInfo *session_info,
                                        ObIArray<ObOptTableStat*> &table_stats,
                                        ObIArray<ObOptColumnStat*> &column_stats,
                                        const bool is_index_stat/*default false*/,
                                        const bool is_online_stat /*default false*/)
{
  int ret = OB_SUCCESS;
  int64_t idx_tab_stat = 0;
  int64_t idx_col_stat = 0;
  //avoid the write stat sql is too long, we split write table stats and column stats:
  //  write 2000 tables and 2000 columns every time.
  LOG_DEBUG("dbms stats write stats", K(table_stats), K(column_stats));
  int64_t current_time = ObTimeUtility::current_time();
  if (OB_ISNULL(session_info)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null", K(ret), K(session_info));
  }
  while (OB_SUCC(ret) &&
        (idx_tab_stat < table_stats.count() || idx_col_stat < column_stats.count())) {
    ObSEArray<ObOptTableStat*, 4> write_table_stats;
    ObSEArray<ObOptColumnStat*, 4> write_column_stats;
    if (OB_UNLIKELY(idx_tab_stat > table_stats.count() || idx_col_stat > column_stats.count())) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("get unexpectd error", K(ret), K(idx_tab_stat), K(table_stats.count()),
                                      K(idx_col_stat), K(column_stats.count()));
    } else {
      for (int64_t i = 0; OB_SUCC(ret) && i < MAX_NUM_OF_WRITE_STATS && idx_tab_stat < table_stats.count(); ++i) {
        if (OB_FAIL(write_table_stats.push_back(table_stats.at(idx_tab_stat++)))) {
          LOG_WARN("failed to push back", K(ret));
        } else {/*do nothing*/}
      }
      int64_t col_stat_cnt = 0;
      int64_t hist_stat_cnt = 0;
      while (OB_SUCC(ret) &&
             col_stat_cnt < MAX_NUM_OF_WRITE_STATS &&
             hist_stat_cnt < MAX_NUM_OF_WRITE_STATS &&
             idx_col_stat < column_stats.count()) {
        ObOptColumnStat *cur_opt_col_stat = column_stats.at(idx_col_stat);
        if (OB_ISNULL(cur_opt_col_stat)) {
          ret = OB_ERR_UNEXPECTED;
          LOG_WARN("get unexpected null", K(ret), K(cur_opt_col_stat));
        } else if (OB_FAIL(write_column_stats.push_back(cur_opt_col_stat))) {
          LOG_WARN("failed to push back", K(ret));
        } else {
          ++ col_stat_cnt;
          ++ idx_col_stat;
          if (cur_opt_col_stat->get_histogram().is_valid()) {
            hist_stat_cnt += cur_opt_col_stat->get_histogram().get_bucket_size();
          }
        }
      }
    }
    if (OB_SUCC(ret)) {
      if (OB_FAIL(ObDbmsStatsUtils::batch_write(schema_guard,
                                                session_info->get_effective_tenant_id(),
                                                conn,
                                                write_table_stats,
                                                write_column_stats,
                                                current_time,
                                                is_index_stat,
                                                is_online_stat,
                                                CREATE_OBJ_PRINT_PARAM(session_info)))) {
        LOG_WARN("failed to batch write stats", K(ret), K(idx_tab_stat), K(idx_col_stat));
      } else {/*do nothing*/}
    }
  }
  return ret;
}

bool ObDbmsStatsUtils::is_subpart_id(const ObIArray<PartInfo> &partition_infos,
                                     const int64_t partition_id,
                                     int64_t &part_id)
{
  bool is_true = false;
  part_id = OB_INVALID_ID;
  for (int64_t i = 0; !is_true && i < partition_infos.count(); ++i) {
    is_true = (partition_infos.at(i).first_part_id_ != OB_INVALID_ID &&
               partition_id == partition_infos.at(i).part_id_);
    if (is_true) {
      part_id = partition_infos.at(i).first_part_id_;
    }
  }
  return is_true;
}

int ObDbmsStatsUtils::get_valid_duration_time(const int64_t start_time,
                                              const int64_t max_duration_time,
                                              int64_t &valid_duration_time)
{
  int ret = OB_SUCCESS;
  const int64_t current_time = ObTimeUtility::current_time();
  if (OB_UNLIKELY(start_time <= 0)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected error", K(start_time), K(ret));
  } else if (max_duration_time == -1) {
    //do nothing
  } else if (OB_UNLIKELY(current_time - start_time >= max_duration_time)) {
    ret = OB_TIMEOUT;
    LOG_WARN("reach the duration time", K(ret), K(current_time), K(start_time), K(max_duration_time));
  } else {
    valid_duration_time = max_duration_time - (current_time - start_time);
  }
  return ret;
}

int ObDbmsStatsUtils::get_dst_partition_by_tablet_id(sql::ObExecContext &ctx,
                                                     const uint64_t tablet_id,
                                                     const ObIArray<PartInfo> &partition_infos,
                                                     int64_t &partition_id)
{
  int ret = OB_SUCCESS;
  partition_id = -1;
  ObTabletID tmp_tablet_id(tablet_id);
  for (int64_t i = 0; partition_id == -1 && i < partition_infos.count(); ++i) {
    if (partition_infos.at(i).tablet_id_ == tmp_tablet_id) {
      partition_id = partition_infos.at(i).part_id_;
    }
  }
  if (OB_UNLIKELY(partition_id == -1)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null", K(ret), K(tablet_id), K(partition_infos));
  } else {
    LOG_TRACE("succeed to get dst partition by tablet id", K(tablet_id), K(partition_infos), K(partition_id));
  }
  return ret;
}

int ObDbmsStatsUtils::calssify_opt_stat(const ObIArray<ObOptStat> &opt_stats,
                                        ObIArray<ObOptTableStat*> &table_stats,
                                        ObIArray<ObOptColumnStat*> &column_stats)
{
  int ret = OB_SUCCESS;
  for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) {
    if (OB_FAIL(table_stats.push_back(opt_stats.at(i).table_stat_))) {
      LOG_WARN("failed to push back table stat", K(ret));
    } else {
      for (int64_t j = 0; OB_SUCC(ret) && j < opt_stats.at(i).column_stats_.count(); ++j) {
        if (OB_ISNULL(opt_stats.at(i).column_stats_.at(j))) {
          ret = OB_ERR_UNEXPECTED;
          LOG_WARN("get unexpected null", K(ret), K(opt_stats.at(i).column_stats_.at(j)));
        } else if (opt_stats.at(i).column_stats_.at(j)->is_valid()) {
          ret = column_stats.push_back(opt_stats.at(i).column_stats_.at(j));
        }
      }
    }
  }
  return ret;
}

// merge history stats and online stats
// for each stats in history, first search wether is in online_stats(map).
//  if not exists, set_refactored.
//  else merge old and new stats;
int ObDbmsStatsUtils::merge_tab_stats(const ObTableStatParam &param,
                                      const TabStatIndMap &online_table_stats,
                                      common::ObIArray<ObOptTableStat *> &old_tab_stats,
                                      common::ObIArray<ObOptTableStat *> &dst_tab_stats)
{
  int ret = OB_SUCCESS;
  ObOptTableStat *tmp_tab_stat = NULL;

  // the map is faster than array traversal
  for (int64_t i = 0; OB_SUCC(ret) && i < old_tab_stats.count(); i++) {
    if (OB_ISNULL(old_tab_stats.at(i))) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("get unexpected error", K(ret), K(old_tab_stats.at(i)));
    } else {
      ObOptTableStat::Key key(param.tenant_id_,
                              old_tab_stats.at(i)->get_table_id(),
                              old_tab_stats.at(i)->get_partition_id());
      if (OB_FAIL(online_table_stats.get_refactored(key, tmp_tab_stat))) {
        if (OB_HASH_NOT_EXIST != ret) {
          LOG_WARN("failed to find in hashmap", K(ret));
        } else {
          ret = OB_SUCCESS;
        }
      } else if (OB_FAIL(tmp_tab_stat->merge_table_stat(*old_tab_stats.at(i)))) {
        //merge
        LOG_WARN("fail to merge new table stat with old table stat", K(ret));
      }
    }
  }

  // put all stats into array for future use.
  FOREACH_X(it, online_table_stats, OB_SUCC(ret)) {
    if (is_part_id_valid(param, it->second->get_partition_id())) {
      if (OB_FAIL(dst_tab_stats.push_back(it->second))) {
        LOG_WARN("fail to push back table stats", K(ret));
      }
    }
  }
  LOG_TRACE("merge mtab stats", K(old_tab_stats), K(dst_tab_stats));

  return ret;
}

int ObDbmsStatsUtils::merge_col_stats(const ObTableStatParam &param,
                                      const ColStatIndMap &online_column_stats,
                                      common::ObIArray<ObOptColumnStat*> &old_col_stats,
                                      common::ObIArray<ObOptColumnStat*> &dst_col_stats)
{
  int ret = OB_SUCCESS;
  ObOptColumnStat *tmp_col_stat = NULL;
  for (int64_t i = 0; OB_SUCC(ret) && i < old_col_stats.count(); i++) {
    if (OB_ISNULL(old_col_stats.at(i))) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("get unexpected error", K(ret), K(old_col_stats.at(i)));
    } else {
      ObOptColumnStat::Key key(param.tenant_id_,
                              old_col_stats.at(i)->get_table_id(),
                              old_col_stats.at(i)->get_partition_id(),
                              old_col_stats.at(i)->get_column_id());
      if (OB_FAIL(online_column_stats.get_refactored(key, tmp_col_stat))) {
        if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) {
          LOG_WARN("failed to find in hashmap", K(ret));
        } else {
          ret = OB_SUCCESS;
        }
      } else if (OB_FAIL(tmp_col_stat->merge_column_stat(*old_col_stats.at(i)))) {
        LOG_WARN("fail to merge new table stat with old table stat", K(ret));
      }
    }
  }

  FOREACH_X(it, online_column_stats, OB_SUCC(ret)) {
    // after merge, we need to re-calc ndv from llc
    bool is_valid = false;
    ObOptColumnStat *col_stat = NULL;
    if (OB_ISNULL(col_stat = it->second)) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("get unexpected null pointer", K(ret));
    } else if (is_part_id_valid(param, col_stat->get_partition_id())) {
      col_stat->set_num_distinct(ObGlobalNdvEval::get_ndv_from_llc(col_stat->get_llc_bitmap()));
      if (OB_UNLIKELY(col_stat->get_num_distinct() < 0)) {
        ret = OB_ERR_UNEXPECTED;
        LOG_WARN("get unexpected error", KPC(col_stat), K(old_col_stats), K(ret));
      } else if (OB_FAIL(dst_col_stats.push_back(col_stat))) {
        LOG_WARN("fail to push back table stats", K(ret));
      }
    }
  }
  LOG_TRACE("merge col stats", K(old_col_stats), K(dst_col_stats));
  return ret;
}

bool ObDbmsStatsUtils::is_part_id_valid(const ObTableStatParam &param,
                                           const ObObjectID part_id)
{
  bool is_valid = false;
  if (param.global_stat_param_.need_modify_) {
    if (part_id == param.global_part_id_) {
      is_valid = true;
    }
  }
  if (!is_valid && param.part_stat_param_.need_modify_) {
    for (int64_t i = 0; !is_valid && i < param.part_infos_.count(); i++) {
      if (part_id == param.part_infos_.at(i).part_id_) {
        is_valid = true;
      }
    }
  }
  if (!is_valid && param.subpart_stat_param_.need_modify_) {
    for (int64_t i = 0; !is_valid && i < param.subpart_infos_.count(); i++) {
      if (part_id == param.subpart_infos_.at(i).part_id_) {
        is_valid = true;
      }
    }
  }
  return is_valid;
}

int ObDbmsStatsUtils::get_part_infos(const ObTableSchema &table_schema,
                                     ObIAllocator &allocator,
                                     ObIArray<PartInfo> &part_infos,
                                     ObIArray<PartInfo> &subpart_infos,
                                     ObIArray<int64_t> &part_ids,
                                     ObIArray<int64_t> &subpart_ids,
                                     OSGPartMap *part_map/*default null*/)
{
  int ret = OB_SUCCESS;
  const ObPartition *part = NULL;
  const bool is_twopart = (table_schema.get_part_level() == share::schema::PARTITION_LEVEL_TWO);
  ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL;
  if (table_schema.is_partitioned_table()) {
    ObPartIterator iter(table_schema, check_partition_mode);
    while (OB_SUCC(ret) && OB_SUCC(iter.next(part))) {
      if (OB_ISNULL(part)) {
        ret = OB_ERR_UNEXPECTED;
        LOG_WARN("get null partition", K(ret), K(part));
      } else {
        PartInfo part_info;
        part_info.part_id_ = part->get_part_id();
        part_info.tablet_id_ = part->get_tablet_id();
        if (OB_FAIL(ob_write_string(allocator, part->get_part_name(), part_info.part_name_))) {
          LOG_WARN("failed to write string", K(ret));
        } else if (OB_NOT_NULL(part_map)) {
          OSGPartInfo part_info;
          part_info.part_id_ = part->get_part_id();
          part_info.tablet_id_ = part->get_tablet_id();
          if (OB_FAIL(part_map->set_refactored(part->get_part_id(), part_info))) {
            LOG_WARN("fail to add part info to hashmap", K(ret), K(part_info), K(part->get_part_id()));
          }
        }
        int64_t origin_cnt = subpart_infos.count();
        if (OB_FAIL(ret)) {
        } else if (OB_FAIL(part_infos.push_back(part_info))) {
          LOG_WARN("failed to push back part info", K(ret));
        } else if (OB_FAIL(part_ids.push_back(part_info.part_id_))) {
          LOG_WARN("failed to push back part id", K(ret));
        } else if (is_twopart &&
                   OB_FAIL(get_subpart_infos(table_schema, part, allocator, subpart_infos, subpart_ids, part_map))) {
          LOG_WARN("failed to get subpart info", K(ret));
        } else {
          part_infos.at(part_infos.count() - 1).subpart_cnt_ = subpart_infos.count() - origin_cnt;
          LOG_TRACE("succeed to get table part infos", K(part_info));
        }
      }
    }
    ret = (ret == OB_ITER_END ? OB_SUCCESS : ret);
  }
  return ret;
}

int ObDbmsStatsUtils::get_subpart_infos(const ObTableSchema &table_schema,
                                        const ObPartition *part,
                                        ObIAllocator &allocator,
                                        ObIArray<PartInfo> &subpart_infos,
                                        ObIArray<int64_t> &subpart_ids,
                                        OSGPartMap *part_map/*default NULL*/)
{
  int ret = OB_SUCCESS;
  ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL;
  if (OB_ISNULL(part) ||
      OB_UNLIKELY(table_schema.get_part_level() != share::schema::PARTITION_LEVEL_TWO)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("partition is null", K(ret), K(table_schema.get_part_level()));
  } else {
    const ObSubPartition *subpart = NULL;
    ObSubPartIterator sub_iter(table_schema, *part, check_partition_mode);
    while (OB_SUCC(ret) && OB_SUCC(sub_iter.next(subpart))) {
      if (OB_ISNULL(subpart)) {
        ret = OB_ERR_UNEXPECTED;
        LOG_WARN("get null subpartition", K(ret));
      } else {
        PartInfo subpart_info;
        subpart_info.part_name_ = subpart->get_part_name();
        subpart_info.part_id_ = subpart->get_sub_part_id(); // means object_id
        subpart_info.tablet_id_ = subpart->get_tablet_id();
        subpart_info.first_part_id_ = part->get_part_id();
        if (OB_FAIL(ob_write_string(allocator, subpart->get_part_name(), subpart_info.part_name_))) {
          LOG_WARN("failed to write string", K(ret));
        } else if (OB_NOT_NULL(part_map)) {
          OSGPartInfo part_info;
          part_info.part_id_ = part->get_part_id();
          part_info.tablet_id_ = subpart->get_tablet_id();
          if (OB_FAIL(part_map->set_refactored(subpart->get_sub_part_id(), part_info))) {
            LOG_WARN("fail to add part info to hashmap", K(ret), K(part_info), K(subpart->get_sub_part_id()));
          }
        }
        if (OB_FAIL(ret)) {
        } else if (OB_FAIL(subpart_infos.push_back(subpart_info))) {
          LOG_WARN("failed to push back subpart_info", K(ret));
        } else if (OB_FAIL(subpart_ids.push_back(subpart_info.part_id_))) {
          LOG_WARN("failed to push back part id", K(ret));
        } else {
          LOG_TRACE("succeed to get table part infos", K(subpart_info)) ;
        }
      }
    }
    ret = (ret == OB_ITER_END ? OB_SUCCESS : ret);
  }
  return ret;
}

int ObDbmsStatsUtils::truncate_string_for_opt_stats(const ObObj *old_obj,
                                                    ObIAllocator &alloc,
                                                    ObObj *&new_obj)
{
  int ret = OB_SUCCESS;
  bool is_truncated = false;
  if (OB_ISNULL(old_obj)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null");
  } else if (ObColumnStatParam::is_valid_opt_col_type(old_obj->get_type()) && old_obj->is_string_type()) {
    if(old_obj->is_lob_storage()) {
      ObObj *tmp_obj = NULL;
      ObString str = old_obj->get_string();
      bool can_reuse = false;
      if (OB_FAIL(check_text_can_reuse(*old_obj, can_reuse))) {
        LOG_WARN("failed to check text obj can reuse", K(ret), K(*old_obj));
      } else if (can_reuse) {
        // do nothing
      } else if (OB_FAIL(sql::ObTextStringHelper::read_prefix_string_data(&alloc, *old_obj, str, OPT_STATS_MAX_VALUE_CHAR_LEN))) {
        LOG_WARN("failed to read prefix string data", K(ret));
      } else if (OB_ISNULL(tmp_obj = static_cast<ObObj*>(alloc.alloc(sizeof(ObObj))))) {
        ret = OB_ALLOCATE_MEMORY_FAILED;
        LOG_WARN("fail to alloc buf", K(ret));
      } else if (OB_FAIL(sql::ObTextStringHelper::str_to_lob_storage_obj(alloc, str, *tmp_obj))) {
        LOG_WARN("failed to convert str to lob", K(ret));
      } else {
        tmp_obj->set_meta_type(old_obj->get_meta());
        new_obj = tmp_obj;
        is_truncated = true;
      }
    } else {
      ObString str;
      if (OB_FAIL(old_obj->get_string(str))) {
        LOG_WARN("failed to get string", K(ret), K(str));
      } else {
        ObObj *tmp_obj = NULL;
        int64_t truncated_str_len = get_truncated_str_len(str, old_obj->get_collation_type());
        if (truncated_str_len == str.length()) {
          //do nothing
        } else if (OB_UNLIKELY(truncated_str_len < 0)) {
          ret = OB_ERR_UNEXPECTED;
          LOG_WARN("get unexpected error", K(ret), K(old_obj), K(str), K(truncated_str_len));
        } else if (OB_ISNULL(tmp_obj = static_cast<ObObj*>(alloc.alloc(sizeof(ObObj))))) {
          ret = OB_ALLOCATE_MEMORY_FAILED;
          LOG_WARN("fail to alloc buf", K(ret));
        } else {
          tmp_obj->set_varchar(str.ptr(), static_cast<int32_t>(truncated_str_len));
          tmp_obj->set_meta_type(old_obj->get_meta());
          new_obj = tmp_obj;
          is_truncated = true;
        }
      }
    }
  }
  if (OB_SUCC(ret) && !is_truncated) {
    new_obj = const_cast<ObObj *>(old_obj);
  }
  LOG_TRACE("Succeed to truncate string obj for opt stats", KPC(old_obj), KPC(new_obj), K(is_truncated));
  return ret;
}


int ObDbmsStatsUtils::truncate_string_for_opt_stats(ObObj &obj, ObIAllocator &allocator)
{
  int ret = OB_SUCCESS;
  if (ObColumnStatParam::is_valid_opt_col_type(obj.get_type()) && obj.is_string_type()) {
    if(obj.is_lob_storage()) {
      ObObjMeta ori_meta = obj.get_meta();
      ObString str = obj.get_string();
      bool can_reuse = false;
      if (OB_FAIL(check_text_can_reuse(obj, can_reuse))) {
        LOG_WARN("failed to check text obj can reuse", K(ret), K(obj));
      } else if (can_reuse) {
        // do nothing
      } else if (OB_FAIL(sql::ObTextStringHelper::read_prefix_string_data(&allocator, obj, str, OPT_STATS_MAX_VALUE_CHAR_LEN))) {
        LOG_WARN("failed to read prefix string data", K(ret));
      } else if (OB_FAIL(sql::ObTextStringHelper::str_to_lob_storage_obj(allocator, str, obj))) {
        LOG_WARN("failed to convert str to lob", K(ret));
      } else {
        obj.set_meta_type(ori_meta);
        LOG_TRACE("Succeed to truncate text obj for opt stats", K(ret), K(obj), K(str));
      }
    } else {
      ObString str = obj.get_string();
      int64_t truncated_str_len = get_truncated_str_len(str, obj.get_collation_type());
      if (OB_UNLIKELY(truncated_str_len < 0)) {
        ret = OB_ERR_UNEXPECTED;
        LOG_WARN("get unexpected error", K(ret), K(obj), K(str), K(truncated_str_len));
      } else if (truncated_str_len == str.length()) {
        // do nothing
      } else {
        str.assign_ptr(str.ptr(), static_cast<int32_t>(truncated_str_len));
        obj.set_common_value(str);
      }
      LOG_TRACE("Succeed to truncate string obj for opt stats", K(ret), K(obj), K(str), K(truncated_str_len));
    }
  }
  return ret;
}


int64_t ObDbmsStatsUtils::get_truncated_str_len(const ObString &str, const ObCollationType cs_type)
{
  int64_t truncated_str_len = 0;
  int64_t mb_len = ObCharset::strlen_char(cs_type, str.ptr(), str.length());
  if (mb_len <= OPT_STATS_MAX_VALUE_CHAR_LEN) {//keep origin str
    truncated_str_len = str.length();
  } else {//get max truncate str len
    truncated_str_len = ObCharset::charpos(cs_type, str.ptr(), str.length(), OPT_STATS_MAX_VALUE_CHAR_LEN);
  }
  LOG_TRACE("Succeed to get truncated str len", K(str), K(cs_type), K(truncated_str_len));
  return truncated_str_len;
}

int64_t ObDbmsStatsUtils::check_text_can_reuse(const ObObj &obj, bool &can_reuse)
{
  int ret = OB_SUCCESS;
  ObString str = obj.get_string();
  ObLobLocatorV2 lob(str, obj.has_lob_header());
  if (lob.has_inrow_data()) {
    if (OB_FAIL(lob.get_inrow_data(str))) {
      LOG_WARN("failed to get inrow data", K(ret));
    } else {
      int64_t mb_len = ObCharset::strlen_char(obj.get_collation_type(), str.ptr(), str.length());
      if (mb_len <= OPT_STATS_MAX_VALUE_CHAR_LEN) {
        can_reuse = true;
      }
    }
  }
  return ret;
}

int ObDbmsStatsUtils::get_current_opt_stats(const ObTableStatParam &param,
                                            ObIArray<ObOptTableStatHandle> &cur_tab_handles,
                                            ObIArray<ObOptColumnStatHandle> &cur_col_handles)
{
  int ret = OB_SUCCESS;
  ObSEArray<int64_t, 4> part_ids;
  ObSEArray<uint64_t, 4> column_ids;
  if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) {
    LOG_WARN("failed to get part ids and column ids", K(ret));
  } else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_, part_ids, column_ids))) {
    LOG_WARN("failed to erase stat cache", K(ret));
  } else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_,
                                                                     param.table_id_,
                                                                     part_ids,
                                                                     cur_tab_handles))) {
    LOG_WARN("failed to get table stat", K(ret));
  } else if (OB_FAIL(ObOptStatManager::get_instance().get_column_stat(param.tenant_id_,
                                                                      param.table_id_,
                                                                      part_ids,
                                                                      column_ids,
                                                                      cur_col_handles))) {
    LOG_WARN("failed to get column stat", K(ret));
  }
  return ret;
}

int ObDbmsStatsUtils::get_part_ids_and_column_ids(const ObTableStatParam &param,
                                                  ObIArray<int64_t> &part_ids,
                                                  ObIArray<uint64_t> &column_ids,
                                                  bool need_stat_column /*default false*/)
{
  int ret = OB_SUCCESS;
  //get part ids
  if (param.global_stat_param_.need_modify_) {
    int64_t part_id = param.global_part_id_;
    if (OB_FAIL(part_ids.push_back(part_id))) {
        LOG_WARN("failed to push back", K(ret));
    } else {/*do nothing*/}
  }

  if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) {
    for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) {
      if (OB_FAIL(part_ids.push_back(param.part_infos_.at(i).part_id_))) {
        LOG_WARN("failed to push back", K(ret));
      } else {/*do nothing*/}
    }
    for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) {
      if (OB_FAIL(part_ids.push_back(param.approx_part_infos_.at(i).part_id_))) {
        LOG_WARN("failed to push back", K(ret));
      } else {/*do nothing*/}
    }
  }

  if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) {
    for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) {
      if (OB_FAIL(part_ids.push_back(param.subpart_infos_.at(i).part_id_))) {
        LOG_WARN("failed to push back", K(ret));
      } else {/*do nothing*/}
    }
  }
  //get column ids
  for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) {
    if (!need_stat_column || param.column_params_.at(i).need_basic_stat()) {
      if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) {
        LOG_WARN("failed to push back", K(ret));
      } else {/*do nothing*/}
    }
  }
  return ret;
}

int ObDbmsStatsUtils::erase_stat_cache(const uint64_t tenant_id,
                                       const uint64_t table_id,
                                       const ObIArray<int64_t> &part_ids,
                                       const ObIArray<uint64_t> &column_ids)
{
  int ret = OB_SUCCESS;
  if (OB_FAIL(ObOptStatManager::get_instance().erase_table_stat(tenant_id, table_id, part_ids))) {
    LOG_WARN("failed to erase table stats", K(ret));
  } else if (OB_FAIL(ObOptStatManager::get_instance().erase_column_stat(tenant_id,
                                                                        table_id,
                                                                        part_ids,
                                                                        column_ids))) {
    LOG_WARN("failed to erase column stats", K(ret));
  } else {/*do nothing*/}
  return ret;
}

bool ObDbmsStatsUtils::find_part(const ObIArray<PartInfo> &part_infos,
                                 const ObString &part_name,
                                 bool is_sensitive_compare,
                                 PartInfo &part)
{
  bool found = false;
  for (int64_t i = 0; !found && i < part_infos.count(); ++i) {
    if ((is_sensitive_compare &&
         ObCharset::case_sensitive_equal(part_name, part_infos.at(i).part_name_)) ||
        (!is_sensitive_compare &&
         ObCharset::case_insensitive_equal(part_name, part_infos.at(i).part_name_))) {
      part = part_infos.at(i);
      found = true;
    }
  }
  return found;
}

int ObDbmsStatsUtils::remove_stat_gather_param_partition_info(int64_t reserved_partition_id,
                                                              ObOptStatGatherParam &param)
{
  int ret = OB_SUCCESS;
  bool found_it = false;
  for (int64_t i = 0; !found_it && i < param.partition_infos_.count(); ++i) {
    if (reserved_partition_id == param.partition_infos_.at(i).part_id_) {
      found_it = true;
      PartInfo tmp_part_info = param.partition_infos_.at(i);
      param.partition_infos_.reset();
      if (OB_FAIL(param.partition_infos_.push_back(tmp_part_info))) {
        LOG_WARN("failed to push back", K(ret));
      }
    }
  }
  if (!found_it) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected error", K(ret), K(param), K(reserved_partition_id));
  }
  return ret;
}

int ObDbmsStatsUtils::prepare_gather_stat_param(const ObTableStatParam &param,
                                                StatLevel stat_level,
                                                const PartitionIdBlockMap *partition_id_block_map,
                                                bool is_split_gather,
                                                int64_t gather_vectorize,
                                                bool use_column_store,
                                                ObOptStatGatherParam &gather_param)
{
  int ret = OB_SUCCESS;
  gather_param.tenant_id_ = param.tenant_id_;
  gather_param.db_name_ = param.db_name_;
  gather_param.tab_name_ = param.tab_name_;
  gather_param.table_id_ = param.table_id_;
  gather_param.stat_level_ = stat_level;
  if (stat_level == SUBPARTITION_LEVEL) {
    gather_param.need_histogram_ = param.subpart_stat_param_.gather_histogram_;
    gather_param.is_specify_partition_ = param.subpart_infos_.count() != param.all_subpart_infos_.count();
  } else if (stat_level == PARTITION_LEVEL) {
    gather_param.need_histogram_ = param.part_stat_param_.gather_histogram_;
    gather_param.is_specify_partition_ = param.part_infos_.count() != param.all_part_infos_.count();
  } else if (stat_level == TABLE_LEVEL) {
    gather_param.need_histogram_ = param.global_stat_param_.gather_histogram_;
  }
  gather_param.sample_info_.is_sample_ = param.sample_info_.is_sample_;
  gather_param.sample_info_.is_block_sample_ = param.sample_info_.is_block_sample_;
  gather_param.sample_info_.sample_type_ = param.sample_info_.sample_type_;
  gather_param.sample_info_.sample_value_ = param.sample_info_.sample_value_;
  gather_param.degree_ = param.degree_;
  gather_param.max_duration_time_ = param.duration_time_;
  gather_param.allocator_ = param.allocator_;
  gather_param.partition_id_block_map_ = partition_id_block_map;
  gather_param.gather_start_time_ = ObTimeUtility::current_time();
  gather_param.stattype_ = param.stattype_;
  gather_param.is_split_gather_ = is_split_gather;
  gather_param.need_approx_ndv_ = param.need_approx_ndv_;
  gather_param.data_table_name_ = param.data_table_name_;
  gather_param.global_part_id_ = param.global_part_id_;
  gather_param.gather_vectorize_ = gather_vectorize;
  gather_param.use_column_store_ = use_column_store;
  return gather_param.column_group_params_.assign(param.column_group_params_);
}

int ObDbmsStatsUtils::get_current_opt_stats(ObIAllocator &allocator,
                                            sqlclient::ObISQLConnection *conn,
                                            const ObTableStatParam &param,
                                            ObIArray<ObOptTableStat *> &table_stats,
                                            ObIArray<ObOptColumnStat *> &column_stats)
{
  int ret = OB_SUCCESS;
  ObSEArray<int64_t, 4> part_ids;
  ObSEArray<uint64_t, 4> column_ids;
  if (OB_ISNULL(conn)) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected error", K(ret));
  } else if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) {
    LOG_WARN("failed to get part ids and column ids", K(ret));
  } else if (OB_FAIL(table_stats.prepare_allocate(part_ids.count()))) {
    LOG_WARN("failed to prepare allocate table stats", K(ret));
  } else if (OB_FAIL(column_stats.prepare_allocate(part_ids.count() * column_ids.count())))  {
    LOG_WARN("failed to prepare allocate column stats", K(ret));
  } else {
    ObOptStatManager &stat_manager = ObOptStatManager::get_instance();
    ObSEArray<ObOptKeyColumnStat, 4> key_column_stats;
    for (int64_t i = 0; OB_SUCC(ret) && i < part_ids.count(); ++i) {
      void *ptr = NULL;
      if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObOptTableStat)))) {
        ret = OB_ALLOCATE_MEMORY_FAILED;
        LOG_WARN("memory is not enough", K(ret), K(ptr));
      } else {
        table_stats.at(i) = new (ptr) ObOptTableStat();
        table_stats.at(i)->set_table_id(param.table_id_);
        table_stats.at(i)->set_partition_id(part_ids.at(i));
        for (int64_t j = 0; OB_SUCC(ret) && j < column_ids.count(); ++j) {
          void *ptr1 = NULL;
          int64_t idx = i * column_ids.count() + j;
          if (OB_UNLIKELY(idx >= column_stats.count())) {
            ret = OB_ERR_UNEXPECTED;
            LOG_WARN("get unexpected error", K(ret), K(idx), K(column_stats.count()));
          } else if (OB_ISNULL(ptr1 = allocator.alloc(sizeof(ObOptColumnStat::Key))) ||
                     OB_ISNULL(column_stats.at(idx) = ObOptColumnStat::malloc_new_column_stat(allocator))) {
            ret = OB_ALLOCATE_MEMORY_FAILED;
            LOG_WARN("memory is not enough", K(ret), K(ptr1), K(column_stats.at(idx)));
          } else {
            ObOptColumnStat::Key *col_key = new (ptr1) ObOptColumnStat::Key(param.tenant_id_,
                                                                            param.table_id_,
                                                                            part_ids.at(i),
                                                                            column_ids.at(j));
            column_stats.at(idx)->set_table_id(param.table_id_);
            column_stats.at(idx)->set_partition_id(part_ids.at(i));
            column_stats.at(idx)->set_column_id(column_ids.at(j));
            if (OB_FAIL(key_column_stats.push_back(ObOptKeyColumnStat(col_key, column_stats.at(idx))))) {
              LOG_WARN("failed to push back", K(ret));
            } else {/*do nothing*/}
          }
        }
      }
    }
    if (OB_SUCC(ret)) {
      if (OB_FAIL(stat_manager.get_stat_service().get_sql_service().batch_fetch_table_stats(conn,
                                                                                            param.tenant_id_,
                                                                                            param.table_id_,
                                                                                            part_ids,
                                                                                            table_stats))) {
        LOG_WARN("failed to batch fetch table stats", K(ret));
      } else if (OB_FAIL(stat_manager.get_stat_service().get_sql_service().fetch_column_stat(param.tenant_id_,
                                                                                             allocator,
                                                                                             key_column_stats,
                                                                                             false,
                                                                                             conn))) {
        LOG_WARN("failed to fetch table stats", K(ret));
      } else {
        for (int64_t i = 0; i < table_stats.count(); ++i) {
          for (int64_t j = 0; j < column_ids.count(); ++j) {
            int64_t idx = i * column_ids.count() + j;
            if (OB_UNLIKELY(idx >= column_stats.count())) {
              ret = OB_ERR_UNEXPECTED;
              LOG_WARN("get unexpected error", K(ret), K(idx), K(column_stats.count()));
            } else if (table_stats.at(i) != NULL && column_stats.at(idx) != NULL && column_stats.at(idx)->get_num_distinct() > 0) {
              column_stats.at(idx)->set_num_not_null(table_stats.at(i)->get_row_count() - column_stats.at(idx)->get_num_null());
              column_stats.at(idx)->set_total_col_len(column_stats.at(idx)->get_num_not_null() * column_stats.at(idx)->get_avg_len());
            }
          }
        }
      }
    }
  }
  return ret;
}

int ObDbmsStatsUtils::merge_split_gather_tab_stats(ObIArray<ObOptTableStat *> &all_tstats,
                                                   ObIArray<ObOptTableStat *> &cur_all_tstats)
{
  int ret = OB_SUCCESS;
  if (all_tstats.empty()) {
    if (OB_FAIL(all_tstats.assign(cur_all_tstats))) {
      LOG_WARN("failed to assign");
    } else {
      cur_all_tstats.reset();
    }
  } else if (OB_UNLIKELY(all_tstats.count() != cur_all_tstats.count())) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected error", K(ret), K(all_tstats), K(cur_all_tstats));
  } else {
    for (int64_t i = 0; OB_SUCC(ret) && i < all_tstats.count(); ++i) {
      if (OB_ISNULL(all_tstats.at(i)) ||
          OB_ISNULL(cur_all_tstats.at(i))) {
        ret = OB_ERR_UNEXPECTED;
        LOG_WARN("get unexpected error", K(ret), KPC(all_tstats.at(i)), K(cur_all_tstats.at(i)));
      } else if (!all_tstats.at(i)->is_valid() && cur_all_tstats.at(i)->is_valid()) {
        *all_tstats.at(i) = *cur_all_tstats.at(i);
        cur_all_tstats.at(i)->~ObOptTableStat();
        cur_all_tstats.at(i) = NULL;
      } else if (OB_UNLIKELY(all_tstats.at(i)->get_table_id() != cur_all_tstats.at(i)->get_table_id() ||
                             all_tstats.at(i)->get_partition_id() != cur_all_tstats.at(i)->get_partition_id())) {
        ret = OB_ERR_UNEXPECTED;
        LOG_WARN("get unexpected error", K(ret), KPC(all_tstats.at(i)), K(cur_all_tstats.at(i)));
      } else {
        all_tstats.at(i)->set_row_count(std::max(all_tstats.at(i)->get_row_count(), cur_all_tstats.at(i)->get_row_count()));
        all_tstats.at(i)->set_avg_row_size(all_tstats.at(i)->get_avg_row_size() + cur_all_tstats.at(i)->get_avg_row_size());
        //free memory
        cur_all_tstats.at(i)->~ObOptTableStat();
        cur_all_tstats.at(i) = NULL;
      }
    }
    cur_all_tstats.reset();
  }
  return ret;
}

int ObDbmsStatsUtils::check_all_cols_range_skew(const ObIArray<ObColumnStatParam> &column_params,
                                                ObIArray<ObOptStat> &opt_stats)
{
  int ret = OB_SUCCESS;
  for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) {
    ObIArray<ObOptColumnStat *> &col_stats = opt_stats.at(i).column_stats_;
    if (OB_UNLIKELY(column_params.count() != col_stats.count())) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("get unexpected error", K(column_params.count()),
                                       K(col_stats.count()), K(ret));
    } else {
      for (int64_t j = 0; OB_SUCC(ret) && j < column_params.count(); ++j) {
        const ObColumnStatParam &col_param = column_params.at(j);
        if (col_param.is_size_skewonly() || col_param.is_size_auto()) {
          if (OB_ISNULL(col_stats.at(j))) {
            ret = OB_ERR_UNEXPECTED;
            LOG_WARN("get unexpected null", K(ret), K(col_stats.at(j)));
          } else {
            ObHistogram &hist = col_stats.at(j)->get_histogram();
            if ((hist.get_type() == ObHistType::FREQUENCY && col_param.is_size_skewonly()) ||
                hist.get_type() == ObHistType::HYBIRD) {
              if (OB_UNLIKELY(hist.get_bucket_size() < 1 || col_param.bucket_num_ < 1)) {
                ret = OB_ERR_UNEXPECTED;
                LOG_WARN("get unexpected error", K(ret), K(hist.get_bucket_size()),
                                                 K(col_param.bucket_num_), K(*col_stats.at(j)));
              } else {
                bool is_even_dist = false;
                int64_t standard_cnt = hist.get_type() == ObHistType::FREQUENCY ?
                                         hist.get_buckets().at(0).endpoint_num_ :
                                         hist.get_sample_size() / col_param.bucket_num_;
                if (OB_FAIL(ObDbmsStatsUtils::check_range_skew(hist.get_type(),
                                                               hist.get_buckets(),
                                                               standard_cnt,
                                                               is_even_dist))) {
                  LOG_WARN("failed to check range skew", K(ret));
                } else if (is_even_dist) {//Evenly distributed, no need to build a histogram.
                  LOG_TRACE("check hist range skew is evenly distributed", K(hist.get_type()));
                  hist.reset();
                }
              }
            } else {/*do nothing*/}
          }
        } else {/*do nothing*/}
      }
    }
  }
  return ret;
}

int ObDbmsStatsUtils::implicit_commit_before_gather_stats(sql::ObExecContext &ctx)
{
  int ret = OB_SUCCESS;
  uint64_t optimizer_features_enable_version = 0;
  if (OB_ISNULL(ctx.get_my_session())) {
    ret = OB_ERR_UNEXPECTED;
    LOG_WARN("get unexpected null", K(ret), K(ctx.get_my_session()));
  } else if (OB_FAIL(ctx.get_my_session()->get_optimizer_features_enable_version(optimizer_features_enable_version))) {
    LOG_WARN("failed to get_optimizer_features_enable_version", K(ret));
  } else if (optimizer_features_enable_version < COMPAT_VERSION_4_2_4 ||
             (optimizer_features_enable_version >= COMPAT_VERSION_4_3_0 &&
              optimizer_features_enable_version < COMPAT_VERSION_4_3_2)) {
    //do nothing
  } else if (OB_FAIL(ObResultSet::implicit_commit_before_cmd_execute(*ctx.get_my_session(), ctx, stmt::T_ANALYZE))) {
    LOG_WARN("failed to implicit commit before cmd execute", K(ret));
  } else {/*do nothing*/}
  return ret;
}

int ObDbmsStatsUtils::scale_col_stats(const uint64_t tenant_id,
                                      const common::ObIArray<ObOptTableStat*> &tab_stats,
                                      common::ObIArray<ObOptColumnStat*> &col_stats)
{
  int ret = OB_SUCCESS;
  TabStatIndMap table_stats;
  ObOptTableStat* table_stat = NULL;
  ObOptColumnStat* col_stat = NULL;
  if (OB_FAIL(table_stats.create(512,
                                 "TabStatBkt",
                                 "TabStatNode",
                                 tenant_id))) {
    LOG_WARN("fail to create table stats map", KR(ret));
  }
  for (int64_t i = 0; OB_SUCC(ret) && i < tab_stats.count(); ++i) {
    if (OB_ISNULL(table_stat = tab_stats.at(i))) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("unexpected table stat is null", KR(ret));
    } else {
      ObOptTableStat::Key key(tenant_id,
                              table_stat->get_table_id(),
                              table_stat->get_partition_id());
      if (OB_FAIL(table_stats.set_refactored(key, table_stat))) {
        LOG_WARN("fail to set table stat", KR(ret), K(key), KPC(table_stat));
        if (OB_HASH_EXIST == ret) {
          ret = OB_ENTRY_EXIST;
        }
      }
    }
  }
  if (OB_SUCC(ret) && OB_FAIL(scale_col_stats(tenant_id,
                                              table_stats,
                                              col_stats))) {
    LOG_WARN("failed to scale col stats", K(ret));
  }
  return ret;
}

int ObDbmsStatsUtils::scale_col_stats(const uint64_t tenant_id,
                                      const TabStatIndMap &table_stats,
                                      common::ObIArray<ObOptColumnStat*> &col_stats)
{
  int ret = OB_SUCCESS;
  ObOptTableStat* table_stat = NULL;
  ObOptColumnStat* col_stat = NULL;
  for (int64_t i = 0; OB_SUCC(ret) && i < col_stats.count(); ++i) {
    if (OB_ISNULL(col_stat = col_stats.at(i))) {
      ret = OB_ERR_UNEXPECTED;
      LOG_WARN("get unexpected error", K(ret), K(col_stat));
    } else {
      ObOptTableStat::Key key(tenant_id,
                              col_stat->get_table_id(),
                              col_stat->get_partition_id());
      if (OB_FAIL(table_stats.get_refactored(key, table_stat))) {
        if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) {
          LOG_WARN("failed to find in hashmap", K(ret));
        } else {
          ret = OB_SUCCESS;
        }
      } else if (table_stat->get_sample_size() < table_stat->get_row_count() &&
                 table_stat->get_row_count() > 0) {
        double sample_value = static_cast<double>(table_stat->get_sample_size()) /
                              static_cast<double>(table_stat->get_row_count());
        if (sample_value >= 0.00000001 && sample_value < 1.0) {
          double num_distinct = ObOptSelectivity::scale_distinct(table_stat->get_row_count(),
                                                                 table_stat->get_sample_size(),
                                                                 col_stat->get_num_distinct());
          int64_t num_null = static_cast<int64_t>(col_stat->get_num_null() / sample_value);
          num_null = num_null > table_stat->get_row_count() ? table_stat->get_row_count() : num_null;
          col_stat->set_num_not_null(table_stat->get_row_count() - num_null);
          col_stat->set_num_null(num_null);
          col_stat->set_num_distinct(num_distinct);
        }
      }
    }
  }
  return ret;
}

int ObDbmsStatsUtils::get_sys_online_estimate_percent(sql::ObExecContext &ctx,
                                                      const uint64_t tenant_id,
                                                      const uint64_t table_id,
                                                      double &percent)
{
  int ret = OB_SUCCESS;
  ObTableStatParam stat_param;
  ObArenaAllocator tmp_alloc("ObDbmsStatsUtil", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID());
  ObSEArray<ObStatPrefs*, 4> stat_prefs;
  ObOnlineEstimatePercentPrefs *tmp_pref = NULL;
  stat_param.allocator_ = &tmp_alloc;
  stat_param.tenant_id_ = tenant_id;
  stat_param.table_id_ = table_id;
  if (OB_FAIL(new_stat_prefs(*stat_param.allocator_, ctx.get_my_session(), ObString(), tmp_pref))) {
    LOG_WARN("failed to new stat prefs", K(ret));
  } else if (OB_FAIL(stat_prefs.push_back(tmp_pref))) {
    LOG_WARN("failed to push back", K(ret));
  } else if (OB_FAIL(ObDbmsStatsPreferences::get_sys_default_stat_options(ctx, stat_prefs, stat_param))) {
    LOG_WARN("failed to get sys default stat options", K(ret));
  } else {
    percent = stat_param.online_sample_percent_;
  }
  return ret;
}

}
}