diff --git a/be/src/common/config.h b/be/src/common/config.h index 78ee4f9fef..61c9cc7d08 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -168,7 +168,7 @@ DECLARE_mString(process_full_gc_size); // used memory and the exec_mem_limit will be canceled. // If false, cancel query when the memory used exceeds exec_mem_limit, same as before. DECLARE_mBool(enable_query_memory_overcommit); - +//waibibabu // gc will release cache, cancel task, and task will wait for gc to release memory, // default gc strategy is conservative, if you want to exclude the interference of gc, let it be true DECLARE_mBool(disable_memory_gc); diff --git a/be/src/common/status.h b/be/src/common/status.h index 44c840a2d0..14b91dd7d0 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -113,7 +113,6 @@ E(NOT_INITIALIZED, -236); E(ALREADY_CANCELLED, -237); E(TOO_MANY_SEGMENTS, -238); E(ALREADY_CLOSED, -239); -E(NEED_SEND_AGAIN, -240); E(CE_CMD_PARAMS_ERROR, -300); E(CE_BUFFER_TOO_SMALL, -301); E(CE_CMD_NOT_VALID, -302); @@ -285,7 +284,6 @@ constexpr bool capture_stacktrace(int code) { && code != ErrorCode::TOO_MANY_VERSION && code != ErrorCode::ALREADY_CANCELLED && code != ErrorCode::ALREADY_CLOSED - && code != ErrorCode::NEED_SEND_AGAIN && code != ErrorCode::PUSH_TRANSACTION_ALREADY_EXIST && code != ErrorCode::BE_NO_SUITABLE_VERSION && code != ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION @@ -429,7 +427,6 @@ public: ERROR_CTOR(DataQualityError, DATA_QUALITY_ERROR) ERROR_CTOR(NotAuthorized, NOT_AUTHORIZED) ERROR_CTOR(HttpError, HTTP_ERROR) - ERROR_CTOR(NeedSendAgain, NEED_SEND_AGAIN) #undef ERROR_CTOR template diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 71ca504d3a..764dddf4e9 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -27,10 +27,7 @@ #include #include -#include -#include "common/exception.h" -#include "common/status.h" #include "olap/tablet_schema.h" #include "runtime/descriptors.h" #include "runtime/large_int_value.h" @@ -40,11 +37,8 @@ #include "util/hash_util.hpp" #include "util/string_parser.hpp" #include "util/string_util.h" -#include "vec/columns/column.h" -#include "vec/columns/column_nullable.h" -#include "vec/common/assert_cast.h" #include "vec/common/string_ref.h" -#include "vec/exprs/vliteral.h" +#include "vec/exprs/vexpr.h" #include "vec/runtime/vdatetime_value.h" namespace doris { @@ -63,61 +57,6 @@ void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const { } } -bool VOlapTablePartKeyComparator::operator()(const BlockRowWithIndicator lhs, - const BlockRowWithIndicator rhs) const { - vectorized::Block* l_block = std::get<0>(lhs); - vectorized::Block* r_block = std::get<0>(rhs); - int32_t l_row = std::get<1>(lhs); - int32_t r_row = std::get<1>(rhs); - bool l_use_new = std::get<2>(lhs); - bool r_use_new = std::get<2>(rhs); - - if (l_row == -1) { - return false; - } else if (r_row == -1) { - return true; - } - - if (_param_locs.empty()) { // no transform, use origin column - for (auto slot_loc : _slot_locs) { - auto res = l_block->get_by_position(slot_loc).column->compare_at( - l_row, r_row, *r_block->get_by_position(slot_loc).column, -1); - if (res != 0) { - return res < 0; - } - } - } else { // use transformed column to compare - DCHECK(_slot_locs.size() == _param_locs.size()) - << _slot_locs.size() << ' ' << _param_locs.size(); - - //TODO: use template to accelerate this for older compiler. - const std::vector* l_index = l_use_new ? &_param_locs : &_slot_locs; - const std::vector* r_index = r_use_new ? &_param_locs : &_slot_locs; - - for (int i = 0; i < _slot_locs.size(); i++) { - vectorized::ColumnPtr l_col = l_block->get_by_position((*l_index)[i]).column; - vectorized::ColumnPtr r_col = r_block->get_by_position((*r_index)[i]).column; - //TODO: when we support any function for transform, maybe the best way is refactor all doris' functions to its essential nullable mode. - if (auto* nullable = - vectorized::check_and_get_column(l_col)) { - l_col = nullable->get_nested_column_ptr(); - } - if (auto* nullable = - vectorized::check_and_get_column(r_col)) { - r_col = nullable->get_nested_column_ptr(); - } - - auto res = l_col->compare_at(l_row, r_row, *r_col, -1); - if (res != 0) { - return res < 0; - } - } - } - - // equal, return false - return false; -} - Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { _db_id = pschema.db_id(); _table_id = pschema.table_id(); @@ -269,23 +208,11 @@ VOlapTablePartitionParam::VOlapTablePartitionParam(std::shared_ptrtuple_desc()->slots()), - _mem_tracker(std::make_unique("OlapTablePartitionParam")), - _part_type(t_param.partition_type) { + _mem_tracker(std::make_unique("OlapTablePartitionParam")) { for (auto slot : _slots) { _partition_block.insert( {slot->get_empty_mutable_column(), slot->get_data_type_ptr(), slot->col_name()}); } - - if (t_param.__isset.enable_automatic_partition && t_param.enable_automatic_partition) { - _is_auto_partiton = true; - Status st = vectorized::VExpr::create_expr_tree(t_param.partition_function_exprs[0], - _part_func_ctx); - if (!st.ok()) { - throw Exception(Status::InternalError("Partition function expr is not valid"), - "Partition function expr is not valid"); - } - _partition_function = _part_func_ctx->root(); - } } VOlapTablePartitionParam::~VOlapTablePartitionParam() { @@ -316,8 +243,8 @@ Status VOlapTablePartitionParam::init() { } _partitions_map.reset( - new std::map( - VOlapTablePartKeyComparator(_partition_slot_locs, _transformed_slot_locs))); + new std::map( + VOlapTablePartKeyComparator(_partition_slot_locs))); if (_t_param.__isset.distributed_columns) { for (auto& col : _t_param.distributed_columns) { RETURN_IF_ERROR(find_slot_locs(col, _distributed_slot_locs, "distributed")); @@ -345,22 +272,67 @@ Status VOlapTablePartitionParam::init() { }; } - // for both auto/non-auto partition table. - _is_in_partition = _part_type == TPartitionType::type::LIST_PARTITIONED; + DCHECK(!_t_param.partitions.empty()) << "must have at least 1 partition"; + _is_in_partition = _t_param.partitions[0].__isset.in_keys; // initial partitions for (int i = 0; i < _t_param.partitions.size(); ++i) { const TOlapTablePartition& t_part = _t_param.partitions[i]; - VOlapTablePartition* part = nullptr; - RETURN_IF_ERROR(generate_partition_from(t_part, part)); + auto part = _obj_pool.add(new VOlapTablePartition(&_partition_block)); + part->id = t_part.id; + part->is_mutable = t_part.is_mutable; + + if (!_is_in_partition) { + if (t_part.__isset.start_keys) { + RETURN_IF_ERROR(_create_partition_keys(t_part.start_keys, &part->start_key)); + } + + if (t_part.__isset.end_keys) { + RETURN_IF_ERROR(_create_partition_keys(t_part.end_keys, &part->end_key)); + } + } else { + for (const auto& keys : t_part.in_keys) { + RETURN_IF_ERROR(_create_partition_keys( + keys, &part->in_keys.emplace_back(&_partition_block, -1))); + } + if (t_part.__isset.is_default_partition && t_part.is_default_partition) { + _default_partition = part; + } + } + + part->num_buckets = t_part.num_buckets; + auto num_indexes = _schema->indexes().size(); + if (t_part.indexes.size() != num_indexes) { + return Status::InternalError( + "number of partition's index is not equal with schema's" + ", num_part_indexes={}, num_schema_indexes={}", + t_part.indexes.size(), num_indexes); + } + part->indexes = t_part.indexes; + std::sort(part->indexes.begin(), part->indexes.end(), + [](const OlapTableIndexTablets& lhs, const OlapTableIndexTablets& rhs) { + return lhs.index_id < rhs.index_id; + }); + // check index + for (int j = 0; j < num_indexes; ++j) { + if (part->indexes[j].index_id != _schema->indexes()[j]->index_id) { + std::stringstream ss; + ss << "partition's index is not equal with schema's" + << ", part_index=" << part->indexes[j].index_id + << ", schema_index=" << _schema->indexes()[j]->index_id; + return Status::InternalError( + "partition's index is not equal with schema's" + ", part_index={}, schema_index={}", + part->indexes[j].index_id, _schema->indexes()[j]->index_id); + } + } _partitions.emplace_back(part); if (_is_in_partition) { for (auto& in_key : part->in_keys) { - _partitions_map->emplace(std::tuple {in_key.first, in_key.second, false}, part); + _partitions_map->emplace(&in_key, part); } } else { - _partitions_map->emplace(std::tuple {part->end_key.first, part->end_key.second, false}, - part); + _partitions_map->emplace(&part->end_key, part); } } @@ -371,32 +343,19 @@ Status VOlapTablePartitionParam::init() { bool VOlapTablePartitionParam::find_partition(BlockRow* block_row, const VOlapTablePartition** partition) const { - // block_row is gave by inserting process. So try to use transformed index. - auto it = - _is_in_partition - ? _partitions_map->find(std::tuple {block_row->first, block_row->second, true}) - : _partitions_map->upper_bound( - std::tuple {block_row->first, block_row->second, true}); + auto it = _is_in_partition ? _partitions_map->find(block_row) + : _partitions_map->upper_bound(block_row); // for list partition it might result in default partition if (_is_in_partition) { *partition = (it != _partitions_map->end()) ? it->second : _default_partition; it = _partitions_map->end(); } - if (it != _partitions_map->end() && - _part_contains(it->second, std::tuple {block_row->first, block_row->second, true})) { + if (it != _partitions_map->end() && _part_contains(it->second, block_row)) { *partition = it->second; } return (*partition != nullptr); } -bool VOlapTablePartitionParam::_part_contains(VOlapTablePartition* part, - BlockRowWithIndicator key) const { - // start_key.second == -1 means only single partition - VOlapTablePartKeyComparator comparator(_partition_slot_locs, _transformed_slot_locs); - return part->start_key.second == -1 || - !comparator(key, std::tuple {part->start_key.first, part->start_key.second, false}); -} - uint32_t VOlapTablePartitionParam::find_tablet(BlockRow* block_row, const VOlapTablePartition& partition) const { return _compute_tablet_index(block_row, partition.num_buckets); @@ -410,61 +369,6 @@ Status VOlapTablePartitionParam::_create_partition_keys(const std::vectorid = t_part.id; - part_result->is_mutable = t_part.is_mutable; - - if (!_is_in_partition) { - if (t_part.__isset.start_keys) { - RETURN_IF_ERROR(_create_partition_keys(t_part.start_keys, &part_result->start_key)); - } - - if (t_part.__isset.end_keys) { - RETURN_IF_ERROR(_create_partition_keys(t_part.end_keys, &part_result->end_key)); - } - } else { - for (const auto& keys : t_part.in_keys) { - RETURN_IF_ERROR(_create_partition_keys( - keys, &part_result->in_keys.emplace_back(&_partition_block, -1))); - } - if (t_part.__isset.is_default_partition && t_part.is_default_partition && - _default_partition == nullptr) { - _default_partition = part_result; - } - } - - part_result->num_buckets = t_part.num_buckets; - auto num_indexes = _schema->indexes().size(); - if (t_part.indexes.size() != num_indexes) { - return Status::InternalError( - "number of partition's index is not equal with schema's" - ", num_part_indexes={}, num_schema_indexes={}", - t_part.indexes.size(), num_indexes); - } - part_result->indexes = t_part.indexes; - std::sort(part_result->indexes.begin(), part_result->indexes.end(), - [](const OlapTableIndexTablets& lhs, const OlapTableIndexTablets& rhs) { - return lhs.index_id < rhs.index_id; - }); - // check index - for (int j = 0; j < num_indexes; ++j) { - if (part_result->indexes[j].index_id != _schema->indexes()[j]->index_id) { - std::stringstream ss; - ss << "partition's index is not equal with schema's" - << ", part_index=" << part_result->indexes[j].index_id - << ", schema_index=" << _schema->indexes()[j]->index_id; - return Status::InternalError( - "partition's index is not equal with schema's" - ", part_index={}, schema_index={}", - part_result->indexes[j].index_id, _schema->indexes()[j]->index_id); - } - } - return Status::OK(); -} - Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, BlockRow* part_key, uint16_t pos) { auto column = std::move(*part_key->first->get_by_position(pos).column).mutate(); @@ -553,72 +457,4 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, return Status::OK(); } -Status VOlapTablePartitionParam::add_partitions( - const std::vector& partitions) { - for (const auto& t_part : partitions) { - auto part = _obj_pool.add(new VOlapTablePartition(&_partition_block)); - part->id = t_part.id; - part->is_mutable = t_part.is_mutable; - - DCHECK(t_part.__isset.start_keys == t_part.__isset.end_keys && - t_part.__isset.start_keys != t_part.__isset.in_keys); - // range partition - if (t_part.__isset.start_keys) { - RETURN_IF_ERROR(_create_partition_keys(t_part.start_keys, &part->start_key)); - } - if (t_part.__isset.end_keys) { - RETURN_IF_ERROR(_create_partition_keys(t_part.end_keys, &part->end_key)); - } - // list partition - we only set 1 value in 1 partition for new created ones - if (t_part.__isset.in_keys) { - for (const auto& keys : t_part.in_keys) { - RETURN_IF_ERROR(_create_partition_keys( - keys, &part->in_keys.emplace_back(&_partition_block, -1))); - } - if (t_part.__isset.is_default_partition && t_part.is_default_partition) { - _default_partition = part; - } - } - - part->num_buckets = t_part.num_buckets; - auto num_indexes = _schema->indexes().size(); - if (t_part.indexes.size() != num_indexes) { - return Status::InternalError( - "number of partition's index is not equal with schema's" - ", num_part_indexes={}, num_schema_indexes={}", - t_part.indexes.size(), num_indexes); - } - part->indexes = t_part.indexes; - std::sort(part->indexes.begin(), part->indexes.end(), - [](const OlapTableIndexTablets& lhs, const OlapTableIndexTablets& rhs) { - return lhs.index_id < rhs.index_id; - }); - // check index - for (int j = 0; j < num_indexes; ++j) { - if (part->indexes[j].index_id != _schema->indexes()[j]->index_id) { - std::stringstream ss; - ss << "partition's index is not equal with schema's" - << ", part_index=" << part->indexes[j].index_id - << ", schema_index=" << _schema->indexes()[j]->index_id; - return Status::InternalError( - "partition's index is not equal with schema's" - ", part_index={}, schema_index={}", - part->indexes[j].index_id, _schema->indexes()[j]->index_id); - } - } - _partitions.emplace_back(part); - // after _creating_partiton_keys - if (_is_in_partition) { - for (auto& in_key : part->in_keys) { - _partitions_map->emplace(std::tuple {in_key.first, in_key.second, false}, part); - } - } else { - _partitions_map->emplace(std::tuple {part->end_key.first, part->end_key.second, false}, - part); - } - } - - return Status::OK(); -} - } // namespace doris diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 3e6ab7b94b..c508b322ee 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -36,8 +35,6 @@ #include "vec/columns/column.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" -#include "vec/exprs/vexpr.h" -#include "vec/exprs/vexpr_context.h" #include "vec/exprs/vexpr_fwd.h" namespace doris { @@ -113,8 +110,7 @@ using OlapTableIndexTablets = TOlapTableIndexTablets; // } using BlockRow = std::pair; -using BlockRowWithIndicator = - std::tuple; // [block, column, is_transformed] +using VecBlock = vectorized::Block; struct VOlapTablePartition { int64_t id = 0; @@ -129,20 +125,32 @@ struct VOlapTablePartition { : start_key {partition_block, -1}, end_key {partition_block, -1} {} }; -// this is only used by tablet_sink. so we can assume it's inited by its' descriptor. class VOlapTablePartKeyComparator { public: - VOlapTablePartKeyComparator(const std::vector& slot_locs, - const std::vector& params_locs) - : _slot_locs(slot_locs), _param_locs(params_locs) {} + VOlapTablePartKeyComparator(const std::vector& slot_locs) : _slot_locs(slot_locs) {} // return true if lhs < rhs - // 'row' is -1 mean maximal boundary - bool operator()(const BlockRowWithIndicator lhs, const BlockRowWithIndicator rhs) const; + // 'row' is -1 mean + bool operator()(const BlockRow* lhs, const BlockRow* rhs) const { + if (lhs->second == -1) { + return false; + } else if (rhs->second == -1) { + return true; + } + + for (auto slot_loc : _slot_locs) { + auto res = lhs->first->get_by_position(slot_loc).column->compare_at( + lhs->second, rhs->second, *rhs->first->get_by_position(slot_loc).column, -1); + if (res != 0) { + return res < 0; + } + } + // equal, return false + return false; + } private: const std::vector& _slot_locs; - const std::vector& _param_locs; }; // store an olap table's tablet information @@ -166,26 +174,6 @@ public: const std::vector& get_partitions() const { return _partitions; } - // it's same with auto now because we only support transformed partition in auto partition. may expand in future - bool is_projection_partition() const { return _is_auto_partiton; } - bool is_auto_partition() const { return _is_auto_partiton; } - - std::vector get_partition_keys() const { return _partition_slot_locs; } - - Status add_partitions(const std::vector& partitions); - - //TODO: use vector when we support multi partition column for auto-partition - vectorized::VExprContextSPtr get_part_func_ctx() { return _part_func_ctx; } - vectorized::VExprSPtr get_partition_function() { return _partition_function; } - - // which will affect _partition_block - Status generate_partition_from(const TOlapTablePartition& t_part, - VOlapTablePartition*& part_result); - - void set_transformed_slots(const std::vector& new_slots) { - _transformed_slot_locs = new_slots; - } - private: Status _create_partition_keys(const std::vector& t_exprs, BlockRow* part_key); @@ -194,7 +182,11 @@ private: std::function _compute_tablet_index; // check if this partition contain this key - bool _part_contains(VOlapTablePartition* part, BlockRowWithIndicator key) const; + bool _part_contains(VOlapTablePartition* part, BlockRow* key) const { + // start_key.second == -1 means only single partition + VOlapTablePartKeyComparator comparator(_partition_slot_locs); + return part->start_key.second == -1 || !comparator(key, &part->start_key); + } // this partition only valid in this schema std::shared_ptr _schema; @@ -202,32 +194,21 @@ private: const std::vector& _slots; std::vector _partition_slot_locs; - std::vector _transformed_slot_locs; std::vector _distributed_slot_locs; ObjectPool _obj_pool; vectorized::Block _partition_block; std::unique_ptr _mem_tracker; std::vector _partitions; - // For all partition value rows saved in this map, indicator is false. whenever we use a value to find in it, the param is true. - // so that we can distinguish which column index to use (origin slots or transformed slots). - std::unique_ptr< - std::map> + std::unique_ptr> _partitions_map; bool _is_in_partition = false; uint32_t _mem_usage = 0; // only works when using list partition, the resource is owned by _partitions VOlapTablePartition* _default_partition = nullptr; - - // for auto partition, now only support 1 column. TODO: use vector to save them when we support multi column auto-partition. - bool _is_auto_partiton = false; - vectorized::VExprContextSPtr _part_func_ctx = nullptr; - vectorized::VExprSPtr _partition_function = nullptr; - TPartitionType::type _part_type; // support list or range }; -// indicate where's the tablet and all its replications (node-wise) using TabletLocation = TTabletLocation; // struct TTabletLocation { // 1: required i64 tablet_id @@ -254,17 +235,9 @@ public: return nullptr; } - void add_locations(std::vector& locations) { - for (auto& location : locations) { - if (_tablets.find(location.tablet_id) == _tablets.end()) { - _tablets[location.tablet_id] = &location; - } - } - } - private: TOlapTableLocationParam _t_param; - // [tablet_id, tablet]. tablet has id, also. + std::unordered_map _tablets; }; @@ -305,15 +278,6 @@ public: return nullptr; } - void add_nodes(const std::vector& t_nodes) { - for (const auto& node : t_nodes) { - auto node_info = find_node(node.id); - if (node_info == nullptr) { - _nodes.emplace(node.id, node); - } - } - } - const std::unordered_map& nodes_info() { return _nodes; } private: diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 4fd0e7b779..3df28bfd37 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -193,7 +193,89 @@ PFilterType get_type(RuntimeFilterType type) { } Status create_literal(const TypeDescriptor& type, const void* data, vectorized::VExprSPtr& expr) { - TExprNode node = create_texpr_node_from(data, type.type, type.precision, type.scale); + TExprNode node; + + switch (type.type) { + case TYPE_BOOLEAN: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_TINYINT: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_SMALLINT: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_INT: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_BIGINT: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_LARGEINT: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_FLOAT: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_DOUBLE: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_DATEV2: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_DATETIMEV2: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_DATE: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_DATETIME: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_DECIMALV2: { + create_texpr_literal_node(data, &node, type.precision, type.scale); + break; + } + case TYPE_DECIMAL32: { + create_texpr_literal_node(data, &node, type.precision, type.scale); + break; + } + case TYPE_DECIMAL64: { + create_texpr_literal_node(data, &node, type.precision, type.scale); + break; + } + case TYPE_DECIMAL128I: { + create_texpr_literal_node(data, &node, type.precision, type.scale); + break; + } + case TYPE_CHAR: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_VARCHAR: { + create_texpr_literal_node(data, &node); + break; + } + case TYPE_STRING: { + create_texpr_literal_node(data, &node); + break; + } + default: + DCHECK(false); + return Status::InvalidArgument("Invalid type!"); + } try { expr = vectorized::VLiteral::create_shared(node); diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h index 8fa603e4a1..3bd3efd6cc 100644 --- a/be/src/exprs/runtime_filter.h +++ b/be/src/exprs/runtime_filter.h @@ -42,7 +42,6 @@ #include "vec/common/string_ref.h" #include "vec/core/types.h" #include "vec/data_types/data_type.h" -#include "vec/exprs/vexpr.h" #include "vec/runtime/vdatetime_value.h" namespace butil { @@ -426,4 +425,143 @@ public: private: WrapperPtr _wrapper; }; + +// copied from expr.h since it is only used in runtime filter + +template +Status create_texpr_literal_node(const void* data, TExprNode* node, int precision = 0, + int scale = 0) { + if constexpr (T == TYPE_BOOLEAN) { + auto origin_value = reinterpret_cast(data); + TBoolLiteral boolLiteral; + (*node).__set_node_type(TExprNodeType::BOOL_LITERAL); + boolLiteral.__set_value(*origin_value); + (*node).__set_bool_literal(boolLiteral); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); + } else if constexpr (T == TYPE_TINYINT) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::INT_LITERAL); + TIntLiteral intLiteral; + intLiteral.__set_value(*origin_value); + (*node).__set_int_literal(intLiteral); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TINYINT)); + } else if constexpr (T == TYPE_SMALLINT) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::INT_LITERAL); + TIntLiteral intLiteral; + intLiteral.__set_value(*origin_value); + (*node).__set_int_literal(intLiteral); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_SMALLINT)); + } else if constexpr (T == TYPE_INT) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::INT_LITERAL); + TIntLiteral intLiteral; + intLiteral.__set_value(*origin_value); + (*node).__set_int_literal(intLiteral); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_INT)); + } else if constexpr (T == TYPE_BIGINT) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::INT_LITERAL); + TIntLiteral intLiteral; + intLiteral.__set_value(*origin_value); + (*node).__set_int_literal(intLiteral); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_BIGINT)); + } else if constexpr (T == TYPE_LARGEINT) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::LARGE_INT_LITERAL); + TLargeIntLiteral large_int_literal; + large_int_literal.__set_value(LargeIntValue::to_string(*origin_value)); + (*node).__set_large_int_literal(large_int_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_LARGEINT)); + } else if constexpr ((T == TYPE_DATE) || (T == TYPE_DATETIME) || (T == TYPE_TIME)) { + auto origin_value = reinterpret_cast(data); + TDateLiteral date_literal; + char convert_buffer[30]; + origin_value->to_string(convert_buffer); + date_literal.__set_value(convert_buffer); + (*node).__set_date_literal(date_literal); + (*node).__set_node_type(TExprNodeType::DATE_LITERAL); + if (origin_value->type() == TimeType::TIME_DATE) { + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATE)); + } else if (origin_value->type() == TimeType::TIME_DATETIME) { + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIME)); + } else if (origin_value->type() == TimeType::TIME_TIME) { + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIME)); + } + } else if constexpr (T == TYPE_DATEV2) { + auto origin_value = + reinterpret_cast*>(data); + TDateLiteral date_literal; + char convert_buffer[30]; + origin_value->to_string(convert_buffer); + date_literal.__set_value(convert_buffer); + (*node).__set_date_literal(date_literal); + (*node).__set_node_type(TExprNodeType::DATE_LITERAL); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATEV2)); + } else if constexpr (T == TYPE_DATETIMEV2) { + auto origin_value = + reinterpret_cast*>( + data); + TDateLiteral date_literal; + char convert_buffer[30]; + origin_value->to_string(convert_buffer); + date_literal.__set_value(convert_buffer); + (*node).__set_date_literal(date_literal); + (*node).__set_node_type(TExprNodeType::DATE_LITERAL); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIMEV2)); + } else if constexpr (T == TYPE_DECIMALV2) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); + TDecimalLiteral decimal_literal; + decimal_literal.__set_value(origin_value->to_string()); + (*node).__set_decimal_literal(decimal_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMALV2, precision, scale)); + } else if constexpr (T == TYPE_DECIMAL32) { + auto origin_value = reinterpret_cast*>(data); + (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); + TDecimalLiteral decimal_literal; + decimal_literal.__set_value(origin_value->to_string(scale)); + (*node).__set_decimal_literal(decimal_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL32, precision, scale)); + } else if constexpr (T == TYPE_DECIMAL64) { + auto origin_value = reinterpret_cast*>(data); + (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); + TDecimalLiteral decimal_literal; + decimal_literal.__set_value(origin_value->to_string(scale)); + (*node).__set_decimal_literal(decimal_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL64, precision, scale)); + } else if constexpr (T == TYPE_DECIMAL128I) { + auto origin_value = reinterpret_cast*>(data); + (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); + TDecimalLiteral decimal_literal; + decimal_literal.__set_value(origin_value->to_string(scale)); + (*node).__set_decimal_literal(decimal_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL128I, precision, scale)); + } else if constexpr (T == TYPE_FLOAT) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::FLOAT_LITERAL); + TFloatLiteral float_literal; + float_literal.__set_value(*origin_value); + (*node).__set_float_literal(float_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_FLOAT)); + } else if constexpr (T == TYPE_DOUBLE) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::FLOAT_LITERAL); + TFloatLiteral float_literal; + float_literal.__set_value(*origin_value); + (*node).__set_float_literal(float_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DOUBLE)); + } else if constexpr ((T == TYPE_STRING) || (T == TYPE_CHAR) || (T == TYPE_VARCHAR)) { + auto origin_value = reinterpret_cast(data); + (*node).__set_node_type(TExprNodeType::STRING_LITERAL); + TStringLiteral string_literal; + string_literal.__set_value(origin_value->to_string()); + (*node).__set_string_literal(string_literal); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_STRING)); + } else { + return Status::InvalidArgument("Invalid argument type!"); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp index 87df03c83b..035db53a27 100644 --- a/be/src/pipeline/pipeline_task.cpp +++ b/be/src/pipeline/pipeline_task.cpp @@ -24,7 +24,6 @@ #include -#include "common/status.h" #include "pipeline/exec/operator.h" #include "pipeline/pipeline.h" #include "pipeline_fragment_context.h" @@ -278,9 +277,6 @@ Status PipelineTask::execute(bool* eos) { if (_block->rows() != 0 || *eos) { SCOPED_TIMER(_sink_timer); auto status = _sink->sink(_state, block, _data_state); - if (status.is()) { - status = _sink->sink(_state, block, _data_state); - } if (!status.is()) { RETURN_IF_ERROR(status); } diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp index bee2f906bc..d39c72352b 100644 --- a/be/src/runtime/load_channel.cpp +++ b/be/src/runtime/load_channel.cpp @@ -86,12 +86,7 @@ Status LoadChannel::open(const PTabletWriterOpenRequest& params) { } } - if (params.is_incremental()) { - // incremental open would ensure not to open tablet repeatedly - RETURN_IF_ERROR(channel->incremental_open(params)); - } else { - RETURN_IF_ERROR(channel->open(params)); - } + RETURN_IF_ERROR(channel->open(params)); _opened = true; _last_updated_time.store(time(nullptr)); diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 1d8db7ce5a..750ff70833 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -39,7 +39,6 @@ #include "common/config.h" #include "common/logging.h" -#include "common/status.h" #include "common/version_internal.h" #include "exec/data_sink.h" #include "exec/exec_node.h" @@ -323,6 +322,7 @@ Status PlanFragmentExecutor::open_vectorized_internal() { while (!eos) { RETURN_IF_CANCELLED(_runtime_state); RETURN_IF_ERROR(get_vectorized_internal(&block, &eos)); + // Collect this plan and sub plan statistics, and send to parent plan. if (_collect_query_statistics_with_every_batch) { _collect_query_statistics(); @@ -330,13 +330,6 @@ Status PlanFragmentExecutor::open_vectorized_internal() { if (!eos || block.rows() > 0) { auto st = _sink->send(runtime_state(), &block); - //TODO: Asynchronisation need refactor this - if (st.is()) { // created partition, do it again. - st = _sink->send(runtime_state(), &block); - if (st.is()) { - LOG(WARNING) << "have to create partition again..."; - } - } if (st.is()) { break; } diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 098f1eb252..2e21ec92c3 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -24,7 +24,6 @@ // IWYU pragma: no_include #include "common/compiler_util.h" // IWYU pragma: keep -#include "common/status.h" // IWYU pragma: no_include #include // IWYU pragma: keep #include @@ -119,73 +118,6 @@ Status TabletsChannel::open(const PTabletWriterOpenRequest& request) { return Status::OK(); } -Status TabletsChannel::incremental_open(const PTabletWriterOpenRequest& params) { - if (_state == kInitialized) { // haven't opened - return open(params); - } - std::lock_guard l(_lock); - std::vector* index_slots = nullptr; - int32_t schema_hash = 0; - for (auto& index : _schema->indexes()) { - if (index->index_id == _index_id) { - index_slots = &index->slots; - schema_hash = index->schema_hash; - break; - } - } - if (index_slots == nullptr) { - Status::InternalError("unknown index id, key={}", _key.to_string()); - } - // update tablets - std::vector tablet_ids; - tablet_ids.reserve(params.tablets_size()); - size_t incremental_tablet_num = 0; - std::stringstream ss; - ss << "LocalTabletsChannel txn_id: " << _txn_id << " load_id: " << print_id(params.id()) - << " incremental open delta writer: "; - - for (auto& tablet : params.tablets()) { - if (_tablet_writers.find(tablet.tablet_id()) != _tablet_writers.end()) { - continue; - } - incremental_tablet_num++; - - WriteRequest wrequest; - wrequest.index_id = params.index_id(); - wrequest.tablet_id = tablet.tablet_id(); - wrequest.schema_hash = schema_hash; - wrequest.txn_id = _txn_id; - wrequest.partition_id = tablet.partition_id(); - wrequest.load_id = params.id(); - wrequest.tuple_desc = _tuple_desc; - wrequest.slots = index_slots; - wrequest.is_high_priority = _is_high_priority; - wrequest.table_schema_param = _schema; - - DeltaWriter* writer = nullptr; - auto st = DeltaWriter::open(&wrequest, &writer, _profile, _load_id); - if (!st.ok()) { - auto err_msg = fmt::format( - "open delta writer failed, tablet_id={}" - ", txn_id={}, partition_id={}, err={}", - tablet.tablet_id(), _txn_id, tablet.partition_id(), st.to_string()); - LOG(WARNING) << err_msg; - return Status::InternalError(err_msg); - } - ss << "[" << tablet.tablet_id() << "]"; - { - std::lock_guard l(_tablet_writers_lock); - _tablet_writers.emplace(tablet.tablet_id(), writer); - } - } - - _s_tablet_writer_count += incremental_tablet_num; - LOG(INFO) << ss.str(); - - _state = kOpened; - return Status::OK(); -} - Status TabletsChannel::close( LoadChannel* parent, int sender_id, int64_t backend_id, bool* finished, const google::protobuf::RepeatedField& partition_ids, @@ -348,7 +280,7 @@ void TabletsChannel::_commit_txn(DeltaWriter* writer, void TabletsChannel::_add_error_tablet( google::protobuf::RepeatedPtrField* tablet_errors, int64_t tablet_id, - Status error) const { + Status error) { PTabletError* tablet_error = tablet_errors->Add(); tablet_error->set_tablet_id(tablet_id); tablet_error->set_msg(error.to_string()); @@ -369,15 +301,10 @@ void TabletsChannel::refresh_profile() { write_mem_usage += write_mem; int64_t flush_mem = it.second->mem_consumption(MemType::FLUSH); flush_mem_usage += flush_mem; - if (write_mem > max_tablet_write_mem_usage) { - max_tablet_write_mem_usage = write_mem; - } - if (flush_mem > max_tablet_flush_mem_usage) { - max_tablet_flush_mem_usage = flush_mem; - } - if (write_mem + flush_mem > max_tablet_mem_usage) { + if (write_mem > max_tablet_write_mem_usage) max_tablet_write_mem_usage = write_mem; + if (flush_mem > max_tablet_flush_mem_usage) max_tablet_flush_mem_usage = flush_mem; + if (write_mem + flush_mem > max_tablet_mem_usage) max_tablet_mem_usage = write_mem + flush_mem; - } } } COUNTER_SET(_memory_usage_counter, write_mem_usage + flush_mem_usage); @@ -414,12 +341,7 @@ Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& request } #endif - int tablet_cnt = 0; for (auto& tablet : request.tablets()) { - if (_tablet_writers.find(tablet.tablet_id()) != _tablet_writers.end()) { - continue; - } - tablet_cnt++; WriteRequest wrequest; wrequest.index_id = request.index_id(); wrequest.tablet_id = tablet.tablet_id(); @@ -448,7 +370,7 @@ Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& request } } _s_tablet_writer_count += _tablet_writers.size(); - DCHECK_EQ(_tablet_writers.size(), tablet_cnt); + DCHECK_EQ(_tablet_writers.size(), request.tablets_size()); return Status::OK(); } @@ -526,7 +448,7 @@ Status TabletsChannel::add_batch(const PTabletWriterAddBlockRequest& request, response->mutable_tablet_errors(); auto tablet_writer_it = _tablet_writers.find(tablet_id); if (tablet_writer_it == _tablet_writers.end()) { - return Status::InternalError("unknown tablet to append data, tablet={}", tablet_id); + return Status::InternalError("unknown tablet to append data, tablet={}"); } Status st = write_func(tablet_writer_it->second); if (!st.ok()) { diff --git a/be/src/runtime/tablets_channel.h b/be/src/runtime/tablets_channel.h index fe9c226829..e3d8d87ec3 100644 --- a/be/src/runtime/tablets_channel.h +++ b/be/src/runtime/tablets_channel.h @@ -90,8 +90,6 @@ public: ~TabletsChannel(); Status open(const PTabletWriterOpenRequest& request); - // open + open writers - Status incremental_open(const PTabletWriterOpenRequest& params); // no-op when this channel has been closed or cancelled Status add_batch(const PTabletWriterAddBlockRequest& request, @@ -130,7 +128,7 @@ private: void _add_broken_tablet(int64_t tablet_id); void _add_error_tablet(google::protobuf::RepeatedPtrField* tablet_errors, - int64_t tablet_id, Status error) const; + int64_t tablet_id, Status error); bool _is_broken_tablet(int64_t tablet_id); void _init_profile(RuntimeProfile* profile); diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 18d73039e5..0fea95a90e 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -757,14 +757,10 @@ Block Block::copy_block(const std::vector& column_offset) const { return columns_with_type_and_name; } -void Block::append_to_block_by_selector(MutableBlock* dst, - const IColumn::Selector& selector) const { +void Block::append_block_by_selector(MutableBlock* dst, const IColumn::Selector& selector) const { DCHECK_EQ(data.size(), dst->mutable_columns().size()); for (size_t i = 0; i < data.size(); i++) { - // FIXME: this is a quickfix. we assume that only partition functions make there some - if (!is_column_const(*data[i].column)) { - data[i].column->append_data_by_selector(dst->mutable_columns()[i], selector); - } + data[i].column->append_data_by_selector(dst->mutable_columns()[i], selector); } } diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h index fbf15a443d..cad45ac237 100644 --- a/be/src/vec/core/block.h +++ b/be/src/vec/core/block.h @@ -283,7 +283,7 @@ public: // copy a new block by the offset column Block copy_block(const std::vector& column_offset) const; - void append_to_block_by_selector(MutableBlock* dst, const IColumn::Selector& selector) const; + void append_block_by_selector(MutableBlock* dst, const IColumn::Selector& selector) const; // need exception safety static void filter_block_internal(Block* block, const std::vector& columns_to_filter, diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index a19dafe439..341125c89e 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -55,93 +55,6 @@ namespace doris { class RowDescriptor; class RuntimeState; -TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, int precision, - int scale) { - TExprNode node; - - switch (type) { - case TYPE_BOOLEAN: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_TINYINT: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_SMALLINT: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_INT: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_BIGINT: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_LARGEINT: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_FLOAT: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_DOUBLE: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_DATEV2: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_DATETIMEV2: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_DATE: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_DATETIME: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_DECIMALV2: { - create_texpr_literal_node(data, &node, precision, scale); - break; - } - case TYPE_DECIMAL32: { - create_texpr_literal_node(data, &node, precision, scale); - break; - } - case TYPE_DECIMAL64: { - create_texpr_literal_node(data, &node, precision, scale); - break; - } - case TYPE_DECIMAL128I: { - create_texpr_literal_node(data, &node, precision, scale); - break; - } - case TYPE_CHAR: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_VARCHAR: { - create_texpr_literal_node(data, &node); - break; - } - case TYPE_STRING: { - create_texpr_literal_node(data, &node); - break; - } - default: - DCHECK(false); - throw std::invalid_argument("Invalid type!"); - } - return node; -} } // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 4bd8917906..b2f0fb9059 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -32,7 +32,6 @@ #include "common/factory_creator.h" #include "common/status.h" #include "runtime/define_primitive_type.h" -#include "runtime/large_int_value.h" #include "runtime/types.h" #include "udf/udf.h" #include "vec/aggregate_functions/aggregate_function.h" @@ -90,7 +89,6 @@ public: /// /// Subclasses overriding this function should call VExpr::Prepare() to recursively call /// Prepare() on the expr tree - /// row_desc used in vslot_ref and some subclass to specify column virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, VExprContext* context); @@ -255,144 +253,4 @@ protected: }; } // namespace vectorized - -template -Status create_texpr_literal_node(const void* data, TExprNode* node, int precision = 0, - int scale = 0) { - if constexpr (T == TYPE_BOOLEAN) { - auto origin_value = reinterpret_cast(data); - TBoolLiteral boolLiteral; - (*node).__set_node_type(TExprNodeType::BOOL_LITERAL); - boolLiteral.__set_value(*origin_value); - (*node).__set_bool_literal(boolLiteral); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); - } else if constexpr (T == TYPE_TINYINT) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::INT_LITERAL); - TIntLiteral intLiteral; - intLiteral.__set_value(*origin_value); - (*node).__set_int_literal(intLiteral); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TINYINT)); - } else if constexpr (T == TYPE_SMALLINT) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::INT_LITERAL); - TIntLiteral intLiteral; - intLiteral.__set_value(*origin_value); - (*node).__set_int_literal(intLiteral); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_SMALLINT)); - } else if constexpr (T == TYPE_INT) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::INT_LITERAL); - TIntLiteral intLiteral; - intLiteral.__set_value(*origin_value); - (*node).__set_int_literal(intLiteral); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_INT)); - } else if constexpr (T == TYPE_BIGINT) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::INT_LITERAL); - TIntLiteral intLiteral; - intLiteral.__set_value(*origin_value); - (*node).__set_int_literal(intLiteral); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_BIGINT)); - } else if constexpr (T == TYPE_LARGEINT) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::LARGE_INT_LITERAL); - TLargeIntLiteral large_int_literal; - large_int_literal.__set_value(LargeIntValue::to_string(*origin_value)); - (*node).__set_large_int_literal(large_int_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_LARGEINT)); - } else if constexpr ((T == TYPE_DATE) || (T == TYPE_DATETIME) || (T == TYPE_TIME)) { - auto origin_value = reinterpret_cast(data); - TDateLiteral date_literal; - char convert_buffer[30]; - origin_value->to_string(convert_buffer); - date_literal.__set_value(convert_buffer); - (*node).__set_date_literal(date_literal); - (*node).__set_node_type(TExprNodeType::DATE_LITERAL); - if (origin_value->type() == TimeType::TIME_DATE) { - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATE)); - } else if (origin_value->type() == TimeType::TIME_DATETIME) { - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIME)); - } else if (origin_value->type() == TimeType::TIME_TIME) { - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIME)); - } - } else if constexpr (T == TYPE_DATEV2) { - auto origin_value = - reinterpret_cast*>(data); - TDateLiteral date_literal; - char convert_buffer[30]; - origin_value->to_string(convert_buffer); - date_literal.__set_value(convert_buffer); - (*node).__set_date_literal(date_literal); - (*node).__set_node_type(TExprNodeType::DATE_LITERAL); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATEV2)); - } else if constexpr (T == TYPE_DATETIMEV2) { - auto origin_value = - reinterpret_cast*>( - data); - TDateLiteral date_literal; - char convert_buffer[30]; - origin_value->to_string(convert_buffer); - date_literal.__set_value(convert_buffer); - (*node).__set_date_literal(date_literal); - (*node).__set_node_type(TExprNodeType::DATE_LITERAL); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIMEV2)); - } else if constexpr (T == TYPE_DECIMALV2) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); - TDecimalLiteral decimal_literal; - decimal_literal.__set_value(origin_value->to_string()); - (*node).__set_decimal_literal(decimal_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMALV2, precision, scale)); - } else if constexpr (T == TYPE_DECIMAL32) { - auto origin_value = reinterpret_cast*>(data); - (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); - TDecimalLiteral decimal_literal; - decimal_literal.__set_value(origin_value->to_string(scale)); - (*node).__set_decimal_literal(decimal_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL32, precision, scale)); - } else if constexpr (T == TYPE_DECIMAL64) { - auto origin_value = reinterpret_cast*>(data); - (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); - TDecimalLiteral decimal_literal; - decimal_literal.__set_value(origin_value->to_string(scale)); - (*node).__set_decimal_literal(decimal_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL64, precision, scale)); - } else if constexpr (T == TYPE_DECIMAL128I) { - auto origin_value = reinterpret_cast*>(data); - (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); - TDecimalLiteral decimal_literal; - decimal_literal.__set_value(origin_value->to_string(scale)); - (*node).__set_decimal_literal(decimal_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL128I, precision, scale)); - } else if constexpr (T == TYPE_FLOAT) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::FLOAT_LITERAL); - TFloatLiteral float_literal; - float_literal.__set_value(*origin_value); - (*node).__set_float_literal(float_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_FLOAT)); - } else if constexpr (T == TYPE_DOUBLE) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::FLOAT_LITERAL); - TFloatLiteral float_literal; - float_literal.__set_value(*origin_value); - (*node).__set_float_literal(float_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DOUBLE)); - } else if constexpr ((T == TYPE_STRING) || (T == TYPE_CHAR) || (T == TYPE_VARCHAR)) { - auto origin_value = reinterpret_cast(data); - (*node).__set_node_type(TExprNodeType::STRING_LITERAL); - TStringLiteral string_literal; - string_literal.__set_value(origin_value->to_string()); - (*node).__set_string_literal(string_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_STRING)); - } else { - return Status::InvalidArgument("Invalid argument type!"); - } - return Status::OK(); -} - -TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, int precision = 0, - int scale = 0); - } // namespace doris diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp index 03d1659eee..3d39a844dc 100644 --- a/be/src/vec/exprs/vliteral.cpp +++ b/be/src/vec/exprs/vliteral.cpp @@ -66,7 +66,6 @@ Status VLiteral::execute(VExprContext* context, vectorized::Block* block, int* r } std::string VLiteral::value() const { - //TODO: dcheck the equality of size with 1. then use string with size to replace the ss. std::stringstream out; for (size_t i = 0; i < _column_ptr->size(); i++) { if (i != 0) { diff --git a/be/src/vec/sink/vtablet_finder.cpp b/be/src/vec/sink/vtablet_finder.cpp index f1a99e2605..2ee9f598b5 100644 --- a/be/src/vec/sink/vtablet_finder.cpp +++ b/be/src/vec/sink/vtablet_finder.cpp @@ -18,9 +18,6 @@ #include "vec/sink/vtablet_finder.h" #include -#include -#include -#include #include #include @@ -31,48 +28,37 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" #include "exec/tablet_info.h" -#include "exprs/runtime_filter.h" -#include "gutil/integral_types.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" #include "vec/core/block.h" -#include "vec/core/columns_with_type_and_name.h" -#include "vec/data_types/data_type.h" -#include "vec/functions/simple_function_factory.h" namespace doris { namespace stream_load { Status OlapTabletFinder::find_tablet(RuntimeState* state, vectorized::Block* block, int row_index, const VOlapTablePartition** partition, uint32_t& tablet_index, - bool& stop_processing, bool& is_continue, - bool* missing_partition) { + bool& stop_processing, bool& is_continue) { Status status = Status::OK(); *partition = nullptr; tablet_index = 0; BlockRow block_row; block_row = {block, row_index}; if (!_vpartition->find_partition(&block_row, partition)) { - if (missing_partition != nullptr) { // auto partition table - *missing_partition = true; - return status; - } else { - RETURN_IF_ERROR(state->append_error_msg_to_file( - []() -> std::string { return ""; }, - [&]() -> std::string { - fmt::memory_buffer buf; - fmt::format_to(buf, "no partition for this tuple. tuple={}", - block->dump_data(row_index, 1)); - return fmt::to_string(buf); - }, - &stop_processing)); - _num_filtered_rows++; - if (stop_processing) { - return Status::EndOfFile("Encountered unqualified data, stop processing"); - } - is_continue = true; - return status; + RETURN_IF_ERROR(state->append_error_msg_to_file( + []() -> std::string { return ""; }, + [&]() -> std::string { + fmt::memory_buffer buf; + fmt::format_to(buf, "no partition for this tuple. tuple={}", + block->dump_data(row_index, 1)); + return fmt::to_string(buf); + }, + &stop_processing)); + _num_filtered_rows++; + if (stop_processing) { + return Status::EndOfFile("Encountered unqualified data, stop processing"); } + is_continue = true; + return status; } if (!(*partition)->is_mutable) { _num_immutable_partition_filtered_rows++; diff --git a/be/src/vec/sink/vtablet_finder.h b/be/src/vec/sink/vtablet_finder.h index 9fe6494430..97282e403a 100644 --- a/be/src/vec/sink/vtablet_finder.h +++ b/be/src/vec/sink/vtablet_finder.h @@ -41,7 +41,7 @@ public: Status find_tablet(RuntimeState* state, vectorized::Block* block, int row_index, const VOlapTablePartition** partition, uint32_t& tablet_index, - bool& filtered, bool& is_continue, bool* missing_partition = nullptr); + bool& filtered, bool& is_continue); bool is_find_tablet_every_sink() { return _find_tablet_mode == FindTabletMode::FIND_TABLET_EVERY_SINK; diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp index 824ed84e03..09bcdc6e25 100644 --- a/be/src/vec/sink/vtablet_sink.cpp +++ b/be/src/vec/sink/vtablet_sink.cpp @@ -25,44 +25,23 @@ #include #include #include -#include -#include -#include -#include #include #include #include #include -#include #include #include #include #include #include -#include -#include -#include -#include #include #include #include -#include #include -#include #include #include -#include "runtime/datetime_value.h" -#include "util/runtime_profile.h" -#include "vec/core/columns_with_type_and_name.h" -#include "vec/data_types/data_type.h" -#include "vec/data_types/data_type_factory.hpp" -#include "vec/data_types/data_type_string.h" -#include "vec/exprs/vexpr_fwd.h" -#include "vec/functions/simple_function_factory.h" -#include "vec/runtime/vdatetime_value.h" - #ifdef DEBUG #include #endif @@ -73,7 +52,6 @@ #include "common/object_pool.h" #include "common/status.h" #include "exec/tablet_info.h" -#include "runtime/client_cache.h" #include "runtime/define_primitive_type.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" @@ -92,7 +70,6 @@ #include "util/telemetry/telemetry.h" #include "util/thread.h" #include "util/threadpool.h" -#include "util/thrift_rpc_helper.h" #include "util/thrift_util.h" #include "util/time.h" #include "util/uid_util.h" @@ -124,7 +101,6 @@ class TExpr; namespace stream_load { -// an IndexChannel is related to specific table and its rollup and mv class IndexChannel { public: IndexChannel(VOlapTableSink* parent, int64_t index_id, @@ -135,7 +111,6 @@ public: } ~IndexChannel() = default; - // allow to init multi times, for incremental open more tablets for one index(table) Status init(RuntimeState* state, const std::vector& tablets); void for_each_node_channel( @@ -207,23 +182,20 @@ private: Status IndexChannel::init(RuntimeState* state, const std::vector& tablets) { SCOPED_CONSUME_MEM_TRACKER(_index_channel_tracker.get()); for (auto& tablet : tablets) { - // First find the location BEs of this tablet - auto tablet_locations = _parent->_location->find_tablet(tablet.tablet_id); - if (tablet_locations == nullptr) { + auto location = _parent->_location->find_tablet(tablet.tablet_id); + if (location == nullptr) { return Status::InternalError("unknown tablet, tablet_id={}", tablet.tablet_id); } std::vector> channels; - // For tablet, deal with its' all replica (in some node). - for (auto& replica_node_id : tablet_locations->node_ids) { + for (auto& node_id : location->node_ids) { std::shared_ptr channel; - auto it = _node_channels.find(replica_node_id); - // when we prepare for TableSink or incremental open tablet, we need init + auto it = _node_channels.find(node_id); if (it == _node_channels.end()) { // NodeChannel is not added to the _parent->_pool. // Because the deconstruction of NodeChannel may take a long time to wait rpc finish. // but the ObjectPool will hold a spin lock to delete objects. - channel = std::make_shared(_parent, this, replica_node_id); - _node_channels.emplace(replica_node_id, channel); + channel = std::make_shared(_parent, this, node_id); + _node_channels.emplace(node_id, channel); } else { channel = it->second; } @@ -235,7 +207,7 @@ Status IndexChannel::init(RuntimeState* state, const std::vector(fmt::format( "NodeChannel:indexID={}:threadId={}", std::to_string(_index_channel->_index_id), thread_context()->get_thread_id())); } VNodeChannel::~VNodeChannel() { - for (auto& closure : _open_closures) { - if (closure != nullptr) { - if (closure->unref()) { - delete closure; - } - closure = nullptr; + if (_open_closure != nullptr) { + if (_open_closure->unref()) { + delete _open_closure; } + _open_closure = nullptr; } if (_add_block_closure != nullptr) { delete _add_block_closure; _add_block_closure = nullptr; } + if (_open_closure != nullptr) { + delete _open_closure; + } static_cast(_cur_add_block_request.release_id()); } @@ -369,12 +338,12 @@ Status VNodeChannel::init(RuntimeState* state) { SCOPED_CONSUME_MEM_TRACKER(_node_channel_tracker.get()); _tuple_desc = _parent->_output_tuple_desc; _state = state; - // get corresponding BE node. auto node = _parent->_nodes_info->find_node(_node_id); if (node == nullptr) { _cancelled = true; return Status::InternalError("unknown node id, id={}", _node_id); } + _node_info = *node; _load_info = "load_id=" + print_id(_parent->_load_id) + @@ -395,9 +364,7 @@ Status VNodeChannel::init(RuntimeState* state) { _timeout_watch.start(); // Initialize _cur_add_block_request - if (!_cur_add_block_request.has_id()) { - _cur_add_block_request.set_allocated_id(&_parent->_load_id); - } + _cur_add_block_request.set_allocated_id(&_parent->_load_id); _cur_add_block_request.set_index_id(_index_channel->_index_id); _cur_add_block_request.set_sender_id(_parent->_sender_id); _cur_add_block_request.set_backend_id(_node_id); @@ -414,22 +381,17 @@ Status VNodeChannel::init(RuntimeState* state) { return Status::OK(); } -void VNodeChannel::_open_internal(bool is_incremental) { +void VNodeChannel::open() { SCOPED_CONSUME_MEM_TRACKER(_node_channel_tracker.get()); PTabletWriterOpenRequest request; request.set_allocated_id(&_parent->_load_id); request.set_index_id(_index_channel->_index_id); request.set_txn_id(_parent->_txn_id); request.set_allocated_schema(_parent->_schema->to_protobuf()); - std::set deduper; for (auto& tablet : _all_tablets) { - if (deduper.contains(tablet.tablet_id)) { - continue; - } auto ptablet = request.add_tablets(); ptablet->set_partition_id(tablet.partition_id); ptablet->set_tablet_id(tablet.tablet_id); - deduper.insert(tablet.tablet_id); } request.set_num_senders(_parent->_num_senders); request.set_need_gen_rollup(false); // Useless but it is a required field in pb @@ -440,75 +402,160 @@ void VNodeChannel::_open_internal(bool is_incremental) { request.set_is_vectorized(true); request.set_backend_id(_node_id); request.set_enable_profile(_state->enable_profile()); - request.set_is_incremental(is_incremental); - auto* open_closure = new RefCountClosure {}; - open_closure->ref(); + _open_closure = new RefCountClosure(); + _open_closure->ref(); - open_closure->ref(); // This ref is for RPC's reference - open_closure->cntl.set_timeout_ms(config::tablet_writer_open_rpc_timeout_sec * 1000); + // This ref is for RPC's reference + _open_closure->ref(); + _open_closure->cntl.set_timeout_ms(config::tablet_writer_open_rpc_timeout_sec * 1000); if (config::tablet_writer_ignore_eovercrowded) { - open_closure->cntl.ignore_eovercrowded(); + _open_closure->cntl.ignore_eovercrowded(); } - // the real transmission here. the corresponding BE's load mgr will open load channel for it. - _stub->tablet_writer_open(&open_closure->cntl, &request, &open_closure->result, open_closure); - + _stub->tablet_writer_open(&_open_closure->cntl, &request, &_open_closure->result, + _open_closure); static_cast(request.release_id()); static_cast(request.release_schema()); } -void VNodeChannel::open() { - _open_internal(false); -} - -void VNodeChannel::incremental_open() { - _open_internal(true); -} - Status VNodeChannel::open_wait() { - Status status; - for (auto& open_closure : _open_closures) { - open_closure->join(); - SCOPED_CONSUME_MEM_TRACKER(_node_channel_tracker.get()); - if (open_closure->cntl.Failed()) { - if (!ExecEnv::GetInstance()->brpc_internal_client_cache()->available( - _stub, _node_info.host, _node_info.brpc_port)) { - ExecEnv::GetInstance()->brpc_internal_client_cache()->erase( - open_closure->cntl.remote_side()); - } + _open_closure->join(); + SCOPED_CONSUME_MEM_TRACKER(_node_channel_tracker.get()); + if (_open_closure->cntl.Failed()) { + if (!ExecEnv::GetInstance()->brpc_internal_client_cache()->available( + _stub, _node_info.host, _node_info.brpc_port)) { + ExecEnv::GetInstance()->brpc_internal_client_cache()->erase( + _open_closure->cntl.remote_side()); + } - _cancelled = true; - auto error_code = open_closure->cntl.ErrorCode(); - auto error_text = open_closure->cntl.ErrorText(); - if (open_closure->unref()) { - delete open_closure; - } - open_closure = nullptr; - return Status::InternalError( - "failed to open tablet writer, error={}, error_text={}, info={}", - berror(error_code), error_text, channel_info()); + _cancelled = true; + auto error_code = _open_closure->cntl.ErrorCode(); + auto error_text = _open_closure->cntl.ErrorText(); + if (_open_closure->unref()) { + delete _open_closure; } - status = Status::create(open_closure->result.status()); - if (open_closure->unref()) { - delete open_closure; - } - open_closure = nullptr; + _open_closure = nullptr; + return Status::InternalError( + "failed to open tablet writer, error={}, error_text={}, info={}", + berror(error_code), error_text, channel_info()); + } + Status status(Status::create(_open_closure->result.status())); + if (_open_closure->unref()) { + delete _open_closure; + } + _open_closure = nullptr; - if (!status.ok()) { - _cancelled = true; - return status; - } + if (!status.ok()) { + _cancelled = true; + return status; } // add block closure _add_block_closure = ReusableClosure::create(); - _add_block_closure->addFailedHandler( - [this](bool is_last_rpc) { _add_block_failed_callback(is_last_rpc); }); + _add_block_closure->addFailedHandler([this](bool is_last_rpc) { + std::lock_guard l(this->_closed_lock); + if (this->_is_closed) { + // if the node channel is closed, no need to call `mark_as_failed`, + // and notice that _index_channel may already be destroyed. + return; + } + SCOPED_ATTACH_TASK(_state); + // If rpc failed, mark all tablets on this node channel as failed + _index_channel->mark_as_failed(this->node_id(), this->host(), + fmt::format("rpc failed, error coed:{}, error text:{}", + _add_block_closure->cntl.ErrorCode(), + _add_block_closure->cntl.ErrorText()), + -1); + Status st = _index_channel->check_intolerable_failure(); + if (!st.ok()) { + _cancel_with_msg(fmt::format("{}, err: {}", channel_info(), st.to_string())); + } else if (is_last_rpc) { + // if this is last rpc, will must set _add_batches_finished. otherwise, node channel's close_wait + // will be blocked. + _add_batches_finished = true; + } + }); - _add_block_closure->addSuccessHandler( - [this](const PTabletWriterAddBlockResult& result, bool is_last_rpc) { - _add_block_success_callback(result, is_last_rpc); - }); + _add_block_closure->addSuccessHandler([this](const PTabletWriterAddBlockResult& result, + bool is_last_rpc) { + std::lock_guard l(this->_closed_lock); + if (this->_is_closed) { + // if the node channel is closed, no need to call the following logic, + // and notice that _index_channel may already be destroyed. + return; + } + SCOPED_ATTACH_TASK(_state); + Status status(Status::create(result.status())); + if (status.ok()) { + // if has error tablet, handle them first + for (auto& error : result.tablet_errors()) { + _index_channel->mark_as_failed(this->node_id(), this->host(), + "tablet error: " + error.msg(), error.tablet_id()); + } + + Status st = _index_channel->check_intolerable_failure(); + if (!st.ok()) { + _cancel_with_msg(st.to_string()); + } else if (is_last_rpc) { + for (auto& tablet : result.tablet_vec()) { + TTabletCommitInfo commit_info; + commit_info.tabletId = tablet.tablet_id(); + commit_info.backendId = _node_id; + _tablet_commit_infos.emplace_back(std::move(commit_info)); + if (tablet.has_received_rows()) { + _tablets_received_rows.emplace_back(tablet.tablet_id(), + tablet.received_rows()); + } + if (tablet.has_num_rows_filtered()) { + _state->update_num_rows_filtered_in_strict_mode_partial_update( + tablet.num_rows_filtered()); + } + VLOG_CRITICAL << "master replica commit info: tabletId=" << tablet.tablet_id() + << ", backendId=" << _node_id + << ", master node id: " << this->node_id() + << ", host: " << this->host() << ", txn_id=" << _parent->_txn_id; + } + if (_parent->_write_single_replica) { + for (auto& tablet_slave_node_ids : result.success_slave_tablet_node_ids()) { + for (auto slave_node_id : tablet_slave_node_ids.second.slave_node_ids()) { + TTabletCommitInfo commit_info; + commit_info.tabletId = tablet_slave_node_ids.first; + commit_info.backendId = slave_node_id; + _tablet_commit_infos.emplace_back(std::move(commit_info)); + VLOG_CRITICAL << "slave replica commit info: tabletId=" + << tablet_slave_node_ids.first + << ", backendId=" << slave_node_id + << ", master node id: " << this->node_id() + << ", host: " << this->host() + << ", txn_id=" << _parent->_txn_id; + } + } + } + _add_batches_finished = true; + } + } else { + _cancel_with_msg(fmt::format("{}, add batch req success but status isn't ok, err: {}", + channel_info(), status.to_string())); + } + + if (result.has_execution_time_us()) { + _add_batch_counter.add_batch_execution_time_us += result.execution_time_us(); + _add_batch_counter.add_batch_wait_execution_time_us += result.wait_execution_time_us(); + _add_batch_counter.add_batch_num++; + } + if (result.has_load_channel_profile()) { + TRuntimeProfileTree tprofile; + const uint8_t* buf = (const uint8_t*)result.load_channel_profile().data(); + uint32_t len = result.load_channel_profile().size(); + auto st = deserialize_thrift_msg(buf, &len, false, &tprofile); + if (st.ok()) { + _state->load_channel_profile()->update(tprofile); + } else { + LOG(WARNING) << "load channel TRuntimeProfileTree deserialize failed, errmsg=" + << st; + } + } + }); return status; } @@ -592,7 +639,7 @@ Status VNodeChannel::add_block(vectorized::Block* block, const Payload* payload, SCOPED_RAW_TIMER(&_stat.append_node_channel_ns); if (is_append) { // Do not split the data of the block by tablets but append it to a single delta writer. - // This is a faster way to send block than append_to_block_by_selector + // This is a faster way to send block than append_block_by_selector // TODO: we could write to local delta writer if single_replica_load is true VLOG_DEBUG << "send whole block by append block"; std::vector tablets(block->rows(), payload->second[0]); @@ -601,12 +648,12 @@ Status VNodeChannel::add_block(vectorized::Block* block, const Payload* payload, columns.reserve(block->columns()); // Hold the reference of block columns to avoid copying for (auto column : block->get_columns()) { - columns.push_back(std::move(*column).mutate()); + columns.push_back(column->assume_mutable()); } *_cur_add_block_request.mutable_tablet_ids() = {tablets.begin(), tablets.end()}; _cur_add_block_request.set_is_single_tablet_block(true); } else { - block->append_to_block_by_selector(_cur_mutable_block.get(), *(payload->first)); + block->append_block_by_selector(_cur_mutable_block.get(), *(payload->first)); for (auto tablet_id : payload->second) { _cur_add_block_request.add_tablet_ids(tablet_id); } @@ -619,8 +666,6 @@ Status VNodeChannel::add_block(vectorized::Block* block, const Payload* payload, std::lock_guard l(_pending_batches_lock); // To simplify the add_row logic, postpone adding block into req until the time of sending req _pending_batches_bytes += _cur_mutable_block->allocated_bytes(); - _cur_add_block_request.set_eos( - false); // for multi-add, only when marking close we set it eos. _pending_blocks.emplace(std::move(_cur_mutable_block), _cur_add_block_request); _pending_batches_num++; VLOG_DEBUG << "VOlapTableSink:" << _parent << " VNodeChannel:" << this @@ -650,7 +695,7 @@ int VNodeChannel::try_send_and_fetch_status(RuntimeState* state, // We are sure that try_send_batch is not running if (_pending_batches_num > 0) { auto s = thread_pool_token->submit_func( - std::bind(&VNodeChannel::try_send_pending_block, this, state)); + std::bind(&VNodeChannel::try_send_block, this, state)); if (!s.ok()) { _cancel_with_msg("submit send_batch task to send_batch_thread_pool failed"); // clear in flight @@ -691,12 +736,13 @@ Status VNodeChannel::none_of(std::initializer_list vars) { return st; } -void VNodeChannel::try_send_pending_block(RuntimeState* state) { +void VNodeChannel::try_send_block(RuntimeState* state) { SCOPED_ATTACH_TASK(state); SCOPED_CONSUME_MEM_TRACKER(_node_channel_tracker); SCOPED_ATOMIC_TIMER(&_actual_consume_ns); AddBlockReq send_block; { + debug::ScopedTSANIgnoreReadsAndWrites ignore_tsan; std::lock_guard l(_pending_batches_lock); DCHECK(!_pending_blocks.empty()); send_block = std::move(_pending_blocks.front()); @@ -821,108 +867,6 @@ void VNodeChannel::try_send_pending_block(RuntimeState* state) { _next_packet_seq++; } -void VNodeChannel::_add_block_success_callback(const PTabletWriterAddBlockResult& result, - bool is_last_rpc) { - std::lock_guard l(this->_closed_lock); - if (this->_is_closed) { - // if the node channel is closed, no need to call the following logic, - // and notice that _index_channel may already be destroyed. - return; - } - SCOPED_ATTACH_TASK(_state); - Status status(Status::create(result.status())); - if (status.ok()) { - // if has error tablet, handle them first - for (auto& error : result.tablet_errors()) { - _index_channel->mark_as_failed(this->node_id(), this->host(), - "tablet error: " + error.msg(), error.tablet_id()); - } - - Status st = _index_channel->check_intolerable_failure(); - if (!st.ok()) { - _cancel_with_msg(st.to_string()); - } else if (is_last_rpc) { - for (auto& tablet : result.tablet_vec()) { - TTabletCommitInfo commit_info; - commit_info.tabletId = tablet.tablet_id(); - commit_info.backendId = _node_id; - _tablet_commit_infos.emplace_back(std::move(commit_info)); - if (tablet.has_received_rows()) { - _tablets_received_rows.emplace_back(tablet.tablet_id(), tablet.received_rows()); - } - if (tablet.has_num_rows_filtered()) { - _state->update_num_rows_filtered_in_strict_mode_partial_update( - tablet.num_rows_filtered()); - } - VLOG_CRITICAL << "master replica commit info: tabletId=" << tablet.tablet_id() - << ", backendId=" << _node_id - << ", master node id: " << this->node_id() - << ", host: " << this->host() << ", txn_id=" << _parent->_txn_id; - } - if (_parent->_write_single_replica) { - for (auto& tablet_slave_node_ids : result.success_slave_tablet_node_ids()) { - for (auto slave_node_id : tablet_slave_node_ids.second.slave_node_ids()) { - TTabletCommitInfo commit_info; - commit_info.tabletId = tablet_slave_node_ids.first; - commit_info.backendId = slave_node_id; - _tablet_commit_infos.emplace_back(std::move(commit_info)); - VLOG_CRITICAL - << "slave replica commit info: tabletId=" - << tablet_slave_node_ids.first << ", backendId=" << slave_node_id - << ", master node id: " << this->node_id() - << ", host: " << this->host() << ", txn_id=" << _parent->_txn_id; - } - } - } - _add_batches_finished = true; - } - } else { - _cancel_with_msg(fmt::format("{}, add batch req success but status isn't ok, err: {}", - channel_info(), status.to_string())); - } - - if (result.has_execution_time_us()) { - _add_batch_counter.add_batch_execution_time_us += result.execution_time_us(); - _add_batch_counter.add_batch_wait_execution_time_us += result.wait_execution_time_us(); - _add_batch_counter.add_batch_num++; - } - if (result.has_load_channel_profile()) { - TRuntimeProfileTree tprofile; - const uint8_t* buf = (const uint8_t*)result.load_channel_profile().data(); - uint32_t len = result.load_channel_profile().size(); - auto st = deserialize_thrift_msg(buf, &len, false, &tprofile); - if (st.ok()) { - _state->load_channel_profile()->update(tprofile); - } else { - LOG(WARNING) << "load channel TRuntimeProfileTree deserialize failed, errmsg=" << st; - } - } -} - -void VNodeChannel::_add_block_failed_callback(bool is_last_rpc) { - std::lock_guard l(this->_closed_lock); - if (this->_is_closed) { - // if the node channel is closed, no need to call `mark_as_failed`, - // and notice that _index_channel may already be destroyed. - return; - } - SCOPED_ATTACH_TASK(_state); - // If rpc failed, mark all tablets on this node channel as failed - _index_channel->mark_as_failed( - this->node_id(), this->host(), - fmt::format("rpc failed, error coed:{}, error text:{}", - _add_block_closure->cntl.ErrorCode(), _add_block_closure->cntl.ErrorText()), - -1); - Status st = _index_channel->check_intolerable_failure(); - if (!st.ok()) { - _cancel_with_msg(fmt::format("{}, err: {}", channel_info(), st.to_string())); - } else if (is_last_rpc) { - // if this is last rpc, will must set _add_batches_finished. otherwise, node channel's close_wait - // will be blocked. - _add_batches_finished = true; - } -} - void VNodeChannel::cancel(const std::string& cancel_msg) { if (_is_closed) { // skip the channels that have been canceled or close_wait. @@ -984,6 +928,7 @@ Status VNodeChannel::close_wait(RuntimeState* state) { // waiting for finished, it may take a long time, so we couldn't set a timeout // In pipeline, is_close_done() is false at this time, will not bock. while (!_add_batches_finished && !_cancelled && !state->is_cancelled()) { + // std::this_thread::sleep_for(std::chrono::milliseconds(1)); bthread_usleep(1000); } _close_time_ms = UnixMillis() - _close_time_ms; @@ -1016,12 +961,12 @@ void VNodeChannel::mark_close() { _cur_add_block_request.set_eos(true); { + debug::ScopedTSANIgnoreReadsAndWrites ignore_tsan; std::lock_guard l(_pending_batches_lock); if (!_cur_mutable_block) { // add a dummy block _cur_mutable_block = vectorized::MutableBlock::create_unique(); } - // when prepare to close, add block to queue so that try_send_pending_block thread will send it. _pending_blocks.emplace(std::move(_cur_mutable_block), _cur_add_block_request); _pending_batches_num++; DCHECK(_pending_blocks.back().second.eos()); @@ -1135,8 +1080,6 @@ Status VOlapTableSink::prepare(RuntimeState* state) { _filter_timer = ADD_CHILD_TIMER(_profile, "FilterTime", "SendDataTime"); _where_clause_timer = ADD_CHILD_TIMER(_profile, "WhereClauseTime", "SendDataTime"); _append_node_channel_timer = ADD_CHILD_TIMER(_profile, "AppendNodeChannelTime", "SendDataTime"); - _add_partition_request_timer = - ADD_CHILD_TIMER(_profile, "AddPartitionRequestTime", "SendDataTime"); _validate_data_timer = ADD_TIMER(_profile, "ValidateDataTime"); _open_timer = ADD_TIMER(_profile, "OpenTime"); _close_timer = ADD_TIMER(_profile, "CloseWaitTime"); @@ -1183,22 +1126,15 @@ Status VOlapTableSink::prepare(RuntimeState* state) { tablets.emplace_back(std::move(tablet_with_partition)); } } - if (tablets.empty() && !_vpartition->is_auto_partition()) { + if (UNLIKELY(tablets.empty())) { LOG(WARNING) << "load job:" << state->load_job_id() << " index: " << index->index_id << " would open 0 tablet"; } _channels.emplace_back(new IndexChannel(this, index->index_id, index->where_clause)); - _index_id_to_channel[index->index_id] = _channels.back(); RETURN_IF_ERROR(_channels.back()->init(state, tablets)); } // Prepare the exprs to run. RETURN_IF_ERROR(vectorized::VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); - // prepare for auto partition functions - if (_vpartition->is_auto_partition()) { - auto [part_ctx, part_func] = _get_partition_function(); - RETURN_IF_ERROR(part_func->prepare(_state, *_output_row_desc, part_ctx.get())); - } - _prepare = true; return Status::OK(); } @@ -1245,7 +1181,6 @@ Status VOlapTableSink::open(RuntimeState* state) { MIN(_send_batch_parallelism, config::max_send_batch_parallelism_per_job); _send_batch_thread_pool_token = state->exec_env()->send_batch_thread_pool()->new_token( ThreadPool::ExecutionMode::CONCURRENT, send_batch_parallelism); - // start to send batch continually if (bthread_start_background(&_sender_thread, nullptr, periodic_send_batch, (void*)this) != 0) { return Status::Error("bthread_start_backgroud failed"); } @@ -1257,12 +1192,7 @@ void VOlapTableSink::_send_batch_process() { SCOPED_TIMER(_non_blocking_send_timer); SCOPED_ATTACH_TASK(_state); SCOPED_CONSUME_MEM_TRACKER(_mem_tracker); - - bool had_effect = false; while (true) { - // incremental open will temporarily make channels into abnormal state. stop checking when this. - std::unique_lock l(_stop_check_channel); - int running_channels_num = 0; for (const auto& index_channel : _channels) { index_channel->for_each_node_channel([&running_channels_num, @@ -1272,14 +1202,11 @@ void VOlapTableSink::_send_batch_process() { }); } - // if there is no channel, maybe auto partition table. so check does there have had running channels ever. - if (running_channels_num == 0 && had_effect) { + if (running_channels_num == 0) { LOG(INFO) << "all node channels are stopped(maybe finished/offending/cancelled), " "sender thread exit. " << print_id(_load_id); return; - } else if (running_channels_num != 0) { - had_effect = true; } bthread_usleep(config::olap_table_sink_send_interval_ms * 1000); } @@ -1293,96 +1220,6 @@ size_t VOlapTableSink::get_pending_bytes() const { return mem_consumption; } -Status VOlapTableSink::_automatic_create_partition() { - SCOPED_TIMER(_add_partition_request_timer); - TCreatePartitionRequest request; - TCreatePartitionResult result; - request.__set_txn_id(_txn_id); - request.__set_db_id(_vpartition->db_id()); - request.__set_table_id(_vpartition->table_id()); - request.__set_partitionValues(_partitions_need_create); - - VLOG(1) << "automatic partition rpc begin request " << request; - TNetworkAddress master_addr = ExecEnv::GetInstance()->master_info()->network_address; - int time_out = _state->execution_timeout() * 1000; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( - master_addr.hostname, master_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->createPartition(result, request); - }, - time_out)); - - Status status(Status::create(result.status)); - VLOG(1) << "automatic partition rpc end response " << result; - if (result.status.status_code == TStatusCode::OK) { - // add new created partitions - RETURN_IF_ERROR(_vpartition->add_partitions(result.partitions)); - - // add new tablet locations. it will use by address. so add to pool - auto* new_locations = _pool->add(new std::vector(result.tablets)); - _location->add_locations(*new_locations); - - // update new node info - _nodes_info->add_nodes(result.nodes); - - // incremental open node channel - RETURN_IF_ERROR(_incremental_open_node_channel(result.partitions)); - } - - return status; -} - -Status VOlapTableSink::_incremental_open_node_channel( - const std::vector& partitions) { - // do what we did in prepare() for partitions. indexes which don't change when we create new partition is orthogonal to partitions. - std::unique_lock _l(_stop_check_channel); - for (int i = 0; i < _schema->indexes().size(); ++i) { - const OlapTableIndexSchema* index = _schema->indexes()[i]; - std::vector tablets; - for (auto& t_part : partitions) { - VOlapTablePartition* part = nullptr; - RETURN_IF_ERROR(_vpartition->generate_partition_from(t_part, part)); - for (const auto& tablet : part->indexes[i].tablets) { - TTabletWithPartition tablet_with_partition; - tablet_with_partition.partition_id = part->id; - tablet_with_partition.tablet_id = tablet; - tablets.emplace_back(std::move(tablet_with_partition)); - } - DCHECK(!tablets.empty()) << "incremental open got nothing!"; - } - // update and reinit for existing channels. - std::shared_ptr channel = _index_id_to_channel[index->index_id]; - DCHECK(channel != nullptr); - RETURN_IF_ERROR(channel->init(_state, tablets)); // add tablets into it - } - - fmt::memory_buffer buf; - for (auto& channel : _channels) { - // incremental open new partition's tablet on storage side - channel->for_each_node_channel( - [](const std::shared_ptr& ch) { ch->incremental_open(); }); - fmt::format_to(buf, "index id:{}", channel->_index_id); - VLOG_DEBUG << "list of open index id = " << fmt::to_string(buf); - - channel->for_each_node_channel([&channel](const std::shared_ptr& ch) { - auto st = ch->open_wait(); - if (!st.ok()) { - // The open() phase is mainly to generate DeltaWriter instances on the nodes corresponding to each node channel. - // This phase will not fail due to a single tablet. - // Therefore, if the open() phase fails, all tablets corresponding to the node need to be marked as failed. - channel->mark_as_failed( - ch->node_id(), ch->host(), - fmt::format("{}, open failed, err: {}", ch->channel_info(), st.to_string()), - -1); - } - }); - - RETURN_IF_ERROR(channel->check_intolerable_failure()); - } - - return Status::OK(); -} - void VOlapTableSink::_generate_row_distribution_payload( ChannelDistributionPayload& channel_to_payload, const VOlapTablePartition* partition, uint32_t tablet_index, int row_idx, size_t row_cnt) { @@ -1456,27 +1293,6 @@ Status VOlapTableSink::_single_partition_generate(RuntimeState* state, vectorize return Status::OK(); } -std::pair -VOlapTableSink::_get_partition_function() { - return {_vpartition->get_part_func_ctx(), _vpartition->get_partition_function()}; -} - -void VOlapTableSink::_save_missing_values(vectorized::ColumnPtr col, - vectorized::DataTypePtr value_type, - std::vector filter) { - _partitions_need_create.clear(); - std::set deduper; - // de-duplication - for (auto row : filter) { - deduper.emplace(value_type->to_string(*col, row)); - } - for (auto& value : deduper) { - TStringLiteral node; - node.value = value; - _partitions_need_create.emplace_back(std::vector {node}); // only 1 partition column now - } -} - Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block, bool eos) { SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get()); Status status = Status::OK(); @@ -1504,6 +1320,9 @@ Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block, RETURN_IF_ERROR(_block_convertor->validate_and_convert_block( state, input_block, block, _output_vexpr_ctxs, rows, eos, has_filtered_rows)); + // clear and release the references of columns + input_block->clear(); + SCOPED_RAW_TIMER(&_send_data_ns); // This is just for passing compilation. bool stop_processing = false; @@ -1513,106 +1332,30 @@ Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block, _row_distribution_watch.start(); auto num_rows = block->rows(); size_t partition_num = _vpartition->get_partitions().size(); - if (!_vpartition->is_auto_partition() && partition_num == 1 && - _tablet_finder->is_find_tablet_every_sink()) { + if (partition_num == 1 && _tablet_finder->is_find_tablet_every_sink()) { RETURN_IF_ERROR(_single_partition_generate(state, block.get(), channel_to_payload, num_rows, has_filtered_rows)); } else { - // if there's projection of partition calc, we need to calc it first. - auto [part_ctx, part_func] = _get_partition_function(); - int result_idx; - if (_vpartition->is_projection_partition()) { - // calc the start value of missing partition ranges. - part_func->execute(part_ctx.get(), block.get(), &result_idx); - VLOG_DEBUG << "Partition-calculated block:" << block->dump_data(); - // change the column to compare to transformed. - _vpartition->set_transformed_slots({(uint16_t)result_idx}); - } - - if (_vpartition->is_auto_partition()) { - std::vector partition_keys = _vpartition->get_partition_keys(); - //TODO: use loop to create missing_vals for multi column. - CHECK(partition_keys.size() == 1) - << "now support only 1 partition column for auto partitions."; - auto partition_col = block->get_by_position(partition_keys[0]); - - std::vector missing_map; // indice of missing values in partition_col - missing_map.reserve(partition_col.column->size()); - - // try to find tablet and save missing value - for (int i = 0; i < num_rows; ++i) { - if (UNLIKELY(has_filtered_rows) && _block_convertor->filter_bitmap().Get(i)) { - continue; - } - const VOlapTablePartition* partition = nullptr; - bool is_continue = false; - uint32_t tablet_index = 0; - bool missing_this = false; - RETURN_IF_ERROR(_tablet_finder->find_tablet(state, block.get(), i, &partition, - tablet_index, stop_processing, - is_continue, &missing_this)); - if (missing_this) { - missing_map.push_back(i); - } else { - _generate_row_distribution_payload(channel_to_payload, partition, tablet_index, - i, 1); - } + for (int i = 0; i < num_rows; ++i) { + if (UNLIKELY(has_filtered_rows) && _block_convertor->filter_bitmap().Get(i)) { + continue; } - missing_map.shrink_to_fit(); - - // for missing partition keys, calc the missing partition and save in _partitions_need_create - auto type = partition_col.type; - if (missing_map.size() > 0) { - auto return_type = part_func->data_type(); - - // expose the data column - vectorized::ColumnPtr range_left_col = block->get_by_position(result_idx).column; - if (auto* nullable = - check_and_get_column(*range_left_col)) { - range_left_col = nullable->get_nested_column_ptr(); - return_type = - assert_cast(return_type.get()) - ->get_nested_type(); - } - // calc the end value and save them. - _save_missing_values(range_left_col, return_type, missing_map); - // then call FE to create it. then FragmentExecutor will redo the load. - RETURN_IF_ERROR(_automatic_create_partition()); - // now we need to rollback the metrics - _number_input_rows -= rows; - state->update_num_rows_load_total(-rows); - state->update_num_bytes_load_total(-bytes); - DorisMetrics::instance()->load_rows->increment(-rows); - DorisMetrics::instance()->load_bytes->increment(-bytes); - // In the next round, we will _generate_row_distribution_payload again to get right payload of new tablet - LOG(INFO) << "Auto created partition. Send block again."; - return Status::NeedSendAgain(""); - } // creating done - } else { // not auto partition - for (int i = 0; i < num_rows; ++i) { - if (UNLIKELY(has_filtered_rows) && _block_convertor->filter_bitmap().Get(i)) { - continue; - } - const VOlapTablePartition* partition = nullptr; - bool is_continue = false; - uint32_t tablet_index = 0; - RETURN_IF_ERROR(_tablet_finder->find_tablet(state, block.get(), i, &partition, - tablet_index, stop_processing, - is_continue)); - if (is_continue) { - continue; - } - // each row - _generate_row_distribution_payload(channel_to_payload, partition, tablet_index, i, - 1); + const VOlapTablePartition* partition = nullptr; + bool is_continue = false; + uint32_t tablet_index = 0; + RETURN_IF_ERROR(_tablet_finder->find_tablet( + state, block.get(), i, &partition, tablet_index, stop_processing, is_continue)); + if (is_continue) { + continue; } + // each row + _generate_row_distribution_payload(channel_to_payload, partition, tablet_index, i, 1); } } _row_distribution_watch.stop(); // Random distribution and the block belongs to a single tablet, we could optimize to append the whole // block into node channel. - bool load_block_to_single_tablet = - !_vpartition->is_auto_partition() && _tablet_finder->is_single_tablet(); + bool load_block_to_single_tablet = _tablet_finder->is_single_tablet(); if (load_block_to_single_tablet) { SCOPED_RAW_TIMER(&_filter_ns); // Filter block @@ -1629,17 +1372,12 @@ Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block, block.get(), filter_col, block->columns())); } } - // FIXME: Before load, we need to projection unuseful column - // auto slots = _schema->tuple_desc()->slots(); - // for (auto desc : slots) { - // desc->col_pos(); - // } // Add block to node channel for (size_t i = 0; i < _channels.size(); i++) { for (const auto& entry : channel_to_payload[i]) { // if this node channel is already failed, this add_row will be skipped auto st = entry.first->add_block( - block.get(), &entry.second, // entry.second is a [row -> tablet] mapping + block.get(), &entry.second, // if it is load single tablet, then append this whole block load_block_to_single_tablet); if (!st.ok()) { @@ -1754,7 +1492,6 @@ Status VOlapTableSink::close(RuntimeState* state, Status exec_status) { SCOPED_TIMER(_profile->total_time_counter()); try_close(state, exec_status); - // If _close_status is not ok, all nodes have been canceled in try_close. if (_close_status.ok()) { auto status = Status::OK(); @@ -1765,43 +1502,44 @@ Status VOlapTableSink::close(RuntimeState* state, Status exec_status) { total_wait_exec_time_ns = 0, max_wait_exec_time_ns = 0, total_add_batch_num = 0, num_node_channels = 0; VNodeChannelStat channel_stat; - - for (const auto& index_channel : _channels) { - if (!status.ok()) { - break; - } - int64_t add_batch_exec_time = 0; - int64_t wait_exec_time = 0; - index_channel->for_each_node_channel( - [this, &index_channel, &status, &state, &node_add_batch_counter_map, - &serialize_batch_ns, &channel_stat, &queue_push_lock_ns, &actual_consume_ns, - &total_add_batch_exec_time_ns, &add_batch_exec_time, &total_wait_exec_time_ns, - &wait_exec_time, - &total_add_batch_num](const std::shared_ptr& ch) { - if (!status.ok() || ch->is_closed()) { - return; - } - // in pipeline, all node channels are done or canceled, will not block. - // no pipeline, close may block waiting. - auto s = ch->close_wait(state); - if (!s.ok()) { - status = this->_cancel_channel_and_check_intolerable_failure( - status, s.to_string(), index_channel, ch); - } - ch->time_report(&node_add_batch_counter_map, &serialize_batch_ns, - &channel_stat, &queue_push_lock_ns, &actual_consume_ns, - &total_add_batch_exec_time_ns, &add_batch_exec_time, - &total_wait_exec_time_ns, &wait_exec_time, - &total_add_batch_num); - }); - num_node_channels += index_channel->num_node_channels(); - if (add_batch_exec_time > max_add_batch_exec_time_ns) { - max_add_batch_exec_time_ns = add_batch_exec_time; - } - if (wait_exec_time > max_wait_exec_time_ns) { - max_wait_exec_time_ns = wait_exec_time; - } - } // end for index channels + { + for (const auto& index_channel : _channels) { + if (!status.ok()) { + break; + } + int64_t add_batch_exec_time = 0; + int64_t wait_exec_time = 0; + index_channel->for_each_node_channel( + [this, &index_channel, &status, &state, &node_add_batch_counter_map, + &serialize_batch_ns, &channel_stat, &queue_push_lock_ns, + &actual_consume_ns, &total_add_batch_exec_time_ns, &add_batch_exec_time, + &total_wait_exec_time_ns, &wait_exec_time, + &total_add_batch_num](const std::shared_ptr& ch) { + if (!status.ok() || ch->is_closed()) { + return; + } + // in pipeline, all node channels are done or canceled, will not block. + // no pipeline, close may block waiting. + auto s = ch->close_wait(state); + if (!s.ok()) { + status = this->_cancel_channel_and_check_intolerable_failure( + status, s.to_string(), index_channel, ch); + } + ch->time_report(&node_add_batch_counter_map, &serialize_batch_ns, + &channel_stat, &queue_push_lock_ns, &actual_consume_ns, + &total_add_batch_exec_time_ns, &add_batch_exec_time, + &total_wait_exec_time_ns, &wait_exec_time, + &total_add_batch_num); + }); + num_node_channels += index_channel->num_node_channels(); + if (add_batch_exec_time > max_add_batch_exec_time_ns) { + max_add_batch_exec_time_ns = add_batch_exec_time; + } + if (wait_exec_time > max_wait_exec_time_ns) { + max_wait_exec_time_ns = wait_exec_time; + } + } // end for index channels + } if (status.ok()) { // TODO need to be improved diff --git a/be/src/vec/sink/vtablet_sink.h b/be/src/vec/sink/vtablet_sink.h index 7b37acf1f9..dc408d392b 100644 --- a/be/src/vec/sink/vtablet_sink.h +++ b/be/src/vec/sink/vtablet_sink.h @@ -20,9 +20,6 @@ #include #include #include -#include -#include -#include #include #include #include @@ -35,7 +32,6 @@ #include // IWYU pragma: no_include #include // IWYU pragma: keep -#include #include #include #include @@ -44,7 +40,6 @@ #include #include #include -#include #include #include #include @@ -159,8 +154,6 @@ public: cid = cntl.call_id(); } - // if _packet_in_flight == false, set it to true. Return true. - // if _packet_in_flight == true, Return false. bool try_set_in_flight() { bool value = false; return _packet_in_flight.compare_exchange_strong(value, true); @@ -218,47 +211,31 @@ public: int64_t append_node_channel_ns = 0; }; -// every NodeChannel keeps a data transmission channel with one BE. for multiple times open, it has a dozen of requests and corresponding closures. class VNodeChannel { public: - VNodeChannel(VOlapTableSink* parent, IndexChannel* index_channel, int64_t node_id, - bool is_incremental = false); + VNodeChannel(VOlapTableSink* parent, IndexChannel* index_channel, int64_t node_id); ~VNodeChannel(); - // called before open, used to add tablet located in this backend. called by IndexChannel::init + // called before open, used to add tablet located in this backend void add_tablet(const TTabletWithPartition& tablet) { _all_tablets.emplace_back(tablet); } - std::string debug_tablets() const { - std::stringstream ss; - for (auto& tab : _all_tablets) { - tab.printTo(ss); - ss << '\n'; - } - return ss.str(); - } void add_slave_tablet_nodes(int64_t tablet_id, const std::vector& slave_nodes) { _slave_tablet_nodes[tablet_id] = slave_nodes; } - // build a request and build corresponding connect to BE. void open(); - // for auto partition, we use this to open more tablet. - void incremental_open(); Status init(RuntimeState* state); - // this will block until all request transmission which were opened or incremental opened finished. Status open_wait(); Status add_block(vectorized::Block* block, const Payload* payload, bool is_append = false); - // @return: unfinished running channels. - // @caller: VOlapTabletSink::_send_batch_process. it's a continual asynchronous process. int try_send_and_fetch_status(RuntimeState* state, std::unique_ptr& thread_pool_token); - // when there's pending block found by try_send_and_fetch_status(), we will awake a thread to send it. - void try_send_pending_block(RuntimeState* state); + + void try_send_block(RuntimeState* state); void clear_all_blocks(); @@ -322,18 +299,10 @@ public: size_t get_pending_bytes() { return _pending_batches_bytes; } - bool is_incremental() const { return _is_incremental; } - protected: - // make a real open request for relative BE's load channel. - void _open_internal(bool is_incremental); - void _close_check(); void _cancel_with_msg(const std::string& msg); - void _add_block_success_callback(const PTabletWriterAddBlockResult& result, bool is_last_rpc); - void _add_block_failed_callback(bool is_last_rpc); - VOlapTableSink* _parent = nullptr; IndexChannel* _index_channel = nullptr; int64_t _node_id = -1; @@ -376,8 +345,7 @@ protected: std::atomic _pending_batches_num {0}; // reuse for vectorized std::shared_ptr _stub = nullptr; - // because we have incremantal open, we should keep one relative closure for one request. it's similarly for adding block. - std::vector*> _open_closures; + RefCountClosure* _open_closure = nullptr; std::vector _all_tablets; // map from tablet_id to node_id where slave replicas locate in @@ -405,7 +373,6 @@ protected: // rows number received per tablet, tablet_id -> rows_num std::vector> _tablets_received_rows; - // build a _cur_mutable_block and push into _pending_blocks. when not building, this block is empty. std::unique_ptr _cur_mutable_block; PTabletWriterAddBlockRequest _cur_add_block_request; @@ -413,8 +380,6 @@ protected: std::pair, PTabletWriterAddBlockRequest>; std::queue _pending_blocks; ReusableClosure* _add_block_closure = nullptr; - - bool _is_incremental; }; // Write block data to Olap Table. @@ -468,16 +433,6 @@ private: void _cancel_all_channel(Status status); - std::pair _get_partition_function(); - - void _save_missing_values(vectorized::ColumnPtr col, vectorized::DataTypePtr value_type, - std::vector filter); - - // create partitions when need for auto-partition table using #_partitions_need_create. - Status _automatic_create_partition(); - - Status _incremental_open_node_channel(const std::vector& partitions); - std::shared_ptr _mem_tracker; ObjectPool* _pool; @@ -509,16 +464,11 @@ private: std::unique_ptr _tablet_finder; // index_channel - std::mutex _stop_check_channel; std::vector> _channels; - std::unordered_map> _index_id_to_channel; bthread_t _sender_thread = 0; std::unique_ptr _send_batch_thread_pool_token; - // support only one partition column now - std::vector> _partitions_need_create; - std::unique_ptr _block_convertor; // Stats for this int64_t _send_data_ns = 0; @@ -536,7 +486,6 @@ private: RuntimeProfile::Counter* _append_node_channel_timer = nullptr; RuntimeProfile::Counter* _filter_timer = nullptr; RuntimeProfile::Counter* _where_clause_timer = nullptr; - RuntimeProfile::Counter* _add_partition_request_timer = nullptr; RuntimeProfile::Counter* _wait_mem_limit_timer = nullptr; RuntimeProfile::Counter* _validate_data_timer = nullptr; RuntimeProfile::Counter* _open_timer = nullptr; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index e33ef112e2..1ae98bedcf 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -68,11 +68,9 @@ public final class FeMetaVersion { public static final int VERSION_123 = 123; // For auto-increment column public static final int VERSION_124 = 124; - // For write/read auto create partition expr - public static final int VERSION_125 = 125; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_125; + public static final int VERSION_CURRENT = VERSION_124; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 3cd7f9fd3f..7b32fefac5 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3228,28 +3228,7 @@ opt_partition ::= | KW_PARTITION KW_BY KW_LIST LPAREN ident_list:columns RPAREN LPAREN opt_all_partition_desc_list:list RPAREN {: - RESULT = new ListPartitionDesc(columns, list); - :} - /* expr range partition */ - | KW_AUTO KW_PARTITION KW_BY KW_RANGE function_call_expr:fnExpr - LPAREN opt_all_partition_desc_list:list RPAREN - {: - ArrayList exprs = new ArrayList(); - exprs.add(fnExpr); - RESULT = RangePartitionDesc.createRangePartitionDesc(exprs, list); - :} - /* expr list partition */ - | KW_AUTO KW_PARTITION KW_BY KW_LIST LPAREN expr_list:exprs RPAREN - LPAREN opt_all_partition_desc_list:list RPAREN - {: - RESULT = ListPartitionDesc.createListPartitionDesc(exprs, list); - :} - | KW_AUTO KW_PARTITION KW_BY KW_LIST function_call_expr:fnExpr - LPAREN opt_all_partition_desc_list:list RPAREN - {: - ArrayList exprs = new ArrayList(); - exprs.add(fnExpr); - RESULT = ListPartitionDesc.createListPartitionDesc(exprs, list); + RESULT = new ListPartitionDesc(columns, list); :} ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java index 0ca97ca960..d0b6bebf05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ListPartitionDesc.java @@ -36,24 +36,6 @@ public class ListPartitionDesc extends PartitionDesc { List allPartitionDescs) throws AnalysisException { super(partitionColNames, allPartitionDescs); type = PartitionType.LIST; - this.isAutoCreatePartitions = false; - } - - public ListPartitionDesc(ArrayList exprs, List partitionColNames, - List allPartitionDescs) throws AnalysisException { - if (exprs != null) { - this.partitionExprs = exprs; - } - this.partitionColNames = partitionColNames; - this.singlePartitionDescs = handleAllPartitionDesc(allPartitionDescs); - this.type = PartitionType.LIST; - this.isAutoCreatePartitions = true; - } - - public static ListPartitionDesc createListPartitionDesc(ArrayList exprs, - List allPartitionDescs) throws AnalysisException { - List colNames = getColNamesFromExpr(exprs, true); - return new ListPartitionDesc(exprs, colNames, allPartitionDescs); } @Override @@ -118,8 +100,7 @@ public class ListPartitionDesc extends PartitionDesc { } } - ListPartitionInfo listPartitionInfo = new ListPartitionInfo(this.isAutoCreatePartitions, this.partitionExprs, - partitionColumns); + ListPartitionInfo listPartitionInfo = new ListPartitionInfo(partitionColumns); for (SinglePartitionDesc desc : singlePartitionDescs) { long partitionId = partitionNameToId.get(desc.getPartitionName()); listPartitionInfo.handleNewSinglePartitionDesc(desc, partitionId, isTemp); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index 8c5ff8b0ef..11cb795fd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -27,14 +27,11 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.qe.ConnectContext; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.lang3.NotImplementedException; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,23 +39,14 @@ import java.util.Set; public class PartitionDesc { protected List partitionColNames; protected List singlePartitionDescs; - protected ArrayList partitionExprs; //eg: auto partition by range date_trunc(column, 'day') - protected boolean isAutoCreatePartitions; + protected PartitionType type; - public static final ImmutableSet RANGE_PARTITION_FUNCTIONS = new ImmutableSortedSet.Builder( - String.CASE_INSENSITIVE_ORDER).add("date_trunc").add("date_ceil").add("date_floor") - .build(); public PartitionDesc() {} public PartitionDesc(List partitionColNames, List allPartitionDescs) throws AnalysisException { this.partitionColNames = partitionColNames; - this.singlePartitionDescs = handleAllPartitionDesc(allPartitionDescs); - } - - public List handleAllPartitionDesc(List allPartitionDescs) - throws AnalysisException { boolean isMultiPartition = false; List tmpList = Lists.newArrayList(); if (allPartitionDescs != null) { @@ -77,7 +65,7 @@ public class PartitionDesc { throw new AnalysisException("multi partition column size except 1 but provided " + partitionColNames.size() + "."); } - return tmpList; + this.singlePartitionDescs = tmpList; } public List getSinglePartitionDescs() { @@ -97,62 +85,6 @@ public class PartitionDesc { return partitionColNames; } - // 1. partition by list (column) : now support one slotRef - // 2. partition by range(column/function(column)) : support slotRef and some - // special function eg: date_trunc, date_floor/ceil - public static List getColNamesFromExpr(ArrayList exprs, boolean isListPartition) - throws AnalysisException { - List colNames = new ArrayList<>(); - for (Expr expr : exprs) { - if ((expr instanceof FunctionCallExpr) && (isListPartition == false)) { - FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; - List paramsExpr = functionCallExpr.getParams().exprs(); - String name = functionCallExpr.getFnName().getFunction(); - if (RANGE_PARTITION_FUNCTIONS.contains(name)) { - for (Expr param : paramsExpr) { - if (param instanceof SlotRef) { - if (colNames.isEmpty()) { - colNames.add(((SlotRef) param).getColumnName()); - } else { - throw new AnalysisException( - "auto create partition only support one slotRef in function expr. " - + expr.toSql()); - } - } - } - } else { - throw new AnalysisException( - "auto create partition only support function call expr is date_trunc/date_floor/date_ceil. " - + expr.toSql()); - } - } else if (expr instanceof SlotRef) { - if (colNames.isEmpty()) { - colNames.add(((SlotRef) expr).getColumnName()); - } else { - throw new AnalysisException( - "auto create partition only support one slotRef in expr. " - + expr.toSql()); - } - } else { - if (!isListPartition) { - throw new AnalysisException( - "auto create partition only support slotRef and date_trunc/date_floor/date_ceil" - + "function in range partitions. " + expr.toSql()); - } else { - throw new AnalysisException( - "auto create partition only support slotRef in list partitions. " - + expr.toSql()); - } - } - } - if (colNames.isEmpty()) { - throw new AnalysisException( - "auto create partition have not find any partition columns. " - + exprs.get(0).toSql()); - } - return colNames; - } - public void analyze(List columnDefs, Map otherProperties) throws AnalysisException { if (partitionColNames == null || partitionColNames.isEmpty()) { throw new AnalysisException("No partition columns."); @@ -196,15 +128,6 @@ public class PartitionDesc { if (this instanceof ListPartitionDesc && columnDef.isAllowNull()) { throw new AnalysisException("The list partition column must be NOT NULL"); } - if (this instanceof RangePartitionDesc && partitionExprs != null) { - if (partitionExprs.get(0) instanceof FunctionCallExpr) { - if (!columnDef.getType().isDatetime() && !columnDef.getType().isDatetimeV2()) { - throw new AnalysisException( - "auto create partition function expr need datetime/datetimev2 type. " - + partitionExprs.get(0).toSql()); - } - } - } found = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java deleted file mode 100644 index dadf74b27c..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ /dev/null @@ -1,193 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.Type; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.thrift.TStringLiteral; - -import com.google.common.collect.Maps; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class PartitionExprUtil { - public static final String DATETIME_FORMATTER = "%04d-%02d-%02d %02d:%02d:%02d"; - public static final String DATE_FORMATTER = "%04d-%02d-%02d"; - public static final String DATETIME_NAME_FORMATTER = "%04d%02d%02d%02d%02d%02d"; - private static final Logger LOG = LogManager.getLogger(PartitionExprUtil.class); - private static final PartitionExprUtil partitionExprUtil = new PartitionExprUtil(); - - public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList partitionExprs, - PartitionType partitionType) throws AnalysisException { - if (partitionType != PartitionType.RANGE) { - return null; - } - if (partitionExprs.size() != 1) { - throw new AnalysisException("now only support one expr in range partition"); - } - - Expr e = partitionExprs.get(0); - if (!(e instanceof FunctionCallExpr)) { - throw new AnalysisException("now range partition only support FunctionCallExpr"); - } - FunctionCallExpr functionCallExpr = (FunctionCallExpr) e; - String fnName = functionCallExpr.getFnName().getFunction(); - String timeUnit; - int interval; - if ("date_trunc".equalsIgnoreCase(fnName)) { - List paramsExprs = functionCallExpr.getParams().exprs(); - if (paramsExprs.size() != 2) { - throw new AnalysisException("date_trunc params exprs size should be 2."); - } - Expr param = paramsExprs.get(1); - if (!(param instanceof StringLiteral)) { - throw new AnalysisException("date_trunc param of time unit is not string literal."); - } - timeUnit = ((StringLiteral) param).getStringValue().toLowerCase(); - interval = 1; - } else { - throw new AnalysisException("now range partition only support date_trunc."); - } - return partitionExprUtil.new FunctionIntervalInfo(timeUnit, interval); - } - - public static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInfo intervalInfo) - throws AnalysisException { - String timeUnit = intervalInfo.timeUnit; - int interval = intervalInfo.interval; - switch (timeUnit) { - case "year": - return beginTime.plusYears(interval); - case "month": - return beginTime.plusMonths(interval); - case "day": - return beginTime.plusDays(interval); - case "hour": - return beginTime.plusHours(interval); - case "minute": - return beginTime.plusMinutes(interval); - case "second": - return beginTime.plusSeconds(interval); - default: - break; - } - return null; - } - - public static Map getAddPartitionClauseFromPartitionValues(OlapTable olapTable, - ArrayList partitionValues, PartitionInfo partitionInfo) - throws AnalysisException { - Map result = Maps.newHashMap(); - ArrayList partitionExprs = partitionInfo.getPartitionExprs(); - PartitionType partitionType = partitionInfo.getType(); - List partiitonColumn = partitionInfo.getPartitionColumns(); - Type partitionColumnType = partiitonColumn.get(0).getType(); - FunctionIntervalInfo intervalInfo = getFunctionIntervalInfo(partitionExprs, partitionType); - Set filterPartitionValues = new HashSet(); - - for (TStringLiteral partitionValue : partitionValues) { - PartitionKeyDesc partitionKeyDesc = null; - String partitionName = "p"; - String value = partitionValue.value; - if (filterPartitionValues.contains(value)) { - continue; - } - filterPartitionValues.add(value); - if (partitionType == PartitionType.RANGE) { - String beginTime = value; - DateLiteral beginDateTime = new DateLiteral(beginTime, Type.DATETIMEV2); - partitionName += String.format(DATETIME_NAME_FORMATTER, - beginDateTime.getYear(), beginDateTime.getMonth(), beginDateTime.getDay(), - beginDateTime.getHour(), beginDateTime.getMinute(), beginDateTime.getSecond()); - DateLiteral endDateTime = getRangeEnd(beginDateTime, intervalInfo); - partitionKeyDesc = createPartitionKeyDescWithRange(beginDateTime, endDateTime, partitionColumnType); - } else if (partitionType == PartitionType.LIST) { - List> listValues = new ArrayList<>(); - // TODO: need to support any type - String pointValue = value; - PartitionValue lowerValue = new PartitionValue(pointValue); - listValues.add(Collections.singletonList(lowerValue)); - partitionKeyDesc = PartitionKeyDesc.createIn( - listValues); - partitionName += lowerValue.getStringValue(); - } else { - throw new AnalysisException("now only support range and list partition"); - } - - Map partitionProperties = Maps.newHashMap(); - DistributionDesc distributionDesc = olapTable.getDefaultDistributionInfo().toDistributionDesc(); - - SinglePartitionDesc singleRangePartitionDesc = new SinglePartitionDesc(true, partitionName, - partitionKeyDesc, partitionProperties); - - AddPartitionClause addPartitionClause = new AddPartitionClause(singleRangePartitionDesc, - distributionDesc, partitionProperties, false); - result.put(partitionName, addPartitionClause); - } - return result; - } - - public static PartitionKeyDesc createPartitionKeyDescWithRange(DateLiteral beginDateTime, - DateLiteral endDateTime, Type partitionColumnType) throws AnalysisException { - String beginTime; - String endTime; - // maybe need check the range in FE also, like getAddPartitionClause. - if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { - beginTime = String.format(DATE_FORMATTER, beginDateTime.getYear(), beginDateTime.getMonth(), - beginDateTime.getDay()); - endTime = String.format(DATE_FORMATTER, endDateTime.getYear(), endDateTime.getMonth(), - endDateTime.getDay()); - } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()) { - beginTime = String.format(DATETIME_FORMATTER, - beginDateTime.getYear(), beginDateTime.getMonth(), beginDateTime.getDay(), - beginDateTime.getHour(), beginDateTime.getMinute(), beginDateTime.getSecond()); - endTime = String.format(DATETIME_FORMATTER, - endDateTime.getYear(), endDateTime.getMonth(), endDateTime.getDay(), - endDateTime.getHour(), endDateTime.getMinute(), endDateTime.getSecond()); - } else { - throw new AnalysisException( - "not support range partition with column type : " + partitionColumnType.toString()); - } - PartitionValue lowerValue = new PartitionValue(beginTime); - PartitionValue upperValue = new PartitionValue(endTime); - return PartitionKeyDesc.createFixed( - Collections.singletonList(lowerValue), - Collections.singletonList(upperValue)); - } - - public class FunctionIntervalInfo { - public String timeUnit; - public int interval; - - public FunctionIntervalInfo(String timeUnit, int interval) { - this.timeUnit = timeUnit; - this.interval = interval; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java index 099e5b0b21..cc18df6299 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java @@ -35,22 +35,6 @@ public class RangePartitionDesc extends PartitionDesc { List allPartitionDescs) throws AnalysisException { super(partitionColNames, allPartitionDescs); type = org.apache.doris.catalog.PartitionType.RANGE; - this.isAutoCreatePartitions = false; - } - - public RangePartitionDesc(ArrayList exprs, List partitionColNames, - List allPartitionDescs) throws AnalysisException { - this.partitionExprs = exprs; - this.partitionColNames = partitionColNames; - this.singlePartitionDescs = handleAllPartitionDesc(allPartitionDescs); - this.type = org.apache.doris.catalog.PartitionType.RANGE; - this.isAutoCreatePartitions = true; - } - - public static RangePartitionDesc createRangePartitionDesc(ArrayList exprs, - List allPartitionDescs) throws AnalysisException { - List colNames = getColNamesFromExpr(exprs, false); - return new RangePartitionDesc(exprs, colNames, allPartitionDescs); } @Override @@ -132,8 +116,7 @@ public class RangePartitionDesc extends PartitionDesc { * [ {10, 100, 1000}, {50, 500, MIN } ) * [ {50, 500, MIN }, {80, MIN, MIN } ) */ - RangePartitionInfo rangePartitionInfo = new RangePartitionInfo(this.isAutoCreatePartitions, this.partitionExprs, - partitionColumns); + RangePartitionInfo rangePartitionInfo = new RangePartitionInfo(partitionColumns); for (SinglePartitionDesc desc : singlePartitionDescs) { long partitionId = partitionNameToId.get(desc.getPartitionName()); rangePartitionInfo.handleNewSinglePartitionDesc(desc, partitionId, isTemp); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java index d657dc7d9d..c7a6b5e5a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java @@ -18,13 +18,11 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.AllPartitionDesc; -import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ListPartitionDesc; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.SinglePartitionDesc; -import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.ListUtil; @@ -56,14 +54,6 @@ public class ListPartitionInfo extends PartitionInfo { this.isMultiColumnPartition = partitionColumns.size() > 1; } - public ListPartitionInfo(boolean isAutoCreatePartitions, ArrayList exprs, List partitionColumns) { - super(PartitionType.LIST, partitionColumns); - this.isAutoCreatePartitions = isAutoCreatePartitions; - if (exprs != null) { - this.partitionExprs.addAll(exprs); - } - } - public static PartitionInfo read(DataInput in) throws IOException { PartitionInfo partitionInfo = new ListPartitionInfo(); partitionInfo.readFields(in); @@ -196,31 +186,16 @@ public class ListPartitionInfo extends PartitionInfo { @Override public String toSql(OlapTable table, List partitionId) { StringBuilder sb = new StringBuilder(); + sb.append("PARTITION BY LIST("); int idx = 0; - if (enableAutomaticPartition()) { - sb.append("AUTO PARTITION BY LIST "); - for (Expr e : partitionExprs) { - boolean isSlotRef = (e instanceof SlotRef); - if (isSlotRef) { - sb.append("("); - } - sb.append(e.toSql()); - if (isSlotRef) { - sb.append(")"); - } + for (Column column : partitionColumns) { + if (idx != 0) { + sb.append(", "); } - sb.append("\n("); - } else { - sb.append("PARTITION BY LIST("); - for (Column column : partitionColumns) { - if (idx != 0) { - sb.append(", "); - } - sb.append("`").append(column.getName()).append("`"); - idx++; - } - sb.append(")\n("); + sb.append("`").append(column.getName()).append("`"); + idx++; } + sb.append(")\n("); // sort list List> entries = new ArrayList<>(this.idToItem.entrySet()); @@ -294,6 +269,6 @@ public class ListPartitionInfo extends PartitionInfo { allPartitionDescs.add(new SinglePartitionDesc(false, partitionName, partitionKeyDesc, properties)); } - return new ListPartitionDesc(this.partitionExprs, partitionColumnNames, allPartitionDescs); + return new ListPartitionDesc(partitionColumnNames, allPartitionDescs); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 0d3ac83f72..b94440225b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -137,7 +137,7 @@ public class OlapTable extends Table { private PartitionInfo partitionInfo; @SerializedName("idToPartition") private Map idToPartition = new HashMap<>(); - private Map nameToPartition = Maps.newTreeMap(); + private Map nameToPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); @SerializedName(value = "distributionInfo") private DistributionInfo defaultDistributionInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index b7ca3c622c..d319882af4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -18,7 +18,6 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.DateLiteral; -import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.MaxLiteral; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionValue; @@ -41,7 +40,6 @@ import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -83,10 +81,6 @@ public class PartitionInfo implements Writable { // so we defer adding meta serialization until memory engine feature is more complete. protected Map idToTabletType; - // the enable automatic partition will hold this, could create partition by expr result - protected ArrayList partitionExprs; - protected boolean isAutoCreatePartitions; - public PartitionInfo() { this.type = PartitionType.UNPARTITIONED; this.idToDataProperty = new HashMap<>(); @@ -94,7 +88,6 @@ public class PartitionInfo implements Writable { this.idToInMemory = new HashMap<>(); this.idToTabletType = new HashMap<>(); this.idToStoragePolicy = new HashMap<>(); - this.partitionExprs = new ArrayList<>(); } public PartitionInfo(PartitionType type) { @@ -104,7 +97,6 @@ public class PartitionInfo implements Writable { this.idToInMemory = new HashMap<>(); this.idToTabletType = new HashMap<>(); this.idToStoragePolicy = new HashMap<>(); - this.partitionExprs = new ArrayList<>(); } public PartitionInfo(PartitionType type, List partitionColumns) { @@ -223,14 +215,6 @@ public class PartitionInfo implements Writable { return null; } - public boolean enableAutomaticPartition() { - return isAutoCreatePartitions; - } - - public ArrayList getPartitionExprs() { - return this.partitionExprs; - } - public void checkPartitionItemListsMatch(List list1, List list2) throws DdlException { } @@ -390,13 +374,6 @@ public class PartitionInfo implements Writable { idToReplicaAllocation.get(entry.getKey()).write(out); out.writeBoolean(idToInMemory.get(entry.getKey())); } - int size = partitionExprs.size(); - out.writeInt(size); - for (int i = 0; i < size; ++i) { - Expr e = this.partitionExprs.get(i); - Expr.writeTo(e, out); - } - out.writeBoolean(isAutoCreatePartitions); } public void readFields(DataInput in) throws IOException { @@ -423,14 +400,6 @@ public class PartitionInfo implements Writable { idToInMemory.put(partitionId, in.readBoolean()); } - if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_125) { - int size = in.readInt(); - for (int i = 0; i < size; ++i) { - Expr e = Expr.readIn(in); - this.partitionExprs.add(e); - } - this.isAutoCreatePartitions = in.readBoolean(); - } } @Override @@ -469,13 +438,12 @@ public class PartitionInfo implements Writable { && Objects.equals(idToTempItem, that.idToTempItem) && Objects.equals(idToDataProperty, that.idToDataProperty) && Objects.equals(idToStoragePolicy, that.idToStoragePolicy) && Objects.equals(idToReplicaAllocation, that.idToReplicaAllocation) && Objects.equals( - idToInMemory, that.idToInMemory) && Objects.equals(idToTabletType, that.idToTabletType) - && Objects.equals(partitionExprs, that.partitionExprs); + idToInMemory, that.idToInMemory) && Objects.equals(idToTabletType, that.idToTabletType); } @Override public int hashCode() { return Objects.hash(type, partitionColumns, idToItem, idToTempItem, idToDataProperty, idToStoragePolicy, - idToReplicaAllocation, isMultiColumnPartition, idToInMemory, idToTabletType, partitionExprs); + idToReplicaAllocation, isMultiColumnPartition, idToInMemory, idToTabletType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java index 952fa88d25..cd3614bec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java @@ -18,12 +18,10 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.AllPartitionDesc; -import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.RangePartitionDesc; import org.apache.doris.analysis.SinglePartitionDesc; -import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.RangeUtils; @@ -56,14 +54,6 @@ public class RangePartitionInfo extends PartitionInfo { this.isMultiColumnPartition = partitionColumns.size() > 1; } - public RangePartitionInfo(boolean isAutoCreatePartitions, ArrayList exprs, List partitionColumns) { - super(PartitionType.RANGE, partitionColumns); - this.isAutoCreatePartitions = isAutoCreatePartitions; - if (exprs != null) { - this.partitionExprs.addAll(exprs); - } - } - @Override public PartitionItem createAndCheckPartitionItem(SinglePartitionDesc desc, boolean isTemp) throws DdlException { Range newRange = null; @@ -262,31 +252,16 @@ public class RangePartitionInfo extends PartitionInfo { @Override public String toSql(OlapTable table, List partitionId) { StringBuilder sb = new StringBuilder(); + sb.append("PARTITION BY RANGE("); int idx = 0; - if (enableAutomaticPartition()) { - sb.append("AUTO PARTITION BY RANGE "); - for (Expr e : partitionExprs) { - boolean isSlotRef = (e instanceof SlotRef); - if (isSlotRef) { - sb.append("("); - } - sb.append(e.toSql()); - if (isSlotRef) { - sb.append(")"); - } + for (Column column : partitionColumns) { + if (idx != 0) { + sb.append(", "); } - sb.append("\n("); - } else { - sb.append("PARTITION BY RANGE("); - for (Column column : partitionColumns) { - if (idx != 0) { - sb.append(", "); - } - sb.append("`").append(column.getName()).append("`"); - idx++; - } - sb.append(")\n("); + sb.append("`").append(column.getName()).append("`"); + idx++; } + sb.append(")\n("); // sort range List> entries = new ArrayList<>(this.idToItem.entrySet()); @@ -350,6 +325,6 @@ public class RangePartitionInfo extends PartitionInfo { allPartitionDescs.add(new SinglePartitionDesc(false, partitionName, partitionKeyDesc, properties)); } - return new RangePartitionDesc(this.partitionExprs, partitionColumnNames, allPartitionDescs); + return new RangePartitionDesc(partitionColumnNames, allPartitionDescs); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index c238dcc262..24e3cf2029 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -131,7 +131,7 @@ public class OlapTableSink extends DataSink { if (partitionIds == null) { partitionIds = dstTable.getPartitionIds(); - if (partitionIds.isEmpty() && dstTable.getPartitionInfo().enableAutomaticPartition() == false) { + if (partitionIds.isEmpty()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_EMPTY_PARTITION_IN_TABLE, dstTable.getName()); } } @@ -178,7 +178,7 @@ public class OlapTableSink extends DataSink { tSink.setNumReplicas(numReplicas); tSink.setNeedGenRollup(dstTable.shouldLoadToNewRollup()); tSink.setSchema(createSchema(tSink.getDbId(), dstTable, analyzer)); - tSink.setPartition(createPartition(tSink.getDbId(), dstTable, analyzer)); + tSink.setPartition(createPartition(tSink.getDbId(), dstTable)); List locationParams = createLocation(dstTable); tSink.setLocation(locationParams.get(0)); if (singleReplicaLoad) { @@ -293,8 +293,7 @@ public class OlapTableSink extends DataSink { return distColumns; } - private TOlapTablePartitionParam createPartition(long dbId, OlapTable table, Analyzer analyzer) - throws UserException { + private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) throws UserException { TOlapTablePartitionParam partitionParam = new TOlapTablePartitionParam(); partitionParam.setDbId(dbId); partitionParam.setTableId(table.getId()); @@ -338,22 +337,6 @@ public class OlapTableSink extends DataSink { } } } - // for auto create partition by function expr, there is no any partition firstly, - // But this is required in thrift struct. - if (partitionIds.isEmpty()) { - partitionParam.setDistributedColumns(getDistColumns(table.getDefaultDistributionInfo())); - partitionParam.setPartitions(new ArrayList()); - } - ArrayList exprs = partitionInfo.getPartitionExprs(); - if (exprs != null && analyzer != null) { - tupleDescriptor.setTable(table); - analyzer.registerTupleDescriptor(tupleDescriptor); - for (Expr e : exprs) { - e.analyze(analyzer); - } - partitionParam.setPartitionFunctionExprs(Expr.treesToThrift(exprs)); - } - partitionParam.setEnableAutomaticPartition(partitionInfo.enableAutomaticPartition()); break; } case UNPARTITIONED: { @@ -379,18 +362,16 @@ public class OlapTableSink extends DataSink { } partitionParam.addToPartitions(tPartition); partitionParam.setDistributedColumns(getDistColumns(partition.getDistributionInfo())); - partitionParam.setEnableAutomaticPartition(false); break; } default: { throw new UserException("unsupported partition for OlapTable, partition=" + partType); } } - partitionParam.setPartitionType(partType.toThrift()); return partitionParam; } - public static void setPartitionKeys(TOlapTablePartition tPartition, PartitionItem partitionItem, int partColNum) { + private void setPartitionKeys(TOlapTablePartition tPartition, PartitionItem partitionItem, int partColNum) { if (partitionItem instanceof RangePartitionItem) { Range range = partitionItem.getItems(); // set start keys @@ -458,10 +439,6 @@ public class OlapTableSink extends DataSink { } } - // for partition by function expr, there is no any partition firstly, But this is required in thrift struct. - if (partitionIds.isEmpty()) { - locationParam.setTablets(new ArrayList()); - } // check if disk capacity reach limit // this is for load process, so use high water mark to check Status st = Env.getCurrentSystemInfo().checkExceedDiskCapacityLimit(allBePathsMap, true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index d22ae9b7f2..af7a04b45f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -19,12 +19,10 @@ package org.apache.doris.service; import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.AddColumnsClause; -import org.apache.doris.analysis.AddPartitionClause; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ColumnDef; import org.apache.doris.analysis.LabelName; import org.apache.doris.analysis.NativeInsertStmt; -import org.apache.doris.analysis.PartitionExprUtil; import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.SetType; import org.apache.doris.analysis.SqlParser; @@ -39,10 +37,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Index; -import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; @@ -55,7 +50,6 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.AuthenticationException; import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.MetaNotFoundException; @@ -78,7 +72,6 @@ import org.apache.doris.master.MasterImpl; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.planner.OlapTableSink; import org.apache.doris.planner.StreamLoadPlanner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectProcessor; @@ -117,8 +110,6 @@ import org.apache.doris.thrift.TCommitTxnRequest; import org.apache.doris.thrift.TCommitTxnResult; import org.apache.doris.thrift.TConfirmUnusedRemoteFilesRequest; import org.apache.doris.thrift.TConfirmUnusedRemoteFilesResult; -import org.apache.doris.thrift.TCreatePartitionRequest; -import org.apache.doris.thrift.TCreatePartitionResult; import org.apache.doris.thrift.TDescribeTableParams; import org.apache.doris.thrift.TDescribeTableResult; import org.apache.doris.thrift.TDescribeTablesParams; @@ -164,9 +155,6 @@ import org.apache.doris.thrift.TMasterOpResult; import org.apache.doris.thrift.TMasterResult; import org.apache.doris.thrift.TMySqlLoadAcquireTokenResult; import org.apache.doris.thrift.TNetworkAddress; -import org.apache.doris.thrift.TNodeInfo; -import org.apache.doris.thrift.TOlapTableIndexTablets; -import org.apache.doris.thrift.TOlapTablePartition; import org.apache.doris.thrift.TPipelineFragmentParams; import org.apache.doris.thrift.TPrivilegeCtrl; import org.apache.doris.thrift.TPrivilegeHier; @@ -192,12 +180,10 @@ import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStreamLoadMultiTablePutResult; import org.apache.doris.thrift.TStreamLoadPutRequest; import org.apache.doris.thrift.TStreamLoadPutResult; -import org.apache.doris.thrift.TStringLiteral; import org.apache.doris.thrift.TTableIndexQueryStats; import org.apache.doris.thrift.TTableMetadataNameIds; import org.apache.doris.thrift.TTableQueryStats; import org.apache.doris.thrift.TTableStatus; -import org.apache.doris.thrift.TTabletLocation; import org.apache.doris.thrift.TTxnParams; import org.apache.doris.thrift.TUpdateExportTaskStatusRequest; import org.apache.doris.thrift.TUpdateFollowerStatsCacheRequest; @@ -214,7 +200,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -3042,124 +3027,4 @@ public class FrontendServiceImpl implements FrontendService.Iface { // Return Ok anyway return new TStatus(TStatusCode.OK); } - - @Override - public TCreatePartitionResult createPartition(TCreatePartitionRequest request) throws TException { - LOG.info("Receive create partition request: {}", request); - long dbId = request.getDbId(); - long tableId = request.getTableId(); - TCreatePartitionResult result = new TCreatePartitionResult(); - TStatus errorStatus = new TStatus(TStatusCode.RUNTIME_ERROR); - - Database db = Env.getCurrentEnv().getInternalCatalog().getDbNullable(dbId); - if (db == null) { - errorStatus.setErrorMsgs(Lists.newArrayList(String.format("dbId=%d is not exists", dbId))); - result.setStatus(errorStatus); - return result; - } - - Table table = db.getTable(tableId).get(); - if (table == null) { - errorStatus.setErrorMsgs( - (Lists.newArrayList(String.format("dbId=%d tableId=%d is not exists", dbId, tableId)))); - result.setStatus(errorStatus); - return result; - } - - if (!(table instanceof OlapTable)) { - errorStatus.setErrorMsgs( - Lists.newArrayList(String.format("dbId=%d tableId=%d is not olap table", dbId, tableId))); - result.setStatus(errorStatus); - return result; - } - - if (request.partitionValues == null) { - errorStatus.setErrorMsgs(Lists.newArrayList("partitionValues should not null.")); - result.setStatus(errorStatus); - return result; - } - - OlapTable olapTable = (OlapTable) table; - PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - ArrayList partitionValues = new ArrayList(); - for (int i = 0; i < request.partitionValues.size(); i++) { - if (request.partitionValues.get(i).size() != 1) { - errorStatus.setErrorMsgs( - Lists.newArrayList("Only support single partition, partitionValues size should equal 1.")); - result.setStatus(errorStatus); - return result; - } - partitionValues.add(request.partitionValues.get(i).get(0)); - } - Map addPartitionClauseMap; - try { - addPartitionClauseMap = PartitionExprUtil.getAddPartitionClauseFromPartitionValues(olapTable, - partitionValues, partitionInfo); - } catch (AnalysisException ex) { - errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); - result.setStatus(errorStatus); - return result; - } - - for (AddPartitionClause addPartitionClause : addPartitionClauseMap.values()) { - try { - // here maybe check and limit created partitions num - Env.getCurrentEnv().addPartition(db, olapTable.getName(), addPartitionClause); - } catch (DdlException e) { - LOG.warn(e); - errorStatus.setErrorMsgs( - Lists.newArrayList(String.format("create partition failed. error:%s", e.getMessage()))); - result.setStatus(errorStatus); - return result; - } - } - - // build partition & tablets - List partitions = Lists.newArrayList(); - List tablets = Lists.newArrayList(); - for (String partitionName : addPartitionClauseMap.keySet()) { - Partition partition = table.getPartition(partitionName); - TOlapTablePartition tPartition = new TOlapTablePartition(); - tPartition.setId(partition.getId()); - int partColNum = partitionInfo.getPartitionColumns().size(); - // set partition keys - OlapTableSink.setPartitionKeys(tPartition, partitionInfo.getItem(partition.getId()), partColNum); - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( - index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); - tPartition.setNumBuckets(index.getTablets().size()); - } - tPartition.setIsMutable(olapTable.getPartitionInfo().getIsMutable(partition.getId())); - partitions.add(tPartition); - // tablet - int quorum = olapTable.getPartitionInfo().getReplicaAllocation(partition.getId()).getTotalReplicaNum() / 2 - + 1; - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - for (Tablet tablet : index.getTablets()) { - // we should ensure the replica backend is alive - // otherwise, there will be a 'unknown node id, id=xxx' error for stream load - // BE id -> path hash - Multimap bePathsMap = tablet.getNormalReplicaBackendPathMap(); - if (bePathsMap.keySet().size() < quorum) { - LOG.warn("auto go quorum exception"); - } - tablets.add(new TTabletLocation(tablet.getId(), Lists.newArrayList(bePathsMap.keySet()))); - } - } - } - result.setPartitions(partitions); - result.setTablets(tablets); - - // build nodes - List nodeInfos = Lists.newArrayList(); - SystemInfoService systemInfoService = Env.getCurrentSystemInfo(); - for (Long id : systemInfoService.getAllBackendIds(false)) { - Backend backend = systemInfoService.getBackend(id); - nodeInfos.add(new TNodeInfo(backend.getId(), 0, backend.getHost(), backend.getBrpcPort())); - } - result.setNodes(nodeInfos); - result.setStatus(new TStatus(TStatusCode.OK)); - LOG.debug("send create partition result: {}", result); - return result; - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java index c13730f32d..0a05d7c618 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java @@ -84,27 +84,26 @@ public class TruncateTableTest { @Test public void testTruncateWithCaseInsensitivePartitionName() throws Exception { - //now in order to support auto create partition, need set partition name is case sensitive Database db = Env.getCurrentInternalCatalog().getDbNullable("default_cluster:test"); OlapTable tbl = db.getOlapTableOrDdlException("case_sensitive_table"); - long p20211006Id = tbl.getPartition("p20211006").getId(); + long p20211006Id = tbl.getPartition("P20211006").getId(); long p20211007Id = tbl.getPartition("P20211007").getId(); - long p20211008Id = tbl.getPartition("P20211008").getId(); + long p20211008Id = tbl.getPartition("p20211008").getId(); // truncate p20211008(real name is P20211008) - String truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION P20211008; \n"; + String truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION p20211008; \n"; TruncateTableStmt truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext); Env.getCurrentEnv().truncateTable(truncateTableStmt); - Assert.assertNotEquals(p20211008Id, tbl.getPartition("P20211008").getId()); + Assert.assertNotEquals(p20211008Id, tbl.getPartition("p20211008").getId()); // 2. truncate P20211007 truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION P20211007; \n"; truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext); Env.getCurrentEnv().truncateTable(truncateTableStmt); Assert.assertEquals(3, tbl.getPartitionInfo().idToDataProperty.size()); - Assert.assertNotEquals(p20211007Id, tbl.getPartition("P20211007").getId()); + Assert.assertNotEquals(p20211007Id, tbl.getPartition("p20211007").getId()); Assert.assertEquals(p20211006Id, tbl.getPartition("p20211006").getId()); Assert.assertNotNull(tbl.getPartition("p20211006")); - Assert.assertNotNull(tbl.getPartition("p20211006")); + Assert.assertNotNull(tbl.getPartition("P20211006")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java index 18ebaf6851..ac9aded5b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java @@ -222,7 +222,7 @@ public class DynamicPartitionUtilTest { List partitionColumnList = Lists.newArrayList(); Column partitionColumn = new Column(); partitionColumn.setType(Type.DATE); - Deencapsulation.setField(rangePartitionInfo, "partitionColumns", partitionColumnList); + Deencapsulation.setField(rangePartitionInfo, partitionColumnList); try { Deencapsulation.invoke(dynamicPartitionUtil, "checkTimeUnit", "HOUR", rangePartitionInfo); Assert.fail(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java deleted file mode 100644 index 9a1264776a..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ /dev/null @@ -1,159 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.service; - - -import org.apache.doris.analysis.CreateDbStmt; -import org.apache.doris.analysis.CreateTableStmt; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.common.Config; -import org.apache.doris.common.FeConstants; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TCreatePartitionRequest; -import org.apache.doris.thrift.TCreatePartitionResult; -import org.apache.doris.thrift.TStatusCode; -import org.apache.doris.thrift.TStringLiteral; -import org.apache.doris.utframe.UtFrameUtils; - -import mockit.Mocked; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -public class FrontendServiceImplTest { - private static String runningDir = "fe/mocked/FrontendServiceImplTest/" + UUID.randomUUID().toString() + "/"; - private static ConnectContext connectContext; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Mocked - ExecuteEnv exeEnv; - - @BeforeClass - public static void beforeClass() throws Exception { - FeConstants.runningUnitTest = true; - FeConstants.default_scheduler_interval_millisecond = 100; - Config.dynamic_partition_enable = true; - Config.dynamic_partition_check_interval_seconds = 1; - UtFrameUtils.createDorisCluster(runningDir); - // create connect context - connectContext = UtFrameUtils.createDefaultCtx(); - // create database - String createDbStmtStr = "create database test;"; - CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext); - Env.getCurrentEnv().createDb(createDbStmt); - } - - @AfterClass - public static void tearDown() { - UtFrameUtils.cleanDorisFeDir(runningDir); - } - - private static void createTable(String sql) throws Exception { - CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); - Env.getCurrentEnv().createTable(createTableStmt); - } - - - @Test - public void testCreatePartitionRange() throws Exception { - String createOlapTblStmt = new String("CREATE TABLE test.partition_range(\n" - + " event_day DATETIME,\n" - + " site_id INT DEFAULT '10',\n" - + " city_code VARCHAR(100)\n" - + ")\n" - + "DUPLICATE KEY(event_day, site_id, city_code)\n" - + "AUTO PARTITION BY range date_trunc( event_day,'day') (\n" - + "\n" - + ")\n" - + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 2\n" - + "PROPERTIES(\"replication_num\" = \"1\");"); - - createTable(createOlapTblStmt); - Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test"); - OlapTable table = (OlapTable) db.getTableOrAnalysisException("partition_range"); - - List> partitionValues = new ArrayList<>(); - List values = new ArrayList<>(); - - TStringLiteral start = new TStringLiteral(); - start.setValue("2023-08-07 00:00:00"); - values.add(start); - - partitionValues.add(values); - - FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); - TCreatePartitionRequest request = new TCreatePartitionRequest(); - request.setDbId(db.getId()); - request.setTableId(table.getId()); - request.setPartitionValues(partitionValues); - TCreatePartitionResult partition = impl.createPartition(request); - - Assert.assertEquals(partition.getStatus().getStatusCode(), TStatusCode.OK); - Partition p20230807 = table.getPartition("p20230807000000"); - Assert.assertNotNull(p20230807); - } - - @Test - public void testCreatePartitionList() throws Exception { - String createOlapTblStmt = new String("CREATE TABLE test.partition_list(\n" - + " event_day DATETIME,\n" - + " site_id INT DEFAULT '10',\n" - + " city_code VARCHAR(100) not null\n" - + ")\n" - + "DUPLICATE KEY(event_day, site_id, city_code)\n" - + "AUTO PARTITION BY list (city_code) (\n" - + "\n" - + ")\n" - + "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 2\n" - + "PROPERTIES(\"replication_num\" = \"1\");"); - - createTable(createOlapTblStmt); - Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test"); - OlapTable table = (OlapTable) db.getTableOrAnalysisException("partition_list"); - - List> partitionValues = new ArrayList<>(); - List values = new ArrayList<>(); - - TStringLiteral start = new TStringLiteral(); - start.setValue("BEIJING"); - values.add(start); - - partitionValues.add(values); - - FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); - TCreatePartitionRequest request = new TCreatePartitionRequest(); - request.setDbId(db.getId()); - request.setTableId(table.getId()); - request.setPartitionValues(partitionValues); - TCreatePartitionResult partition = impl.createPartition(request); - - Assert.assertEquals(partition.getStatus().getStatusCode(), TStatusCode.OK); - Partition pbeijing = table.getPartition("pBEIJING"); - Assert.assertNotNull(pbeijing); - } -} diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 11a1d747bb..5a462987d2 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -94,7 +94,6 @@ message PTabletWriterOpenRequest { optional bool is_vectorized = 12 [default = false]; optional int64 backend_id = 13 [default = -1]; optional bool enable_profile = 14 [default = false]; - optional bool is_incremental = 15 [default = false]; }; message PTabletWriterOpenResult { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 1b9f85e2b7..3020570876 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -20,7 +20,6 @@ namespace java org.apache.doris.thrift include "Types.thrift" include "Exprs.thrift" -include "Partitions.thrift" struct TColumn { 1: required string column_name @@ -192,9 +191,6 @@ struct TOlapTablePartitionParam { 6: required list partitions 7: optional list partition_columns - 8: optional list partition_function_exprs - 9: optional bool enable_automatic_partition - 10: optional Partitions.TPartitionType partition_type } struct TOlapTableIndex { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index e6215cb1ca..c0ef12dadb 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1130,21 +1130,6 @@ struct TAutoIncrementRangeResult { 3: optional i64 length } -struct TCreatePartitionRequest { - 1: optional i64 txn_id - 2: optional i64 db_id - 3: optional i64 table_id - // for each partition column's partition values. [missing_rows, partition_keys]->Left bound(for range) or Point(for list) - 4: optional list> partitionValues -} - -struct TCreatePartitionResult { - 1: optional Status.TStatus status - 2: optional list partitions - 3: optional list tablets - 4: optional list nodes -} - service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) @@ -1214,6 +1199,4 @@ service FrontendService { Status.TStatus updateStatsCache(1: TUpdateFollowerStatsCacheRequest request) TAutoIncrementRangeResult getAutoIncrementRange(1: TAutoIncrementRangeRequest request) - - TCreatePartitionResult createPartition(1: TCreatePartitionRequest request) } diff --git a/regression-test/data/partition_p0/auto_partition/auto_partition_stream_load1.csv b/regression-test/data/partition_p0/auto_partition/auto_partition_stream_load1.csv deleted file mode 100644 index ada7d3c6af..0000000000 --- a/regression-test/data/partition_p0/auto_partition/auto_partition_stream_load1.csv +++ /dev/null @@ -1,10 +0,0 @@ -1,2001-12-12 12:12:12.123,2001-12-12 12:12:12.123456 -2,2002-12-12 12:12:12.123,2001-12-12 12:12:12.123456 -3,2002-12-12 12:12:12.123,2001-12-12 12:12:12.123456 -4,2002-12-12 12:12:12.123,2001-12-12 12:12:12.123456 -5,2003-12-12 12:12:12.123,2001-12-13 12:12:12.123456 -6,2004-12-12 12:12:12.123,2001-12-14 12:12:12.123456 -7,2005-12-12 12:12:12.123,2001-11-12 12:12:12.123456 -8,2006-12-12 12:12:12.123,2001-11-12 12:12:12.123456 -9,2006-12-12 12:12:12.123,2001-11-13 12:12:12.123456 -10,2007-12-12 12:12:12.123,2001-11-14 12:12:12.123456 \ No newline at end of file diff --git a/regression-test/data/partition_p0/auto_partition/auto_partition_stream_load2.csv b/regression-test/data/partition_p0/auto_partition/auto_partition_stream_load2.csv deleted file mode 100644 index 16e3fae491..0000000000 --- a/regression-test/data/partition_p0/auto_partition/auto_partition_stream_load2.csv +++ /dev/null @@ -1,10 +0,0 @@ -1,Beijing,2001-12-12 12:12:12.123456 -2,BJ,2001-12-12 12:12:12.123456 -3,bj,2001-12-12 12:12:12.123456 -4,bJ,2001-12-12 12:12:12.123456 -5,Chengdu,2001-12-13 12:12:12.123456 -6,XIAN,2001-12-14 12:12:12.123456 -7,Chengdu,2001-11-12 12:12:12.123456 -8,chengDU,2001-11-12 12:12:12.123456 -9,xian,2001-11-13 12:12:12.123456 -10,beiJing,2001-11-14 12:12:12.123456 \ No newline at end of file diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out b/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out deleted file mode 100644 index 36a6418f3f..0000000000 --- a/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out +++ /dev/null @@ -1,41 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql1 -- -Abc -Beijing -Beijing -XXX -xxx - --- !sql2 -- -Abc -Abc -Beijing -Beijing -Beijing -Beijing -XXX -XXX -new -xxx -xxx - --- !sql3 -- -Abc -Beijing -Beijing -XXX -xxx - --- !sql4 -- -Abc -Abc -Beijing -Beijing -Beijing -Beijing -XXX -XXX -new -xxx -xxx - diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_partition_load.out b/regression-test/data/partition_p0/auto_partition/test_auto_partition_load.out deleted file mode 100644 index 7e1dd673f6..0000000000 --- a/regression-test/data/partition_p0/auto_partition/test_auto_partition_load.out +++ /dev/null @@ -1,25 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select1 -- -1 2001-12-12T12:12:12 2001-12-12T12:12:12.123456 -2 2002-12-12T12:12:12 2001-12-12T12:12:12.123456 -3 2002-12-12T12:12:12 2001-12-12T12:12:12.123456 -4 2002-12-12T12:12:12 2001-12-12T12:12:12.123456 -5 2003-12-12T12:12:12 2001-12-13T12:12:12.123456 -6 2004-12-12T12:12:12 2001-12-14T12:12:12.123456 -7 2005-12-12T12:12:12 2001-11-12T12:12:12.123456 -8 2006-12-12T12:12:12 2001-11-12T12:12:12.123456 -9 2006-12-12T12:12:12 2001-11-13T12:12:12.123456 -10 2007-12-12T12:12:12 2001-11-14T12:12:12.123456 - --- !select2 -- -1 Beijing 2001-12-12T12:12:12.123456 -2 BJ 2001-12-12T12:12:12.123456 -3 bj 2001-12-12T12:12:12.123456 -4 bJ 2001-12-12T12:12:12.123456 -5 Chengdu 2001-12-13T12:12:12.123456 -6 XIAN 2001-12-14T12:12:12.123456 -7 Chengdu 2001-11-12T12:12:12.123456 -8 chengDU 2001-11-12T12:12:12.123456 -9 xian 2001-11-13T12:12:12.123456 -10 beiJing 2001-11-14T12:12:12.123456 - diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_range_partition.out b/regression-test/data/partition_p0/auto_partition/test_auto_range_partition.out deleted file mode 100644 index f359c996ec..0000000000 --- a/regression-test/data/partition_p0/auto_partition/test_auto_range_partition.out +++ /dev/null @@ -1,74 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select00 -- -2022-12-14T00:00 -2022-12-15T00:00 -2022-12-16T00:00 -2022-12-17T00:00 -2022-12-18T00:00 -2022-12-19T00:00 -2022-12-20T00:00 -2122-12-14T00:00 -2122-12-15T00:00 -2122-12-16T00:00 -2122-12-17T00:00 -2122-12-18T00:00 -2122-12-19T00:00 -2122-12-20T00:00 - --- !select01 -- -2022-12-15T00:00 - --- !select02 -- -2022-12-16T00:00 -2022-12-17T00:00 -2022-12-18T00:00 -2022-12-19T00:00 -2022-12-20T00:00 -2122-12-14T00:00 -2122-12-15T00:00 -2122-12-16T00:00 -2122-12-17T00:00 -2122-12-18T00:00 -2122-12-19T00:00 -2122-12-20T00:00 - --- !select10 -- -2022-11-14T22:22:22.222 -2022-11-15T22:22:22.222 -2022-11-16T22:22:22.222 -2022-11-17T22:22:22.222 -2022-11-18T22:22:22.222 -2022-11-19T22:22:22.222 -2022-11-20T22:22:22.222 -2022-12-14T22:22:22.222 -2022-12-15T22:22:22.222 -2022-12-16T22:22:22.222 -2022-12-17T22:22:22.222 -2022-12-18T22:22:22.222 -2022-12-19T22:22:22.222 -2022-12-20T22:22:22.222 -2122-12-14T22:22:22.222 -2122-12-15T22:22:22.222 -2122-12-16T22:22:22.222 -2122-12-17T22:22:22.222 -2122-12-18T22:22:22.222 -2122-12-19T22:22:22.222 -2122-12-20T22:22:22.222 - --- !select11 -- -2022-12-15T22:22:22.222 - --- !select12 -- -2022-12-16T22:22:22.222 -2022-12-17T22:22:22.222 -2022-12-18T22:22:22.222 -2022-12-19T22:22:22.222 -2022-12-20T22:22:22.222 -2122-12-14T22:22:22.222 -2122-12-15T22:22:22.222 -2122-12-16T22:22:22.222 -2122-12-17T22:22:22.222 -2122-12-18T22:22:22.222 -2122-12-19T22:22:22.222 -2122-12-20T22:22:22.222 - diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy deleted file mode 100644 index 405b2a1fc5..0000000000 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_auto_list_partition") { - def tblName1 = "list_table1" - sql "drop table if exists ${tblName1}" - sql """ - CREATE TABLE `${tblName1}` ( - `str` varchar not null - ) ENGINE=OLAP - DUPLICATE KEY(`str`) - COMMENT 'OLAP' - AUTO PARTITION BY LIST (`str`) - ( - ) - DISTRIBUTED BY HASH(`str`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ insert into ${tblName1} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc") """ - qt_sql1 """ select * from ${tblName1} order by `str` """ - result11 = sql "show partitions from ${tblName1}" - assertEquals(result11.size(), 4) - sql """ insert into ${tblName1} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc"), ("new") """ - qt_sql2 """ select * from ${tblName1} order by `str` """ - result12 = sql "show partitions from ${tblName1}" - assertEquals(result12.size(), 5) - - def tblName2 = "list_table2" - sql "drop table if exists ${tblName2}" - sql """ - CREATE TABLE `${tblName2}` ( - `str` varchar not null - ) ENGINE=OLAP - DUPLICATE KEY(`str`) - COMMENT 'OLAP' - AUTO PARTITION BY LIST (`str`) - ( - ) - DISTRIBUTED BY HASH(`str`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ insert into ${tblName2} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc") """ - qt_sql3 """ select * from ${tblName2} order by `str` """ - result21 = sql "show partitions from ${tblName2}" - assertEquals(result21.size(), 4) - sql """ insert into ${tblName2} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc"), ("new") """ - qt_sql4 """ select * from ${tblName2} order by `str` """ - result22 = sql "show partitions from ${tblName2}" - assertEquals(result22.size(), 5) - - def tblName3 = "list_table3" - sql "drop table if exists ${tblName3}" - sql """ - CREATE TABLE `${tblName3}` ( - `k1` INT, - `k2` VARCHAR(50) not null, - `k3` DATETIMEV2(6) - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - AUTO PARTITION BY LIST (`k2`) - ( - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 16 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ insert into ${tblName3} values (1, 'ABC', '2000-01-01 12:12:12.123456'), (2, 'AAA', '2000-01-01'), (3, 'aaa', '2000-01-01'), (3, 'AaA', '2000-01-01') """ - result3 = sql "show partitions from ${tblName3}" - logger.info("${result3}") - assertEquals(result3.size(), 4) -} diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy deleted file mode 100644 index 0cf2eaf9c1..0000000000 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_auto_partition_load") { - def tblName1 = "load_table1" - sql "drop table if exists ${tblName1}" - sql """ - CREATE TABLE `${tblName1}` ( - `k1` INT, - `k2` DATETIME, - `k3` DATETIMEV2(6) - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - AUTO PARTITION BY RANGE date_trunc(`k2`, 'year') - ( - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 16 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - streamLoad { - table "${tblName1}" - set 'column_separator', ',' - file "auto_partition_stream_load1.csv" - time 20000 - } - - qt_select1 "select * from ${tblName1} order by k1" - result1 = sql "show partitions from ${tblName1}" - logger.info("${result1}") - assertEquals(result1.size(), 7) - - - def tblName2 = "load_table2" - sql "drop table if exists ${tblName2}" - sql """ - CREATE TABLE `${tblName2}` ( - `k1` INT, - `k2` VARCHAR(50) not null, - `k3` DATETIMEV2(6) - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - AUTO PARTITION BY LIST (`k2`) - ( - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 16 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - streamLoad { - table "${tblName2}" - set 'column_separator', ',' - file "auto_partition_stream_load2.csv" - time 20000 - } - - qt_select2 "select * from ${tblName2} order by k1" - result2 = sql "show partitions from ${tblName2}" - logger.info("${result2}") - assertEquals(result2.size(), 9) -} diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy deleted file mode 100644 index 874704ea8f..0000000000 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_auto_range_partition") { - def tblName1 = "range_table1" - sql "drop table if exists ${tblName1}" - // not support datev2 now. need impl date_trunc(datev2) - sql """ - CREATE TABLE `${tblName1}` ( - `TIME_STAMP` datetimev2 NOT NULL COMMENT '采集日期' - ) ENGINE=OLAP - DUPLICATE KEY(`TIME_STAMP`) - COMMENT 'OLAP' - AUTO PARTITION BY RANGE date_trunc(`TIME_STAMP`, 'day') - ( - ) - DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ insert into ${tblName1} values ('2022-12-14'), ('2022-12-15'), ('2022-12-16'), ('2022-12-17'), ('2022-12-18'), ('2022-12-19'), ('2022-12-20') """ - sql """ insert into ${tblName1} values ('2122-12-14'), ('2122-12-15'), ('2122-12-16'), ('2122-12-17'), ('2122-12-18'), ('2122-12-19'), ('2122-12-20') """ - - qt_select00 """ select * from ${tblName1} order by TIME_STAMP """ - qt_select01 """ select * from ${tblName1} WHERE TIME_STAMP = '2022-12-15' order by TIME_STAMP """ - qt_select02 """ select * from ${tblName1} WHERE TIME_STAMP > '2022-12-15' order by TIME_STAMP """ - - def tblName2 = "range_table2" - sql "drop table if exists ${tblName2}" - sql """ - CREATE TABLE `${tblName2}` ( - `TIME_STAMP` datetimev2(3) NOT NULL COMMENT '采集日期' - ) ENGINE=OLAP - DUPLICATE KEY(`TIME_STAMP`) - COMMENT 'OLAP' - AUTO PARTITION BY RANGE date_trunc(`TIME_STAMP`, 'day') - ( - ) - DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ insert into ${tblName2} values ('2022-12-14 22:22:22.222'), ('2022-12-15 22:22:22.222'), ('2022-12-16 22:22:22.222'), ('2022-12-17 22:22:22.222'), ('2022-12-18 22:22:22.222'), ('2022-12-19 22:22:22.222'), ('2022-12-20 22:22:22.222') """ - sql """ insert into ${tblName2} values ('2122-12-14 22:22:22.222'), ('2122-12-15 22:22:22.222'), ('2122-12-16 22:22:22.222'), ('2122-12-17 22:22:22.222'), ('2122-12-18 22:22:22.222'), ('2122-12-19 22:22:22.222'), ('2122-12-20 22:22:22.222') """ - sql """ insert into ${tblName2} values ('2022-11-14 22:22:22.222'), ('2022-11-15 22:22:22.222'), ('2022-11-16 22:22:22.222'), ('2022-11-17 22:22:22.222'), ('2022-11-18 22:22:22.222'), ('2022-11-19 22:22:22.222'), ('2022-11-20 22:22:22.222') """ - - - qt_select10 """ select * from ${tblName2} order by TIME_STAMP """ - qt_select11 """ select * from ${tblName2} WHERE TIME_STAMP = '2022-12-15 22:22:22.222' order by TIME_STAMP """ - qt_select12 """ select * from ${tblName2} WHERE TIME_STAMP > '2022-12-15 22:22:22.222' order by TIME_STAMP """ - - def tblName3 = "range_table3" - sql "drop table if exists ${tblName3}" - sql """ - CREATE TABLE `${tblName3}` ( - `k1` INT, - `k2` DATETIMEV2(3), - `k3` DATETIMEV2(6) - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - AUTO PARTITION BY RANGE date_trunc(`k2`, 'day') - ( - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 16 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """ insert into ${tblName3} values (1, '1990-01-01', '2000-01-01 12:12:12.123456'), (2, '1991-02-01', '2000-01-01'), (3, '1991-01-01', '2000-01-01'), (3, '1991-01-01', '2000-01-01') """ - result1 = sql "show partitions from ${tblName3}" - logger.info("${result1}") - assertEquals(result1.size(), 3) -} diff --git a/regression-test/suites/partition_p0/test_datev2_partition.groovy b/regression-test/suites/partition_p0/test_datev2_partition.groovy index 63852bb4e2..600b820684 100644 --- a/regression-test/suites/partition_p0/test_datev2_partition.groovy +++ b/regression-test/suites/partition_p0/test_datev2_partition.groovy @@ -43,6 +43,7 @@ suite("test_datev2_partition") { qt_select """ select * from ${tblName1} order by TIME_STAMP """ qt_select """ select * from ${tblName1} WHERE TIME_STAMP = '2022-12-15' order by TIME_STAMP """ qt_select """ select * from ${tblName1} WHERE TIME_STAMP > '2022-12-15' order by TIME_STAMP """ + sql "drop table ${tblName1}" def tblName2 = "test_datev2_partition2" sql "drop table if exists ${tblName2}" @@ -71,4 +72,5 @@ suite("test_datev2_partition") { qt_select """ select * from ${tblName2} order by TIME_STAMP """ qt_select """ select * from ${tblName2} WHERE TIME_STAMP = '2022-12-15 22:22:22.222' order by TIME_STAMP """ qt_select """ select * from ${tblName2} WHERE TIME_STAMP > '2022-12-15 22:22:22.222' order by TIME_STAMP """ + sql "drop table ${tblName2}" }