diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 29436ecf05..877b76fb8d 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -23,7 +23,9 @@ #include "exprs/expr_context.h" #include "gen_cpp/PaloInternalService_types.h" #include "olap/decimal12.h" +#include "olap/field.h" #include "olap/storage_engine.h" +#include "olap/tablet_schema.h" #include "olap/uint24.h" #include "olap_scan_node.h" #include "olap_utils.h" @@ -86,6 +88,14 @@ Status OlapScanner::prepare( LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } + _tablet_schema = _tablet->tablet_schema(); + if (!_parent->_olap_scan_node.columns_desc.empty() && + _parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) { + _tablet_schema.clear_columns(); + for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) { + _tablet_schema.append_column(TabletColumn(column_desc)); + } + } { std::shared_lock rdlock(_tablet->get_header_lock()); const RowsetSharedPtr rowset = _tablet->rowset_with_max_version(); @@ -170,6 +180,7 @@ Status OlapScanner::_init_tablet_reader_params( RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode)); _tablet_reader_params.tablet = _tablet; + _tablet_reader_params.tablet_schema = &_tablet_schema; _tablet_reader_params.reader_type = READER_QUERY; _tablet_reader_params.aggregation = _aggregation; _tablet_reader_params.version = Version(0, _version); @@ -210,7 +221,7 @@ Status OlapScanner::_init_tablet_reader_params( _tablet_reader_params.return_columns.push_back(i); } for (auto index : _return_columns) { - if (_tablet->tablet_schema().column(index).is_key()) { + if (_tablet_schema.column(index).is_key()) { continue; } else { _tablet_reader_params.return_columns.push_back(index); @@ -219,13 +230,12 @@ Status OlapScanner::_init_tablet_reader_params( } // use _tablet_reader_params.return_columns, because reader use this to merge sort - Status res = - _read_row_cursor.init(_tablet->tablet_schema(), _tablet_reader_params.return_columns); + Status res = _read_row_cursor.init(_tablet_schema, _tablet_reader_params.return_columns); if (!res.ok()) { LOG(WARNING) << "fail to init row cursor.res = " << res; return Status::InternalError("failed to initialize storage read row cursor"); } - _read_row_cursor.allocate_memory_for_string_type(_tablet->tablet_schema()); + _read_row_cursor.allocate_memory_for_string_type(_tablet_schema); // If a agg node is this scan node direct parent // we will not call agg object finalize method in scan node, @@ -244,7 +254,9 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) { if (!slot->is_materialized()) { continue; } - int32_t index = _tablet->field_index(slot->col_name()); + int32_t index = slot->col_unique_id() >= 0 + ? _tablet_schema.field_index(slot->col_unique_id()) + : _tablet_schema.field_index(slot->col_name()); if (index < 0) { std::stringstream ss; ss << "field name is invalid. field=" << slot->col_name(); @@ -252,7 +264,7 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) { return Status::InternalError(ss.str()); } _return_columns.push_back(index); - if (slot->is_nullable() && !_tablet->tablet_schema().column(index).is_nullable()) + if (slot->is_nullable() && !_tablet_schema.column(index).is_nullable()) _tablet_columns_convert_to_null_set.emplace(index); _query_slots.push_back(slot); } @@ -261,13 +273,13 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) { if (_tablet->tablet_schema().has_sequence_col() && need_seq_col) { bool has_replace_col = false; for (auto col : _return_columns) { - if (_tablet->tablet_schema().column(col).aggregation() == + if (_tablet_schema.column(col).aggregation() == FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) { has_replace_col = true; break; } } - if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx(); + if (auto sequence_col_idx = _tablet_schema.sequence_col_idx(); has_replace_col && std::find(_return_columns.begin(), _return_columns.end(), sequence_col_idx) == _return_columns.end()) { _return_columns.push_back(sequence_col_idx); diff --git a/be/src/exec/olap_scanner.h b/be/src/exec/olap_scanner.h index 65086eaceb..a128026217 100644 --- a/be/src/exec/olap_scanner.h +++ b/be/src/exec/olap_scanner.h @@ -145,6 +145,8 @@ protected: MonotonicStopWatch _watcher; std::shared_ptr _mem_tracker; + + TabletSchema _tablet_schema; }; } // namespace doris diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 71999bb3bb..097b667c3e 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -31,6 +31,9 @@ void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const { for (auto slot : slots) { pindex->add_columns(slot->col_name()); } + for (auto column : columns) { + column->to_schema_pb(pindex->add_columns_desc()); + } } Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { @@ -57,6 +60,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { } index->slots.emplace_back(it->second); } + for (auto& pcolumn_desc : p_index.columns_desc()) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_pb(pcolumn_desc); + index->columns.emplace_back(tc); + } _indexes.emplace_back(index); } @@ -90,6 +98,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { } index->slots.emplace_back(it->second); } + for (auto& tcolumn_desc : t_index.columns_desc) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_thrift(tcolumn_desc); + index->columns.emplace_back(tc); + } _indexes.emplace_back(index); } diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 40279a70fe..f287fcfa0d 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -27,6 +27,7 @@ #include "common/status.h" #include "gen_cpp/Descriptors_types.h" #include "gen_cpp/descriptors.pb.h" +#include "olap/tablet_schema.h" #include "runtime/descriptors.h" #include "runtime/raw_value.h" #include "runtime/tuple.h" @@ -41,6 +42,7 @@ struct OlapTableIndexSchema { int64_t index_id; std::vector slots; int32_t schema_hash; + std::vector columns; void to_protobuf(POlapTableIndexSchema* pindex) const; }; diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 00b7dc30ae..cb2e7561f2 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -72,4 +72,15 @@ void BaseTablet::_gen_tablet_path() { } } +bool BaseTablet::set_tablet_schema_into_rowset_meta() { + bool flag = false; + for (RowsetMetaSharedPtr rowset_meta : _tablet_meta->all_mutable_rs_metas()) { + if (!rowset_meta->get_rowset_pb().has_tablet_schema()) { + rowset_meta->set_tablet_schema(&_schema); + flag = true; + } + } + return flag; +} + } /* namespace doris */ diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 69a092b84a..99c7d1cac3 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -66,7 +66,9 @@ public: } // properties encapsulated in TabletSchema - const TabletSchema& tablet_schema() const; + virtual const TabletSchema& tablet_schema() const; + + bool set_tablet_schema_into_rowset_meta(); protected: void _gen_tablet_path(); diff --git a/be/src/olap/collect_iterator.cpp b/be/src/olap/collect_iterator.cpp index cf7b079004..de163eb3b0 100644 --- a/be/src/olap/collect_iterator.cpp +++ b/be/src/olap/collect_iterator.cpp @@ -55,9 +55,9 @@ Status CollectIterator::add_child(RowsetReaderSharedPtr rs_reader) { // then merged with the base rowset. void CollectIterator::build_heap(const std::vector& rs_readers) { DCHECK(rs_readers.size() == _children.size()); - _reverse = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS; - SortType sort_type = _reader->_tablet->tablet_schema().sort_type(); - int sort_col_num = _reader->_tablet->tablet_schema().sort_col_num(); + _reverse = _reader->_tablet_schema->keys_type() == KeysType::UNIQUE_KEYS; + SortType sort_type = _reader->_tablet_schema->sort_type(); + int sort_col_num = _reader->_tablet_schema->sort_col_num(); if (_children.empty()) { _inner_iter.reset(nullptr); return; @@ -200,7 +200,7 @@ CollectIterator::Level0Iterator::Level0Iterator(RowsetReaderSharedPtr rs_reader, CollectIterator::Level0Iterator::~Level0Iterator() = default; Status CollectIterator::Level0Iterator::init() { - RETURN_NOT_OK_LOG(_row_cursor.init(_reader->_tablet->tablet_schema(), _reader->_seek_columns), + RETURN_NOT_OK_LOG(_row_cursor.init(*_reader->_tablet_schema, _reader->_seek_columns), "failed to init row cursor"); return (this->*_refresh_current_row)(); } diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index a54329fb8a..e06e5a9b00 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -17,7 +17,10 @@ #include "olap/compaction.h" +#include "common/status.h" #include "gutil/strings/substitute.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/tablet.h" #include "util/time.h" #include "util/trace.h" @@ -141,8 +144,10 @@ Status Compaction::do_compaction_impl(int64_t permits) { LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name() << ", output_version=" << _output_version << ", permits: " << permits; + // get cur schema if rowset schema exist, rowset schema must be newer than tablet schema + const TabletSchema cur_tablet_schema = _tablet->tablet_schema(); - RETURN_NOT_OK(construct_output_rowset_writer()); + RETURN_NOT_OK(construct_output_rowset_writer(&cur_tablet_schema)); RETURN_NOT_OK(construct_input_rowset_readers()); TRACE("prepare finished"); @@ -152,11 +157,11 @@ Status Compaction::do_compaction_impl(int64_t permits) { Status res; if (use_vectorized_compaction) { - res = Merger::vmerge_rowsets(_tablet, compaction_type(), _input_rs_readers, - _output_rs_writer.get(), &stats); + res = Merger::vmerge_rowsets(_tablet, compaction_type(), &cur_tablet_schema, + _input_rs_readers, _output_rs_writer.get(), &stats); } else { - res = Merger::merge_rowsets(_tablet, compaction_type(), _input_rs_readers, - _output_rs_writer.get(), &stats); + res = Merger::merge_rowsets(_tablet, compaction_type(), &cur_tablet_schema, + _input_rs_readers, _output_rs_writer.get(), &stats); } if (!res.ok()) { @@ -219,8 +224,8 @@ Status Compaction::do_compaction_impl(int64_t permits) { return Status::OK(); } -Status Compaction::construct_output_rowset_writer() { - return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, +Status Compaction::construct_output_rowset_writer(const TabletSchema* schema) { + return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, schema, _oldest_write_timestamp, _newest_write_timestamp, &_output_rs_writer); } diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index bd76ded20a..648ff78935 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -67,7 +67,7 @@ protected: Status modify_rowsets(); void gc_output_rowset(); - Status construct_output_rowset_writer(); + Status construct_output_rowset_writer(const TabletSchema* schema); Status construct_input_rowset_readers(); Status check_version_continuity(const std::vector& rowsets); diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 55530420b3..d63d792acf 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -485,6 +485,11 @@ Status DataDir::load() { << " schema hash: " << rowset_meta->tablet_schema_hash() << " for txn: " << rowset_meta->txn_id(); } + if (!rowset_meta->get_rowset_pb().has_tablet_schema()) { + rowset_meta->set_tablet_schema(&tablet->tablet_schema()); + RowsetMetaManager::save(_meta, rowset_meta->tablet_uid(), rowset_meta->rowset_id(), + rowset_meta->get_rowset_pb()); + } } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE && rowset_meta->tablet_uid() == tablet->tablet_uid()) { Status publish_status = tablet->add_rowset(rowset); @@ -506,6 +511,15 @@ Status DataDir::load() { ++invalid_rowset_counter; } } + + for (int64_t tablet_id : tablet_ids) { + TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet && tablet->set_tablet_schema_into_rowset_meta()) { + TabletMetaManager::save(this, tablet->tablet_id(), tablet->schema_hash(), + tablet->tablet_meta()); + } + } + // At startup, we only count these invalid rowset, but do not actually delete it. // The actual delete operation is in StorageEngine::_clean_unused_rowset_metas, // which is cleaned up uniformly by the background cleanup thread. diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index b088ef50cc..5a1a5c5c06 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -40,7 +40,7 @@ DeltaWriter::DeltaWriter(WriteRequest* req, StorageEngine* storage_engine, bool _tablet(nullptr), _cur_rowset(nullptr), _rowset_writer(nullptr), - _tablet_schema(nullptr), + _tablet_schema(new TabletSchema), _delta_written_success(false), _storage_engine(storage_engine), _is_vec(is_vec) {} @@ -121,10 +121,11 @@ Status DeltaWriter::init() { RETURN_NOT_OK(_storage_engine->txn_manager()->prepare_txn(_req.partition_id, _tablet, _req.txn_id, _req.load_id)); } + // build tablet schema in request level + _build_current_tablet_schema(_req.index_id, _req.ptable_schema_param, _tablet->tablet_schema()); RETURN_NOT_OK(_tablet->create_rowset_writer(_req.txn_id, _req.load_id, PREPARED, OVERLAPPING, - &_rowset_writer)); - _tablet_schema = &(_tablet->tablet_schema()); + _tablet_schema.get(), &_rowset_writer)); _schema.reset(new Schema(*_tablet_schema)); _reset_mem_table(); @@ -172,7 +173,6 @@ Status DeltaWriter::write(const RowBatch* row_batch, const std::vector& row if (_is_cancelled) { return Status::OLAPInternalError(OLAP_ERR_ALREADY_CANCELLED); } - for (const auto& row_idx : row_idxs) { _mem_table->insert(row_batch->get_row(row_idx)->get_tuple(0)); } @@ -266,9 +266,9 @@ Status DeltaWriter::wait_flush() { } void DeltaWriter::_reset_mem_table() { - _mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema, _req.slots, - _req.tuple_desc, _tablet->keys_type(), _rowset_writer.get(), - _mem_tracker, _is_vec)); + _mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema.get(), + _req.slots, _req.tuple_desc, _tablet->keys_type(), + _rowset_writer.get(), _mem_tracker, _is_vec)); } Status DeltaWriter::close() { @@ -367,4 +367,17 @@ int64_t DeltaWriter::partition_id() const { return _req.partition_id; } +void DeltaWriter::_build_current_tablet_schema(int64_t index_id, + const POlapTableSchemaParam& ptable_schema_param, + const TabletSchema& ori_tablet_schema) { + *_tablet_schema = ori_tablet_schema; + //new tablet schame if new table + if (ptable_schema_param.indexes_size() > 0 && + ptable_schema_param.indexes(0).columns_desc_size() != 0 && + ptable_schema_param.indexes(0).columns_desc(0).unique_id() >= 0) { + _tablet_schema->build_current_tablet_schema(index_id, ptable_schema_param, + ori_tablet_schema); + } +} + } // namespace doris diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h index 1e2e9b6ed0..432f0a0b76 100644 --- a/be/src/olap/delta_writer.h +++ b/be/src/olap/delta_writer.h @@ -47,6 +47,8 @@ struct WriteRequest { // slots are in order of tablet's schema const std::vector* slots; bool is_high_priority = false; + POlapTableSchemaParam ptable_schema_param; + int64_t index_id; }; // Writer for a particular (load, index, tablet). @@ -107,6 +109,10 @@ private: void _reset_mem_table(); + void _build_current_tablet_schema(int64_t index_id, + const POlapTableSchemaParam& table_schema_param, + const TabletSchema& ori_tablet_schema); + bool _is_init = false; bool _is_cancelled = false; WriteRequest _req; @@ -116,7 +122,11 @@ private: // TODO: Recheck the lifetime of _mem_table, Look should use unique_ptr std::shared_ptr _mem_table; std::unique_ptr _schema; - const TabletSchema* _tablet_schema; + //const TabletSchema* _tablet_schema; + // tablet schema owned by delta writer, all write will use this tablet schema + // it's build from tablet_schema（stored when create tablet） and OlapTableSchema + // every request will have it's own tablet schema so simple schema change can work + std::unique_ptr _tablet_schema; bool _delta_written_success; StorageEngine* _storage_engine; diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index c9fec74b25..c9cba8ca1d 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -30,6 +30,7 @@ namespace doris { Status Merger::merge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, + const TabletSchema* cur_tablet_schema, const std::vector& src_rowset_readers, RowsetWriter* dst_rowset_writer, Merger::Statistics* stats_output) { TRACE_COUNTER_SCOPE_LATENCY_US("merge_rowsets_latency_us"); @@ -40,13 +41,15 @@ Status Merger::merge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, reader_params.reader_type = reader_type; reader_params.rs_readers = src_rowset_readers; reader_params.version = dst_rowset_writer->version(); + + reader_params.tablet_schema = cur_tablet_schema; RETURN_NOT_OK(reader.init(reader_params)); RowCursor row_cursor; RETURN_NOT_OK_LOG( - row_cursor.init(tablet->tablet_schema()), + row_cursor.init(*cur_tablet_schema), "failed to init row cursor when merging rowsets of tablet " + tablet->full_name()); - row_cursor.allocate_memory_for_string_type(tablet->tablet_schema()); + row_cursor.allocate_memory_for_string_type(*cur_tablet_schema); std::unique_ptr mem_pool(new MemPool("Merger:merge_rowsets")); @@ -88,6 +91,7 @@ Status Merger::merge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, } Status Merger::vmerge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, + const TabletSchema* cur_tablet_schema, const std::vector& src_rowset_readers, RowsetWriter* dst_rowset_writer, Statistics* stats_output) { TRACE_COUNTER_SCOPE_LATENCY_US("merge_rowsets_latency_us"); @@ -98,8 +102,9 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, reader_params.reader_type = reader_type; reader_params.rs_readers = src_rowset_readers; reader_params.version = dst_rowset_writer->version(); + reader_params.tablet_schema = cur_tablet_schema; - const auto& schema = tablet->tablet_schema(); + const auto& schema = *cur_tablet_schema; reader_params.return_columns.resize(schema.num_columns()); std::iota(reader_params.return_columns.begin(), reader_params.return_columns.end(), 0); reader_params.origin_return_columns = &reader_params.return_columns; diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h index 5f258779fb..4d37e6ccd6 100644 --- a/be/src/olap/merger.h +++ b/be/src/olap/merger.h @@ -36,10 +36,12 @@ public: // return OLAP_SUCCESS and set statistics into `*stats_output`. // return others on error static Status merge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, + const TabletSchema* cur_tablet_schema, const std::vector& src_rowset_readers, RowsetWriter* dst_rowset_writer, Statistics* stats_output); static Status vmerge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, + const TabletSchema* cur_tablet_schema, const std::vector& src_rowset_readers, RowsetWriter* dst_rowset_writer, Statistics* stats_output); }; diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 5b3d1b2cbc..64e936739e 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -116,12 +116,16 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR DeletePredicatePB del_pred; DeleteConditionHandler del_cond_handler; - { - std::shared_lock rdlock(tablet_var.tablet->get_header_lock()); - res = del_cond_handler.generate_delete_predicate( - tablet_var.tablet->tablet_schema(), request.delete_conditions, &del_pred); - del_preds.push(del_pred); + auto tablet_schema = tablet_var.tablet->tablet_schema(); + if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { + tablet_schema.clear_columns(); + for (const auto& column_desc : request.columns_desc) { + tablet_schema.append_column(TabletColumn(column_desc)); + } } + res = del_cond_handler.generate_delete_predicate(tablet_schema, + request.delete_conditions, &del_pred); + del_preds.push(del_pred); if (!res.ok()) { LOG(WARNING) << "fail to generate delete condition. res=" << res << ", tablet=" << tablet_var.tablet->full_name(); @@ -139,14 +143,24 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR return Status::OLAPInternalError(OLAP_ERR_TOO_MANY_VERSION); } - // write + auto tablet_schema = tablet_vars->at(0).tablet->tablet_schema(); + if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { + tablet_schema.clear_columns(); + for (const auto& column_desc : request.columns_desc) { + tablet_schema.append_column(TabletColumn(column_desc)); + } + } + + // writes if (push_type == PUSH_NORMAL_V2) { res = _convert_v2(tablet_vars->at(0).tablet, tablet_vars->at(1).tablet, - &(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add)); + &(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add), + &tablet_schema); } else { res = _convert(tablet_vars->at(0).tablet, tablet_vars->at(1).tablet, - &(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add)); + &(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add), + &tablet_schema); } if (!res.ok()) { LOG(WARNING) << "fail to convert tmp file when realtime push. res=" << res @@ -205,7 +219,8 @@ void PushHandler::_get_tablet_infos(const std::vector& tablet_vars, } Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet, - RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset) { + RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, + const TabletSchema* tablet_schema) { Status res = Status::OK(); uint32_t num_rows = 0; PUniqueId load_id; @@ -217,13 +232,13 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_ // 1. init RowsetBuilder of cur_tablet for current push VLOG_NOTICE << "init rowset builder. tablet=" << cur_tablet->full_name() - << ", block_row_size=" << cur_tablet->num_rows_per_row_block(); + << ", block_row_size=" << tablet_schema->num_rows_per_row_block(); // although the spark load output files are fully sorted, // but it depends on thirparty implementation, so we conservatively // set this value to OVERLAP_UNKNOWN std::unique_ptr rowset_writer; res = cur_tablet->create_rowset_writer(_request.transaction_id, load_id, PREPARED, - OVERLAP_UNKNOWN, &rowset_writer); + OVERLAP_UNKNOWN, tablet_schema, &rowset_writer); if (!res.ok()) { LOG(WARNING) << "failed to init rowset writer, tablet=" << cur_tablet->full_name() << ", txn_id=" << _request.transaction_id << ", res=" << res; @@ -245,7 +260,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_ } // init schema - std::unique_ptr schema(new (std::nothrow) Schema(cur_tablet->tablet_schema())); + std::unique_ptr schema(new (std::nothrow) Schema(*tablet_schema)); if (schema == nullptr) { LOG(WARNING) << "fail to create schema. tablet=" << cur_tablet->full_name(); res = Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); @@ -313,8 +328,8 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_ // 5. Convert data for schema change tables VLOG_TRACE << "load to related tables of schema_change if possible."; if (new_tablet != nullptr) { - res = SchemaChangeHandler::schema_version_convert(cur_tablet, new_tablet, cur_rowset, - new_rowset, *_desc_tbl); + res = SchemaChangeHandler::schema_version_convert( + cur_tablet, new_tablet, cur_rowset, new_rowset, *_desc_tbl, tablet_schema); if (!res.ok()) { LOG(WARNING) << "failed to change schema version for delta." << "[res=" << res << " new_tablet='" << new_tablet->full_name() @@ -329,7 +344,8 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_ } Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet, - RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset) { + RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, + const TabletSchema* tablet_schema) { Status res = Status::OK(); RowCursor row; BinaryFile raw_file; @@ -376,7 +392,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab } // init BinaryReader - if (!(res = reader->init(cur_tablet, &raw_file))) { + if (!(res = reader->init(tablet_schema, &raw_file))) { LOG(WARNING) << "fail to init reader. res=" << res << ", tablet=" << cur_tablet->full_name() << ", file=" << _request.http_file_path; @@ -388,7 +404,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab // 2. init RowsetBuilder of cur_tablet for current push std::unique_ptr rowset_writer; res = cur_tablet->create_rowset_writer(_request.transaction_id, load_id, PREPARED, - OVERLAP_UNKNOWN, &rowset_writer); + OVERLAP_UNKNOWN, tablet_schema, &rowset_writer); if (!res.ok()) { LOG(WARNING) << "failed to init rowset writer, tablet=" << cur_tablet->full_name() << ", txn_id=" << _request.transaction_id << ", res=" << res; @@ -400,7 +416,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab << ", block_row_size=" << cur_tablet->num_rows_per_row_block(); // 4. Init RowCursor - if (!(res = row.init(cur_tablet->tablet_schema()))) { + if (!(res = row.init(*tablet_schema))) { LOG(WARNING) << "fail to init rowcursor. res=" << res; break; } @@ -453,8 +469,8 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab // 7. Convert data for schema change tables VLOG_TRACE << "load to related tables of schema_change if possible."; if (new_tablet != nullptr) { - res = SchemaChangeHandler::schema_version_convert(cur_tablet, new_tablet, cur_rowset, - new_rowset, *_desc_tbl); + res = SchemaChangeHandler::schema_version_convert( + cur_tablet, new_tablet, cur_rowset, new_rowset, *_desc_tbl, tablet_schema); if (!res.ok()) { LOG(WARNING) << "failed to change schema version for delta." << "[res=" << res << " new_tablet='" << new_tablet->full_name() @@ -500,13 +516,13 @@ IBinaryReader* IBinaryReader::create(bool need_decompress) { BinaryReader::BinaryReader() : _row_buf(nullptr), _row_buf_size(0) {} -Status BinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) { +Status BinaryReader::init(const TabletSchema* tablet_schema, BinaryFile* file) { Status res = Status::OK(); do { _file = file; _content_len = _file->file_length() - _file->header_size(); - _row_buf_size = tablet->row_size(); + _row_buf_size = _tablet_schema->row_size(); _row_buf = new (std::nothrow) char[_row_buf_size]; if (_row_buf == nullptr) { @@ -521,7 +537,7 @@ Status BinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) { break; } - _tablet = tablet; + _tablet_schema = tablet_schema; _ready = true; } while (false); @@ -545,10 +561,10 @@ Status BinaryReader::next(RowCursor* row) { return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); } - const TabletSchema& schema = _tablet->tablet_schema(); + const TabletSchema& schema = *_tablet_schema; size_t offset = 0; size_t field_size = 0; - size_t num_null_bytes = (_tablet->num_null_columns() + 7) / 8; + size_t num_null_bytes = (schema.num_null_columns() + 7) / 8; if (!(res = _file->read(_row_buf + offset, num_null_bytes))) { LOG(WARNING) << "read file for one row fail. res=" << res; @@ -642,7 +658,7 @@ LzoBinaryReader::LzoBinaryReader() _row_num(0), _next_row_start(0) {} -Status LzoBinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) { +Status LzoBinaryReader::init(const TabletSchema* tablet_schema, BinaryFile* file) { Status res = Status::OK(); do { @@ -663,7 +679,7 @@ Status LzoBinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) { break; } - _tablet = tablet; + _tablet_schema = tablet_schema; _ready = true; } while (false); @@ -696,10 +712,10 @@ Status LzoBinaryReader::next(RowCursor* row) { } } - const TabletSchema& schema = _tablet->tablet_schema(); + const TabletSchema& schema = *_tablet_schema; size_t offset = 0; size_t field_size = 0; - size_t num_null_bytes = (_tablet->num_null_columns() + 7) / 8; + size_t num_null_bytes = (schema.num_null_columns() + 7) / 8; size_t p = 0; for (size_t i = 0; i < schema.num_columns(); ++i) { @@ -787,7 +803,7 @@ Status LzoBinaryReader::_next_block() { SAFE_DELETE_ARRAY(_row_buf); _max_row_num = _row_num; - _max_row_buf_size = _max_row_num * _tablet->row_size(); + _max_row_buf_size = _max_row_num * _tablet_schema->row_size(); _row_buf = new (std::nothrow) char[_max_row_buf_size]; if (_row_buf == nullptr) { LOG(WARNING) << "fail to malloc rows buf. size=" << _max_row_buf_size; diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h index a290eb01c7..70c134c983 100644 --- a/be/src/olap/push_handler.h +++ b/be/src/olap/push_handler.h @@ -60,11 +60,13 @@ public: private: Status _convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet_vec, - RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset); + RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, + const TabletSchema* tablet_schema); // Convert local data file to internal formatted delta, // return new delta's SegmentGroup Status _convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet_vec, - RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset); + RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset, + const TabletSchema* tablet_schema); // Only for debug std::string _debug_version_list(const Versions& versions) const; @@ -112,7 +114,7 @@ public: static IBinaryReader* create(bool need_decompress); virtual ~IBinaryReader() = default; - virtual Status init(TabletSharedPtr tablet, BinaryFile* file) = 0; + virtual Status init(const TabletSchema* tablet_schema, BinaryFile* file) = 0; virtual Status finalize() = 0; virtual Status next(RowCursor* row) = 0; @@ -131,7 +133,7 @@ protected: _ready(false) {} BinaryFile* _file; - TabletSharedPtr _tablet; + const TabletSchema* _tablet_schema; size_t _content_len; size_t _curr; uint32_t _adler_checksum; @@ -144,7 +146,7 @@ public: explicit BinaryReader(); ~BinaryReader() override { finalize(); } - Status init(TabletSharedPtr tablet, BinaryFile* file) override; + Status init(const TabletSchema* tablet_schema, BinaryFile* file) override; Status finalize() override; Status next(RowCursor* row) override; @@ -161,7 +163,7 @@ public: explicit LzoBinaryReader(); ~LzoBinaryReader() override { finalize(); } - Status init(TabletSharedPtr tablet, BinaryFile* file) override; + Status init(const TabletSchema* tablet_schema, BinaryFile* file) override; Status finalize() override; Status next(RowCursor* row) override; diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index 3d8a3f398f..a3286f4b33 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -188,7 +188,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params, bool need_ordered_result = true; if (read_params.reader_type == READER_QUERY) { - if (_tablet->tablet_schema().keys_type() == DUP_KEYS) { + if (_tablet_schema->keys_type() == DUP_KEYS) { // duplicated keys are allowed, no need to merge sort keys in rowset need_ordered_result = false; } @@ -200,7 +200,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params, } _reader_context.reader_type = read_params.reader_type; - _reader_context.tablet_schema = &_tablet->tablet_schema(); + _reader_context.tablet_schema = _tablet_schema; _reader_context.need_ordered_result = need_ordered_result; _reader_context.return_columns = &_return_columns; _reader_context.seek_columns = &_seek_columns; @@ -235,6 +235,7 @@ Status TabletReader::_init_params(const ReaderParams& read_params) { _need_agg_finalize = read_params.need_agg_finalize; _reader_type = read_params.reader_type; _tablet = read_params.tablet; + _tablet_schema = read_params.tablet_schema; _init_conditions_param(read_params); _init_load_bf_columns(read_params); @@ -259,8 +260,8 @@ Status TabletReader::_init_params(const ReaderParams& read_params) { _init_seek_columns(); - if (_tablet->tablet_schema().has_sequence_col()) { - auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx(); + if (_tablet_schema->has_sequence_col()) { + auto sequence_col_idx = _tablet_schema->sequence_col_idx(); DCHECK_NE(sequence_col_idx, -1); for (auto col : _return_columns) { // query has sequence col @@ -292,16 +293,16 @@ Status TabletReader::_init_return_columns(const ReaderParams& read_params) { } } for (auto id : read_params.return_columns) { - if (_tablet->tablet_schema().column(id).is_key()) { + if (_tablet_schema->column(id).is_key()) { _key_cids.push_back(id); } else { _value_cids.push_back(id); } } } else if (read_params.return_columns.empty()) { - for (size_t i = 0; i < _tablet->tablet_schema().num_columns(); ++i) { + for (size_t i = 0; i < _tablet_schema->num_columns(); ++i) { _return_columns.push_back(i); - if (_tablet->tablet_schema().column(i).is_key()) { + if (_tablet_schema->column(i).is_key()) { _key_cids.push_back(i); } else { _value_cids.push_back(i); @@ -314,7 +315,7 @@ Status TabletReader::_init_return_columns(const ReaderParams& read_params) { !read_params.return_columns.empty()) { _return_columns = read_params.return_columns; for (auto id : read_params.return_columns) { - if (_tablet->tablet_schema().column(id).is_key()) { + if (_tablet_schema->column(id).is_key()) { _key_cids.push_back(id); } else { _value_cids.push_back(id); @@ -323,7 +324,7 @@ Status TabletReader::_init_return_columns(const ReaderParams& read_params) { } else if (read_params.reader_type == READER_CHECKSUM) { _return_columns = read_params.return_columns; for (auto id : read_params.return_columns) { - if (_tablet->tablet_schema().column(id).is_key()) { + if (_tablet_schema->column(id).is_key()) { _key_cids.push_back(id); } else { _value_cids.push_back(id); @@ -353,7 +354,7 @@ void TabletReader::_init_seek_columns() { max_key_column_count = std::max(max_key_column_count, key.field_count()); } - for (size_t i = 0; i < _tablet->tablet_schema().num_columns(); i++) { + for (size_t i = 0; i < _tablet_schema->num_columns(); i++) { if (i < max_key_column_count || column_set.find(i) != column_set.end()) { _seek_columns.push_back(i); } @@ -373,19 +374,18 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) { std::vector(start_key_size).swap(_keys_param.start_keys); size_t scan_key_size = read_params.start_key.front().size(); - if (scan_key_size > _tablet->tablet_schema().num_columns()) { + if (scan_key_size > _tablet_schema->num_columns()) { LOG(WARNING) << "Input param are invalid. Column count is bigger than num_columns of schema. " << "column_count=" << scan_key_size - << ", schema.num_columns=" << _tablet->tablet_schema().num_columns(); + << ", schema.num_columns=" << _tablet_schema->num_columns(); return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); } std::vector columns(scan_key_size); std::iota(columns.begin(), columns.end(), 0); - std::shared_ptr schema = - std::make_shared(_tablet->tablet_schema().columns(), columns); + std::shared_ptr schema = std::make_shared(_tablet_schema->columns(), columns); for (size_t i = 0; i < start_key_size; ++i) { if (read_params.start_key[i].size() != scan_key_size) { @@ -396,12 +396,11 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) { } Status res = _keys_param.start_keys[i].init_scan_key( - _tablet->tablet_schema(), read_params.start_key[i].values(), schema); + *_tablet_schema, read_params.start_key[i].values(), schema); if (!res.ok()) { LOG(WARNING) << "fail to init row cursor. res = " << res; return res; } - res = _keys_param.start_keys[i].from_tuple(read_params.start_key[i]); if (!res.ok()) { LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << "key_index=" << i; @@ -419,7 +418,7 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) { return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); } - Status res = _keys_param.end_keys[i].init_scan_key(_tablet->tablet_schema(), + Status res = _keys_param.end_keys[i].init_scan_key(*_tablet_schema, read_params.end_key[i].values(), schema); if (!res.ok()) { LOG(WARNING) << "fail to init row cursor. res = " << res; @@ -439,13 +438,12 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) { } void TabletReader::_init_conditions_param(const ReaderParams& read_params) { - _conditions.set_tablet_schema(&_tablet->tablet_schema()); - _all_conditions.set_tablet_schema(&_tablet->tablet_schema()); + _conditions.set_tablet_schema(_tablet_schema); + _all_conditions.set_tablet_schema(_tablet_schema); for (const auto& condition : read_params.conditions) { ColumnPredicate* predicate = _parse_to_predicate(condition); if (predicate != nullptr) { - if (_tablet->tablet_schema() - .column(_tablet->field_index(condition.column_name)) + if (_tablet_schema->column(_tablet_schema->field_index(condition.column_name)) .aggregation() != FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) { _value_col_predicates.push_back(predicate); } else { @@ -576,11 +574,11 @@ COMPARISON_PREDICATE_CONDITION_VALUE(ge, GreaterEqualPredicate) ColumnPredicate* TabletReader::_parse_to_predicate( const std::pair>& bloom_filter) { - int32_t index = _tablet->field_index(bloom_filter.first); + int32_t index = _tablet_schema->field_index(bloom_filter.first); if (index < 0) { return nullptr; } - const TabletColumn& column = _tablet->tablet_schema().column(index); + const TabletColumn& column = _tablet_schema->column(index); return BloomFilterColumnPredicateFactory::create_column_predicate(index, bloom_filter.second, column.type()); } @@ -588,12 +586,12 @@ ColumnPredicate* TabletReader::_parse_to_predicate( ColumnPredicate* TabletReader::_parse_to_predicate(const TCondition& condition, bool opposite) const { // TODO: not equal and not in predicate is not pushed down - int32_t index = _tablet->field_index(condition.column_name); + int32_t index = _tablet_schema->field_index(condition.column_name); if (index < 0) { return nullptr; } - const TabletColumn& column = _tablet->tablet_schema().column(index); + const TabletColumn& column = _tablet_schema->column(index); ColumnPredicate* predicate = nullptr; if ((condition.condition_op == "*=" || condition.condition_op == "!*=" || @@ -795,7 +793,7 @@ void TabletReader::_init_load_bf_columns(const ReaderParams& read_params, Condit std::set* load_bf_columns) { // add all columns with condition to load_bf_columns for (const auto& cond_column : conditions->columns()) { - if (!_tablet->tablet_schema().column(cond_column.first).is_bf_column()) { + if (!_tablet_schema->column(cond_column.first).is_bf_column()) { continue; } for (const auto& cond : cond_column.second->conds()) { @@ -807,7 +805,7 @@ void TabletReader::_init_load_bf_columns(const ReaderParams& read_params, Condit } // remove columns which have same value between start_key and end_key - int min_scan_key_len = _tablet->tablet_schema().num_columns(); + int min_scan_key_len = _tablet_schema->num_columns(); for (const auto& start_key : read_params.start_key) { min_scan_key_len = std::min(min_scan_key_len, static_cast(start_key.size())); } @@ -838,7 +836,7 @@ void TabletReader::_init_load_bf_columns(const ReaderParams& read_params, Condit if (max_equal_index == -1) { return; } - FieldType type = _tablet->tablet_schema().column(max_equal_index).type(); + FieldType type = _tablet_schema->column(max_equal_index).type(); if ((type != OLAP_FIELD_TYPE_VARCHAR && type != OLAP_FIELD_TYPE_STRING) || max_equal_index + 1 > _tablet->num_short_key_columns()) { load_bf_columns->erase(max_equal_index); @@ -858,7 +856,7 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { } auto delete_init = [&]() -> Status { - return _delete_handler.init(_tablet->tablet_schema(), _tablet->delete_predicates(), + return _delete_handler.init(*_tablet_schema, _tablet->delete_predicates(), read_params.version.second, this); }; diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h index c0fb42ebf8..2ca738f8fd 100644 --- a/be/src/olap/reader.h +++ b/be/src/olap/reader.h @@ -27,6 +27,7 @@ #include "olap/row_cursor.h" #include "olap/rowset/rowset_reader.h" #include "olap/tablet.h" +#include "olap/tablet_schema.h" #include "util/runtime_profile.h" namespace doris { @@ -57,6 +58,7 @@ public: // mainly include tablet, data version and fetch range. struct ReaderParams { TabletSharedPtr tablet; + const TabletSchema* tablet_schema; ReaderType reader_type = READER_QUERY; bool direct_mode = false; bool aggregation = false; @@ -171,6 +173,7 @@ protected: std::set* load_bf_columns); TabletSharedPtr tablet() { return _tablet; } + const TabletSchema& tablet_schema() { return *_tablet_schema; } std::unique_ptr _predicate_mem_pool; std::set _load_bf_columns; @@ -183,6 +186,7 @@ protected: TabletSharedPtr _tablet; RowsetReaderContext _reader_context; + const TabletSchema* _tablet_schema; KeysParam _keys_param; std::vector _is_lower_keys_included; std::vector _is_upper_keys_included; diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index d8ef290cda..14fdeee317 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -69,7 +69,8 @@ Status BetaRowset::do_load(bool /*use_cache*/) { return Status::OK(); } -Status BetaRowset::load_segments(std::vector* segments) { +Status BetaRowset::load_segments(std::vector* segments, + const TabletSchema* read_tablet_schema) { auto fs = _rowset_meta->fs(); if (!fs) { return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); @@ -77,7 +78,7 @@ Status BetaRowset::load_segments(std::vector* segm for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { auto seg_path = segment_file_path(seg_id); std::shared_ptr segment; - auto s = segment_v2::Segment::open(fs, seg_path, seg_id, _schema, &segment); + auto s = segment_v2::Segment::open(fs, seg_path, seg_id, read_tablet_schema, &segment); if (!s.ok()) { LOG(WARNING) << "failed to open segment. " << seg_path << " under rowset " << unique_id() << " : " << s.to_string(); diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 9ccaa7b2b9..e1e963f6ba 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -70,7 +70,8 @@ public: bool check_file_exist() override; - Status load_segments(std::vector* segments); + Status load_segments(std::vector* segments, + const TabletSchema* read_tablet_schema); protected: BetaRowset(const TabletSchema* schema, const std::string& tablet_path, diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index b5ff00e077..5dd7c12483 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -86,7 +86,7 @@ Status BetaRowsetReader::init(RowsetReaderContext* read_context) { // load segments RETURN_NOT_OK(SegmentLoader::instance()->load_segments( - _rowset, &_segment_cache_handle, + _rowset, &_segment_cache_handle, read_context->tablet_schema, read_context->reader_type == ReaderType::READER_QUERY)); // create iterator for each segment diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 439327ef79..87a4ba8b4c 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -88,6 +88,8 @@ Status BetaRowsetWriter::init(const RowsetWriterContext& rowset_writer_context) _rowset_meta->set_newest_write_timestamp(_context.newest_write_timestamp); } _rowset_meta->set_tablet_uid(_context.tablet_uid); + _rowset_meta->set_tablet_schema(_context.tablet_schema); + return Status::OK(); } diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp index 55fe81ffd3..009ce832bf 100644 --- a/be/src/olap/rowset/rowset.cpp +++ b/be/src/olap/rowset/rowset.cpp @@ -23,10 +23,7 @@ namespace doris { Rowset::Rowset(const TabletSchema* schema, const std::string& tablet_path, RowsetMetaSharedPtr rowset_meta) - : _schema(schema), - _tablet_path(tablet_path), - _rowset_meta(std::move(rowset_meta)), - _refs_by_reader(0) { + : _tablet_path(tablet_path), _rowset_meta(std::move(rowset_meta)), _refs_by_reader(0) { _is_pending = !_rowset_meta->has_version(); if (_is_pending) { _is_cumulative = false; @@ -34,6 +31,8 @@ Rowset::Rowset(const TabletSchema* schema, const std::string& tablet_path, Version version = _rowset_meta->version(); _is_cumulative = version.first != version.second; } + // build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema + _schema = _rowset_meta->tablet_schema() != nullptr ? _rowset_meta->tablet_schema() : schema; } Status Rowset::load(bool use_cache) { diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index ec2a39652b..3488ed2b54 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -138,6 +138,7 @@ public: // publish rowset to make it visible to read void make_visible(Version version); + const TabletSchema* tablet_schema() { return _schema; } // helper class to access RowsetMeta int64_t start_version() const { return rowset_meta()->version().first; } diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index c91fe0469d..d8f2c98249 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -31,6 +31,7 @@ #include "json2pb/json_to_pb.h" #include "json2pb/pb_to_json.h" #include "olap/olap_common.h" +#include "olap/tablet_schema.h" namespace doris { @@ -326,6 +327,14 @@ public: int64_t oldest_write_timestamp() const { return _rowset_meta_pb.oldest_write_timestamp(); } int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); } + void set_tablet_schema(const TabletSchema* tablet_schema) { + TabletSchemaPB* ts_pb = _rowset_meta_pb.mutable_tablet_schema(); + tablet_schema->to_schema_pb(ts_pb); + CHECK(_schema == nullptr); + _schema = std::make_shared(*tablet_schema); + } + + const TabletSchema* tablet_schema() { return _schema.get(); } private: friend class AlphaRowsetMeta; @@ -369,6 +378,10 @@ private: } set_num_segments(num_segments); } + if (_rowset_meta_pb.has_tablet_schema()) { + _schema = std::make_shared(); + _schema->init_from_pb(_rowset_meta_pb.tablet_schema()); + } } friend bool operator==(const RowsetMeta& a, const RowsetMeta& b) { @@ -384,6 +397,7 @@ private: private: RowsetMetaPB _rowset_meta_pb; + std::shared_ptr _schema = nullptr; RowsetId _rowset_id; io::FileSystemPtr _fs; bool _is_removed_from_rowset_meta = false; diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index fcf1d58da6..e18a000905 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -47,7 +47,7 @@ Status Segment::open(io::FileSystem* fs, const std::string& path, uint32_t segme Segment::Segment(io::FileSystem* fs, const std::string& path, uint32_t segment_id, const TabletSchema* tablet_schema) - : _fs(fs), _path(path), _segment_id(segment_id), _tablet_schema(tablet_schema) { + : _fs(fs), _path(path), _segment_id(segment_id), _tablet_schema(*tablet_schema) { #ifndef BE_TEST _mem_tracker = StorageEngine::instance()->tablet_mem_tracker(); #else @@ -177,16 +177,16 @@ Status Segment::_create_column_readers() { _column_id_to_footer_ordinal.emplace(column_pb.unique_id(), ordinal); } - _column_readers.resize(_tablet_schema->columns().size()); - for (uint32_t ordinal = 0; ordinal < _tablet_schema->num_columns(); ++ordinal) { - auto& column = _tablet_schema->columns()[ordinal]; + _column_readers.resize(_tablet_schema.columns().size()); + for (uint32_t ordinal = 0; ordinal < _tablet_schema.num_columns(); ++ordinal) { + auto& column = _tablet_schema.columns()[ordinal]; auto iter = _column_id_to_footer_ordinal.find(column.unique_id()); if (iter == _column_id_to_footer_ordinal.end()) { continue; } ColumnReaderOptions opts; - opts.kept_in_memory = _tablet_schema->is_in_memory(); + opts.kept_in_memory = _tablet_schema.is_in_memory(); std::unique_ptr reader; RETURN_IF_ERROR(ColumnReader::create(opts, _footer.columns(iter->second), _footer.num_rows(), _fs, _path, &reader)); @@ -197,7 +197,7 @@ Status Segment::_create_column_readers() { Status Segment::new_column_iterator(uint32_t cid, ColumnIterator** iter) { if (_column_readers[cid] == nullptr) { - const TabletColumn& tablet_column = _tablet_schema->column(cid); + const TabletColumn& tablet_column = _tablet_schema.column(cid); if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) { return Status::InternalError("invalid nonexistent column without default value."); } diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index 11925e35e9..2c9cb83df1 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -76,7 +76,7 @@ public: Status new_bitmap_index_iterator(uint32_t cid, BitmapIndexIterator** iter); - size_t num_short_keys() const { return _tablet_schema->num_short_key_columns(); } + size_t num_short_keys() const { return _tablet_schema.num_short_key_columns(); } uint32_t num_rows_per_block() const { DCHECK(_load_index_once.has_called() && _load_index_once.stored_result().ok()); @@ -121,7 +121,7 @@ private: std::string _path; uint32_t _segment_id; - const TabletSchema* _tablet_schema; + TabletSchema _tablet_schema; // This mem tracker is only for tracking memory use by segment meta data such as footer or index page. // The memory consumed by querying is tracked in segment iterator. diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 0e81609005..cedc0a9c6f 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -143,7 +143,7 @@ Status SegmentIterator::_init(bool is_vec) { RETURN_IF_ERROR(_init_return_column_iterators()); RETURN_IF_ERROR(_init_bitmap_index_iterators()); // z-order can not use prefix index - if (_segment->_tablet_schema->sort_type() != SortType::ZORDER) { + if (_segment->_tablet_schema.sort_type() != SortType::ZORDER) { RETURN_IF_ERROR(_get_row_ranges_by_keys()); } RETURN_IF_ERROR(_get_row_ranges_by_column_conditions()); diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 7f282d0517..ba0609e03e 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1632,8 +1632,8 @@ bool SchemaChangeWithSorting::_internal_sorting( std::unique_ptr rowset_writer; if (!new_tablet->create_rowset_writer(version, VISIBLE, segments_overlap, - oldest_write_timestamp, newest_write_timestamp, - &rowset_writer)) { + &new_tablet->tablet_schema(), oldest_write_timestamp, + newest_write_timestamp, &rowset_writer)) { return false; } @@ -1658,9 +1658,9 @@ Status VSchemaChangeWithSorting::_internal_sorting( MultiBlockMerger merger(new_tablet); std::unique_ptr rowset_writer; - RETURN_IF_ERROR(new_tablet->create_rowset_writer(version, VISIBLE, segments_overlap, - oldest_write_timestamp, newest_write_timestamp, - &rowset_writer)); + RETURN_IF_ERROR(new_tablet->create_rowset_writer( + version, VISIBLE, segments_overlap, &new_tablet->tablet_schema(), + oldest_write_timestamp, newest_write_timestamp, &rowset_writer)); Defer defer {[&]() { new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + @@ -1687,10 +1687,16 @@ bool SchemaChangeWithSorting::_external_sorting(vector& src_row } rs_readers.push_back(rs_reader); } + // get cur schema if rowset schema exist, rowset schema must be newer than tablet schema + auto max_version_rowset = src_rowsets.back(); + const TabletSchema* cur_tablet_schema = max_version_rowset->rowset_meta()->tablet_schema(); + if (cur_tablet_schema == nullptr) { + cur_tablet_schema = &(new_tablet->tablet_schema()); + } Merger::Statistics stats; - auto res = Merger::merge_rowsets(new_tablet, READER_ALTER_TABLE, rs_readers, rowset_writer, - &stats); + auto res = Merger::merge_rowsets(new_tablet, READER_ALTER_TABLE, cur_tablet_schema, rs_readers, + rowset_writer, &stats); if (!res) { LOG(WARNING) << "failed to merge rowsets. tablet=" << new_tablet->full_name() << ", version=" << rowset_writer->version().first << "-" @@ -1713,8 +1719,9 @@ Status VSchemaChangeWithSorting::_external_sorting(vector& src_ } Merger::Statistics stats; - RETURN_IF_ERROR(Merger::vmerge_rowsets(new_tablet, READER_ALTER_TABLE, rs_readers, - rowset_writer, &stats)); + RETURN_IF_ERROR(Merger::vmerge_rowsets(new_tablet, READER_ALTER_TABLE, + &new_tablet->tablet_schema(), rs_readers, rowset_writer, + &stats)); _add_merged_rows(stats.merged_rows); _add_filtered_rows(stats.filtered_rows); @@ -1801,6 +1808,13 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& // delete handlers for new tablet DeleteHandler delete_handler; std::vector return_columns; + auto base_tablet_schema = base_tablet->tablet_schema(); + if (!request.columns.empty() && request.columns[0].col_unique_id >= 0) { + base_tablet_schema.clear_columns(); + for (const auto& column : request.columns) { + base_tablet_schema.append_column(TabletColumn(column)); + } + } // begin to find deltas to convert from base tablet to new tablet so that // obtain base tablet and new tablet's push lock and header write lock to prevent loading data @@ -1811,7 +1825,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& std::lock_guard new_tablet_wlock(new_tablet->get_header_lock()); // check if the tablet has alter task // if it has alter task, it means it is under old alter process - size_t num_cols = base_tablet->tablet_schema().num_columns(); + size_t num_cols = base_tablet_schema.num_columns(); return_columns.resize(num_cols); for (int i = 0; i < num_cols; ++i) { return_columns[i] = i; @@ -1821,7 +1835,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& // with rs_readers RowsetReaderContext reader_context; reader_context.reader_type = READER_ALTER_TABLE; - reader_context.tablet_schema = &base_tablet->tablet_schema(); + reader_context.tablet_schema = &base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.return_columns = &return_columns; @@ -1907,7 +1921,8 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& reader_params.tablet = base_tablet; reader_params.reader_type = READER_ALTER_TABLE; reader_params.rs_readers = rs_readers; - const auto& schema = base_tablet->tablet_schema(); + reader_params.tablet_schema = &base_tablet_schema; + const auto& schema = base_tablet_schema; reader_params.return_columns.resize(schema.num_columns()); std::iota(reader_params.return_columns.begin(), reader_params.return_columns.end(), 0); reader_params.origin_return_columns = &reader_params.return_columns; @@ -1915,8 +1930,8 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& // BlockReader::init will call base_tablet->get_header_lock(), but this lock we already get at outer layer, so we just call TabletReader::init RETURN_NOT_OK(reader.TabletReader::init(reader_params)); - res = delete_handler.init(base_tablet->tablet_schema(), - base_tablet->delete_predicates(), end_version, &reader); + res = delete_handler.init(base_tablet_schema, base_tablet->delete_predicates(), + end_version, &reader); if (!res) { LOG(WARNING) << "init delete handler failed. base_tablet=" << base_tablet->full_name() << ", end_version=" << end_version; @@ -1947,6 +1962,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& sc_params.new_tablet = new_tablet; sc_params.ref_rowset_readers = rs_readers; sc_params.delete_handler = &delete_handler; + sc_params.base_tablet_schema = &base_tablet_schema; if (request.__isset.materialized_view_params) { for (auto item : request.materialized_view_params) { AlterMaterializedViewParam mv_param; @@ -2025,7 +2041,8 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, TabletSharedPtr new_tablet, RowsetSharedPtr* base_rowset, RowsetSharedPtr* new_rowset, - DescriptorTbl desc_tbl) { + DescriptorTbl desc_tbl, + const TabletSchema* base_schema_change) { Status res = Status::OK(); LOG(INFO) << "begin to convert delta version for schema changing. " << "base_tablet=" << base_tablet->full_name() @@ -2039,7 +2056,7 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, const std::unordered_map materialized_function_map; if (res = _parse_request(base_tablet, new_tablet, &rb_changer, &sc_sorting, &sc_directly, - materialized_function_map, desc_tbl); + materialized_function_map, desc_tbl, base_schema_change); !res) { LOG(WARNING) << "failed to parse the request. res=" << res; return res; @@ -2053,7 +2070,7 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, // c. Convert data DeleteHandler delete_handler; std::vector return_columns; - size_t num_cols = base_tablet->tablet_schema().num_columns(); + size_t num_cols = base_schema_change->num_columns(); return_columns.resize(num_cols); for (int i = 0; i < num_cols; ++i) { return_columns[i] = i; @@ -2061,7 +2078,7 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, RowsetReaderContext reader_context; reader_context.reader_type = READER_ALTER_TABLE; - reader_context.tablet_schema = &base_tablet->tablet_schema(); + reader_context.tablet_schema = base_schema_change; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.return_columns = &return_columns; @@ -2077,9 +2094,10 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet, load_id.set_hi((*base_rowset)->load_id().hi()); load_id.set_lo((*base_rowset)->load_id().lo()); std::unique_ptr rowset_writer; - RETURN_NOT_OK(new_tablet->create_rowset_writer( - (*base_rowset)->txn_id(), load_id, PREPARED, - (*base_rowset)->rowset_meta()->segments_overlap(), &rowset_writer)); + RETURN_NOT_OK( + new_tablet->create_rowset_writer((*base_rowset)->txn_id(), load_id, PREPARED, + (*base_rowset)->rowset_meta()->segments_overlap(), + reader_context.tablet_schema, &rowset_writer)); auto schema_version_convert_error = [&]() -> Status { if (*new_rowset != nullptr) { @@ -2160,9 +2178,9 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams bool sc_directly = false; // a.Parse the Alter request and convert it into an internal representation - Status res = - _parse_request(sc_params.base_tablet, sc_params.new_tablet, &rb_changer, &sc_sorting, - &sc_directly, sc_params.materialized_params_map, *sc_params.desc_tbl); + Status res = _parse_request(sc_params.base_tablet, sc_params.new_tablet, &rb_changer, + &sc_sorting, &sc_directly, sc_params.materialized_params_map, + *sc_params.desc_tbl, sc_params.base_tablet_schema); auto process_alter_exit = [&]() -> Status { { @@ -2204,8 +2222,8 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams Status status = new_tablet->create_rowset_writer( rs_reader->version(), VISIBLE, rs_reader->rowset()->rowset_meta()->segments_overlap(), - rs_reader->oldest_write_timestamp(), rs_reader->newest_write_timestamp(), - &rowset_writer); + &new_tablet->tablet_schema(), rs_reader->oldest_write_timestamp(), + rs_reader->newest_write_timestamp(), &rowset_writer); if (!status.ok()) { res = Status::OLAPInternalError(OLAP_ERR_ROWSET_BUILDER_INIT); return process_alter_exit(); @@ -2267,7 +2285,7 @@ Status SchemaChangeHandler::_parse_request( bool* sc_sorting, bool* sc_directly, const std::unordered_map& materialized_function_map, - DescriptorTbl desc_tbl) { + DescriptorTbl desc_tbl, const TabletSchema* base_tablet_schema) { // set column mapping for (int i = 0, new_schema_size = new_tablet->tablet_schema().num_columns(); i < new_schema_size; ++i) { @@ -2276,7 +2294,7 @@ Status SchemaChangeHandler::_parse_request( ColumnMapping* column_mapping = rb_changer->get_mutable_column_mapping(i); if (new_column.has_reference_column()) { - int32_t column_index = base_tablet->field_index(new_column.referenced_column()); + int32_t column_index = base_tablet_schema->field_index(new_column.referenced_column()); if (column_index < 0) { LOG(WARNING) << "referenced column was missing. " @@ -2295,7 +2313,7 @@ Status SchemaChangeHandler::_parse_request( auto mvParam = materialized_function_map.find(column_name)->second; column_mapping->materialized_function = mvParam.mv_expr; column_mapping->expr = mvParam.expr; - int32_t column_index = base_tablet->field_index(mvParam.origin_column_name); + int32_t column_index = base_tablet_schema->field_index(mvParam.origin_column_name); if (column_index >= 0) { column_mapping->ref_column = column_index; continue; @@ -2307,7 +2325,7 @@ Status SchemaChangeHandler::_parse_request( } } - int32_t column_index = base_tablet->field_index(column_name); + int32_t column_index = base_tablet_schema->field_index(column_name); if (column_index >= 0) { column_mapping->ref_column = column_index; continue; @@ -2316,7 +2334,7 @@ Status SchemaChangeHandler::_parse_request( // Newly added column go here column_mapping->ref_column = -1; - if (i < base_tablet->num_short_key_columns()) { + if (i < base_tablet_schema->num_short_key_columns()) { *sc_directly = true; } RETURN_IF_ERROR( @@ -2345,7 +2363,7 @@ Status SchemaChangeHandler::_parse_request( } } - const TabletSchema& ref_tablet_schema = base_tablet->tablet_schema(); + const TabletSchema& ref_tablet_schema = *base_tablet_schema; const TabletSchema& new_tablet_schema = new_tablet->tablet_schema(); if (ref_tablet_schema.keys_type() != new_tablet_schema.keys_type()) { // only when base table is dup and mv is agg @@ -2362,7 +2380,7 @@ Status SchemaChangeHandler::_parse_request( // old keys: A B C D // new keys: A B if (new_tablet_schema.keys_type() != KeysType::DUP_KEYS && - new_tablet->num_key_columns() < base_tablet->num_key_columns()) { + new_tablet->num_key_columns() < base_tablet_schema->num_key_columns()) { // this is a table with aggregate key type, and num of key columns in new schema // is less, which means the data in new tablet should be more aggregated. // so we use sorting schema change to sort and merge the data. @@ -2370,7 +2388,7 @@ Status SchemaChangeHandler::_parse_request( return Status::OK(); } - if (base_tablet->num_short_key_columns() != new_tablet->num_short_key_columns()) { + if (base_tablet_schema->num_short_key_columns() != new_tablet->num_short_key_columns()) { // the number of short_keys changed, can't do linked schema change *sc_directly = true; return Status::OK(); diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index 4e186cae15..42909d459a 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -251,7 +251,8 @@ class SchemaChangeHandler { public: static Status schema_version_convert(TabletSharedPtr base_tablet, TabletSharedPtr new_tablet, RowsetSharedPtr* base_rowset, RowsetSharedPtr* new_rowset, - DescriptorTbl desc_tbl); + DescriptorTbl desc_tbl, + const TabletSchema* base_schema_change); // schema change v2, it will not set alter task in base tablet static Status process_alter_tablet_v2(const TAlterTabletReqV2& request); @@ -305,6 +306,7 @@ private: AlterTabletType alter_tablet_type; TabletSharedPtr base_tablet; TabletSharedPtr new_tablet; + TabletSchema* base_tablet_schema = nullptr; std::vector ref_rowset_readers; DeleteHandler* delete_handler = nullptr; std::unordered_map materialized_params_map; @@ -323,7 +325,7 @@ private: RowBlockChanger* rb_changer, bool* sc_sorting, bool* sc_directly, const std::unordered_map& materialized_function_map, - DescriptorTbl desc_tbl); + DescriptorTbl desc_tbl, const TabletSchema* base_tablet_schema); // Initialization Settings for creating a default value static Status _init_column_mapping(ColumnMapping* column_mapping, diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp index a105336f6a..6741fcf5a3 100644 --- a/be/src/olap/segment_loader.cpp +++ b/be/src/olap/segment_loader.cpp @@ -18,6 +18,7 @@ #include "olap/segment_loader.h" #include "olap/rowset/rowset.h" +#include "olap/tablet_schema.h" #include "util/stopwatch.hpp" namespace doris { @@ -58,16 +59,17 @@ void SegmentLoader::_insert(const SegmentLoader::CacheKey& key, SegmentLoader::C } Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset, - SegmentCacheHandle* cache_handle, bool use_cache) { - SegmentLoader::CacheKey cache_key(rowset->rowset_id()); - if (_lookup(cache_key, cache_handle)) { + SegmentCacheHandle* cache_handle, + const TabletSchema* read_tablet_schema, bool use_cache) { + SegmentLoader::CacheKey cache_key(rowset->rowset_id(), *read_tablet_schema); + if (use_cache && _lookup(cache_key, cache_handle)) { cache_handle->owned = false; return Status::OK(); } cache_handle->owned = !use_cache; std::vector segments; - RETURN_NOT_OK(rowset->load_segments(&segments)); + RETURN_NOT_OK(rowset->load_segments(&segments, read_tablet_schema)); if (use_cache) { // memory of SegmentLoader::CacheValue will be handled by SegmentLoader diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h index 8541ae6ed3..2932314285 100644 --- a/be/src/olap/segment_loader.h +++ b/be/src/olap/segment_loader.h @@ -25,6 +25,7 @@ #include "olap/lru_cache.h" #include "olap/olap_common.h" // for rowset id #include "olap/rowset/beta_rowset.h" +#include "olap/tablet_schema.h" #include "util/time.h" namespace doris { @@ -48,11 +49,17 @@ class SegmentLoader { public: // The cache key or segment lru cache struct CacheKey { - CacheKey(RowsetId rowset_id_) : rowset_id(rowset_id_) {} + CacheKey(RowsetId rowset_id_, const TabletSchema& tablet_schema) + : rowset_id(rowset_id_), tablet_schema(tablet_schema) {} RowsetId rowset_id; + TabletSchema tablet_schema; // Encode to a flat binary which can be used as LRUCache's key - std::string encode() const { return rowset_id.to_string(); } + std::string encode() const { + TabletSchemaPB tablet_schema_pb; + tablet_schema.to_schema_pb(&tablet_schema_pb); + return rowset_id.to_string() + tablet_schema_pb.SerializeAsString(); + } }; // The cache value of segment lru cache. @@ -82,7 +89,7 @@ public: // Load segments of "rowset", return the "cache_handle" which contains segments. // If use_cache is true, it will be loaded from _cache. Status load_segments(const BetaRowsetSharedPtr& rowset, SegmentCacheHandle* cache_handle, - bool use_cache = false); + const TabletSchema* read_tablet_schema, bool use_cache = false); // Try to prune the segment cache if expired. Status prune(); diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 3765008a40..436769bc3a 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -232,7 +232,8 @@ Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb, context.tablet_schema_hash = org_rowset_meta->tablet_schema_hash(); context.rowset_type = org_rowset_meta->rowset_type(); context.tablet_path = new_tablet_path; - context.tablet_schema = &tablet_schema; + context.tablet_schema = + org_rowset_meta->tablet_schema() ? org_rowset_meta->tablet_schema() : &tablet_schema; context.rowset_state = org_rowset_meta->rowset_state(); context.version = org_rowset_meta->version(); context.oldest_write_timestamp = org_rowset_meta->oldest_write_timestamp(); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 49c07adc0c..17188991da 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -375,6 +375,15 @@ const RowsetSharedPtr Tablet::rowset_with_max_version() const { return iter->second; } +const RowsetMetaSharedPtr Tablet::rowset_meta_with_max_schema_version( + const std::vector& rowset_metas) { + return *std::max_element(rowset_metas.begin(), rowset_metas.end(), + [](const RowsetMetaSharedPtr& a, const RowsetMetaSharedPtr& b) { + return a->tablet_schema()->schema_version() < + b->tablet_schema()->schema_version(); + }); +} + RowsetSharedPtr Tablet::_rowset_with_largest_size() { RowsetSharedPtr largest_rowset = nullptr; for (auto& it : _rs_version_map) { @@ -1553,7 +1562,9 @@ Status Tablet::create_initial_rowset(const int64_t req_version) { do { // there is no data in init rowset, so overlapping info is unknown. std::unique_ptr rs_writer; - res = create_rowset_writer(version, VISIBLE, OVERLAP_UNKNOWN, -1, -1, &rs_writer); + res = create_rowset_writer(version, VISIBLE, OVERLAP_UNKNOWN, + &_tablet_meta->tablet_schema(), -1, -1, &rs_writer); + if (!res.ok()) { LOG(WARNING) << "failed to init rowset writer for tablet " << full_name(); break; @@ -1584,6 +1595,7 @@ Status Tablet::create_initial_rowset(const int64_t req_version) { Status Tablet::create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, + const doris::TabletSchema* tablet_schema, int64_t oldest_write_timestamp, int64_t newest_write_timestamp, std::unique_ptr* rowset_writer) { RowsetWriterContext context; @@ -1592,6 +1604,7 @@ Status Tablet::create_rowset_writer(const Version& version, const RowsetStatePB& context.segments_overlap = overlap; context.oldest_write_timestamp = oldest_write_timestamp; context.newest_write_timestamp = newest_write_timestamp; + context.tablet_schema = tablet_schema; _init_context_common_fields(context); return RowsetFactory::create_rowset_writer(context, rowset_writer); } @@ -1599,6 +1612,7 @@ Status Tablet::create_rowset_writer(const Version& version, const RowsetStatePB& Status Tablet::create_rowset_writer(const int64_t& txn_id, const PUniqueId& load_id, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, + const doris::TabletSchema* tablet_schema, std::unique_ptr* rowset_writer) { RowsetWriterContext context; context.txn_id = txn_id; @@ -1607,6 +1621,7 @@ Status Tablet::create_rowset_writer(const int64_t& txn_id, const PUniqueId& load context.segments_overlap = overlap; context.oldest_write_timestamp = -1; context.newest_write_timestamp = -1; + context.tablet_schema = tablet_schema; _init_context_common_fields(context); return RowsetFactory::create_rowset_writer(context, rowset_writer); } @@ -1625,7 +1640,6 @@ void Tablet::_init_context_common_fields(RowsetWriterContext& context) { context.rowset_type = StorageEngine::instance()->default_rowset_type(); } context.tablet_path = tablet_path(); - context.tablet_schema = &(tablet_schema()); context.data_dir = data_dir(); } @@ -1797,4 +1811,15 @@ void Tablet::remove_all_remote_rowsets() { } } +const TabletSchema& Tablet::tablet_schema() const { + std::shared_lock wrlock(_meta_lock); + _tablet_meta->all_rs_metas(); + const RowsetMetaSharedPtr rowset_meta = + rowset_meta_with_max_schema_version(_tablet_meta->all_rs_metas()); + if (rowset_meta->tablet_schema() == nullptr) { + return _schema; + } + return *rowset_meta->tablet_schema(); +} + } // namespace doris diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 3da956cb63..83f5e3d2e2 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -120,6 +120,9 @@ public: const RowsetSharedPtr rowset_with_max_version() const; + static const RowsetMetaSharedPtr rowset_meta_with_max_schema_version( + const std::vector& rowset_metas); + Status add_inc_rowset(const RowsetSharedPtr& rowset); /// Delete stale rowset by timing. This delete policy uses now() minutes /// config::tablet_rowset_expired_stale_sweep_time_sec to compute the deadline of expired rowset @@ -274,13 +277,16 @@ public: return _tablet_meta->all_beta(); } + const TabletSchema& tablet_schema() const override; + Status create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, - const SegmentsOverlapPB& overlap, int64_t oldest_write_timestamp, - int64_t newest_write_timestamp, + const SegmentsOverlapPB& overlap, const TabletSchema* tablet_schema, + int64_t oldest_write_timestamp, int64_t newest_write_timestamp, std::unique_ptr* rowset_writer); Status create_rowset_writer(const int64_t& txn_id, const PUniqueId& load_id, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, + const TabletSchema* tablet_schema, std::unique_ptr* rowset_writer); Status create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 3a0490c370..5c1be52076 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -139,8 +139,13 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id bool has_bf_columns = false; for (TColumn tcolumn : tablet_schema.columns) { ColumnPB* column = schema->add_column(); - uint32_t unique_id = col_ordinal_to_unique_id.at(col_ordinal++); - _init_column_from_tcolumn(unique_id, tcolumn, column); + uint32_t unique_id = -1; + if (tcolumn.col_unique_id >= 0) { + unique_id = tcolumn.col_unique_id; + } else { + unique_id = col_ordinal_to_unique_id.at(col_ordinal++); + } + init_column_from_tcolumn(unique_id, tcolumn, column); if (column->is_key()) { ++key_count; @@ -200,11 +205,12 @@ TabletMeta::TabletMeta(const TabletMeta& b) _storage_medium(b._storage_medium), _cooldown_resource(b._cooldown_resource) {}; -void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, - ColumnPB* column) { +void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, + ColumnPB* column) { column->set_unique_id(unique_id); + column->set_col_unique_id(tcolumn.col_unique_id); column->set_name(tcolumn.column_name); - column->set_has_bitmap_index(false); + column->set_has_bitmap_index(tcolumn.has_bitmap_index); string data_type; EnumToString(TPrimitiveType, tcolumn.column_type.type, data_type); column->set_type(data_type); @@ -243,7 +249,7 @@ void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn& tc } if (tcolumn.column_type.type == TPrimitiveType::ARRAY) { ColumnPB* children_column = column->add_children_columns(); - _init_column_from_tcolumn(0, tcolumn.children_column[0], children_column); + init_column_from_tcolumn(0, tcolumn.children_column[0], children_column); } } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 6e558d996a..26163a15e4 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -149,6 +149,7 @@ public: TabletSchema* mutable_tablet_schema(); const std::vector& all_rs_metas() const; + std::vector& all_mutable_rs_metas(); Status add_rs_meta(const RowsetMetaSharedPtr& rs_meta); void delete_rs_meta_by_version(const Version& version, std::vector* deleted_rs_metas); @@ -199,12 +200,13 @@ public: << _cooldown_resource << " to " << resource; _cooldown_resource = std::move(resource); } + static void init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, + ColumnPB* column); DeleteBitmap& delete_bitmap() { return *_delete_bitmap; } private: Status _save_meta(DataDir* data_dir); - void _init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, ColumnPB* column); // _del_pred_array is ignored to compare. friend bool operator==(const TabletMeta& a, const TabletMeta& b); @@ -473,6 +475,10 @@ inline const std::vector& TabletMeta::all_rs_metas() const return _rs_metas; } +inline std::vector& TabletMeta::all_mutable_rs_metas() { + return _rs_metas; +} + inline const std::vector& TabletMeta::all_stale_rs_metas() const { return _stale_rs_metas; } diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index cda1370b7f..a8659fc1e3 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -17,6 +17,7 @@ #include "olap/tablet_schema.h" +#include "gen_cpp/descriptors.pb.h" #include "tablet_meta.h" #include "vec/aggregate_functions/aggregate_function_reader.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" @@ -307,8 +308,24 @@ TabletColumn::TabletColumn(FieldAggregationMethod agg, FieldType filed_type, boo _length = length; } +TabletColumn::TabletColumn(const ColumnPB& column) { + init_from_pb(column); +} + +TabletColumn::TabletColumn(const TColumn& column) { + init_from_thrift(column); +} + +void TabletColumn::init_from_thrift(const TColumn& tcolumn) { + _unique_id = tcolumn.col_unique_id; + ColumnPB column_pb; + TabletMeta::init_column_from_tcolumn(_unique_id, tcolumn, &column_pb); + init_from_pb(column_pb); +} + void TabletColumn::init_from_pb(const ColumnPB& column) { _unique_id = column.unique_id(); + _col_unique_id = column.col_unique_id(); _col_name = column.name(); _type = TabletColumn::get_field_type_by_string(column.type()); _is_key = column.is_key(); @@ -358,9 +375,10 @@ void TabletColumn::init_from_pb(const ColumnPB& column) { } } -void TabletColumn::to_schema_pb(ColumnPB* column) { +void TabletColumn::to_schema_pb(ColumnPB* column) const { column->set_unique_id(_unique_id); column->set_name(_col_name); + column->set_col_unique_id(_col_unique_id); column->set_type(get_string_by_field_type(_type)); column->set_is_key(_is_key); column->set_is_nullable(_is_nullable); @@ -423,6 +441,28 @@ vectorized::AggregateFunctionPtr TabletColumn::get_aggregate_function( agg_name, argument_types, {}, argument_types.back()->is_nullable()); } +void TabletSchema::append_column(TabletColumn column) { + if (column.is_key()) { + _num_key_columns++; + } + if (column.is_nullable()) { + _num_null_columns++; + } + _field_name_to_index[column.name()] = _num_columns; + _field_id_to_index[column.col_unique_id()] = _num_columns; + _cols.push_back(std::move(column)); + _num_columns++; +} + +void TabletSchema::clear_columns() { + _field_name_to_index.clear(); + _field_id_to_index.clear(); + _num_columns = 0; + _num_null_columns = 0; + _num_key_columns = 0; + _cols.clear(); +} + void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _keys_type = schema.keys_type(); _num_columns = 0; @@ -440,6 +480,9 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _num_null_columns++; } _field_name_to_index[column.name()] = _num_columns; + if (column.col_unique_id() >= 0) { + _field_id_to_index[column.col_unique_id()] = _num_columns; + } _cols.emplace_back(std::move(column)); _num_columns++; } @@ -460,27 +503,87 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _sort_type = schema.sort_type(); _sort_col_num = schema.sort_col_num(); _compression_type = schema.compression_type(); + _schema_version = schema.schema_version(); } -void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_meta_pb) { - tablet_meta_pb->set_keys_type(_keys_type); +void TabletSchema::build_current_tablet_schema(int64_t index_id, + const POlapTableSchemaParam& ptable_schema_param, + const TabletSchema& ori_tablet_schema) { + // copy from ori_tablet_schema + _keys_type = ori_tablet_schema.keys_type(); + _num_short_key_columns = ori_tablet_schema.num_short_key_columns(); + _num_rows_per_row_block = ori_tablet_schema.num_rows_per_row_block(); + _compress_kind = ori_tablet_schema.compress_kind(); + + // todo(yixiu): unique_id + _next_column_unique_id = ori_tablet_schema.next_column_unique_id(); + _is_in_memory = ori_tablet_schema.is_in_memory(); + _delete_sign_idx = ori_tablet_schema.delete_sign_idx(); + _sequence_col_idx = ori_tablet_schema.sequence_col_idx(); + _sort_type = ori_tablet_schema.sort_type(); + _sort_col_num = ori_tablet_schema.sort_col_num(); + + // copy from table_schema_param + _num_columns = 0; + _num_key_columns = 0; + _num_null_columns = 0; + bool has_bf_columns = false; + _cols.clear(); + _field_name_to_index.clear(); + _field_id_to_index.clear(); + + for (const POlapTableIndexSchema& index : ptable_schema_param.indexes()) { + if (index.id() == index_id) { + for (auto& pcolumn : index.columns_desc()) { + TabletColumn column; + column.init_from_pb(pcolumn); + if (column.is_key()) { + _num_key_columns++; + } + if (column.is_nullable()) { + _num_null_columns++; + } + if (column.is_bf_column()) { + has_bf_columns = true; + } + _field_name_to_index[column.name()] = _num_columns; + _field_id_to_index[column.col_unique_id()] = _num_columns; + _cols.emplace_back(std::move(column)); + _num_columns++; + } + break; + } + } + if (has_bf_columns) { + _has_bf_fpp = true; + _bf_fpp = ori_tablet_schema.bloom_filter_fpp(); + } else { + _has_bf_fpp = false; + _bf_fpp = BLOOM_FILTER_DEFAULT_FPP; + } + _schema_version = ptable_schema_param.version(); +} + +void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { + tablet_schema_pb->set_keys_type(_keys_type); for (auto& col : _cols) { - ColumnPB* column = tablet_meta_pb->add_column(); + ColumnPB* column = tablet_schema_pb->add_column(); col.to_schema_pb(column); } - tablet_meta_pb->set_num_short_key_columns(_num_short_key_columns); - tablet_meta_pb->set_num_rows_per_row_block(_num_rows_per_row_block); - tablet_meta_pb->set_compress_kind(_compress_kind); + tablet_schema_pb->set_num_short_key_columns(_num_short_key_columns); + tablet_schema_pb->set_num_rows_per_row_block(_num_rows_per_row_block); + tablet_schema_pb->set_compress_kind(_compress_kind); if (_has_bf_fpp) { - tablet_meta_pb->set_bf_fpp(_bf_fpp); + tablet_schema_pb->set_bf_fpp(_bf_fpp); } - tablet_meta_pb->set_next_column_unique_id(_next_column_unique_id); - tablet_meta_pb->set_is_in_memory(_is_in_memory); - tablet_meta_pb->set_delete_sign_idx(_delete_sign_idx); - tablet_meta_pb->set_sequence_col_idx(_sequence_col_idx); - tablet_meta_pb->set_sort_type(_sort_type); - tablet_meta_pb->set_sort_col_num(_sort_col_num); - tablet_meta_pb->set_compression_type(_compression_type); + tablet_schema_pb->set_next_column_unique_id(_next_column_unique_id); + tablet_schema_pb->set_is_in_memory(_is_in_memory); + tablet_schema_pb->set_delete_sign_idx(_delete_sign_idx); + tablet_schema_pb->set_sequence_col_idx(_sequence_col_idx); + tablet_schema_pb->set_sort_type(_sort_type); + tablet_schema_pb->set_sort_col_num(_sort_col_num); + tablet_schema_pb->set_schema_version(_schema_version); + tablet_schema_pb->set_compression_type(_compression_type); } uint32_t TabletSchema::mem_size() const { @@ -511,6 +614,11 @@ int32_t TabletSchema::field_index(const std::string& field_name) const { return (found == _field_name_to_index.end()) ? -1 : found->second; } +int32_t TabletSchema::field_index(int32_t col_unique_id) const { + const auto& found = _field_id_to_index.find(col_unique_id); + return (found == _field_id_to_index.end()) ? -1 : found->second; +} + const std::vector& TabletSchema::columns() const { return _cols; } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index f0a5b72157..1ae27b8cb3 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -31,18 +31,24 @@ namespace vectorized { class Block; } +class POlapTableSchemaParam; + class TabletColumn { public: TabletColumn(); + TabletColumn(const ColumnPB& column); + TabletColumn(const TColumn& column); TabletColumn(FieldAggregationMethod agg, FieldType type); TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable); TabletColumn(FieldAggregationMethod agg, FieldType filed_type, bool is_nullable, int32_t unique_id, size_t length); void init_from_pb(const ColumnPB& column); - void to_schema_pb(ColumnPB* column); + void init_from_thrift(const TColumn& column); + void to_schema_pb(ColumnPB* column) const; uint32_t mem_size() const; int32_t unique_id() const { return _unique_id; } + int32_t col_unique_id() const { return _col_unique_id; } std::string name() const { return _col_name; } void set_name(std::string col_name) { _col_name = col_name; } FieldType type() const { return _type; } @@ -114,6 +120,7 @@ private: TabletColumn* _parent = nullptr; std::vector _sub_columns; uint32_t _sub_column_count = 0; + int32_t _col_unique_id = -1; }; bool operator==(const TabletColumn& a, const TabletColumn& b); @@ -126,11 +133,13 @@ public: // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema). TabletSchema() = default; void init_from_pb(const TabletSchemaPB& schema); - void to_schema_pb(TabletSchemaPB* tablet_meta_pb); + void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const; + void append_column(TabletColumn column); uint32_t mem_size() const; size_t row_size() const; int32_t field_index(const std::string& field_name) const; + int32_t field_index(int32_t col_unique_id) const; const TabletColumn& column(size_t ordinal) const; const std::vector& columns() const; size_t num_columns() const { return _num_columns; } @@ -143,6 +152,7 @@ public: size_t sort_col_num() const { return _sort_col_num; } CompressKind compress_kind() const { return _compress_kind; } size_t next_column_unique_id() const { return _next_column_unique_id; } + bool has_bf_fpp() const { return _has_bf_fpp; } double bloom_filter_fpp() const { return _bf_fpp; } bool is_in_memory() const { return _is_in_memory; } void set_is_in_memory(bool is_in_memory) { _is_in_memory = is_in_memory; } @@ -152,11 +162,17 @@ public: int32_t sequence_col_idx() const { return _sequence_col_idx; } segment_v2::CompressionTypePB compression_type() const { return _compression_type; } + int32_t schema_version() const { return _schema_version; } + void clear_columns(); vectorized::Block create_block( const std::vector& return_columns, const std::unordered_set* tablet_columns_need_convert_null = nullptr) const; vectorized::Block create_block() const; + void build_current_tablet_schema(int64_t index_id, + const POlapTableSchemaParam& ptable_schema_param, + const TabletSchema& out_tablet_schema); + private: // Only for unit test. void init_field_index_for_test(); @@ -170,6 +186,7 @@ private: size_t _sort_col_num = 0; std::vector _cols; std::unordered_map _field_name_to_index; + std::unordered_map _field_id_to_index; size_t _num_columns = 0; size_t _num_key_columns = 0; size_t _num_null_columns = 0; @@ -184,6 +201,7 @@ private: bool _is_in_memory = false; int32_t _delete_sign_idx = -1; int32_t _sequence_col_idx = -1; + int32_t _schema_version = -1; }; bool operator==(const TabletSchema& a, const TabletSchema& b); diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index 4b81fc2cfe..e035068ef2 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -55,6 +55,7 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc) _tuple_offset(tdesc.byteOffset), _null_indicator_offset(tdesc.nullIndicatorByte, tdesc.nullIndicatorBit), _col_name(tdesc.colName), + _col_unique_id(tdesc.col_unique_id), _slot_idx(tdesc.slotIdx), _slot_size(_type.get_slot_size()), _field_idx(-1), @@ -68,6 +69,7 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc) _tuple_offset(pdesc.byte_offset()), _null_indicator_offset(pdesc.null_indicator_byte(), pdesc.null_indicator_bit()), _col_name(pdesc.col_name()), + _col_unique_id(-1), _slot_idx(pdesc.slot_idx()), _slot_size(_type.get_slot_size()), _field_idx(-1), diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index ee18f8a450..85f5bea18a 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -113,6 +113,8 @@ public: doris::vectorized::DataTypePtr get_data_type_ptr() const; + int32_t col_unique_id() const { return _col_unique_id; } + private: friend class DescriptorTbl; friend class TupleDescriptor; @@ -127,6 +129,8 @@ private: const NullIndicatorOffset _null_indicator_offset; const std::string _col_name; + const int32_t _col_unique_id; + // the idx of the slot in the tuple descriptor (0-based). // this is provided by the FE const int _slot_idx; diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 9733ff0bc7..2ee5ea2dcd 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -225,6 +225,7 @@ Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& request } for (auto& tablet : request.tablets()) { WriteRequest wrequest; + wrequest.index_id = request.index_id(); wrequest.tablet_id = tablet.tablet_id(); wrequest.schema_hash = schema_hash; wrequest.write_type = WriteType::LOAD; @@ -234,6 +235,7 @@ Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& request wrequest.tuple_desc = _tuple_desc; wrequest.slots = index_slots; wrequest.is_high_priority = _is_high_priority; + wrequest.ptable_schema_param = request.schema(); DeltaWriter* writer = nullptr; auto st = DeltaWriter::open(&wrequest, &writer, _is_vec); diff --git a/be/src/vec/exec/volap_scanner.cpp b/be/src/vec/exec/volap_scanner.cpp index 2b3093ce8b..cc063a5978 100644 --- a/be/src/vec/exec/volap_scanner.cpp +++ b/be/src/vec/exec/volap_scanner.cpp @@ -74,6 +74,18 @@ Status VOlapScanner::prepare( LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } + _tablet_schema = _tablet->tablet_schema(); + if (!_parent->_olap_scan_node.columns_desc.empty() && + _parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) { + // Originally scanner get TabletSchema from tablet object in BE. + // To support lightweight schema change for adding / dropping columns, + // tabletschema is bounded to rowset and tablet's schema maybe outdated, + // so we have to use schema from a query plan witch FE puts it in query plans. + _tablet_schema.clear_columns(); + for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) { + _tablet_schema.append_column(TabletColumn(column_desc)); + } + } { std::shared_lock rdlock(_tablet->get_header_lock()); const RowsetSharedPtr rowset = _tablet->rowset_with_max_version(); @@ -156,6 +168,7 @@ Status VOlapScanner::_init_tablet_reader_params( RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode)); _tablet_reader_params.tablet = _tablet; + _tablet_reader_params.tablet_schema = &_tablet_schema; _tablet_reader_params.reader_type = READER_QUERY; _tablet_reader_params.aggregation = _aggregation; _tablet_reader_params.version = Version(0, _version); @@ -192,11 +205,11 @@ Status VOlapScanner::_init_tablet_reader_params( _tablet_reader_params.return_columns = _return_columns; } else { // we need to fetch all key columns to do the right aggregation on storage engine side. - for (size_t i = 0; i < _tablet->num_key_columns(); ++i) { + for (size_t i = 0; i < _tablet_schema.num_key_columns(); ++i) { _tablet_reader_params.return_columns.push_back(i); } for (auto index : _return_columns) { - if (_tablet->tablet_schema().column(index).is_key()) { + if (_tablet_schema.column(index).is_key()) { continue; } else { _tablet_reader_params.return_columns.push_back(index); @@ -221,7 +234,9 @@ Status VOlapScanner::_init_return_columns(bool need_seq_col) { if (!slot->is_materialized()) { continue; } - int32_t index = _tablet->field_index(slot->col_name()); + int32_t index = slot->col_unique_id() >= 0 + ? _tablet_schema.field_index(slot->col_unique_id()) + : _tablet_schema.field_index(slot->col_name()); if (index < 0) { std::stringstream ss; ss << "field name is invalid. field=" << slot->col_name(); @@ -229,21 +244,21 @@ Status VOlapScanner::_init_return_columns(bool need_seq_col) { return Status::InternalError(ss.str()); } _return_columns.push_back(index); - if (slot->is_nullable() && !_tablet->tablet_schema().column(index).is_nullable()) + if (slot->is_nullable() && !_tablet_schema.column(index).is_nullable()) _tablet_columns_convert_to_null_set.emplace(index); } // expand the sequence column - if (_tablet->tablet_schema().has_sequence_col() && need_seq_col) { + if (_tablet_schema.has_sequence_col() && need_seq_col) { bool has_replace_col = false; for (auto col : _return_columns) { - if (_tablet->tablet_schema().column(col).aggregation() == + if (_tablet_schema.column(col).aggregation() == FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) { has_replace_col = true; break; } } - if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx(); + if (auto sequence_col_idx = _tablet_schema.sequence_col_idx(); has_replace_col && std::find(_return_columns.begin(), _return_columns.end(), sequence_col_idx) == _return_columns.end()) { _return_columns.push_back(sequence_col_idx); diff --git a/be/src/vec/exec/volap_scanner.h b/be/src/vec/exec/volap_scanner.h index d4c40c13c1..6fd3ee89d9 100644 --- a/be/src/vec/exec/volap_scanner.h +++ b/be/src/vec/exec/volap_scanner.h @@ -143,6 +143,8 @@ private: VExprContext* _vconjunct_ctx = nullptr; bool _need_to_close = false; + + TabletSchema _tablet_schema; }; } // namespace vectorized diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp index 681b6d4e55..ab187367f9 100644 --- a/be/src/vec/olap/block_reader.cpp +++ b/be/src/vec/olap/block_reader.cpp @@ -79,7 +79,7 @@ void BlockReader::_init_agg_state(const ReaderParams& read_params) { _stored_has_null_tag.resize(_stored_data_columns.size()); _stored_has_string_tag.resize(_stored_data_columns.size()); - auto& tablet_schema = tablet()->tablet_schema(); + auto& tablet_schema = *_tablet_schema; for (auto idx : _agg_columns_idx) { AggregateFunctionPtr function = tablet_schema diff --git a/be/src/vec/olap/vcollect_iterator.cpp b/be/src/vec/olap/vcollect_iterator.cpp index 2b68dc7a38..81eb20f3a2 100644 --- a/be/src/vec/olap/vcollect_iterator.cpp +++ b/be/src/vec/olap/vcollect_iterator.cpp @@ -55,7 +55,7 @@ Status VCollectIterator::add_child(RowsetReaderSharedPtr rs_reader) { // then merged with the base rowset. Status VCollectIterator::build_heap(std::vector& rs_readers) { DCHECK(rs_readers.size() == _children.size()); - _skip_same = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS; + _skip_same = _reader->_tablet_schema->keys_type() == KeysType::UNIQUE_KEYS; if (_children.empty()) { _inner_iter.reset(nullptr); return Status::OK(); diff --git a/be/src/vec/olap/vcollect_iterator.h b/be/src/vec/olap/vcollect_iterator.h index 6cae36dcc3..4c8002d885 100644 --- a/be/src/vec/olap/vcollect_iterator.h +++ b/be/src/vec/olap/vcollect_iterator.h @@ -73,7 +73,7 @@ private: // then merged with other rowset readers. class LevelIterator { public: - LevelIterator(TabletReader* reader) : _schema(reader->tablet()->tablet_schema()) {}; + LevelIterator(TabletReader* reader) : _schema(reader->tablet_schema()) {}; virtual Status init() = 0; diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp index a71b2497d5..01e994fb84 100644 --- a/be/test/olap/rowset/beta_rowset_test.cpp +++ b/be/test/olap/rowset/beta_rowset_test.cpp @@ -433,6 +433,7 @@ TEST_F(BetaRowsetTest, ReadTest) { Aws::SDKOptions aws_options = Aws::SDKOptions {}; Aws::InitAPI(aws_options); + TabletSchema dummy_schema; // failed to head object { Aws::Auth::AWSCredentials aws_cred("ak", "sk"); @@ -446,7 +447,7 @@ TEST_F(BetaRowsetTest, ReadTest) { rowset.rowset_meta()->set_fs(fs); std::vector segments; - Status st = rowset.load_segments(&segments); + Status st = rowset.load_segments(&segments, &dummy_schema); ASSERT_FALSE(st.ok()); } @@ -461,7 +462,7 @@ TEST_F(BetaRowsetTest, ReadTest) { rowset.rowset_meta()->set_fs(fs); std::vector segments; - Status st = rowset.load_segments(&segments); + Status st = rowset.load_segments(&segments, &dummy_schema); ASSERT_FALSE(st.ok()); } @@ -476,7 +477,7 @@ TEST_F(BetaRowsetTest, ReadTest) { rowset.rowset_meta()->set_fs(fs); std::vector segments; - Status st = rowset.load_segments(&segments); + Status st = rowset.load_segments(&segments, &dummy_schema); ASSERT_FALSE(st.ok()); } diff --git a/be/test/olap/test_data/header_without_inc_rs.txt b/be/test/olap/test_data/header_without_inc_rs.txt index 61fc42e419..040b37b244 100644 --- a/be/test/olap/test_data/header_without_inc_rs.txt +++ b/be/test/olap/test_data/header_without_inc_rs.txt @@ -19,7 +19,8 @@ "is_nullable": false, "length": 8, "index_length": 8, - "visible": true + "visible": true, + "col_unique_id": -1 }, { "unique_id": 1, @@ -31,7 +32,8 @@ "default_value": "MA==", "length": 16387, "index_length": 16, - "visible": true + "visible": true, + "col_unique_id": -1 }, { "unique_id": 2, @@ -42,7 +44,8 @@ "is_nullable": false, "length": 4, "index_length": 4, - "visible": true + "visible": true, + "col_unique_id": -1 } ], "num_short_key_columns": 1, @@ -54,7 +57,8 @@ "sequence_col_idx": -1, "sort_type": "LEXICAL", "sort_col_num": 0, - "compression_type": "LZ4F" + "compression_type": "LZ4F", + "schema_version": 0 }, "rs_metas": [ { diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 1a5d6da4ce..7ecc5eb1ff 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -42,8 +42,10 @@ public final class FeMetaVersion { public static final int VERSION_110 = 110; // add catalog PrivTable in PaloAuth to support unified privilege management public static final int VERSION_111 = 111; + // add row policy and add maxColUniqueId for olapTable + public static final int VERSION_112 = 112; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_111; + public static final int VERSION_CURRENT = VERSION_112; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java index 89dce3e169..9cfe420f34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -111,6 +111,10 @@ public abstract class AlterJobV2 implements Writable { return jobState; } + public void setJobState(JobState jobState) { + this.jobState = jobState; + } + public JobType getType() { return type; } @@ -143,6 +147,10 @@ public abstract class AlterJobV2 implements Writable { return finishedTimeMs; } + public void setFinishedTimeMs(long finishedTimeMs) { + this.finishedTimeMs = finishedTimeMs; + } + /** * The keyword 'synchronized' only protects 2 methods: * run() and cancel() diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 4fe0c9854e..bcd73e37be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -383,7 +383,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable { AlterReplicaTask rollupTask = new AlterReplicaTask(rollupReplica.getBackendId(), dbId, tableId, partitionId, rollupIndexId, baseIndexId, rollupTabletId, baseTabletId, rollupReplica.getId(), rollupSchemaHash, baseSchemaHash, visibleVersion, jobId, - JobType.ROLLUP, defineExprs, descTable); + JobType.ROLLUP, defineExprs, descTable, null); rollupBatchTask.addTask(rollupTask); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 574c908803..c5b5151882 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -53,6 +53,7 @@ import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.AnalysisException; @@ -70,6 +71,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.RemoveAlterJobV2OperationLog; +import org.apache.doris.persist.TableAddOrDropColumnsInfo; import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTaskExecutor; @@ -101,6 +103,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.function.IntSupplier; public class SchemaChangeHandler extends AlterHandler { private static final Logger LOG = LogManager.getLogger(SchemaChangeHandler.class); @@ -125,8 +128,16 @@ public class SchemaChangeHandler extends AlterHandler { super("schema change", Config.default_schema_change_scheduler_interval_millisecond); } - private void processAddColumn(AddColumnClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + /** + * @param alterClause + * @param olapTable + * @param indexSchemaMap + * @param colUniqueIdSupplier for multi add columns clause, we need stash middle state of maxColUniqueId + * @return true: can light schema change, false: cannot light schema change + * @throws DdlException + */ + private boolean processAddColumn(AddColumnClause alterClause, OlapTable olapTable, + Map> indexSchemaMap, IntSupplier colUniqueIdSupplier) throws DdlException { Column column = alterClause.getColumn(); ColumnPosition columnPos = alterClause.getColPos(); String targetIndexName = alterClause.getRollupName(); @@ -142,12 +153,18 @@ public class SchemaChangeHandler extends AlterHandler { } Set newColNameSet = Sets.newHashSet(column.getName()); - addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, - indexSchemaMap, newColNameSet); + + //only new table generate ColUniqueId, exist table do not. + if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) { + column.setUniqueId(colUniqueIdSupplier.getAsInt()); + } + + return addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, indexSchemaMap, + newColNameSet, false); } - private void processAddColumn(AddColumnClause alterClause, - Table externalTable, List newSchema) throws DdlException { + private void processAddColumn(AddColumnClause alterClause, Table externalTable, List newSchema) + throws DdlException { Column column = alterClause.getColumn(); ColumnPosition columnPos = alterClause.getColPos(); Set newColNameSet = Sets.newHashSet(column.getName()); @@ -155,8 +172,8 @@ public class SchemaChangeHandler extends AlterHandler { addColumnInternal(column, columnPos, newSchema, newColNameSet); } - private void processAddColumns(AddColumnsClause alterClause, - Table externalTable, List newSchema) throws DdlException { + private void processAddColumns(AddColumnsClause alterClause, Table externalTable, List newSchema) + throws DdlException { List columns = alterClause.getColumns(); Set newColNameSet = Sets.newHashSet(); for (Column column : alterClause.getColumns()) { @@ -168,8 +185,18 @@ public class SchemaChangeHandler extends AlterHandler { } } - private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + /** + * @param alterClause + * @param olapTable + * @param indexSchemaMap + * @param ignoreSameColumn + * @param colUniqueIdSupplier for multi add columns clause, we need stash middle state of maxColUniqueId + * @return true: can light schema change, false: cannot light schema change + * @throws DdlException + */ + public boolean processAddColumns(AddColumnsClause alterClause, OlapTable olapTable, + Map> indexSchemaMap, boolean ignoreSameColumn, IntSupplier colUniqueIdSupplier) + throws DdlException { List columns = alterClause.getColumns(); String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); @@ -188,14 +215,26 @@ public class SchemaChangeHandler extends AlterHandler { targetIndexId = olapTable.getIndexIdByName(targetIndexName); } - for (Column column : columns) { - addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, - indexSchemaMap, newColNameSet); + //for new table calculate column unique id + if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) { + for (Column column : columns) { + column.setUniqueId(colUniqueIdSupplier.getAsInt()); + } } + + boolean ligthSchemaChange = true; + for (Column column : columns) { + boolean result = addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, indexSchemaMap, + newColNameSet, ignoreSameColumn); + if (!result) { + ligthSchemaChange = false; + } + } + return ligthSchemaChange; } - private void processDropColumn(DropColumnClause alterClause, - Table externalTable, List newSchema) throws DdlException { + private void processDropColumn(DropColumnClause alterClause, Table externalTable, List newSchema) + throws DdlException { String dropColName = alterClause.getColName(); // find column in base index and remove it @@ -208,8 +247,9 @@ public class SchemaChangeHandler extends AlterHandler { baseIter.remove(); found = true; } else { - throw new DdlException("Do not allow remove last column of table: " + externalTable.getName() - + " column: " + dropColName); + throw new DdlException( + "Do not allow remove last column of table: " + externalTable.getName() + " column: " + + dropColName); } break; } @@ -220,8 +260,23 @@ public class SchemaChangeHandler extends AlterHandler { } } - private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable, + /** + * @param alterClause + * @param olapTable + * @param indexSchemaMap + * @param indexes + * @return true: can light schema change, false: cannot + * @throws DdlException + */ + private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTable, Map> indexSchemaMap, List indexes) throws DdlException { + + boolean ligthSchemaChange = false; + if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) { + //assume can light schema change. + ligthSchemaChange = true; + } + String dropColName = alterClause.getColName(); String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); @@ -241,6 +296,7 @@ public class SchemaChangeHandler extends AlterHandler { boolean isKey = false; for (Column column : baseSchema) { if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { + ligthSchemaChange = false; isKey = true; break; } @@ -260,6 +316,7 @@ public class SchemaChangeHandler extends AlterHandler { for (Column column : baseSchema) { if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { isKey = true; + ligthSchemaChange = false; } else if (AggregateType.REPLACE == column.getAggregationType() || AggregateType.REPLACE_IF_NOT_NULL == column.getAggregationType()) { hasReplaceColumn = true; @@ -279,6 +336,7 @@ public class SchemaChangeHandler extends AlterHandler { for (Column column : targetIndexSchema) { if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { isKey = true; + ligthSchemaChange = false; } else if (AggregateType.REPLACE == column.getAggregationType() || AggregateType.REPLACE_IF_NOT_NULL == column.getAggregationType()) { hasReplaceColumn = true; @@ -289,6 +347,15 @@ public class SchemaChangeHandler extends AlterHandler { "Can not drop key column when rollup has value column with REPLACE aggregation method"); } } + } else if (KeysType.DUP_KEYS == olapTable.getKeysType()) { + long baseIndexId = olapTable.getBaseIndexId(); + List baseSchema = indexSchemaMap.get(baseIndexId); + for (Column column : baseSchema) { + if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { + ligthSchemaChange = false; + break; + } + } } Iterator it = indexes.iterator(); @@ -332,12 +399,14 @@ public class SchemaChangeHandler extends AlterHandler { while (iter.hasNext()) { Column column = iter.next(); if (column.getName().equalsIgnoreCase(dropColName)) { + ligthSchemaChange = false; iter.remove(); break; } } } // end for index names } else { + ligthSchemaChange = false; // if specify rollup index, only drop column from specified rollup index long targetIndexId = olapTable.getIndexIdByName(targetIndexName); // find column @@ -356,11 +425,12 @@ public class SchemaChangeHandler extends AlterHandler { throw new DdlException("Column does not exists: " + dropColName); } } + return ligthSchemaChange; } // User can modify column type and column position - private void processModifyColumn(ModifyColumnClause alterClause, - Table externalTable, List newSchema) throws DdlException { + private void processModifyColumn(ModifyColumnClause alterClause, Table externalTable, List newSchema) + throws DdlException { Column modColumn = alterClause.getColumn(); ColumnPosition columnPos = alterClause.getColPos(); @@ -427,7 +497,7 @@ public class SchemaChangeHandler extends AlterHandler { // User can modify column type and column position private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + Map> indexSchemaMap) throws DdlException { Column modColumn = alterClause.getColumn(); if (KeysType.AGG_KEYS == olapTable.getKeysType()) { if (modColumn.isKey() && null != modColumn.getAggregationType()) { @@ -438,16 +508,17 @@ public class SchemaChangeHandler extends AlterHandler { } } else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { if (null != modColumn.getAggregationType()) { - throw new DdlException("Can not assign aggregation method" - + " on column in Unique data model table: " + modColumn.getName()); + throw new DdlException("Can not assign aggregation method" + " on column in Unique data model table: " + + modColumn.getName()); } if (!modColumn.isKey()) { modColumn.setAggregationType(AggregateType.REPLACE, true); } } else { if (null != modColumn.getAggregationType()) { - throw new DdlException("Can not assign aggregation method" - + " on column in Duplicate data model table: " + modColumn.getName()); + throw new DdlException( + "Can not assign aggregation method" + " on column in Duplicate data model table: " + + modColumn.getName()); } if (!modColumn.isKey()) { modColumn.setAggregationType(AggregateType.NONE, true); @@ -518,6 +589,7 @@ public class SchemaChangeHandler extends AlterHandler { Column oriColumn = schemaForFinding.get(modColIndex); // retain old column name modColumn.setName(oriColumn.getName()); + modColumn.setUniqueId(oriColumn.getUniqueId()); // handle the move operation in 'indexForFindingColumn' if has if (hasColPos) { @@ -614,8 +686,8 @@ public class SchemaChangeHandler extends AlterHandler { } } - private void processReorderColumn(ReorderColumnsClause alterClause, - Table externalTable, List newSchema) throws DdlException { + private void processReorderColumn(ReorderColumnsClause alterClause, Table externalTable, List newSchema) + throws DdlException { List orderedColNames = alterClause.getColumnsByPos(); newSchema.clear(); @@ -648,7 +720,7 @@ public class SchemaChangeHandler extends AlterHandler { } private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + Map> indexSchemaMap) throws DdlException { List orderedColNames = alterClause.getColumnsByPos(); String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); @@ -663,7 +735,7 @@ public class SchemaChangeHandler extends AlterHandler { long targetIndexId = olapTable.getIndexIdByName(targetIndexName); LinkedList newSchema = new LinkedList(); - LinkedList targetIndexSchema = indexSchemaMap.get(targetIndexId); + List targetIndexSchema = indexSchemaMap.get(targetIndexId); // check and create new ordered column list Set colNameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); @@ -703,9 +775,8 @@ public class SchemaChangeHandler extends AlterHandler { * Add 'newColumn' to specified index. * Modified schema will be saved in 'indexSchemaMap' */ - private void addColumnInternal(Column newColumn, ColumnPosition columnPos, - List modIndexSchema, - Set newColNameSet) throws DdlException { + private void addColumnInternal(Column newColumn, ColumnPosition columnPos, List modIndexSchema, + Set newColNameSet) throws DdlException { String newColName = newColumn.getName(); int posIndex = -1; boolean hasPos = (columnPos != null && !columnPos.isFirst()); @@ -755,16 +826,33 @@ public class SchemaChangeHandler extends AlterHandler { } } - /* - * Add 'newColumn' to specified index. - * Modified schema will be saved in 'indexSchemaMap' + /** + * @param olapTable + * @param newColumn Add 'newColumn' to specified index. + * @param columnPos + * @param targetIndexId + * @param baseIndexId + * @param indexSchemaMap Modified schema will be saved in 'indexSchemaMap' + * @param newColNameSet + * @param ignoreSameColumn + * @return true: can light schema change, false: cannot + * @throws DdlException */ - private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos, - long targetIndexId, long baseIndexId, - Map> indexSchemaMap, - Set newColNameSet) throws DdlException { + private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos, + long targetIndexId, long baseIndexId, Map> indexSchemaMap, + Set newColNameSet, boolean ignoreSameColumn) throws DdlException { + //only new table generate ColUniqueId, exist table do not. + boolean ligthSchemaChange = olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE; String newColName = newColumn.getName(); + + //make sure olapTable has locked + if (newColumn.getUniqueId() > Integer.MAX_VALUE) { + throw new DdlException("schema change add column times overflow: " + newColName); + } + LOG.debug("table: {}, newColumn: {}, uniqueId: {}", olapTable.getName(), newColumn.getName(), + newColumn.getUniqueId()); + // check the validation of aggregation method on column. // also fill the default aggregation method if not specified. if (KeysType.AGG_KEYS == olapTable.getKeysType()) { @@ -772,30 +860,31 @@ public class SchemaChangeHandler extends AlterHandler { throw new DdlException("Can not assign aggregation method on key column: " + newColName); } else if (null == newColumn.getAggregationType()) { newColumn.setIsKey(true); - } else if (newColumn.getAggregationType() == AggregateType.SUM - && newColumn.getDefaultValue() != null && !newColumn.getDefaultValue().equals("0")) { - throw new DdlException("The default value of '" - + newColName + "' with SUM aggregation function must be zero"); + } else if (newColumn.getAggregationType() == AggregateType.SUM && newColumn.getDefaultValue() != null + && !newColumn.getDefaultValue().equals("0")) { + throw new DdlException( + "The default value of '" + newColName + "' with SUM aggregation function must be zero"); } else if (olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo) { if (newColumn.getAggregationType() == AggregateType.REPLACE || newColumn.getAggregationType() == AggregateType.REPLACE_IF_NOT_NULL) { - throw new DdlException("Can not add value column with aggregation type " - + newColumn.getAggregationType() + " for olap table with random distribution : " - + newColName); + throw new DdlException( + "Can not add value column with aggregation type " + newColumn.getAggregationType() + + " for olap table with random distribution : " + newColName); } } } else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { if (newColumn.getAggregationType() != null) { - throw new DdlException("Can not assign aggregation method" - + " on column in Unique data model table: " + newColName); + throw new DdlException( + "Can not assign aggregation method" + " on column in Unique data model table: " + newColName); } if (!newColumn.isKey()) { newColumn.setAggregationType(AggregateType.REPLACE, true); } } else { if (newColumn.getAggregationType() != null) { - throw new DdlException("Can not assign aggregation method" - + " on column in Duplicate data model table: " + newColName); + throw new DdlException( + "Can not assign aggregation method" + " on column in Duplicate data model table: " + + newColName); } if (!newColumn.isKey()) { if (targetIndexId != -1L @@ -816,13 +905,20 @@ public class SchemaChangeHandler extends AlterHandler { throw new DdlException("BITMAP_UNION must be used in AGG_KEYS"); } + //type key column do not allow light schema change. + if (newColumn.isKey()) { + ligthSchemaChange = false; + } + // check if the new column already exist in base schema. // do not support adding new column which already exist in base schema. List baseSchema = olapTable.getBaseSchema(true); boolean found = false; + Column foundColumn = null; for (Column column : baseSchema) { if (column.getName().equalsIgnoreCase(newColName)) { found = true; + foundColumn = column; break; } } @@ -832,7 +928,11 @@ public class SchemaChangeHandler extends AlterHandler { } else if (newColName.equalsIgnoreCase(Column.SEQUENCE_COL)) { throw new DdlException("Can not enable sequence column support, already supported sequence column."); } else { - throw new DdlException("Can not add column which already exists in base table: " + newColName); + if (ignoreSameColumn && newColumn.equals(foundColumn)) { + //for add columns rpc, allow add same type column. + } else { + throw new DdlException("Can not add column which already exists in base table: " + newColName); + } } } @@ -864,9 +964,10 @@ public class SchemaChangeHandler extends AlterHandler { modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true); if (targetIndexId == -1L) { - return; + return ligthSchemaChange; } // 2. add to rollup + ligthSchemaChange = false; modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); } @@ -876,9 +977,10 @@ public class SchemaChangeHandler extends AlterHandler { List modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true); // no specified target index. return - return; + return ligthSchemaChange; } else { // add to rollup index + ligthSchemaChange = false; List modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); @@ -902,13 +1004,15 @@ public class SchemaChangeHandler extends AlterHandler { if (targetIndexId == -1L) { // no specified target index. return - return; + return ligthSchemaChange; } + ligthSchemaChange = false; // 2. add to rollup index modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); } + return ligthSchemaChange; } /* @@ -921,7 +1025,7 @@ public class SchemaChangeHandler extends AlterHandler { * So that k1 will be added to base index 'twice', and we just ignore this repeat adding. */ private void checkAndAddColumn(List modIndexSchema, Column newColumn, ColumnPosition columnPos, - Set newColNameSet, boolean isBaseIndex) throws DdlException { + Set newColNameSet, boolean isBaseIndex) throws DdlException { int posIndex = -1; int lastVisibleIdx = -1; String newColName = newColumn.getName(); @@ -987,8 +1091,8 @@ public class SchemaChangeHandler extends AlterHandler { private void checkIndexExists(OlapTable olapTable, String targetIndexName) throws DdlException { if (targetIndexName != null && !olapTable.hasMaterializedIndex(targetIndexName)) { - throw new DdlException("Index[" + targetIndexName + "] does not exist in table[" + olapTable.getName() - + "]"); + throw new DdlException( + "Index[" + targetIndexName + "] does not exist in table[" + olapTable.getName() + "]"); } } @@ -1002,7 +1106,7 @@ public class SchemaChangeHandler extends AlterHandler { } private void createJob(long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes) throws UserException { + Map propertyMap, List indexes) throws UserException { if (olapTable.getState() == OlapTableState.ROLLUP) { throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); } @@ -1022,8 +1126,8 @@ public class SchemaChangeHandler extends AlterHandler { if (key.endsWith(PropertyAnalyzer.PROPERTIES_SHORT_KEY)) { // short key String[] keyArray = key.split("#"); - if (keyArray.length != 2 || keyArray[0].isEmpty() - || !keyArray[1].equals(PropertyAnalyzer.PROPERTIES_SHORT_KEY)) { + if (keyArray.length != 2 || keyArray[0].isEmpty() || !keyArray[1].equals( + PropertyAnalyzer.PROPERTIES_SHORT_KEY)) { throw new DdlException("Invalid alter table property: " + key); } @@ -1117,8 +1221,8 @@ public class SchemaChangeHandler extends AlterHandler { // create job Catalog catalog = Catalog.getCurrentCatalog(); long jobId = catalog.getNextId(); - SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, dbId, - olapTable.getId(), olapTable.getName(), timeoutSecond * 1000); + SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, dbId, olapTable.getId(), olapTable.getName(), + timeoutSecond * 1000); schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp); schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes); @@ -1209,8 +1313,9 @@ public class SchemaChangeHandler extends AlterHandler { boolean hasKey = false; for (Column column : alterSchema) { if (column.isKey() && meetValue) { - throw new DdlException("Invalid column order. value should be after key. index[" - + olapTable.getIndexNameById(alterIndexId) + "]"); + throw new DdlException( + "Invalid column order. value should be after key. index[" + olapTable.getIndexNameById( + alterIndexId) + "]"); } if (!column.isKey()) { meetValue = true; @@ -1244,9 +1349,9 @@ public class SchemaChangeHandler extends AlterHandler { if (alterColumn.nameEquals(partitionCol.getName(), true)) { // 2.1 partition column cannot be modified if (needAlterColumns.contains(alterColumn) && !alterColumn.equals(partitionCol)) { - throw new DdlException("Can not modify partition column[" - + partitionCol.getName() + "]. index[" - + olapTable.getIndexNameById(alterIndexId) + "]"); + throw new DdlException( + "Can not modify partition column[" + partitionCol.getName() + "]. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); } found = true; break; @@ -1255,8 +1360,9 @@ public class SchemaChangeHandler extends AlterHandler { if (!found && alterIndexId == olapTable.getBaseIndexId()) { // 2.1 partition column cannot be deleted. - throw new DdlException("Partition column[" + partitionCol.getName() - + "] cannot be dropped. index[" + olapTable.getIndexNameById(alterIndexId) + "]"); + throw new DdlException( + "Partition column[" + partitionCol.getName() + "] cannot be dropped. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); // ATTN. partition columns' order also need remaining unchanged. // for now, we only allow one partition column, so no need to check order. } @@ -1273,9 +1379,9 @@ public class SchemaChangeHandler extends AlterHandler { if (alterColumn.nameEquals(distributionCol.getName(), true)) { // 3.1 distribution column cannot be modified if (needAlterColumns.contains(alterColumn) && !alterColumn.equals(distributionCol)) { - throw new DdlException("Can not modify distribution column[" - + distributionCol.getName() + "]. index[" - + olapTable.getIndexNameById(alterIndexId) + "]"); + throw new DdlException( + "Can not modify distribution column[" + distributionCol.getName() + "]. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); } found = true; break; @@ -1284,8 +1390,9 @@ public class SchemaChangeHandler extends AlterHandler { if (!found && alterIndexId == olapTable.getBaseIndexId()) { // 2.2 distribution column cannot be deleted. - throw new DdlException("Distribution column[" + distributionCol.getName() - + "] cannot be dropped. index[" + olapTable.getIndexNameById(alterIndexId) + "]"); + throw new DdlException( + "Distribution column[" + distributionCol.getName() + "] cannot be dropped. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); } } // end for distributionCols } @@ -1340,8 +1447,8 @@ public class SchemaChangeHandler extends AlterHandler { // index state is SHADOW MaterializedIndex shadowIndex = new MaterializedIndex(shadowIndexId, IndexState.SHADOW); MaterializedIndex originIndex = partition.getIndex(originIndexId); - TabletMeta shadowTabletMeta = new TabletMeta(dbId, tableId, partitionId, - shadowIndexId, newSchemaHash, medium); + TabletMeta shadowTabletMeta = new TabletMeta(dbId, tableId, partitionId, shadowIndexId, newSchemaHash, + medium); ReplicaAllocation replicaAlloc = olapTable.getPartitionInfo().getReplicaAllocation(partitionId); Short totalReplicaNum = replicaAlloc.getTotalReplicaNum(); for (Tablet originTablet : originIndex.getTablets()) { @@ -1398,8 +1505,8 @@ public class SchemaChangeHandler extends AlterHandler { schemaChangeJob.addPartitionShadowIndex(partitionId, shadowIndexId, shadowIndex); } // end for partition - schemaChangeJob.addIndexSchema(shadowIndexId, originIndexId, newIndexName, - newSchemaVersion, newSchemaHash, newShortKeyColumnCount, entry.getValue()); + schemaChangeJob.addIndexSchema(shadowIndexId, originIndexId, newIndexName, newSchemaVersion, newSchemaHash, + newShortKeyColumnCount, entry.getValue()); } // end for index // set table state @@ -1425,25 +1532,24 @@ public class SchemaChangeHandler extends AlterHandler { } private void runAlterJobV2() { - runnableSchemaChangeJobV2.values().forEach( - alterJobsV2 -> { - if (!alterJobsV2.isDone() && !activeSchemaChangeJobsV2.containsKey(alterJobsV2.getJobId()) - && activeSchemaChangeJobsV2.size() < MAX_ACTIVE_SCHEMA_CHANGE_JOB_V2_SIZE) { - if (FeConstants.runningUnitTest) { - alterJobsV2.run(); - } else { - schemaChangeThreadPool.submit(() -> { - if (activeSchemaChangeJobsV2.putIfAbsent(alterJobsV2.getJobId(), alterJobsV2) == null) { - try { - alterJobsV2.run(); - } finally { - activeSchemaChangeJobsV2.remove(alterJobsV2.getJobId()); - } - } - }); + runnableSchemaChangeJobV2.values().forEach(alterJobsV2 -> { + if (!alterJobsV2.isDone() && !activeSchemaChangeJobsV2.containsKey(alterJobsV2.getJobId()) + && activeSchemaChangeJobsV2.size() < MAX_ACTIVE_SCHEMA_CHANGE_JOB_V2_SIZE) { + if (FeConstants.runningUnitTest) { + alterJobsV2.run(); + } else { + schemaChangeThreadPool.submit(() -> { + if (activeSchemaChangeJobsV2.putIfAbsent(alterJobsV2.getJobId(), alterJobsV2) == null) { + try { + alterJobsV2.run(); + } finally { + activeSchemaChangeJobsV2.remove(alterJobsV2.getJobId()); + } } - } - }); + }); + } + } + }); } @Override @@ -1465,8 +1571,8 @@ public class SchemaChangeHandler extends AlterHandler { continue; } if (ctx != null) { - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv( - ctx, db.getFullName(), alterJob.getTableName(), PrivPredicate.ALTER)) { + if (!Catalog.getCurrentCatalog().getAuth() + .checkTblPriv(ctx, db.getFullName(), alterJob.getTableName(), PrivPredicate.ALTER)) { continue; } } @@ -1483,6 +1589,19 @@ public class SchemaChangeHandler extends AlterHandler { throws UserException { olapTable.writeLockOrDdlException(); try { + //alterClauses can or cannot light schema change + boolean ligthSchemaChange = true; + //for multi add colmuns clauses + IntSupplier colUniqueIdSupplier = new IntSupplier() { + public int pendingMaxColUniqueId = olapTable.getMaxColUniqueId(); + + @Override + public int getAsInt() { + pendingMaxColUniqueId++; + return pendingMaxColUniqueId; + } + }; + // index id -> index schema Map> indexSchemaMap = new HashMap<>(); for (Map.Entry> entry : olapTable.getIndexIdToSchema(true).entrySet()) { @@ -1509,8 +1628,8 @@ public class SchemaChangeHandler extends AlterHandler { } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_TYPE)) { String distributionType = properties.get(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_TYPE); if (!distributionType.equalsIgnoreCase("random")) { - throw new DdlException("Only support modifying distribution type of table from" - + " hash to random"); + throw new DdlException( + "Only support modifying distribution type of table from" + " hash to random"); } Catalog.getCurrentCatalog().convertDistributionType(db, olapTable); return; @@ -1523,14 +1642,13 @@ public class SchemaChangeHandler extends AlterHandler { } else if (DynamicPartitionUtil.checkDynamicPartitionPropertiesExist(properties)) { if (!olapTable.dynamicPartitionExists()) { try { - DynamicPartitionUtil.checkInputDynamicPartitionProperties( - properties, olapTable.getPartitionInfo()); + DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, + olapTable.getPartitionInfo()); } catch (DdlException e) { // This table is not a dynamic partition table // and didn't supply all dynamic partition properties - throw new DdlException("Table " + db.getFullName() + "." - + olapTable.getName() + " is not a dynamic partition table." - + " Use command `HELP ALTER TABLE` " + throw new DdlException("Table " + db.getFullName() + "." + olapTable.getName() + + " is not a dynamic partition table." + " Use command `HELP ALTER TABLE` " + "to see how to change a normal table to a dynamic partition table."); } } @@ -1538,8 +1656,8 @@ public class SchemaChangeHandler extends AlterHandler { return; } else if (properties.containsKey( "default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) { - Preconditions.checkNotNull(properties.get("default." - + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)); + Preconditions.checkNotNull( + properties.get("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)); Catalog.getCurrentCatalog().modifyTableDefaultReplicaAllocation(db, olapTable, properties); return; } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) { @@ -1559,36 +1677,63 @@ public class SchemaChangeHandler extends AlterHandler { if (alterClause instanceof AddColumnClause) { // add column - processAddColumn((AddColumnClause) alterClause, olapTable, indexSchemaMap); + boolean clauseCanLigthSchemaChange = processAddColumn((AddColumnClause) alterClause, olapTable, + indexSchemaMap, colUniqueIdSupplier); + if (clauseCanLigthSchemaChange == false) { + ligthSchemaChange = false; + } } else if (alterClause instanceof AddColumnsClause) { // add columns - processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap); + boolean clauseCanLigthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable, + indexSchemaMap, false, colUniqueIdSupplier); + if (clauseCanLigthSchemaChange == false) { + ligthSchemaChange = false; + } } else if (alterClause instanceof DropColumnClause) { // drop column and drop indexes on this column - processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap, newIndexes); + boolean clauseCanLigthSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable, + indexSchemaMap, newIndexes); + if (clauseCanLigthSchemaChange == false) { + ligthSchemaChange = false; + } } else if (alterClause instanceof ModifyColumnClause) { // modify column processModifyColumn((ModifyColumnClause) alterClause, olapTable, indexSchemaMap); + ligthSchemaChange = false; } else if (alterClause instanceof ReorderColumnsClause) { // reorder column processReorderColumn((ReorderColumnsClause) alterClause, olapTable, indexSchemaMap); + ligthSchemaChange = false; } else if (alterClause instanceof ModifyTablePropertiesClause) { // modify table properties // do nothing, properties are already in propertyMap + ligthSchemaChange = false; } else if (alterClause instanceof CreateIndexClause) { if (processAddIndex((CreateIndexClause) alterClause, olapTable, newIndexes)) { return; } + ligthSchemaChange = false; } else if (alterClause instanceof DropIndexClause) { if (processDropIndex((DropIndexClause) alterClause, olapTable, newIndexes)) { return; } + ligthSchemaChange = false; } else { Preconditions.checkState(false); } } // end for alter clauses - createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); + LOG.debug("processAddColumns, table: {}({}), ligthSchemaChange: {}", olapTable.getName(), olapTable.getId(), + ligthSchemaChange); + + if (ligthSchemaChange) { + long jobId = Catalog.getCurrentCatalog().getNextId(); + //for schema change add/drop value column optimize, direct modify table meta. + modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, newIndexes, jobId, false); + return; + } else { + createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); + } } finally { olapTable.writeUnlock(); } @@ -1690,9 +1835,8 @@ public class SchemaChangeHandler extends AlterHandler { /** * Update some specified partitions' in-memory property of table */ - public void updatePartitionsInMemoryMeta(Database db, - String tableName, List partitionNames, Map properties) - throws DdlException, MetaNotFoundException { + public void updatePartitionsInMemoryMeta(Database db, String tableName, List partitionNames, + Map properties) throws DdlException, MetaNotFoundException { OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); boolean isInMemory = Boolean.parseBoolean(properties.get(PropertyAnalyzer.PROPERTIES_INMEMORY)); if (isInMemory == olapTable.isInMemory()) { @@ -1714,10 +1858,8 @@ public class SchemaChangeHandler extends AlterHandler { * Update one specified partition's in-memory property by partition name of table * This operation may return partial successfully, with a exception to inform user to retry */ - public void updatePartitionInMemoryMeta(Database db, - String tableName, - String partitionName, - boolean isInMemory) throws UserException { + public void updatePartitionInMemoryMeta(Database db, String tableName, String partitionName, boolean isInMemory) + throws UserException { // be id -> Map>> beIdToTabletIdWithHash = Maps.newHashMap(); OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); @@ -1733,8 +1875,8 @@ public class SchemaChangeHandler extends AlterHandler { int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { - Set> tabletIdWithHash = - beIdToTabletIdWithHash.computeIfAbsent(replica.getBackendId(), k -> Sets.newHashSet()); + Set> tabletIdWithHash = beIdToTabletIdWithHash.computeIfAbsent( + replica.getBackendId(), k -> Sets.newHashSet()); tabletIdWithHash.add(new Pair<>(tablet.getId(), schemaHash)); } } @@ -1748,16 +1890,16 @@ public class SchemaChangeHandler extends AlterHandler { AgentBatchTask batchTask = new AgentBatchTask(); for (Map.Entry>> kv : beIdToTabletIdWithHash.entrySet()) { countDownLatch.addMark(kv.getKey(), kv.getValue()); - UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), - isInMemory, countDownLatch); + UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), isInMemory, + countDownLatch); batchTask.addTask(task); } if (!FeConstants.runningUnitTest) { // send all tasks and wait them finished AgentTaskQueue.addBatchTask(batchTask); AgentTaskExecutor.submit(batchTask); - LOG.info("send update tablet meta task for table {}, partitions {}, number: {}", - tableName, partitionName, batchTask.getTaskNum()); + LOG.info("send update tablet meta task for table {}, partitions {}, number: {}", tableName, partitionName, + batchTask.getTaskNum()); // estimate timeout long timeout = Config.tablet_create_timeout_second * 1000L * totalTaskNum; @@ -1779,8 +1921,8 @@ public class SchemaChangeHandler extends AlterHandler { } else { List>>> unfinishedMarks = countDownLatch.getLeftMarks(); // only show at most 3 results - List>>> subList - = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); + List>>> subList = unfinishedMarks.subList(0, + Math.min(unfinishedMarks.size(), 3)); if (!subList.isEmpty()) { errMsg += " Unfinished mark: " + Joiner.on(", ").join(subList); } @@ -1815,11 +1957,11 @@ public class SchemaChangeHandler extends AlterHandler { // find from new alter jobs first List schemaChangeJobV2List = getUnfinishedAlterJobV2ByTableId(olapTable.getId()); // current schemaChangeJob job doesn't support batch operation,so just need to get one job - schemaChangeJobV2 = schemaChangeJobV2List.size() == 0 - ? null : Iterables.getOnlyElement(schemaChangeJobV2List); + schemaChangeJobV2 = schemaChangeJobV2List.size() == 0 ? null + : Iterables.getOnlyElement(schemaChangeJobV2List); if (schemaChangeJobV2 == null) { - throw new DdlException("Table[" + tableName + "] is under schema change state" - + " but could not find related job"); + throw new DdlException( + "Table[" + tableName + "] is under schema change state" + " but could not find related job"); } } finally { olapTable.writeUnlock(); @@ -1851,8 +1993,8 @@ public class SchemaChangeHandler extends AlterHandler { for (Index existedIdx : existedIndexes) { if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) { if (indexDef.isSetIfNotExists()) { - LOG.info("create index[{}] which already exists on table[{}]", - indexDef.getIndexName(), olapTable.getName()); + LOG.info("create index[{}] which already exists on table[{}]", indexDef.getIndexName(), + olapTable.getName()); return true; } throw new DdlException("index `" + indexDef.getIndexName() + "` already exist."); @@ -1882,8 +2024,8 @@ public class SchemaChangeHandler extends AlterHandler { * Returns true if the index does not exist, there is no need to create the job to drop the index. * Otherwise return false, there is need to create a job to drop the index. */ - private boolean processDropIndex(DropIndexClause alterClause, OlapTable olapTable, - List indexes) throws DdlException { + private boolean processDropIndex(DropIndexClause alterClause, OlapTable olapTable, List indexes) + throws DdlException { String indexName = alterClause.getIndexName(); List existedIndexes = olapTable.getIndexes(); Index found = null; @@ -1943,4 +2085,172 @@ public class SchemaChangeHandler extends AlterHandler { } super.replayAlterJobV2(alterJob); } + + // the invoker should keep table's write lock + public void modifyTableAddOrDropColumns(Database db, OlapTable olapTable, + Map> indexSchemaMap, + List indexes, long jobId, boolean isReplay) throws DdlException { + + LOG.debug("indexSchemaMap:{}, indexes:{}", indexSchemaMap, indexes); + if (olapTable.getState() == OlapTableState.ROLLUP) { + throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); + } + + // for now table's state can only be NORMAL + Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); + + // for bitmapIndex + boolean hasIndexChange = false; + Set newSet = new HashSet<>(indexes); + Set oriSet = new HashSet<>(olapTable.getIndexes()); + if (!newSet.equals(oriSet)) { + hasIndexChange = true; + } + + // begin checking each table + // ATTN: DO NOT change any meta in this loop + Map> changedIndexIdToSchema = Maps.newHashMap(); + for (Long alterIndexId : indexSchemaMap.keySet()) { + // Must get all columns including invisible columns. + // Because in alter process, all columns must be considered. + List alterSchema = indexSchemaMap.get(alterIndexId); + + LOG.debug("index[{}] is changed. start checking...", alterIndexId); + // 1. check order: a) has key; b) value after key + boolean meetValue = false; + boolean hasKey = false; + for (Column column : alterSchema) { + if (column.isKey() && meetValue) { + throw new DdlException( + "Invalid column order. value should be after key. index[" + olapTable.getIndexNameById( + alterIndexId) + "]"); + } + if (!column.isKey()) { + meetValue = true; + } else { + hasKey = true; + } + } + if (!hasKey) { + throw new DdlException("No key column left. index[" + olapTable.getIndexNameById(alterIndexId) + "]"); + } + + // 2. check partition key + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { + List partitionColumns = partitionInfo.getPartitionColumns(); + for (Column partitionCol : partitionColumns) { + boolean found = false; + for (Column alterColumn : alterSchema) { + if (alterColumn.nameEquals(partitionCol.getName(), true)) { + found = true; + break; + } + } // end for alterColumns + + if (!found && alterIndexId == olapTable.getBaseIndexId()) { + // 2.1 partition column cannot be deleted. + throw new DdlException( + "Partition column[" + partitionCol.getName() + "] cannot be dropped. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); + } + } // end for partitionColumns + } + + // 3. check distribution key: + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + if (distributionInfo.getType() == DistributionInfoType.HASH) { + List distributionColumns = ((HashDistributionInfo) distributionInfo).getDistributionColumns(); + for (Column distributionCol : distributionColumns) { + boolean found = false; + for (Column alterColumn : alterSchema) { + if (alterColumn.nameEquals(distributionCol.getName(), true)) { + found = true; + break; + } + } // end for alterColumns + if (!found && alterIndexId == olapTable.getBaseIndexId()) { + // 2.2 distribution column cannot be deleted. + throw new DdlException( + "Distribution column[" + distributionCol.getName() + "] cannot be dropped. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); + } + } // end for distributionCols + } + + // 5. store the changed columns for edit log + changedIndexIdToSchema.put(alterIndexId, alterSchema); + + LOG.debug("schema change[{}-{}-{}] check pass.", db.getId(), olapTable.getId(), alterIndexId); + } // end for indices + + if (changedIndexIdToSchema.isEmpty() && !hasIndexChange) { + throw new DdlException("Nothing is changed. please check your alter stmt."); + } + + //update base index schema + long baseIndexId = olapTable.getBaseIndexId(); + List indexIds = new ArrayList(); + indexIds.add(baseIndexId); + indexIds.addAll(olapTable.getIndexIdListExceptBaseIndex()); + for (int i = 0; i < indexIds.size(); i++) { + List indexSchema = indexSchemaMap.get(indexIds.get(i)); + MaterializedIndexMeta currentIndexMeta = olapTable.getIndexMetaByIndexId(indexIds.get(i)); + currentIndexMeta.setSchema(indexSchema); + + int currentSchemaVersion = currentIndexMeta.getSchemaVersion(); + int newSchemaVersion = currentSchemaVersion + 1; + currentIndexMeta.setSchemaVersion(newSchemaVersion); + } + olapTable.setIndexes(indexes); + olapTable.rebuildFullSchema(); + + //update max column unique id + int maxColUniqueId = olapTable.getMaxColUniqueId(); + for (Column column : indexSchemaMap.get(olapTable.getBaseIndexId())) { + if (column.getUniqueId() > maxColUniqueId) { + maxColUniqueId = column.getUniqueId(); + } + } + olapTable.setMaxColUniqueId(maxColUniqueId); + + if (!isReplay) { + TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(db.getId(), olapTable.getId(), + indexSchemaMap, indexes, jobId); + LOG.debug("logModifyTableAddOrDropColumns info:{}", info); + Catalog.getCurrentCatalog().getEditLog().logModifyTableAddOrDropColumns(info); + } + + //for compatibility, we need create a finished state schema change job v2 + + SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, db.getId(), olapTable.getId(), + olapTable.getName(), 1000); + schemaChangeJob.setJobState(AlterJobV2.JobState.FINISHED); + schemaChangeJob.setFinishedTimeMs(System.currentTimeMillis()); + this.addAlterJobV2(schemaChangeJob); + + LOG.info("finished modify table's add or drop columns. table: {}, is replay: {}", olapTable.getName(), + isReplay); + } + + public void replayModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info) throws MetaNotFoundException { + LOG.debug("info:{}", info); + long dbId = info.getDbId(); + long tableId = info.getTableId(); + Map> indexSchemaMap = info.getIndexSchemaMap(); + List indexes = info.getIndexes(); + long jobId = info.getJobId(); + + Database db = Catalog.getCurrentCatalog().getInternalDataSource().getDbOrMetaException(dbId); + OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); + olapTable.writeLock(); + try { + modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, indexes, jobId, true); + } catch (DdlException e) { + // should not happen + LOG.warn("failed to replay modify table add or drop columns", e); + } finally { + olapTable.writeUnlock(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 1c112d6d28..44ff3cdfae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -435,7 +435,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { long originIdxId = indexIdMap.get(shadowIdxId); int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash; int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId)); - + List originSchemaColumns = tbl.getSchemaByIndexId(originIdxId); for (Tablet shadowTablet : shadowIdx.getTablets()) { long shadowTabletId = shadowTablet.getId(); long originTabletId = partitionIndexTabletMap.get(partitionId, shadowIdxId).get(shadowTabletId); @@ -444,7 +444,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { AlterReplicaTask rollupTask = new AlterReplicaTask(shadowReplica.getBackendId(), dbId, tableId, partitionId, shadowIdxId, originIdxId, shadowTabletId, originTabletId, shadowReplica.getId(), shadowSchemaHash, originSchemaHash, visibleVersion, jobId, - JobType.SCHEMA_CHANGE, defineExprs, descTable); + JobType.SCHEMA_CHANGE, defineExprs, descTable, originSchemaColumns); schemaChangeBatchTask.addTask(rollupTask); } } @@ -641,6 +641,16 @@ public class SchemaChangeJobV2 extends AlterJobV2 { tbl.setStorageFormat(storageFormat); } + //update max column unique id + int maxColUniqueId = tbl.getMaxColUniqueId(); + for (Column column : tbl.getFullSchema()) { + if (column.getUniqueId() > maxColUniqueId) { + maxColUniqueId = column.getUniqueId(); + } + } + tbl.setMaxColUniqueId(maxColUniqueId); + LOG.debug("fullSchema:{}, maxColUniqueId:{}", tbl.getFullSchema(), maxColUniqueId); + tbl.setState(OlapTableState.NORMAL); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index b1b471c603..75b3290030 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -400,7 +400,7 @@ public class ColumnDef { public Column toColumn() { return new Column(name, typeDef.getType(), isKey, aggregateType, isAllowNull, defaultValue.value, comment, - visible, defaultValue.defaultValueExprDef); + visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 4f041dfe01..8c405011ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -29,11 +29,14 @@ import org.apache.doris.thrift.TSlotDescriptor; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Collections; import java.util.List; public class SlotDescriptor { + private static final Logger LOG = LogManager.getLogger(SlotDescriptor.class); private final SlotId id; private final TupleDescriptor parent; private Type type; @@ -278,27 +281,26 @@ public class SlotDescriptor { // TODO public TSlotDescriptor toThrift() { - if (originType != null) { - return new TSlotDescriptor(id.asInt(), parent.getId().asInt(), originType.toThrift(), -1, - byteOffset, nullIndicatorByte, - nullIndicatorBit, ((column != null) ? column.getName() : ""), slotIdx, isMaterialized); - } else { - return new TSlotDescriptor(id.asInt(), parent.getId().asInt(), type.toThrift(), -1, - byteOffset, nullIndicatorByte, - nullIndicatorBit, ((column != null) ? column.getName() : ""), slotIdx, isMaterialized); + + TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(), + (originType != null ? originType.toThrift() : type.toThrift()), -1, byteOffset, nullIndicatorByte, + nullIndicatorBit, ((column != null) ? column.getName() : ""), slotIdx, isMaterialized); + + if (column != null) { + LOG.debug("column name:{}, column unique id:{}", column.getName(), column.getUniqueId()); + tSlotDescriptor.setColUniqueId(column.getUniqueId()); } + return tSlotDescriptor; } public String debugString() { String colStr = (column == null ? "null" : column.getName()); String typeStr = (type == null ? "null" : type.toString()); String parentTupleId = (parent == null) ? "null" : parent.getId().toString(); - return MoreObjects.toStringHelper(this).add("id", id.asInt()).add("parent", parentTupleId) - .add("col", colStr).add("type", typeStr).add("materialized", isMaterialized) - .add("byteSize", byteSize).add("byteOffset", byteOffset) - .add("nullIndicatorByte", nullIndicatorByte) - .add("nullIndicatorBit", nullIndicatorBit) - .add("slotIdx", slotIdx).toString(); + return MoreObjects.toStringHelper(this).add("id", id.asInt()).add("parent", parentTupleId).add("col", colStr) + .add("type", typeStr).add("materialized", isMaterialized).add("byteSize", byteSize) + .add("byteOffset", byteOffset).add("nullIndicatorByte", nullIndicatorByte) + .add("nullIndicatorBit", nullIndicatorBit).add("slotIdx", slotIdx).toString(); } @Override @@ -311,10 +313,8 @@ public class SlotDescriptor { String colStr = (column == null ? "null" : column.getName()); String typeStr = (type == null ? "null" : type.toString()); String parentTupleId = (parent == null) ? "null" : parent.getId().toString(); - builder.append(prefix).append("SlotDescriptor{") - .append("id=").append(id) - .append(", col=").append(colStr) - .append(", type=").append(typeStr).append("}\n"); + builder.append(prefix).append("SlotDescriptor{").append("id=").append(id).append(", col=").append(colStr) + .append(", type=").append(typeStr).append("}\n"); prefix += " "; builder.append(prefix).append("parent=").append(parentTupleId).append("\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index 6a0976404b..0bd5470ed0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -71,7 +71,7 @@ public class SlotRef extends Expr { public SlotRef(SlotDescriptor desc) { super(); this.tblName = null; - this.col = null; + this.col = desc.getColumn() != null ? desc.getColumn().getName() : null; this.desc = desc; this.type = desc.getType(); // TODO(zc): label is meaningful diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 4020ffa72f..add27b1430 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -71,6 +71,7 @@ import org.apache.doris.analysis.ReplacePartitionClause; import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.RollupRenameClause; import org.apache.doris.analysis.ShowAlterStmt.AlterType; +import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRenameClause; import org.apache.doris.analysis.TruncateTableStmt; import org.apache.doris.analysis.UninstallPluginStmt; @@ -251,6 +252,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -567,21 +569,19 @@ public class Catalog { this.metaContext.setThreadLocalInfo(); this.stat = new TabletSchedulerStat(); - this.tabletScheduler = new TabletScheduler(this, systemInfo, - tabletInvertedIndex, stat, Config.tablet_rebalancer_type); + this.tabletScheduler = new TabletScheduler(this, systemInfo, tabletInvertedIndex, stat, + Config.tablet_rebalancer_type); this.tabletChecker = new TabletChecker(this, systemInfo, tabletScheduler, stat); // The pendingLoadTaskScheduler's queue size should not less than Config.desired_max_waiting_jobs. // So that we can guarantee that all submitted load jobs can be scheduled without being starved. this.pendingLoadTaskScheduler = new MasterTaskExecutor("pending_load_task_scheduler", - Config.async_pending_load_task_pool_size, - Config.desired_max_waiting_jobs, !isCheckpointCatalog); + Config.async_pending_load_task_pool_size, Config.desired_max_waiting_jobs, !isCheckpointCatalog); // The loadingLoadTaskScheduler's queue size is unlimited, so that it can receive all loading tasks // created after pending tasks finish. And don't worry about the high concurrency, because the // concurrency is limited by Config.desired_max_waiting_jobs and Config.async_loading_load_task_pool_size. this.loadingLoadTaskScheduler = new MasterTaskExecutor("loading_load_task_scheduler", - Config.async_loading_load_task_pool_size, - Integer.MAX_VALUE, !isCheckpointCatalog); + Config.async_loading_load_task_pool_size, Integer.MAX_VALUE, !isCheckpointCatalog); this.loadJobScheduler = new LoadJobScheduler(); this.loadManager = new LoadManager(loadJobScheduler); @@ -792,8 +792,8 @@ public class Catalog { // 1. check and create dirs and files File meta = new File(metaDir); if (!meta.exists()) { - LOG.warn("Doris' meta dir {} does not exist." - + " You need to create it before starting FE", meta.getAbsolutePath()); + LOG.warn("Doris' meta dir {} does not exist." + " You need to create it before starting FE", + meta.getAbsolutePath()); throw new Exception(meta.getAbsolutePath() + " does not exist, will exit"); } @@ -866,8 +866,7 @@ public class Catalog { } // check file integrity, if has. - if ((roleFile.exists() && !versionFile.exists()) - || (!roleFile.exists() && versionFile.exists())) { + if ((roleFile.exists() && !versionFile.exists()) || (!roleFile.exists() && versionFile.exists())) { throw new IOException("role file and version file must both exist or both not exist. " + "please specific one helper node to recover. will exit."); } @@ -914,12 +913,12 @@ public class Catalog { // But is metadata_failure_recovery is true, // we will not check it because this may be a FE migration. String[] split = nodeName.split("_"); - if (Config.metadata_failure_recovery.equals("false") - && !selfNode.first.equalsIgnoreCase(split[0])) { - throw new IOException("the self host " + selfNode.first - + " does not equal to the host in ROLE" - + " file " + split[0] + ". You need to set 'priority_networks' config" - + " in fe.conf to match the host " + split[0]); + if (Config.metadata_failure_recovery.equals("false") && !selfNode.first.equalsIgnoreCase( + split[0])) { + throw new IOException( + "the self host " + selfNode.first + " does not equal to the host in ROLE" + " file " + + split[0] + ". You need to set 'priority_networks' config" + + " in fe.conf to match the host " + split[0]); } } } @@ -988,8 +987,8 @@ public class Catalog { if (!versionFile.exists()) { // If the version file doesn't exist, download it from helper node if (!getVersionFileFromHelper(rightHelperNode)) { - throw new IOException("fail to download version file from " - + rightHelperNode.first + " will exit."); + throw new IOException( + "fail to download version file from " + rightHelperNode.first + " will exit."); } // NOTE: cluster_id will be init when Storage object is constructed, @@ -1028,8 +1027,8 @@ public class Catalog { Preconditions.checkNotNull(token); Preconditions.checkNotNull(remoteToken); if (!token.equals(remoteToken)) { - throw new IOException("token is not equal with helper node " - + rightHelperNode.first + ". will exit."); + throw new IOException( + "token is not equal with helper node " + rightHelperNode.first + ". will exit."); } } } catch (Exception e) { @@ -1051,8 +1050,7 @@ public class Catalog { } Preconditions.checkState(helperNodes.size() == 1); - LOG.info("finished to get cluster id: {}, role: {} and node name: {}", - clusterId, role.name(), nodeName); + LOG.info("finished to get cluster id: {}, role: {} and node name: {}", clusterId, role.name(), nodeName); } public static String genFeNodeName(String host, int port, boolean isOldStyle) { @@ -1072,13 +1070,13 @@ public class Catalog { Pair rightHelperNode = null; for (Pair helperNode : helperNodes) { try { - URL url = new URL("http://" + helperNode.first + ":" + Config.http_port - + "/role?host=" + selfNode.first + "&port=" + selfNode.second); + URL url = new URL("http://" + helperNode.first + ":" + Config.http_port + "/role?host=" + selfNode.first + + "&port=" + selfNode.second); HttpURLConnection conn = null; conn = (HttpURLConnection) url.openConnection(); if (conn.getResponseCode() != 200) { - LOG.warn("failed to get fe node type from helper node: {}. response code: {}", - helperNode, conn.getResponseCode()); + LOG.warn("failed to get fe node type from helper node: {}. response code: {}", helperNode, + conn.getResponseCode()); continue; } @@ -1166,9 +1164,8 @@ public class Catalog { * In this case, some errors have caused users to be troubled. * So here directly exit the program and inform the user to avoid unnecessary trouble. */ - throw new AnalysisException( - "Do not specify the helper node to FE itself. " - + "Please specify it to the existing running Master or Follower FE"); + throw new AnalysisException("Do not specify the helper node to FE itself. " + + "Please specify it to the existing running Master or Follower FE"); } helperNodes.add(helperHostPort); } @@ -1188,8 +1185,7 @@ public class Catalog { // 1. check if this is the first time to start up File roleFile = new File(this.imageDir, Storage.ROLE_FILE); File versionFile = new File(this.imageDir, Storage.VERSION_FILE); - if ((roleFile.exists() && !versionFile.exists()) - || (!roleFile.exists() && versionFile.exists())) { + if ((roleFile.exists() && !versionFile.exists()) || (!roleFile.exists() && versionFile.exists())) { throw new Exception("role file and version file must both exist or both not exist. " + "please specific one helper node to recover. will exit."); } @@ -1459,7 +1455,8 @@ public class Catalog { if (Config.lower_case_table_names != GlobalVariable.lowerCaseTableNames) { LOG.error("The configuration of \'lower_case_table_names\' does not support modification, " + "the expected value is {}, but the actual value is {}", - GlobalVariable.lowerCaseTableNames, Config.lower_case_table_names); + GlobalVariable.lowerCaseTableNames, + Config.lower_case_table_names); System.exit(-1); } LOG.info("lower_case_table_names is {}", GlobalVariable.lowerCaseTableNames); @@ -1513,8 +1510,7 @@ public class Catalog { StorageInfo info = getStorageInfo(infoUrl); long version = info.getImageSeq(); if (version > localImageVersion) { - String url = "http://" + helperNode.first + ":" + Config.http_port - + "/image?version=" + version; + String url = "http://" + helperNode.first + ":" + Config.http_port + "/image?version=" + version; String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); MetaHelper.getRemoteFile(url, HTTP_TIMEOUT_SECOND * 1000, MetaHelper.getOutputStream(filename, dir)); @@ -1554,8 +1550,8 @@ public class Catalog { connection.setReadTimeout(HTTP_TIMEOUT_SECOND * 1000); String response; - try (BufferedReader bufferedReader - = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + try (BufferedReader bufferedReader = new BufferedReader( + new InputStreamReader(connection.getInputStream()))) { String line; StringBuilder sb = new StringBuilder(); while ((line = bufferedReader.readLine()) != null) { @@ -2030,8 +2026,8 @@ public class Catalog { public long saveExportJob(CountingDataOutputStream dos, long checksum) throws IOException { long curTime = System.currentTimeMillis(); - List jobs = exportMgr.getJobs().stream() - .filter(t -> !t.isExpired(curTime)).collect(Collectors.toList()); + List jobs = exportMgr.getJobs().stream().filter(t -> !t.isExpired(curTime)) + .collect(Collectors.toList()); int size = jobs.size(); checksum ^= size; dos.writeInt(size); @@ -2617,11 +2613,11 @@ public class Catalog { * 4. set table id and base index id * 5. set bloom filter columns * 6. set and build TableProperty includes: - * 6.1. dynamicProperty - * 6.2. replicationNum - * 6.3. inMemory - * 6.4. storageFormat - * 6.5. compressionType + * 6.1. dynamicProperty + * 6.2. replicationNum + * 6.3. inMemory + * 6.4. storageFormat + * 6.5. compressionType * 7. set index meta * 8. check colocation properties * 9. create tablet in BE @@ -2692,10 +2688,8 @@ public class Catalog { // 1.2 other table type sb.append("CREATE "); - if (table.getType() == TableType.ODBC - || table.getType() == TableType.MYSQL - || table.getType() == TableType.ELASTICSEARCH - || table.getType() == TableType.BROKER + if (table.getType() == TableType.ODBC || table.getType() == TableType.MYSQL + || table.getType() == TableType.ELASTICSEARCH || table.getType() == TableType.BROKER || table.getType() == TableType.HIVE) { sb.append("EXTERNAL "); } @@ -2834,8 +2828,7 @@ public class Catalog { Preconditions.checkState(partitionId.size() == 1); partition = olapTable.getPartition(partitionId.get(0)); } - sb.append(partition.getVisibleVersion()) - .append("\""); + sb.append(partition.getVisibleVersion()).append("\""); } // colocateTable @@ -2897,8 +2890,8 @@ public class Catalog { sb.append("\"password\" = \"").append(hidePassword ? "" : mysqlTable.getPasswd()).append("\",\n"); sb.append("\"charset\" = \"").append(mysqlTable.getCharset()).append("\",\n"); } else { - sb.append("\"odbc_catalog_resource\" = \"") - .append(mysqlTable.getOdbcCatalogResourceName()).append("\",\n"); + sb.append("\"odbc_catalog_resource\" = \"").append(mysqlTable.getOdbcCatalogResourceName()) + .append("\",\n"); } sb.append("\"database\" = \"").append(mysqlTable.getMysqlDatabaseName()).append("\",\n"); sb.append("\"table\" = \"").append(mysqlTable.getMysqlTableName()).append("\"\n"); @@ -2918,8 +2911,8 @@ public class Catalog { sb.append("\"driver\" = \"").append(odbcTable.getOdbcDriver()).append("\",\n"); sb.append("\"odbc_type\" = \"").append(odbcTable.getOdbcTableTypeName()).append("\",\n"); } else { - sb.append("\"odbc_catalog_resource\" = \"") - .append(odbcTable.getOdbcCatalogResourceName()).append("\",\n"); + sb.append("\"odbc_catalog_resource\" = \"").append(odbcTable.getOdbcCatalogResourceName()) + .append("\",\n"); } sb.append("\"database\" = \"").append(odbcTable.getOdbcDatabaseName()).append("\",\n"); sb.append("\"table\" = \"").append(odbcTable.getOdbcTableName()).append("\"\n"); @@ -3038,8 +3031,7 @@ public class Catalog { sb.append(")"); } sb.append("(\"version_info\" = \""); - sb.append(partition.getVisibleVersion()) - .append("\""); + sb.append(partition.getVisibleVersion()).append("\""); sb.append(");"); addPartitionStmt.add(sb.toString()); } @@ -3160,8 +3152,7 @@ public class Catalog { LOG.error(frontend.toString() + " does not exist."); return; } - if (removedFe.getRole() == FrontendNodeType.FOLLOWER - || removedFe.getRole() == FrontendNodeType.REPLICA) { + if (removedFe.getRole() == FrontendNodeType.FOLLOWER || removedFe.getRole() == FrontendNodeType.REPLICA) { helperNodes.remove(Pair.create(removedFe.getHost(), removedFe.getEditLogPort())); } @@ -3217,8 +3208,8 @@ public class Catalog { for (Partition partition : olapTable.getAllPartitions()) { long partitionId = partition.getId(); DataProperty dataProperty = partitionInfo.getDataProperty(partition.getId()); - Preconditions.checkNotNull(dataProperty, partition.getName() - + ", pId:" + partitionId + ", db: " + dbId + ", tbl: " + tableId); + Preconditions.checkNotNull(dataProperty, + partition.getName() + ", pId:" + partitionId + ", db: " + dbId + ", tbl: " + tableId); if (dataProperty.getStorageMedium() == TStorageMedium.SSD && dataProperty.getCooldownTimeMs() < currentTimeMs) { // expire. change to HDD. @@ -3257,8 +3248,8 @@ public class Catalog { // use try lock to avoid blocking a long time. // if block too long, backend report rpc will timeout. if (!olapTable.tryWriteLockIfExist(Table.TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { - LOG.warn("try get table {} writelock but failed" - + " when checking backend storage medium", table.getName()); + LOG.warn("try get table {} writelock but failed" + " when checking backend storage medium", + table.getName()); continue; } Preconditions.checkState(olapTable.isWriteLockHeldByCurrentThread()); @@ -3278,16 +3269,13 @@ public class Catalog { DataProperty hddProperty = new DataProperty(TStorageMedium.HDD); partitionInfo.setDataProperty(partition.getId(), hddProperty); storageMediumMap.put(partitionId, TStorageMedium.HDD); - LOG.debug("partition[{}-{}-{}] storage medium changed from SSD to HDD", - dbId, tableId, partitionId); + LOG.debug("partition[{}-{}-{}] storage medium changed from SSD to HDD", dbId, tableId, + partitionId); // log - ModifyPartitionInfo info = - new ModifyPartitionInfo(db.getId(), olapTable.getId(), - partition.getId(), - hddProperty, - ReplicaAllocation.NOT_SET, - partitionInfo.getIsInMemory(partition.getId())); + ModifyPartitionInfo info = new ModifyPartitionInfo(db.getId(), olapTable.getId(), + partition.getId(), hddProperty, ReplicaAllocation.NOT_SET, + partitionInfo.getIsInMemory(partition.getId())); editLog.logModifyPartition(info); } } // end for partitions @@ -3721,15 +3709,15 @@ public class Catalog { if (bucketsNum == -1) { bucketsNum = partition.getDistributionInfo().getBucketNum(); } else if (bucketsNum != partition.getDistributionInfo().getBucketNum()) { - throw new DdlException("Partitions in table " + table.getName() - + " have different buckets number"); + throw new DdlException( + "Partitions in table " + table.getName() + " have different buckets number"); } if (replicaAlloc == null) { replicaAlloc = partitionInfo.getReplicaAllocation(partition.getId()); } else if (!replicaAlloc.equals(partitionInfo.getReplicaAllocation(partition.getId()))) { - throw new DdlException("Partitions in table " + table.getName() - + " have different replica allocation."); + throw new DdlException( + "Partitions in table " + table.getName() + " have different replica allocation."); } } } @@ -3778,8 +3766,7 @@ public class Catalog { TablePropertyInfo info = new TablePropertyInfo(table.getId(), groupId, properties); editLog.logModifyTableColocate(info); } - LOG.info("finished modify table's colocation property. table: {}, is replay: {}", - table.getName(), isReplay); + LOG.info("finished modify table's colocation property. table: {}, is replay: {}", table.getName(), isReplay); } public void replayModifyTableColocate(TablePropertyInfo info) throws MetaNotFoundException { @@ -3871,8 +3858,8 @@ public class Catalog { if (table.getPartitionInfo().getType() != PartitionType.RANGE && table.getPartitionInfo().getType() != PartitionType.LIST) { - throw new DdlException("Table[" + table.getName() + "] is single partitioned. " - + "no need to rename partition name."); + throw new DdlException( + "Table[" + table.getName() + "] is single partitioned. " + "no need to rename partition name."); } String partitionName = renameClause.getPartitionName(); @@ -3940,17 +3927,17 @@ public class Catalog { // Merge the new properties with origin properties, and then analyze them Map origDynamicProperties = tableProperty.getOriginDynamicPartitionProperty(); origDynamicProperties.putAll(properties); - Map analyzedDynamicPartition = DynamicPartitionUtil - .analyzeDynamicPartition(origDynamicProperties, table.getPartitionInfo()); + Map analyzedDynamicPartition = DynamicPartitionUtil.analyzeDynamicPartition( + origDynamicProperties, table.getPartitionInfo()); tableProperty.modifyTableProperties(analyzedDynamicPartition); tableProperty.buildDynamicProperty(); } DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), table, false); - dynamicPartitionScheduler.createOrUpdateRuntimeInfo( - table.getId(), DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime()); - ModifyTablePropertyOperationLog info - = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), logProperties); + dynamicPartitionScheduler.createOrUpdateRuntimeInfo(table.getId(), DynamicPartitionScheduler.LAST_UPDATE_TIME, + TimeUtils.getCurrentFormatTime()); + ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), + logProperties); editLog.logDynamicPartition(info); } @@ -3964,22 +3951,24 @@ public class Catalog { /** * Set replication number for unpartitioned table. + * * @param db * @param table * @param properties * @throws DdlException */ // The caller need to hold the table write lock - public void modifyTableReplicaAllocation(Database db, OlapTable table, - Map properties) throws UserException { + public void modifyTableReplicaAllocation(Database db, OlapTable table, Map properties) + throws UserException { Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread()); String defaultReplicationNumName = "default." + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM; PartitionInfo partitionInfo = table.getPartitionInfo(); if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { - throw new DdlException("This is a partitioned table, you should specify partitions" - + " with MODIFY PARTITION clause." - + " If you want to set default replication number, please use '" + defaultReplicationNumName - + "' instead of '" + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM + "' to escape misleading."); + throw new DdlException( + "This is a partitioned table, you should specify partitions" + " with MODIFY PARTITION clause." + + " If you want to set default replication number, please use '" + defaultReplicationNumName + + "' instead of '" + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM + + "' to escape misleading."); } String partitionName = table.getName(); Partition partition = table.getPartition(partitionName); @@ -3997,13 +3986,14 @@ public class Catalog { ModifyPartitionInfo info = new ModifyPartitionInfo(db.getId(), table.getId(), partition.getId(), newDataProperty, replicaAlloc, isInMemory); editLog.logModifyPartition(info); - LOG.debug("modify partition[{}-{}-{}] replica allocation to {}", db.getId(), table.getId(), - partition.getName(), replicaAlloc.toCreateStmt()); + LOG.debug("modify partition[{}-{}-{}] replica allocation to {}", db.getId(), table.getId(), partition.getName(), + replicaAlloc.toCreateStmt()); } /** * Set default replication allocation for a specified table. * You can see the default replication allocation by executing Show Create Table stmt. + * * @param db * @param table * @param properties @@ -4022,8 +4012,8 @@ public class Catalog { tableProperty.buildReplicaAllocation(); // log - ModifyTablePropertyOperationLog info - = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), properties); + ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), + properties); editLog.logModifyReplicationNum(info); LOG.debug("modify table[{}] replication num to {}", table.getName(), properties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM)); @@ -4045,13 +4035,13 @@ public class Catalog { table.getPartitionInfo().setIsInMemory(partition.getId(), tableProperty.isInMemory()); } - ModifyTablePropertyOperationLog info - = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), properties); + ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), + properties); editLog.logModifyInMemory(info); } - public void replayModifyTableProperty(short opCode, - ModifyTablePropertyOperationLog info) throws MetaNotFoundException { + public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperationLog info) + throws MetaNotFoundException { long dbId = info.getDbId(); long tableId = info.getTableId(); Map properties = info.getProperties(); @@ -4099,17 +4089,18 @@ public class Catalog { DistributionInfo distributionInfo = distributionDesc.toDistributionInfo(baseSchema); // for now. we only support modify distribution's bucket num if (distributionInfo.getType() != defaultDistributionInfo.getType()) { - throw new DdlException("Cannot change distribution type when modify" - + " default distribution bucket num"); + throw new DdlException( + "Cannot change distribution type when modify" + " default distribution bucket num"); } if (distributionInfo.getType() == DistributionInfoType.HASH) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List newDistriCols = hashDistributionInfo.getDistributionColumns(); - List defaultDistriCols = ((HashDistributionInfo) defaultDistributionInfo) - .getDistributionColumns(); + List defaultDistriCols + = ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns(); if (!newDistriCols.equals(defaultDistriCols)) { - throw new DdlException("Cannot assign hash distribution with different distribution cols. " - + "default is: " + defaultDistriCols); + throw new DdlException( + "Cannot assign hash distribution with different distribution cols. " + "default is: " + + defaultDistriCols); } } @@ -4163,8 +4154,8 @@ public class Catalog { // Switch catalog of this sesseion. public void changeCatalog(ConnectContext ctx, String catalogName) throws DdlException { if (dataSourceMgr.getCatalogNullable(catalogName) == null) { - throw new DdlException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg( - catalogName), ErrorCode.ERR_UNKNOWN_CATALOG); + throw new DdlException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(catalogName), + ErrorCode.ERR_UNKNOWN_CATALOG); } ctx.changeDefaultCatalog(catalogName); } @@ -4596,8 +4587,8 @@ public class Catalog { Replica replica = tablet.getReplicaById(info.getReplicaId()); if (replica != null) { replica.setBad(true); - LOG.debug("get replica {} of tablet {} on backend {} to bad when replaying", - info.getReplicaId(), info.getTabletId(), info.getBackendId()); + LOG.debug("get replica {} of tablet {} on backend {} to bad when replaying", info.getReplicaId(), + info.getTabletId(), info.getBackendId()); } } finally { olapTable.writeUnlock(); @@ -4674,8 +4665,8 @@ public class Catalog { ReplacePartitionOperationLog info = new ReplacePartitionOperationLog(db.getId(), olapTable.getId(), partitionNames, tempPartitionNames, isStrictRange, useTempPartitionName); editLog.logReplaceTempPartition(info); - LOG.info("finished to replace partitions {} with temp partitions {} from table: {}", - clause.getPartitionNames(), clause.getTempPartitionNames(), olapTable.getName()); + LOG.info("finished to replace partitions {} with temp partitions {} from table: {}", clause.getPartitionNames(), + clause.getTempPartitionNames(), olapTable.getName()); } public void replayReplaceTempPartition(ReplacePartitionOperationLog replaceTempPartitionLog) @@ -4687,8 +4678,7 @@ public class Catalog { olapTable.writeLock(); try { olapTable.replaceTempPartitions(replaceTempPartitionLog.getPartitions(), - replaceTempPartitionLog.getTempPartitions(), - replaceTempPartitionLog.isStrictRange(), + replaceTempPartitionLog.getTempPartitions(), replaceTempPartitionLog.isStrictRange(), replaceTempPartitionLog.useTempPartitionName()); } catch (DdlException e) { throw new MetaNotFoundException(e); @@ -4764,8 +4754,8 @@ public class Catalog { setReplicaStatusInternal(log.getTabletId(), log.getBackendId(), log.getReplicaStatus(), true); } - private void setReplicaStatusInternal(long tabletId, long backendId, - ReplicaStatus status, boolean isReplay) throws MetaNotFoundException { + private void setReplicaStatusInternal(long tabletId, long backendId, ReplicaStatus status, boolean isReplay) + throws MetaNotFoundException { try { TabletMeta meta = tabletInvertedIndex.getTabletMeta(tabletId); if (meta == null) { @@ -4782,12 +4772,12 @@ public class Catalog { if (status == ReplicaStatus.BAD || status == ReplicaStatus.OK) { if (replica.setBad(status == ReplicaStatus.BAD)) { if (!isReplay) { - SetReplicaStatusOperationLog log - = new SetReplicaStatusOperationLog(backendId, tabletId, status); + SetReplicaStatusOperationLog log = new SetReplicaStatusOperationLog(backendId, tabletId, + status); getEditLog().logSetReplicaStatus(log); } - LOG.info("set replica {} of tablet {} on backend {} as {}. is replay: {}", - replica.getId(), tabletId, backendId, status, isReplay); + LOG.info("set replica {} of tablet {} on backend {} as {}. is replay: {}", replica.getId(), + tabletId, backendId, status, isReplay); } } } finally { @@ -4944,4 +4934,16 @@ public class Catalog { } return count; } + + public TableName getTableNameByTableId(Long tableId) { + for (String dbName : getInternalDataSource().getDbNames()) { + DatabaseIf db = getInternalDataSource().getDbNullable(dbName); + Optional table = db.getTable(tableId); + if (table.isPresent()) { + return new TableName(InternalDataSource.INTERNAL_DS_NAME, + db.getFullName(), table.get().getName()); + } + } + return null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 3badbcab4b..25cd47f628 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -20,6 +20,7 @@ package org.apache.doris.catalog; import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.common.AnalysisException; @@ -55,6 +56,7 @@ public class Column implements Writable { public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__"; public static final String SEQUENCE_COL = "__DORIS_SEQUENCE_COL__"; private static final String COLUMN_ARRAY_CHILDREN = "item"; + public static final int COLUMN_UNIQUE_ID_INIT_VALUE = -1; @SerializedName(value = "name") private String name; @@ -93,6 +95,9 @@ public class Column implements Writable { @SerializedName(value = "defaultValueExprDef") private DefaultValueExprDef defaultValueExprDef; // used for default value + @SerializedName(value = "uniqueId") + private int uniqueId; + public Column() { this.name = ""; this.type = Type.NULL; @@ -102,6 +107,7 @@ public class Column implements Writable { this.visible = true; this.defineExpr = null; this.children = new ArrayList<>(Type.MAX_NESTING_DEPTH); + this.uniqueId = -1; } public Column(String name, PrimitiveType dataType) { @@ -123,11 +129,13 @@ public class Column implements Writable { public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment) { - this(name, type, isKey, aggregateType, isAllowNull, defaultValue, comment, true, null); + this(name, type, isKey, aggregateType, isAllowNull, defaultValue, comment, true, null, + COLUMN_UNIQUE_ID_INIT_VALUE); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, - String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef) { + String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, + int colUniqueId) { this.name = name; if (this.name == null) { this.name = ""; @@ -149,6 +157,7 @@ public class Column implements Writable { this.visible = visible; this.children = new ArrayList<>(Type.MAX_NESTING_DEPTH); createChildrenColumn(this.type, this); + this.uniqueId = colUniqueId; } public Column(Column column) { @@ -164,6 +173,7 @@ public class Column implements Writable { this.stats = column.getStats(); this.visible = column.visible; this.children = column.getChildren(); + this.uniqueId = column.getUniqueId(); } public void createChildrenColumn(Type type, Column column) { @@ -366,6 +376,7 @@ public class Column implements Writable { tColumn.setVisible(visible); toChildrenThrift(this, tColumn); + tColumn.setColUniqueId(uniqueId); // ATTN: // Currently, this `toThrift()` method is only used from CreateReplicaTask. // And CreateReplicaTask does not need `defineExpr` field. @@ -688,4 +699,23 @@ public class Column implements Writable { sb.append(defaultValue == null ? "" : defaultValue); return sb.toString(); } + + public void setUniqueId(int colUniqueId) { + this.uniqueId = colUniqueId; + } + + public int getUniqueId() { + return this.uniqueId; + } + + public void setIndexFlag(TColumn tColumn, List indexes) { + for (Index index : indexes) { + if (index.getIndexType() == IndexDef.IndexType.BITMAP) { + List columns = index.getColumns(); + if (tColumn.getColumnName().equals(columns.get(0))) { + tColumn.setHasBitmapIndex(true); + } + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index ede7c32a17..8e600c5991 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -104,6 +104,14 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable { } } + public void setSchema(List newSchema) { + this.schema = newSchema; + } + + public void setSchemaHash(int newSchemaHash) { + this.schemaHash = newSchemaHash; + } + public int getSchemaHash() { return schemaHash; } @@ -112,6 +120,10 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable { return shortKeyColumnCount; } + public void setSchemaVersion(int newSchemaVersion) { + this.schemaVersion = newSchemaVersion; + } + public int getSchemaVersion() { return schemaVersion; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 80ab60064d..9e23ebe374 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -38,6 +38,7 @@ import org.apache.doris.clone.TabletScheduler; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.io.DeepCopy; @@ -141,6 +142,8 @@ public class OlapTable extends Table { private TableProperty tableProperty; + private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE; + public OlapTable() { // for persist super(TableType.OLAP); @@ -191,6 +194,20 @@ public class OlapTable extends Table { return this.tableProperty; } + //take care: only use at create olap table. + public int incAndGetMaxColUniqueId() { + this.maxColUniqueId++; + return this.maxColUniqueId; + } + + public int getMaxColUniqueId() { + return this.maxColUniqueId; + } + + public void setMaxColUniqueId(int maxColUniqueId) { + this.maxColUniqueId = maxColUniqueId; + } + public boolean dynamicPartitionExists() { return tableProperty != null && tableProperty.getDynamicPartitionProperty() != null @@ -1144,6 +1161,7 @@ public class OlapTable extends Table { } tempPartitions.write(out); + out.writeInt(maxColUniqueId); } @Override @@ -1235,6 +1253,10 @@ public class OlapTable extends Table { } } tempPartitions.unsetPartitionInfo(); + + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_112) { + maxColUniqueId = in.readInt(); + } // In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV. // After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns. // So, here we need to rebuild the fullSchema to ensure the correctness of the properties. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 124c453261..887ae74c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -81,7 +81,7 @@ public class Table extends MetaObject implements Writable, TableIf { * Schema change (c3 to bigint) * When OlapTable is changing schema, the fullSchema is (c1 int, c2 int, c3 int, SHADOW_NAME_PRFIX_c3 bigint) * The fullSchema of OlapTable is mainly used by Scanner of Load job. - * + *

* If you want to get the mv columns, you should call getIndexToSchema in Subclass OlapTable. */ protected List fullSchema; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java index 332fa85078..634bca7079 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java @@ -340,8 +340,8 @@ public class InternalDataSource implements DataSourceIf { Collection allPartitions = olapTable.getAllPartitions(); for (Partition partition : allPartitions) { long partitionId = partition.getId(); - TStorageMedium medium = - olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId) + .getStorageMedium(); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); @@ -475,8 +475,8 @@ public class InternalDataSource implements DataSourceIf { fullNameToDb.remove(db.getAttachDb()); db.setDbState(DbState.NORMAL); info.setUpdateDbState(DbState.NORMAL); - final Cluster cluster = - nameToCluster.get(ClusterNamespace.getClusterNameFromFullName(db.getAttachDb())); + final Cluster cluster = nameToCluster.get( + ClusterNamespace.getClusterNameFromFullName(db.getAttachDb())); final BaseParam param = new BaseParam(); param.addStringParam(db.getAttachDb()); param.addLongParam(db.getId()); @@ -845,8 +845,7 @@ public class InternalDataSource implements DataSourceIf { OlapTable olapTable = (OlapTable) table; if ((olapTable.getState() != OlapTableState.NORMAL)) { throw new DdlException("The table [" + tableName + "]'s state is " + olapTable.getState() - + ", cannot be dropped." - + " please cancel the operation on olap table firstly." + + ", cannot be dropped." + " please cancel the operation on olap table firstly." + " If you want to forcibly drop(cannot be recovered)," + " please use \"DROP table FORCE\"."); } @@ -925,10 +924,9 @@ public class InternalDataSource implements DataSourceIf { schemaHash = olapTable.getSchemaHashByIndexId(info.getIndexId()); } - Replica replica = - new Replica(info.getReplicaId(), info.getBackendId(), info.getVersion(), schemaHash, info.getDataSize(), - info.getRowCount(), ReplicaState.NORMAL, info.getLastFailedVersion(), - info.getLastSuccessVersion()); + Replica replica = new Replica(info.getReplicaId(), info.getBackendId(), info.getVersion(), schemaHash, + info.getDataSize(), info.getRowCount(), ReplicaState.NORMAL, info.getLastFailedVersion(), + info.getLastSuccessVersion()); tablet.addReplica(replica); } @@ -1091,8 +1089,8 @@ public class InternalDataSource implements DataSourceIf { } finally { table.readUnlock(); } - CreateTableStmt parsedCreateTableStmt = - (CreateTableStmt) SqlParserUtils.parseAndAnalyzeStmt(createTableStmt.get(0), ConnectContext.get()); + CreateTableStmt parsedCreateTableStmt = (CreateTableStmt) SqlParserUtils.parseAndAnalyzeStmt( + createTableStmt.get(0), ConnectContext.get()); parsedCreateTableStmt.setTableName(stmt.getTableName()); parsedCreateTableStmt.setIfNotExists(stmt.isIfNotExists()); createTable(parsedCreateTableStmt); @@ -1178,8 +1176,8 @@ public class InternalDataSource implements DataSourceIf { long tableId = table.getId(); for (Partition partition : olapTable.getAllPartitions()) { long partitionId = partition.getId(); - TStorageMedium medium = - olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId) + .getStorageMedium(); for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = mIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); @@ -1263,8 +1261,8 @@ public class InternalDataSource implements DataSourceIf { if (distributionInfo.getType() == DistributionInfoType.HASH) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List newDistriCols = hashDistributionInfo.getDistributionColumns(); - List defaultDistriCols = - ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns(); + List defaultDistriCols + = ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns(); if (!newDistriCols.equals(defaultDistriCols)) { throw new DdlException( "Cannot assign hash distribution with different distribution cols. " + "default is: " @@ -1437,9 +1435,8 @@ public class InternalDataSource implements DataSourceIf { for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = - new TabletMeta(info.getDbId(), info.getTableId(), partition.getId(), index.getId(), - schemaHash, info.getDataProperty().getStorageMedium()); + TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), partition.getId(), + index.getId(), schemaHash, info.getDataProperty().getStorageMedium()); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); invertedIndex.addTablet(tabletId, tabletMeta); @@ -1599,11 +1596,10 @@ public class InternalDataSource implements DataSourceIf { long backendId = replica.getBackendId(); long replicaId = replica.getId(); countDownLatch.addMark(backendId, tabletId); - CreateReplicaTask task = - new CreateReplicaTask(backendId, dbId, tableId, partitionId, indexId, tabletId, replicaId, - shortKeyColumnCount, schemaHash, version, keysType, storageType, storageMedium, - schema, bfColumns, bfFpp, countDownLatch, indexes, isInMemory, tabletType, - dataSortInfo, compressionType); + CreateReplicaTask task = new CreateReplicaTask(backendId, dbId, tableId, partitionId, indexId, + tabletId, replicaId, shortKeyColumnCount, schemaHash, version, keysType, storageType, + storageMedium, schema, bfColumns, bfFpp, countDownLatch, indexes, isInMemory, tabletType, + dataSortInfo, compressionType); task.setStorageFormat(storageFormat); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. @@ -1700,9 +1696,15 @@ public class InternalDataSource implements DataSourceIf { // create table long tableId = Catalog.getCurrentCatalog().getNextId(); - OlapTable olapTable = - new OlapTable(tableId, tableName, baseSchema, keysType, partitionInfo, defaultDistributionInfo, - indexes); + OlapTable olapTable = new OlapTable(tableId, tableName, baseSchema, keysType, partitionInfo, + defaultDistributionInfo, indexes); + + for (Column column : baseSchema) { + column.setUniqueId(olapTable.incAndGetMaxColUniqueId()); + LOG.debug("table: {}, newColumn: {}, uniqueId: {}", olapTable.getName(), column.getName(), + column.getUniqueId()); + } + olapTable.setComment(stmt.getComment()); // set base index id @@ -1732,8 +1734,8 @@ public class InternalDataSource implements DataSourceIf { olapTable.setCompressionType(compressionType); // check data sort properties - DataSortInfo dataSortInfo = - PropertyAnalyzer.analyzeDataSortInfo(properties, keysType, keysDesc.keysColumnSize(), storageFormat); + DataSortInfo dataSortInfo = PropertyAnalyzer.analyzeDataSortInfo(properties, keysType, + keysDesc.keysColumnSize(), storageFormat); olapTable.setDataSortInfo(dataSortInfo); // analyze bloom filter columns @@ -1765,8 +1767,8 @@ public class InternalDataSource implements DataSourceIf { olapTable.setReplicationAllocation(replicaAlloc); // set in memory - boolean isInMemory = - PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_INMEMORY, false); + boolean isInMemory = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_INMEMORY, + false); olapTable.setIsInMemory(isInMemory); // set remote storage @@ -1789,8 +1791,8 @@ public class InternalDataSource implements DataSourceIf { long partitionId = partitionNameToId.get(tableName); DataProperty dataProperty = null; try { - dataProperty = - PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), DataProperty.DEFAULT_DATA_PROPERTY); + dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), + DataProperty.DEFAULT_DATA_PROPERTY); } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } @@ -1858,8 +1860,8 @@ public class InternalDataSource implements DataSourceIf { // set rollup index meta to olap table List rollupColumns = Catalog.getCurrentCatalog().getMaterializedViewHandler() .checkAndPrepareMaterializedView(addRollupClause, olapTable, baseRollupIndex, false); - short rollupShortKeyColumnCount = - Catalog.calcShortKeyColumnCount(rollupColumns, alterClause.getProperties()); + short rollupShortKeyColumnCount = Catalog.calcShortKeyColumnCount(rollupColumns, + alterClause.getProperties()); int rollupSchemaHash = Util.generateSchemaHash(); long rollupIndexId = Catalog.getCurrentCatalog().getNextId(); olapTable.setIndexMeta(rollupIndexId, addRollupClause.getRollupName(), rollupColumns, schemaVersion, @@ -1991,10 +1993,10 @@ public class InternalDataSource implements DataSourceIf { // we have added these index to memory, only need to persist here if (Catalog.getCurrentColocateIndex().isColocateTable(tableId)) { GroupId groupId = Catalog.getCurrentColocateIndex().getGroup(tableId); - Map>> backendsPerBucketSeq = - Catalog.getCurrentColocateIndex().getBackendsPerBucketSeq(groupId); - ColocatePersistInfo info = - ColocatePersistInfo.createForAddTable(groupId, tableId, backendsPerBucketSeq); + Map>> backendsPerBucketSeq = Catalog.getCurrentColocateIndex() + .getBackendsPerBucketSeq(groupId); + ColocatePersistInfo info = ColocatePersistInfo.createForAddTable(groupId, tableId, + backendsPerBucketSeq); Catalog.getCurrentCatalog().getEditLog().logColocateAddTable(info); } LOG.info("successfully create table[{};{}]", tableName, tableId); @@ -2111,8 +2113,8 @@ public class InternalDataSource implements DataSourceIf { HiveTable hiveTable = new HiveTable(tableId, tableName, columns, stmt.getProperties()); hiveTable.setComment(stmt.getComment()); // check hive table whether exists in hive database - HiveMetaStoreClient hiveMetaStoreClient = - HiveMetaStoreClientHelper.getClient(hiveTable.getHiveProperties().get(HiveTable.HIVE_METASTORE_URIS)); + HiveMetaStoreClient hiveMetaStoreClient = HiveMetaStoreClientHelper.getClient( + hiveTable.getHiveProperties().get(HiveTable.HIVE_METASTORE_URIS)); if (!HiveMetaStoreClientHelper.tableExists(hiveMetaStoreClient, hiveTable.getHiveDb(), hiveTable.getHiveTable())) { throw new DdlException(String.format("Table [%s] dose not exist in Hive.", hiveTable.getHiveDbTable())); @@ -2140,9 +2142,8 @@ public class InternalDataSource implements DataSourceIf { throw new DdlException( String.format("Table [%s] dose not exist in Hive Metastore.", hudiTable.getHmsTableIdentifer())); } - org.apache.hadoop.hive.metastore.api.Table hiveTable = - HiveMetaStoreClientHelper.getTable(hudiTable.getHmsDatabaseName(), hudiTable.getHmsTableName(), - metastoreUris); + org.apache.hadoop.hive.metastore.api.Table hiveTable = HiveMetaStoreClientHelper.getTable( + hudiTable.getHmsDatabaseName(), hudiTable.getHmsTableName(), metastoreUris); if (!HudiUtils.isHudiTable(hiveTable)) { throw new DdlException(String.format("Table [%s] is not a hudi table.", hudiTable.getHmsTableIdentifer())); } @@ -2229,8 +2230,8 @@ public class InternalDataSource implements DataSourceIf { for (List backendIds : chosenBackendIds.values()) { for (long backendId : backendIds) { long replicaId = Catalog.getCurrentCatalog().getNextId(); - Replica replica = - new Replica(replicaId, backendId, replicaState, version, tabletMeta.getOldSchemaHash()); + Replica replica = new Replica(replicaId, backendId, replicaState, version, + tabletMeta.getOldSchemaHash()); tablet.addReplica(replica); totalReplicaNum++; } @@ -2403,8 +2404,8 @@ public class InternalDataSource implements DataSourceIf { truncateTableInternal(olapTable, newPartitions, truncateEntireTable); // write edit log - TruncateTableInfo info = - new TruncateTableInfo(db.getId(), olapTable.getId(), newPartitions, truncateEntireTable); + TruncateTableInfo info = new TruncateTableInfo(db.getId(), olapTable.getId(), newPartitions, + truncateEntireTable); Catalog.getCurrentCatalog().getEditLog().logTruncateTable(info); } finally { olapTable.writeUnlock(); @@ -2449,13 +2450,13 @@ public class InternalDataSource implements DataSourceIf { TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); for (Partition partition : info.getPartitions()) { long partitionId = partition.getId(); - TStorageMedium medium = - olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId) + .getStorageMedium(); for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = mIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = - new TabletMeta(db.getId(), olapTable.getId(), partitionId, indexId, schemaHash, medium); + TabletMeta tabletMeta = new TabletMeta(db.getId(), olapTable.getId(), partitionId, indexId, + schemaHash, medium); for (Tablet tablet : mIndex.getTablets()) { long tabletId = tablet.getId(); invertedIndex.addTablet(tabletId, tabletMeta); @@ -2508,8 +2509,8 @@ public class InternalDataSource implements DataSourceIf { if (nameToCluster.containsKey(clusterName)) { ErrorReport.reportDdlException(ErrorCode.ERR_CLUSTER_HAS_EXIST, clusterName); } else { - List backendList = - Catalog.getCurrentSystemInfo().createCluster(clusterName, stmt.getInstanceNum()); + List backendList = Catalog.getCurrentSystemInfo() + .createCluster(clusterName, stmt.getInstanceNum()); // 1: BE returned is less than requested, throws DdlException. // 2: BE returned is more than or equal to 0, succeeds. if (backendList != null || stmt.getInstanceNum() == 0) { @@ -2967,8 +2968,8 @@ public class InternalDataSource implements DataSourceIf { if (db.getDbState() == DbState.MOVE) { int tabletTotal = 0; int tabletQuorum = 0; - final Set beIds = - Sets.newHashSet(Catalog.getCurrentSystemInfo().getClusterBackendIds(db.getClusterName())); + final Set beIds = Sets.newHashSet( + Catalog.getCurrentSystemInfo().getClusterBackendIds(db.getClusterName())); final Set tableNames = db.getTableNamesWithLock(); for (String tableName : tableNames) { @@ -2981,8 +2982,8 @@ public class InternalDataSource implements DataSourceIf { olapTable.readLock(); try { for (Partition partition : olapTable.getPartitions()) { - ReplicaAllocation replicaAlloc = - olapTable.getPartitionInfo().getReplicaAllocation(partition.getId()); + ReplicaAllocation replicaAlloc = olapTable.getPartitionInfo() + .getReplicaAllocation(partition.getId()); short totalReplicaNum = replicaAlloc.getTotalReplicaNum(); for (MaterializedIndex materializedIndex : partition.getMaterializedIndices( IndexExtState.ALL)) { @@ -3071,8 +3072,8 @@ public class InternalDataSource implements DataSourceIf { public void initDefaultCluster() { final List backendList = Lists.newArrayList(); - final List defaultClusterBackends = - Catalog.getCurrentSystemInfo().getClusterBackends(SystemInfoService.DEFAULT_CLUSTER); + final List defaultClusterBackends = Catalog.getCurrentSystemInfo() + .getClusterBackends(SystemInfoService.DEFAULT_CLUSTER); for (Backend backend : defaultClusterBackends) { backendList.add(backend.getId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 78d200b9f7..9803f83cc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -87,6 +87,7 @@ import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.persist.ReplicaPersistInfo; import org.apache.doris.persist.RoutineLoadOperation; import org.apache.doris.persist.SetReplicaStatusOperationLog; +import org.apache.doris.persist.TableAddOrDropColumnsInfo; import org.apache.doris.persist.TableInfo; import org.apache.doris.persist.TablePropertyInfo; import org.apache.doris.persist.TruncateTableInfo; @@ -662,6 +663,11 @@ public class JournalEntity implements Writable { isRead = true; break; } + case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS: { + data = TableAddOrDropColumnsInfo.read(in); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 649e030a50..0ca0793475 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -65,6 +65,7 @@ import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.PushTask; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TPriority; import org.apache.doris.thrift.TPushType; import org.apache.doris.thrift.TTaskType; @@ -238,6 +239,11 @@ public class DeleteHandler implements Writable { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); + List columnsDesc = new ArrayList(); + for (Column column : olapTable.getSchemaByIndexId(indexId)) { + columnsDesc.add(column.toThrift()); + } + for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -260,7 +266,8 @@ public class DeleteHandler implements Writable { TTaskType.REALTIME_PUSH, transactionId, Catalog.getCurrentGlobalTransactionMgr() - .getTransactionIDGenerator().getNextTransactionId()); + .getTransactionIDGenerator().getNextTransactionId(), + columnsDesc); pushTask.setIsSchemaChanging(false); pushTask.setCountDownLatch(countDownLatch); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java b/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java index 0faf356029..954db1d971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java @@ -18,6 +18,7 @@ package org.apache.doris.load; import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; @@ -41,6 +42,7 @@ import org.apache.doris.task.HadoopLoadPendingTask; import org.apache.doris.task.MasterTask; import org.apache.doris.task.MasterTaskExecutor; import org.apache.doris.task.PushTask; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TPriority; import org.apache.doris.thrift.TPushType; import org.apache.doris.thrift.TTaskType; @@ -434,6 +436,10 @@ public class LoadChecker extends MasterDaemon { } int schemaHash = tableLoadInfo.getIndexSchemaHash(indexId); + List columnsDesc = new ArrayList(); + for (Column column : table.getSchemaByIndexId(indexId)) { + columnsDesc.add(column.toThrift()); + } short quorumNum = (short) (replicationNum / 2 + 1); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -471,7 +477,7 @@ public class LoadChecker extends MasterDaemon { job.getId(), type, job.getConditions(), needDecompress, job.getPriority(), TTaskType.REALTIME_PUSH, job.getTransactionId(), Catalog.getCurrentGlobalTransactionMgr() - .getTransactionIDGenerator().getNextTransactionId()); + .getTransactionIDGenerator().getNextTransactionId(), columnsDesc); pushTask.setIsSchemaChanging(autoLoadToTwoTablet); if (AgentTaskQueue.addTask(pushTask)) { batchTask.addTask(pushTask); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java index b9ec64ea7f..59a03c5753 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java @@ -73,6 +73,7 @@ import org.apache.doris.task.PushTask; import org.apache.doris.thrift.TBrokerRangeDesc; import org.apache.doris.thrift.TBrokerScanRange; import org.apache.doris.thrift.TBrokerScanRangeParams; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDescriptorTable; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileType; @@ -101,6 +102,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -109,7 +111,7 @@ import java.util.Set; * There are 4 steps in SparkLoadJob: * Step1: SparkLoadPendingTask will be created by unprotectedExecuteJob method and submit spark etl job. * Step2: LoadEtlChecker will check spark etl job status periodically - * and send push tasks to be when spark etl job is finished. + * and send push tasks to be when spark etl job is finished. * Step3: LoadLoadingChecker will check loading status periodically and commit transaction when push tasks are finished. * Step4: PublishVersionDaemon will send publish version tasks to be and finish transaction. */ @@ -149,9 +151,8 @@ public class SparkLoadJob extends BulkLoadJob { super(EtlJobType.SPARK); } - public SparkLoadJob(long dbId, String label, ResourceDesc resourceDesc, - OriginStatement originStmt, UserIdentity userInfo) - throws MetaNotFoundException { + public SparkLoadJob(long dbId, String label, ResourceDesc resourceDesc, OriginStatement originStmt, + UserIdentity userInfo) throws MetaNotFoundException { super(EtlJobType.SPARK, dbId, label, originStmt, userInfo); this.resourceDesc = resourceDesc; } @@ -166,6 +167,7 @@ public class SparkLoadJob extends BulkLoadJob { /** * merge system conf with load stmt + * * @throws DdlException */ private void setResourceInfo() throws DdlException { @@ -210,8 +212,8 @@ public class SparkLoadJob extends BulkLoadJob { } // create pending task - LoadTask task = new SparkLoadPendingTask(this, fileGroupAggInfo.getAggKeyToFileGroups(), - sparkResource, brokerDesc); + LoadTask task = new SparkLoadPendingTask(this, fileGroupAggInfo.getAggKeyToFileGroups(), sparkResource, + brokerDesc); task.init(); idToTasks.put(task.getSignature(), task); Catalog.getCurrentCatalog().getPendingLoadTaskScheduler().submit(task); @@ -229,19 +231,16 @@ public class SparkLoadJob extends BulkLoadJob { try { // check if job has been cancelled if (isTxnDone()) { - LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id) - .add("state", state) - .add("error_msg", "this task will be ignored when job is: " + state) - .build()); + LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id).add("state", state) + .add("error_msg", "this task will be ignored when job is: " + state).build()); return; } if (finishedTaskIds.contains(attachment.getTaskId())) { - LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id) - .add("task_id", attachment.getTaskId()) - .add("error_msg", "this is a duplicated callback of pending task " - + "when broker already has loading task") - .build()); + LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id).add("task_id", attachment.getTaskId()).add("error_msg", + "this is a duplicated callback of pending task " + + "when broker already has loading task") + .build()); return; } @@ -286,7 +285,7 @@ public class SparkLoadJob extends BulkLoadJob { * 1. RUNNING, update etl job progress * 2. CANCELLED, cancel load job * 3. FINISHED, get the etl output file paths, update job state to LOADING and log job update info - * + *

* Send push tasks if job state changed to LOADING */ public void updateEtlStatus() throws Exception { @@ -296,8 +295,8 @@ public class SparkLoadJob extends BulkLoadJob { // get etl status SparkEtlJobHandler handler = new SparkEtlJobHandler(); - EtlStatus status = handler.getEtlJobStatus(sparkLoadAppHandle, - appId, id, etlOutputPath, sparkResource, brokerDesc); + EtlStatus status = handler.getEtlJobStatus(sparkLoadAppHandle, appId, id, etlOutputPath, sparkResource, + brokerDesc); writeLock(); try { switch (status.getState()) { @@ -364,8 +363,8 @@ public class SparkLoadJob extends BulkLoadJob { unprotectedPrepareLoadingInfos(); } - private void unprotectedUpdateToLoadingState(EtlStatus etlStatus, - Map filePathToSize) throws LoadException { + private void unprotectedUpdateToLoadingState(EtlStatus etlStatus, Map filePathToSize) + throws LoadException { try { for (Map.Entry entry : filePathToSize.entrySet()) { String filePath = entry.getKey(); @@ -428,8 +427,8 @@ public class SparkLoadJob extends BulkLoadJob { AgentBatchTask batchTask = new AgentBatchTask(); boolean hasLoadPartitions = false; Set totalTablets = Sets.newHashSet(); - List tableList = - db.getTablesOnIdOrderOrThrowException(Lists.newArrayList(tableToLoadPartitions.keySet())); + List tableList = db.getTablesOnIdOrderOrThrowException( + Lists.newArrayList(tableToLoadPartitions.keySet())); MetaLockUtils.readLockTables(tableList); try { writeLock(); @@ -454,20 +453,26 @@ public class SparkLoadJob extends BulkLoadJob { } hasLoadPartitions = true; - int quorumReplicaNum = olapTable.getPartitionInfo() - .getReplicaAllocation(partitionId).getTotalReplicaNum() / 2 + 1; + int quorumReplicaNum = + olapTable.getPartitionInfo().getReplicaAllocation(partitionId).getTotalReplicaNum() / 2 + + 1; List indexes = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : indexes) { long indexId = index.getId(); int schemaHash = indexToSchemaHash.get(indexId); + List columnsDesc = new ArrayList(); + for (Column column : olapTable.getSchemaByIndexId(indexId)) { + columnsDesc.add(column.toThrift()); + } + int bucket = 0; for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); totalTablets.add(tabletId); String tabletMetaStr = String.format("%d.%d.%d.%d.%d", olapTable.getId(), partitionId, - indexId, bucket++, schemaHash); + indexId, bucket++, schemaHash); Set tabletAllReplicas = Sets.newHashSet(); Set tabletFinishedReplicas = Sets.newHashSet(); for (Replica replica : tablet.getReplicas()) { @@ -482,8 +487,8 @@ public class SparkLoadJob extends BulkLoadJob { PushBrokerReaderParams params = getPushBrokerReaderParams(olapTable, indexId); // deep copy TBrokerScanRange because filePath and fileSize will be updated // in different tablet push task - TBrokerScanRange tBrokerScanRange - = new TBrokerScanRange(params.tBrokerScanRange); + TBrokerScanRange tBrokerScanRange = new TBrokerScanRange( + params.tBrokerScanRange); // update filePath fileSize TBrokerRangeDesc tBrokerRangeDesc = tBrokerScanRange.getRanges().get(0); tBrokerRangeDesc.setPath(""); @@ -497,22 +502,21 @@ public class SparkLoadJob extends BulkLoadJob { // update broker address Backend backend = Catalog.getCurrentCatalog().getCurrentSystemInfo() .getBackend(backendId); - FsBroker fsBroker = Catalog.getCurrentCatalog().getBrokerMgr().getBroker( - brokerDesc.getName(), backend.getHost()); - tBrokerScanRange.getBrokerAddresses().add( - new TNetworkAddress(fsBroker.ip, fsBroker.port)); + FsBroker fsBroker = Catalog.getCurrentCatalog().getBrokerMgr() + .getBroker(brokerDesc.getName(), backend.getHost()); + tBrokerScanRange.getBrokerAddresses() + .add(new TNetworkAddress(fsBroker.ip, fsBroker.port)); LOG.debug("push task for replica {}, broker {}:{}," + " backendId {}, filePath {}, fileSize {}", - replicaId, fsBroker.ip, fsBroker.port, backendId, - tBrokerRangeDesc.path, + replicaId, fsBroker.ip, + fsBroker.port, backendId, tBrokerRangeDesc.path, tBrokerRangeDesc.file_size); - PushTask pushTask = new PushTask( - backendId, dbId, olapTable.getId(), partitionId, indexId, tabletId, - replicaId, schemaHash, 0, id, TPushType.LOAD_V2, - TPriority.NORMAL, transactionId, taskSignature, - tBrokerScanRange, params.tDescriptorTable); + PushTask pushTask = new PushTask(backendId, dbId, olapTable.getId(), + partitionId, indexId, tabletId, replicaId, schemaHash, 0, id, + TPushType.LOAD_V2, TPriority.NORMAL, transactionId, taskSignature, + tBrokerScanRange, params.tDescriptorTable, columnsDesc); if (AgentTaskQueue.addTask(pushTask)) { batchTask.addTask(pushTask); if (!tabletToSentReplicaPushTask.containsKey(tabletId)) { @@ -548,11 +552,8 @@ public class SparkLoadJob extends BulkLoadJob { } if (!hasLoadPartitions) { - String errMsg = new LogBuilder(LogKey.LOAD_JOB, id) - .add("database_id", dbId) - .add("label", label) - .add("error_msg", "all partitions have no load data") - .build(); + String errMsg = new LogBuilder(LogKey.LOAD_JOB, id).add("database_id", dbId).add("label", label) + .add("error_msg", "all partitions have no load data").build(); throw new LoadException(errMsg); } @@ -632,19 +633,16 @@ public class SparkLoadJob extends BulkLoadJob { } private void tryCommitJob() throws UserException { - LOG.info(new LogBuilder(LogKey.LOAD_JOB, id) - .add("txn_id", transactionId) - .add("msg", "Load job try to commit txn") - .build()); + LOG.info(new LogBuilder(LogKey.LOAD_JOB, id).add("txn_id", transactionId) + .add("msg", "Load job try to commit txn").build()); Database db = getDb(); List

tableList = db.getTablesOnIdOrderOrThrowException( Lists.newArrayList(tableToLoadPartitions.keySet())); MetaLockUtils.writeLockTablesOrMetaException(tableList); try { - Catalog.getCurrentGlobalTransactionMgr().commitTransaction( - dbId, tableList, transactionId, commitInfos, - new LoadJobFinalOperation(id, loadingStatus, progress, loadStartTimestamp, - finishTimestamp, state, failMsg)); + Catalog.getCurrentGlobalTransactionMgr().commitTransaction(dbId, tableList, transactionId, commitInfos, + new LoadJobFinalOperation(id, loadingStatus, progress, loadStartTimestamp, finishTimestamp, state, + failMsg)); } catch (TabletQuorumFailedException e) { // retry in next loop } finally { @@ -785,9 +783,8 @@ public class SparkLoadJob extends BulkLoadJob { * log load job update info when job state changed to etl or loading */ private void unprotectedLogUpdateStateInfo() { - SparkLoadJobStateUpdateInfo info = new SparkLoadJobStateUpdateInfo( - id, state, transactionId, sparkLoadAppHandle, etlStartTimestamp, appId, etlOutputPath, - loadStartTimestamp, tabletMetaToFileInfo); + SparkLoadJobStateUpdateInfo info = new SparkLoadJobStateUpdateInfo(id, state, transactionId, sparkLoadAppHandle, + etlStartTimestamp, appId, etlOutputPath, loadStartTimestamp, tabletMetaToFileInfo); Catalog.getCurrentCatalog().getEditLog().logUpdateLoadJob(info); } @@ -809,8 +806,8 @@ public class SparkLoadJob extends BulkLoadJob { unprotectedPrepareLoadingInfos(); break; default: - LOG.warn("replay update load job state info failed. error: wrong state. job id: {}, state: {}", - id, state); + LOG.warn("replay update load job state info failed. error: wrong state. job id: {}, state: {}", id, + state); break; } } @@ -830,10 +827,9 @@ public class SparkLoadJob extends BulkLoadJob { @SerializedName(value = "tabletMetaToFileInfo") private Map> tabletMetaToFileInfo; - public SparkLoadJobStateUpdateInfo( - long jobId, JobState state, long transactionId, SparkLoadAppHandle sparkLoadAppHandle, - long etlStartTimestamp, String appId, String etlOutputPath, long loadStartTimestamp, - Map> tabletMetaToFileInfo) { + public SparkLoadJobStateUpdateInfo(long jobId, JobState state, long transactionId, + SparkLoadAppHandle sparkLoadAppHandle, long etlStartTimestamp, String appId, String etlOutputPath, + long loadStartTimestamp, Map> tabletMetaToFileInfo) { super(jobId, state, transactionId, loadStartTimestamp); this.sparkLoadAppHandle = sparkLoadAppHandle; this.etlStartTimestamp = etlStartTimestamp; @@ -867,7 +863,7 @@ public class SparkLoadJob extends BulkLoadJob { * Params for be push broker reader * 1. TBrokerScanRange: file path and size, broker address, tranform expr * 2. TDescriptorTable: src and dest SlotDescriptors, src and dest tupleDescriptors - * + *

* These params are sent to Be through push task */ private static class PushBrokerReaderParams { @@ -896,7 +892,7 @@ public class SparkLoadJob extends BulkLoadJob { } private void initTBrokerScanRange(DescriptorTable descTable, TupleDescriptor destTupleDesc, - List columns, BrokerDesc brokerDesc) throws AnalysisException { + List columns, BrokerDesc brokerDesc) throws AnalysisException { // scan range params TBrokerScanRangeParams params = new TBrokerScanRangeParams(); params.setStrictMode(false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 3e0b531970..03bc694ae3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -180,47 +180,45 @@ public class EditLog { } case OperationType.OP_CREATE_TABLE: { CreateTableInfo info = (CreateTableInfo) journal.getData(); - LOG.info("Begin to unprotect create table. db = " - + info.getDbName() + " table = " + info.getTable().getId()); + LOG.info("Begin to unprotect create table. db = " + info.getDbName() + " table = " + info.getTable() + .getId()); catalog.replayCreateTable(info.getDbName(), info.getTable()); break; } case OperationType.OP_ALTER_EXTERNAL_TABLE_SCHEMA: { RefreshExternalTableInfo info = (RefreshExternalTableInfo) journal.getData(); - LOG.info("Begin to unprotect alter external table schema. db = " - + info.getDbName() + " table = " + info.getTableName()); + LOG.info("Begin to unprotect alter external table schema. db = " + info.getDbName() + " table = " + + info.getTableName()); catalog.replayAlterExternalTableSchema(info.getDbName(), info.getTableName(), info.getNewSchema()); break; } case OperationType.OP_DROP_TABLE: { DropInfo info = (DropInfo) journal.getData(); - Database db = - Catalog.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId()); - LOG.info("Begin to unprotect drop table. db = " - + db.getFullName() + " table = " + info.getTableId()); + Database db = Catalog.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId()); + LOG.info("Begin to unprotect drop table. db = " + db.getFullName() + " table = " + + info.getTableId()); catalog.replayDropTable(db, info.getTableId(), info.isForceDrop()); break; } case OperationType.OP_ADD_PARTITION: { PartitionPersistInfo info = (PartitionPersistInfo) journal.getData(); - LOG.info("Begin to unprotect add partition. db = " + info.getDbId() - + " table = " + info.getTableId() - + " partitionName = " + info.getPartition().getName()); + LOG.info( + "Begin to unprotect add partition. db = " + info.getDbId() + " table = " + info.getTableId() + + " partitionName = " + info.getPartition().getName()); catalog.replayAddPartition(info); break; } case OperationType.OP_DROP_PARTITION: { DropPartitionInfo info = (DropPartitionInfo) journal.getData(); - LOG.info("Begin to unprotect drop partition. db = " + info.getDbId() - + " table = " + info.getTableId() - + " partitionName = " + info.getPartitionName()); + LOG.info("Begin to unprotect drop partition. db = " + info.getDbId() + " table = " + + info.getTableId() + " partitionName = " + info.getPartitionName()); catalog.replayDropPartition(info); break; } case OperationType.OP_MODIFY_PARTITION: { ModifyPartitionInfo info = (ModifyPartitionInfo) journal.getData(); - LOG.info("Begin to unprotect modify partition. db = " + info.getDbId() - + " table = " + info.getTableId() + " partitionId = " + info.getPartitionId()); + LOG.info("Begin to unprotect modify partition. db = " + info.getDbId() + " table = " + + info.getTableId() + " partitionId = " + info.getPartitionId()); catalog.getAlterInstance().replayModifyPartition(info); break; } @@ -285,8 +283,9 @@ public class EditLog { case OperationType.OP_BATCH_DROP_ROLLUP: { BatchDropInfo batchDropInfo = (BatchDropInfo) journal.getData(); for (long indexId : batchDropInfo.getIndexIdSet()) { - catalog.getMaterializedViewHandler().replayDropRollup(new DropInfo(batchDropInfo.getDbId(), - batchDropInfo.getTableId(), indexId, false), catalog); + catalog.getMaterializedViewHandler().replayDropRollup( + new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(), indexId, false), + catalog); } break; } @@ -846,6 +845,11 @@ public class EditLog { catalog.getDataSourceMgr().replayAlterCatalogProps(log); break; } + case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS: { + final TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) journal.getData(); + catalog.getSchemaChangeHandler().replayModifyTableAddOrDropColumns(info); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); @@ -935,13 +939,13 @@ public class EditLog { } if (LOG.isDebugEnabled()) { - LOG.debug("nextId = {}, numTransactions = {}, totalTimeTransactions = {}, op = {}", - txId, numTransactions, totalTimeTransactions, op); + LOG.debug("nextId = {}, numTransactions = {}, totalTimeTransactions = {}, op = {}", txId, numTransactions, + totalTimeTransactions, op); } if (txId >= Config.edit_log_roll_num) { - LOG.info("txId {} is equal to or larger than edit_log_roll_num {}, will roll edit.", - txId, Config.edit_log_roll_num); + LOG.info("txId {} is equal to or larger than edit_log_roll_num {}, will roll edit.", txId, + Config.edit_log_roll_num); rollEditLog(); txId = 0; } @@ -1472,4 +1476,8 @@ public class EditLog { public Journal getJournal() { return this.journal; } + + public void logModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info) { + logEdit(OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS, info); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 79d3cf63a0..047714d0f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -69,6 +69,9 @@ public class OperationType { public static final short OP_MODIFY_COMMENT = 126; public static final short OP_MODIFY_TABLE_ENGINE = 127; + //schema change for add and drop columns + public static final short OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS = 128; + // 30~39 130~139 230~239 ... // load job for only hadoop load public static final short OP_LOAD_START = 30; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java new file mode 100644 index 0000000000..6e644f6a3b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * PersistInfo for Table properties + */ +public class TableAddOrDropColumnsInfo implements Writable { + @SerializedName(value = "dbId") + private long dbId; + @SerializedName(value = "tableId") + private long tableId; + @SerializedName(value = "indexSchemaMap") + private Map> indexSchemaMap; + @SerializedName(value = "indexes") + private List indexes; + @SerializedName(value = "jobId") + private long jobId; + + public TableAddOrDropColumnsInfo(long dbId, long tableId, + Map> indexSchemaMap, List indexes, long jobId) { + this.dbId = dbId; + this.tableId = tableId; + this.indexSchemaMap = indexSchemaMap; + this.indexes = indexes; + this.jobId = jobId; + } + + public long getDbId() { + return dbId; + } + + public long getTableId() { + return tableId; + } + + public Map> getIndexSchemaMap() { + return indexSchemaMap; + } + + public List getIndexes() { + return indexes; + } + + public long getJobId() { + return jobId; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static TableAddOrDropColumnsInfo read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), TableAddOrDropColumnsInfo.class); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (!(obj instanceof TableAddOrDropColumnsInfo)) { + return false; + } + + TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) obj; + + return (dbId == info.dbId && tableId == tableId + && indexSchemaMap.equals(info.indexSchemaMap) && indexes.equals(info.indexes) + && jobId == info.jobId); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(" dbId: ").append(dbId); + sb.append(" tableId: ").append(tableId); + sb.append(" indexSchemaMap: ").append(indexSchemaMap); + sb.append(" indexes: ").append(indexes); + sb.append(" jobId: ").append(jobId); + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 7df0cc38de..17f1898437 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -56,6 +56,7 @@ import org.apache.doris.resource.Tag; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TOlapScanNode; @@ -95,8 +96,10 @@ public class OlapScanNode extends ScanNode { private List result = new ArrayList<>(); /* - * When the field value is ON, the storage engine can return the data directly without pre-aggregation. - * When the field value is OFF, the storage engine needs to aggregate the data before returning to scan node. + * When the field value is ON, the storage engine can return the data directly + * without pre-aggregation. + * When the field value is OFF, the storage engine needs to aggregate the data + * before returning to scan node. * For example: * Aggregate table: k1, k2, v1 sum * Field value is ON @@ -109,20 +112,26 @@ public class OlapScanNode extends ScanNode { * This aggregation info is null. * Query2: select k1, min(v1) from table group by k1 * This aggregation function in query is min which different from the schema. - * So the data stored in storage engine need to be merged firstly before returning to scan node. + * So the data stored in storage engine need to be merged firstly before + * returning to scan node. * * There are currently two places to modify this variable: * 1. The turnOffPreAgg() method of SingleNodePlanner. - * This method will only be called on the left deepest OlapScanNode the plan tree, - * while other nodes are false by default (because the Aggregation operation is executed after Join, - * we cannot judge whether other OlapScanNodes can close the pre-aggregation). - * So even the Duplicate key table, if it is not the left deepest node, it will remain false too. + * This method will only be called on the left deepest OlapScanNode the plan + * tree, + * while other nodes are false by default (because the Aggregation operation is + * executed after Join, + * we cannot judge whether other OlapScanNodes can close the pre-aggregation). + * So even the Duplicate key table, if it is not the left deepest node, it will + * remain false too. * - * 2. After MaterializedViewSelector selects the materialized view, the updateScanRangeInfoByNewMVSelector()\ - * method of OlapScanNode may be called to update this variable. - * This call will be executed on all ScanNodes in the plan tree. In this step, - * for the DuplicateKey table, the variable will be set to true. - * See comment of "isPreAggregation" variable in MaterializedViewSelector for details. + * 2. After MaterializedViewSelector selects the materialized view, the + * updateScanRangeInfoByNewMVSelector()\ + * method of OlapScanNode may be called to update this variable. + * This call will be executed on all ScanNodes in the plan tree. In this step, + * for the DuplicateKey table, the variable will be set to true. + * See comment of "isPreAggregation" variable in MaterializedViewSelector for + * details. */ private boolean isPreAggregation = false; private String reasonOfPreAggregation = null; @@ -142,7 +151,8 @@ public class OlapScanNode extends ScanNode { private HashSet scanBackendIds = new HashSet<>(); private Map tabletId2BucketSeq = Maps.newHashMap(); - // a bucket seq may map to many tablets, and each tablet has a TScanRangeLocations. + // a bucket seq may map to many tablets, and each tablet has a + // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); // Constructs node to scan given data files of table 'tbl'. @@ -203,36 +213,45 @@ public class OlapScanNode extends ScanNode { } /** - * The function is used to directly select the index id of the base table as the selectedIndexId. - * It makes sure that the olap scan node must scan the base data rather than scan the materialized view data. + * The function is used to directly select the index id of the base table as the + * selectedIndexId. + * It makes sure that the olap scan node must scan the base data rather than + * scan the materialized view data. * * This function is mainly used to update stmt. * Update stmt also needs to scan data like normal queries. * But its syntax is different from ordinary queries, - * so planner cannot use the logic of query to automatically match the best index id. - * So, here it need to manually specify the index id to scan the base table directly. + * so planner cannot use the logic of query to automatically match the best + * index id. + * So, here it need to manually specify the index id to scan the base table + * directly. */ public void useBaseIndexId() { this.selectedIndexId = olapTable.getBaseIndexId(); } /** - * This method is mainly used to update scan range info in OlapScanNode by the new materialized selector. + * This method is mainly used to update scan range info in OlapScanNode by the + * new materialized selector. * Situation1: - * If the new scan range is same as the old scan range which determined by the old materialized selector, + * If the new scan range is same as the old scan range which determined by the + * old materialized selector, * the scan range will not be changed. *

* Situation2: Scan range is difference. The type of table is duplicated. * The new scan range is used directly. - * The reason is that the old selector does not support SPJ<->SPJG, so the result of old one must be incorrect. + * The reason is that the old selector does not support SPJ<->SPJG, so the + * result of old one must be incorrect. *

* Situation3: Scan range is difference. The type of table is aggregated. * The new scan range is different from the old one. * If the test_materialized_view is set to true, an error will be reported. * The query will be cancelled. *

- * Situation4: Scan range is difference. The type of table is aggregated. `test_materialized_view` is set to false. - * The result of the old version selector will be selected. Print the warning log + * Situation4: Scan range is difference. The type of table is aggregated. + * `test_materialized_view` is set to false. + * The result of the old version selector will be selected. Print the warning + * log * * @param selectedIndexId * @param isPreAggregation @@ -255,8 +274,7 @@ public class OlapScanNode extends ScanNode { String scanRangeInfo = stringBuilder.toString(); String situation; boolean update; - CHECK: - { // CHECKSTYLE IGNORE THIS LINE + CHECK: { // CHECKSTYLE IGNORE THIS LINE if (olapTable.getKeysType() == KeysType.DUP_KEYS) { situation = "The key type of table is duplicate."; update = true; @@ -296,12 +314,14 @@ public class OlapScanNode extends ScanNode { /** * In some situation, the column type between base and mv is different. - * If mv selector selects the mv index, the type of column should be changed to the type of mv column. + * If mv selector selects the mv index, the type of column should be changed to + * the type of mv column. * For example: * base table: k1 int, k2 int * mv table: k1 int, k2 bigint sum * The type of `k2` column between base and mv is different. - * When mv selector selects the mv table to scan, the type of column should be changed to bigint in here. + * When mv selector selects the mv table to scan, the type of column should be + * changed to bigint in here. * Currently, only `SUM` aggregate type could match this changed. */ private void updateColumnType() { @@ -346,9 +366,12 @@ public class OlapScanNode extends ScanNode { /** * Compute InAccurate cardinality before mv selector and tablet pruning. - * - Accurate statistical information relies on the selector of materialized views and bucket reduction. - * - However, Those both processes occur after the reorder algorithm is completed. - * - When Join reorder is turned on, the cardinality must be calculated before the reorder algorithm. + * - Accurate statistical information relies on the selector of materialized + * views and bucket reduction. + * - However, Those both processes occur after the reorder algorithm is + * completed. + * - When Join reorder is turned on, the cardinality must be calculated before + * the reorder algorithm. * - So only an inaccurate cardinality can be calculated here. */ mockRowCountInStatistic(); @@ -388,7 +411,8 @@ public class OlapScanNode extends ScanNode { public void finalize(Analyzer analyzer) throws UserException { LOG.debug("OlapScanNode get scan range locations. Tuple: {}", desc); /** - * If JoinReorder is turned on, it will be calculated init(), and this value is not accurate. + * If JoinReorder is turned on, it will be calculated init(), and this value is + * not accurate. * In the following logic, cardinality will be accurately calculated again. * So here we need to reset the value of cardinality. */ @@ -400,7 +424,8 @@ public class OlapScanNode extends ScanNode { } catch (AnalysisException e) { throw new UserException(e.getMessage()); } - // Relatively accurate cardinality according to ScanRange in getScanRangeLocations + // Relatively accurate cardinality according to ScanRange in + // getScanRangeLocations computeStats(analyzer); computeNumNodes(); } @@ -418,7 +443,8 @@ public class OlapScanNode extends ScanNode { avgRowSize = totalBytes / (float) cardinality * COMPRESSION_RATIO; capCardinalityAtLimit(); } - // when node scan has no data, cardinality should be 0 instead of a invalid value after computeStats() + // when node scan has no data, cardinality should be 0 instead of a invalid + // value after computeStats() cardinality = cardinality == -1 ? 0 : cardinality; // update statsDeriveResult for real statistics @@ -433,7 +459,8 @@ public class OlapScanNode extends ScanNode { if (cardinality > 0) { numNodes = scanBackendIds.size(); } - // even current node scan has no data,at least on backend will be assigned when the fragment actually execute + // even current node scan has no data,at least on backend will be assigned when + // the fragment actually execute numNodes = numNodes <= 0 ? 1 : numNodes; } @@ -470,10 +497,10 @@ public class OlapScanNode extends ScanNode { } else if (partitionInfo.getType() == PartitionType.LIST) { if (analyzer.partitionPruneV2Enabled()) { partitionPruner = new ListPartitionPrunerV2(keyItemMap, partitionInfo.getPartitionColumns(), - columnNameToRange); + columnNameToRange); } else { partitionPruner = new ListPartitionPruner(keyItemMap, - partitionInfo.getPartitionColumns(), columnFilters); + partitionInfo.getPartitionColumns(), columnFilters); } } return partitionPruner.prune(); @@ -502,7 +529,7 @@ public class OlapScanNode extends ScanNode { } private void addScanRangeLocations(Partition partition, - List tablets) throws UserException { + List tablets) throws UserException { long visibleVersion = partition.getVisibleVersion(); String visibleVersionStr = String.valueOf(visibleVersion); @@ -613,8 +640,8 @@ public class OlapScanNode extends ScanNode { } } else { selectedPartitionIds = selectedPartitionIds.stream() - .filter(id -> olapTable.getPartition(id).hasData()) - .collect(Collectors.toList()); + .filter(id -> olapTable.getPartition(id).hasData()) + .collect(Collectors.toList()); } selectedPartitionNum = selectedPartitionIds.size(); @@ -633,9 +660,10 @@ public class OlapScanNode extends ScanNode { // Step2: select best rollup long start = System.currentTimeMillis(); if (olapTable.getKeysType() == KeysType.DUP_KEYS) { - //This function is compatible with the INDEX selection logic of ROLLUP, - //so the Duplicate table here returns base index directly - //and the selection logic of materialized view is selected in "MaterializedViewSelector" + // This function is compatible with the INDEX selection logic of ROLLUP, + // so the Duplicate table here returns base index directly + // and the selection logic of materialized view is selected in + // "MaterializedViewSelector" selectedIndexId = olapTable.getBaseIndexId(); LOG.debug("The best index will be selected later in mv selector"); return; @@ -757,18 +785,23 @@ public class OlapScanNode extends ScanNode { protected void toThrift(TPlanNode msg) { List keyColumnNames = new ArrayList(); List keyColumnTypes = new ArrayList(); + List columnsDesc = new ArrayList(); + if (selectedIndexId != -1) { - for (Column col : olapTable.getSchemaByIndexId(selectedIndexId)) { - if (!col.isKey()) { - break; + for (Column col : olapTable.getSchemaByIndexId(selectedIndexId, true)) { + TColumn tColumn = col.toThrift(); + col.setIndexFlag(tColumn, olapTable.getIndexes()); + columnsDesc.add(tColumn); + if ((Util.showHiddenColumns() || (!Util.showHiddenColumns() && col.isVisible())) && col.isKey()) { + keyColumnNames.add(col.getName()); + keyColumnTypes.add(col.getDataType().toThrift()); } - keyColumnNames.add(col.getName()); - keyColumnTypes.add(col.getDataType().toThrift()); } } + msg.node_type = TPlanNodeType.OLAP_SCAN_NODE; - msg.olap_scan_node = - new TOlapScanNode(desc.getId().asInt(), keyColumnNames, keyColumnTypes, isPreAggregation); + msg.olap_scan_node = new TOlapScanNode(desc.getId().asInt(), keyColumnNames, keyColumnTypes, isPreAggregation, + columnsDesc); if (null != sortColumn) { msg.olap_scan_node.setSortColumn(sortColumn); } @@ -833,7 +866,6 @@ public class OlapScanNode extends ScanNode { return desc.getId(); } - private boolean isEquivalenceExpr(Expr expr) { if (expr instanceof InPredicate) { return true; @@ -914,10 +946,10 @@ public class OlapScanNode extends ScanNode { } /* - Although sometimes the scan range only involves one instance, - the data distribution cannot be set to UNPARTITIONED here. - The reason is that @coordinator will not set the scan range for the fragment, - when data partition of fragment is UNPARTITIONED. + * Although sometimes the scan range only involves one instance, + * the data distribution cannot be set to UNPARTITIONED here. + * The reason is that @coordinator will not set the scan range for the fragment, + * when data partition of fragment is UNPARTITIONED. */ public DataPartition constructInputPartitionByDistributionInfo() throws UserException { ColocateTableIndex colocateTableIndex = Catalog.getCurrentColocateIndex(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 3b462f6d20..1c69425ec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -45,6 +45,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TExplainLevel; @@ -93,8 +94,8 @@ public class OlapTableSink extends DataSink { this.partitionIds = partitionIds; } - public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeoutS, - int sendBatchParallelism, boolean loadToSingleTablet) throws AnalysisException { + public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeoutS, int sendBatchParallelism, + boolean loadToSingleTablet) throws AnalysisException { TOlapTableSink tSink = new TOlapTableSink(); tSink.setLoadId(loadId); tSink.setTxnId(txnId); @@ -102,8 +103,8 @@ public class OlapTableSink extends DataSink { tSink.setLoadChannelTimeoutS(loadChannelTimeoutS); tSink.setSendBatchParallelism(sendBatchParallelism); if (loadToSingleTablet && !(dstTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo)) { - throw new AnalysisException("if load_to_single_tablet set to true," - + " the olap table must be with random distribution"); + throw new AnalysisException( + "if load_to_single_tablet set to true," + " the olap table must be with random distribution"); } tSink.setLoadToSingleTablet(loadToSingleTablet); tDataSink = new TDataSink(TDataSinkType.OLAP_TABLE_SINK); @@ -177,7 +178,7 @@ public class OlapTableSink extends DataSink { TOlapTableSchemaParam schemaParam = new TOlapTableSchemaParam(); schemaParam.setDbId(dbId); schemaParam.setTableId(table.getId()); - schemaParam.setVersion(0); + schemaParam.setVersion(table.getIndexMetaByIndexId(table.getBaseIndexId()).getSchemaVersion()); schemaParam.tuple_desc = tupleDescriptor.toThrift(); for (SlotDescriptor slotDesc : tupleDescriptor.getSlots()) { @@ -187,9 +188,15 @@ public class OlapTableSink extends DataSink { for (Map.Entry pair : table.getIndexIdToMeta().entrySet()) { MaterializedIndexMeta indexMeta = pair.getValue(); List columns = Lists.newArrayList(); + List columnsDesc = Lists.newArrayList(); columns.addAll(indexMeta.getSchema().stream().map(Column::getName).collect(Collectors.toList())); + for (Column column : indexMeta.getSchema()) { + TColumn tColumn = column.toThrift(); + column.setIndexFlag(tColumn, table.getIndexes()); + columnsDesc.add(tColumn); + } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, - indexMeta.getSchemaHash()); + indexMeta.getSchemaHash(), columnsDesc); schemaParam.addToIndexes(indexSchema); } return schemaParam; @@ -276,8 +283,7 @@ public class OlapTableSink extends DataSink { tPartition.setNumBuckets(index.getTablets().size()); } partitionParam.addToPartitions(tPartition); - partitionParam.setDistributedColumns( - getDistColumns(partition.getDistributionInfo())); + partitionParam.setDistributedColumns(getDistColumns(partition.getDistributionInfo())); break; } default: { @@ -328,13 +334,12 @@ public class OlapTableSink extends DataSink { for (Tablet tablet : index.getTablets()) { Multimap bePathsMap = tablet.getNormalReplicaBackendPathMap(); if (bePathsMap.keySet().size() < quorum) { - throw new UserException(InternalErrorCode.REPLICA_FEW_ERR, "tablet " + tablet.getId() - + " has few replicas: " + bePathsMap.keySet().size() - + ", alive backends: [" + StringUtils.join(bePathsMap.keySet(), ",") - + "]"); + throw new UserException(InternalErrorCode.REPLICA_FEW_ERR, + "tablet " + tablet.getId() + " has few replicas: " + bePathsMap.keySet().size() + + ", alive backends: [" + StringUtils.join(bePathsMap.keySet(), ",") + "]"); } - locationParam.addToTablets(new TTabletLocation(tablet.getId(), - Lists.newArrayList(bePathsMap.keySet()))); + locationParam.addToTablets( + new TTabletLocation(tablet.getId(), Lists.newArrayList(bePathsMap.keySet()))); allBePathsMap.putAll(bePathsMap); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 8f62a135ec..e5eed8c250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -211,8 +211,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { if (db != null) { for (String tableName : db.getTableNamesWithLock()) { LOG.debug("get table: {}, wait to check", tableName); - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, params.db, - tableName, PrivPredicate.SHOW)) { + if (!Catalog.getCurrentCatalog().getAuth() + .checkTblPriv(currentUser, params.db, tableName, PrivPredicate.SHOW)) { continue; } @@ -270,14 +270,14 @@ public class FrontendServiceImpl implements FrontendService.Iface { } } for (TableIf table : tables) { - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, params.db, - table.getName(), PrivPredicate.SHOW)) { + if (!Catalog.getCurrentCatalog().getAuth() + .checkTblPriv(currentUser, params.db, table.getName(), PrivPredicate.SHOW)) { continue; } table.readLock(); try { - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, params.db, - table.getName(), PrivPredicate.SHOW)) { + if (!Catalog.getCurrentCatalog().getAuth() + .checkTblPriv(currentUser, params.db, table.getName(), PrivPredicate.SHOW)) { continue; } @@ -386,8 +386,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(currentUser, params.db, - params.getTableName(), PrivPredicate.SHOW)) { + if (!Catalog.getCurrentCatalog().getAuth() + .checkTblPriv(currentUser, params.db, params.getTableName(), PrivPredicate.SHOW)) { return result; } @@ -504,7 +504,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { } private void checkPasswordAndPrivs(String cluster, String user, String passwd, String db, String tbl, - String clientIp, PrivPredicate predicate) throws AuthenticationException { + String clientIp, PrivPredicate predicate) throws AuthenticationException { final String fullUserName = ClusterNamespace.getFullName(cluster, user); final String fullDbName = ClusterNamespace.getFullName(cluster, db); @@ -533,8 +533,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { result.setTxnId(tmpRes.getTxnId()).setDbId(tmpRes.getDbId()); } catch (DuplicatedRequestException e) { // this is a duplicate request, just return previous txn id - LOG.warn("duplicate request for stream load. request id: {}, txn: {}", - e.getDuplicatedRequestId(), e.getTxnId()); + LOG.warn("duplicate request for stream load. request id: {}, txn: {}", e.getDuplicatedRequestId(), + e.getTxnId()); result.setTxnId(e.getTxnId()); } catch (LabelAlreadyUsedException e) { status.setStatusCode(TStatusCode.LABEL_ALREADY_EXISTS); @@ -560,8 +560,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { } if (Strings.isNullOrEmpty(request.getAuthCodeUuid())) { - checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), - request.getTbl(), request.getUserIp(), PrivPredicate.LOAD); + checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), request.getTbl(), + request.getUserIp(), PrivPredicate.LOAD); } // check label @@ -584,10 +584,10 @@ public class FrontendServiceImpl implements FrontendService.Iface { // begin long timeoutSecond = request.isSetTimeout() ? request.getTimeout() : Config.stream_load_default_timeout_second; MetricRepo.COUNTER_LOAD_ADD.increase(1L); - long txnId = Catalog.getCurrentGlobalTransactionMgr().beginTransaction( - db.getId(), Lists.newArrayList(table.getId()), request.getLabel(), request.getRequestId(), - new TxnCoordinator(TxnSourceType.BE, clientIp), - TransactionState.LoadJobSourceType.BACKEND_STREAMING, -1, timeoutSecond); + long txnId = Catalog.getCurrentGlobalTransactionMgr() + .beginTransaction(db.getId(), Lists.newArrayList(table.getId()), request.getLabel(), + request.getRequestId(), new TxnCoordinator(TxnSourceType.BE, clientIp), + TransactionState.LoadJobSourceType.BACKEND_STREAMING, -1, timeoutSecond); if (!Strings.isNullOrEmpty(request.getAuthCodeUuid())) { Catalog.getCurrentGlobalTransactionMgr().getTransactionState(db.getId(), txnId) .setAuthCode(request.getAuthCodeUuid()); @@ -631,8 +631,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else if (request.isSetAuthCodeUuid()) { checkAuthCodeUuid(request.getDb(), request.getTxnId(), request.getAuthCodeUuid()); } else { - checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), - request.getTbl(), request.getUserIp(), PrivPredicate.LOAD); + checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), request.getTbl(), + request.getUserIp(), PrivPredicate.LOAD); } // get database @@ -713,8 +713,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { List

tableList = database.getTablesOnIdOrderOrThrowException(tableIdList); for (Table table : tableList) { // check auth - checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), - table.getName(), request.getUserIp(), PrivPredicate.LOAD); + checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), table.getName(), + request.getUserIp(), PrivPredicate.LOAD); } String txnOperation = request.getOperation().trim(); @@ -767,8 +767,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else if (request.isSetAuthCodeUuid()) { checkAuthCodeUuid(request.getDb(), request.getTxnId(), request.getAuthCodeUuid()); } else { - checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), - request.getTbl(), request.getUserIp(), PrivPredicate.LOAD); + checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), request.getTbl(), + request.getUserIp(), PrivPredicate.LOAD); } // get database @@ -835,8 +835,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else if (request.isSetAuthCodeUuid()) { checkAuthCodeUuid(request.getDb(), request.getTxnId(), request.getAuthCodeUuid()); } else { - checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), - request.getTbl(), request.getUserIp(), PrivPredicate.LOAD); + checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), request.getDb(), request.getTbl(), + request.getUserIp(), PrivPredicate.LOAD); } String dbName = ClusterNamespace.getFullName(cluster, request.getDb()); Database db; @@ -896,8 +896,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { long timeoutMs = request.isSetThriftRpcTimeoutMs() ? request.getThriftRpcTimeoutMs() : 5000; Table table = db.getTableOrMetaException(request.getTbl(), TableType.OLAP); if (!table.tryReadLock(timeoutMs, TimeUnit.MILLISECONDS)) { - throw new UserException("get table read lock timeout, database=" - + fullDbName + ",table=" + table.getName()); + throw new UserException( + "get table read lock timeout, database=" + fullDbName + ",table=" + table.getName()); } try { StreamLoadTask streamLoadTask = StreamLoadTask.fromTStreamLoadPutRequest(request); @@ -918,8 +918,9 @@ public class FrontendServiceImpl implements FrontendService.Iface { @Override public TStatus snapshotLoaderReport(TSnapshotLoaderReportRequest request) throws TException { - if (Catalog.getCurrentCatalog().getBackupHandler().report(request.getTaskType(), request.getJobId(), - request.getTaskId(), request.getFinishedNum(), request.getTotalNum())) { + if (Catalog.getCurrentCatalog().getBackupHandler() + .report(request.getTaskType(), request.getJobId(), request.getTaskId(), request.getFinishedNum(), + request.getTotalNum())) { return new TStatus(TStatusCode.OK); } return new TStatus(TStatusCode.CANCELLED); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java index 4235ce0ded..d9ac6f0bb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java @@ -21,12 +21,15 @@ import org.apache.doris.alter.AlterJobV2; import org.apache.doris.analysis.DescriptorTable; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.catalog.Column; import org.apache.doris.thrift.TAlterMaterializedViewParam; import org.apache.doris.thrift.TAlterTabletReqV2; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TTaskType; import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -48,6 +51,7 @@ public class AlterReplicaTask extends AgentTask { private Map defineExprs; private DescriptorTable descTable; + private List baseSchemaColumns; /** * AlterReplicaTask constructor. @@ -56,7 +60,7 @@ public class AlterReplicaTask extends AgentTask { public AlterReplicaTask(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId, long baseIndexId, long rollupTabletId, long baseTabletId, long newReplicaId, int newSchemaHash, int baseSchemaHash, long version, long jobId, AlterJobV2.JobType jobType, Map defineExprs, - DescriptorTable descTable) { + DescriptorTable descTable, List baseSchemaColumns) { super(null, backendId, TTaskType.ALTER, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); this.baseTabletId = baseTabletId; @@ -71,6 +75,7 @@ public class AlterReplicaTask extends AgentTask { this.jobType = jobType; this.defineExprs = defineExprs; this.descTable = descTable; + this.baseSchemaColumns = baseSchemaColumns; } public long getBaseTabletId() { @@ -115,6 +120,14 @@ public class AlterReplicaTask extends AgentTask { } } req.setDescTbl(descTable.toThrift()); + + if (baseSchemaColumns != null) { + List columns = new ArrayList(); + for (Column column : baseSchemaColumns) { + columns.add(column.toThrift()); + } + req.setColumns(columns); + } return req; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java index 1c32c9680b..fb61fe356d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.Predicate; import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.thrift.TBrokerScanRange; +import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TCondition; import org.apache.doris.thrift.TDescriptorTable; import org.apache.doris.thrift.TPriority; @@ -69,11 +70,13 @@ public class PushTask extends AgentTask { private TBrokerScanRange tBrokerScanRange; private TDescriptorTable tDescriptorTable; - public PushTask(TResourceInfo resourceInfo, long backendId, long dbId, long tableId, long partitionId, - long indexId, long tabletId, long replicaId, int schemaHash, long version, - String filePath, long fileSize, int timeoutSecond, long loadJobId, TPushType pushType, - List conditions, boolean needDecompress, TPriority priority, TTaskType taskType, - long transactionId, long signature) { + // for light schema change + private List columnsDesc = null; + + public PushTask(TResourceInfo resourceInfo, long backendId, long dbId, long tableId, long partitionId, long indexId, + long tabletId, long replicaId, int schemaHash, long version, String filePath, long fileSize, + int timeoutSecond, long loadJobId, TPushType pushType, List conditions, boolean needDecompress, + TPriority priority, TTaskType taskType, long transactionId, long signature, List columnsDesc) { super(resourceInfo, backendId, taskType, dbId, tableId, partitionId, indexId, tabletId, signature); this.replicaId = replicaId; this.schemaHash = schemaHash; @@ -92,17 +95,17 @@ public class PushTask extends AgentTask { this.transactionId = transactionId; this.tBrokerScanRange = null; this.tDescriptorTable = null; + this.columnsDesc = columnsDesc; } // for load v2 (SparkLoadJob) public PushTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, - long replicaId, int schemaHash, int timeoutSecond, long loadJobId, TPushType pushType, - TPriority priority, long transactionId, long signature, - TBrokerScanRange tBrokerScanRange, TDescriptorTable tDescriptorTable) { - this(null, backendId, dbId, tableId, partitionId, indexId, - tabletId, replicaId, schemaHash, -1, null, - 0, timeoutSecond, loadJobId, pushType, null, false, - priority, TTaskType.REALTIME_PUSH, transactionId, signature); + long replicaId, int schemaHash, int timeoutSecond, long loadJobId, TPushType pushType, TPriority priority, + long transactionId, long signature, TBrokerScanRange tBrokerScanRange, TDescriptorTable tDescriptorTable, + List columnsDesc) { + this(null, backendId, dbId, tableId, partitionId, indexId, tabletId, replicaId, schemaHash, -1, null, 0, + timeoutSecond, loadJobId, pushType, null, false, priority, TTaskType.REALTIME_PUSH, transactionId, + signature, columnsDesc); this.tBrokerScanRange = tBrokerScanRange; this.tDescriptorTable = tDescriptorTable; } @@ -171,6 +174,7 @@ public class PushTask extends AgentTask { LOG.warn("unknown push type. type: " + pushType.name()); break; } + request.setColumnsDesc(columnsDesc); return request; } @@ -182,8 +186,8 @@ public class PushTask extends AgentTask { public void countDownLatch(long backendId, long tabletId) { if (this.latch != null) { if (latch.markedCountDown(backendId, tabletId)) { - LOG.debug("pushTask current latch count: {}. backend: {}, tablet:{}", - latch.getCount(), backendId, tabletId); + LOG.debug("pushTask current latch count: {}. backend: {}, tablet:{}", latch.getCount(), backendId, + tabletId); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 48973f1871..9fcc04dc98 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -17,24 +17,341 @@ package org.apache.doris.alter; +import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.ColumnPosition; +import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import mockit.Expectations; import mockit.Injectable; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -public class SchemaChangeHandlerTest { +import java.util.Map; + +public class SchemaChangeHandlerTest extends TestWithFeService { + private static final Logger LOG = LogManager.getLogger(SchemaChangeHandlerTest.class); + private int jobSize = 0; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.default_scheduler_interval_millisecond = 10; + //create database db1 + createDatabase("test"); + + //create tables + String createAggTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_agg (\n" + "user_id LARGEINT NOT NULL,\n" + + "date DATE NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" + + "last_visit_date DATETIME REPLACE DEFAULT '1970-01-01 00:00:00',\n" + "cost BIGINT SUM DEFAULT '0',\n" + + "max_dwell_time INT MAX DEFAULT '0',\n" + "min_dwell_time INT MIN DEFAULT '99999')\n" + + "AGGREGATE KEY(user_id, date, city, age, sex)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1');"; + createTable(createAggTblStmtStr); + + String createUniqTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_uniq (\n" + "user_id LARGEINT NOT NULL,\n" + + "username VARCHAR(50) NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" + + "phone LARGEINT,\n" + "address VARCHAR(500),\n" + "register_time DATETIME)\n" + + "UNIQUE KEY(user_id, username)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1');"; + createTable(createUniqTblStmtStr); + + String createDupTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_dup (\n" + "timestamp DATETIME,\n" + + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" + + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1');"; + createTable(createDupTblStmtStr); + } + + private void waitAlterJobDone(Map alterJobs) throws Exception { + for (AlterJobV2 alterJobV2 : alterJobs.values()) { + while (!alterJobV2.getJobState().isFinalState()) { + LOG.info("alter job {} is running. state: {}", alterJobV2.getJobId(), alterJobV2.getJobState()); + Thread.sleep(1000); + } + LOG.info("alter job {} is done. state: {}", alterJobV2.getJobId(), alterJobV2.getJobState()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); + + Database db = Catalog.getCurrentInternalCatalog().getDbOrMetaException(alterJobV2.getDbId()); + OlapTable tbl = (OlapTable) db.getTableOrMetaException(alterJobV2.getTableId(), Table.TableType.OLAP); + while (tbl.getState() != OlapTable.OlapTableState.NORMAL) { + Thread.sleep(1000); + } + } + } + + @Test + public void testAggAddOrDropColumn() throws Exception { + LOG.info("dbName: {}", Catalog.getCurrentInternalCatalog().getDbNames()); + + Database db = Catalog.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + OlapTable tbl = (OlapTable) db.getTableOrMetaException("sc_agg", Table.TableType.OLAP); + tbl.readLock(); + try { + Assertions.assertNotNull(tbl); + System.out.println(tbl.getName()); + Assertions.assertEquals("Doris", tbl.getEngine()); + Assertions.assertEquals(9, tbl.getBaseSchema().size()); + } finally { + tbl.readUnlock(); + } + + //process agg add value column schema change + String addValColStmtStr = "alter table test.sc_agg add column new_v1 int MAX default '0'"; + AlterTableStmt addValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(addValColStmt); + jobSize++; + //check alter job, do not create job + Map alterJobs = Catalog.getCurrentCatalog().getSchemaChangeHandler().getAlterJobsV2(); + Assertions.assertEquals(jobSize, alterJobs.size()); + + tbl.readLock(); + try { + Assertions.assertEquals(10, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + + //process agg add key column schema change + String addKeyColStmtStr = "alter table test.sc_agg add column new_k1 int default '1'"; + AlterTableStmt addKeyColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addKeyColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(addKeyColStmt); + + //check alter job + jobSize++; + Assertions.assertEquals(jobSize, alterJobs.size()); + waitAlterJobDone(alterJobs); + + tbl.readLock(); + try { + Assertions.assertEquals(11, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + + //process agg drop value column schema change + String dropValColStmtStr = "alter table test.sc_agg drop column new_v1"; + AlterTableStmt dropValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(dropValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(dropValColStmt); + jobSize++; + //check alter job, do not create job + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + + tbl.readLock(); + try { + Assertions.assertEquals(10, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + + try { + //process agg drop key column with replace schema change, expect exception. + String dropKeyColStmtStr = "alter table test.sc_agg drop column new_k1"; + AlterTableStmt dropKeyColStmt = (AlterTableStmt) parseAndAnalyzeStmt(dropKeyColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(dropKeyColStmt); + Assert.fail(); + } catch (Exception e) { + LOG.info(e.getMessage()); + } + + LOG.info("getIndexIdToSchema 1: {}", tbl.getIndexIdToSchema(true)); + + String addRollUpStmtStr = "alter table test.sc_agg add rollup agg_rollup(user_id, max_dwell_time);"; + AlterTableStmt addRollUpStmt = (AlterTableStmt) parseAndAnalyzeStmt(addRollUpStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(addRollUpStmt); + // 2. check alter job + Map materializedViewAlterJobs = Catalog.getCurrentCatalog().getMaterializedViewHandler() + .getAlterJobsV2(); + waitAlterJobDone(materializedViewAlterJobs); + Assertions.assertEquals(1, materializedViewAlterJobs.size()); + + LOG.info("getIndexIdToSchema 2: {}", tbl.getIndexIdToSchema(true)); + + //process agg drop value column with rollup schema change + String dropRollUpValColStmtStr = "alter table test.sc_agg drop column max_dwell_time"; + AlterTableStmt dropRollUpValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(dropRollUpValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(dropRollUpValColStmt); + jobSize++; + //check alter job, need create job + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + waitAlterJobDone(materializedViewAlterJobs); + + tbl.readLock(); + try { + Assertions.assertEquals(9, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + + //process agg add mul value column schema change + String addMultiValColStmtStr + = "alter table test.sc_agg add column new_v2 int MAX default '0', add column new_v3 int MAX default '1';"; + AlterTableStmt addMultiValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addMultiValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(addMultiValColStmt); + jobSize++; + //check alter job, do not create job + Assertions.assertEquals(jobSize, alterJobs.size()); + + tbl.readLock(); + try { + Assertions.assertEquals(11, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + Assertions.assertEquals(12, tbl.getMaxColUniqueId()); + } finally { + tbl.readUnlock(); + } + } + + @Test + public void testUniqAddOrDropColumn() throws Exception { + + LOG.info("dbName: {}", Catalog.getCurrentInternalCatalog().getDbNames()); + + Database db = Catalog.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + OlapTable tbl = (OlapTable) db.getTableOrMetaException("sc_uniq", Table.TableType.OLAP); + tbl.readLock(); + try { + Assertions.assertNotNull(tbl); + System.out.println(tbl.getName()); + Assertions.assertEquals("Doris", tbl.getEngine()); + Assertions.assertEquals(8, tbl.getBaseSchema().size()); + } finally { + tbl.readUnlock(); + } + + //process uniq add value column schema change + String addValColStmtStr = "alter table test.sc_uniq add column new_v1 int default '0'"; + AlterTableStmt addValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(addValColStmt); + jobSize++; + //check alter job, do not create job + Map alterJobs = Catalog.getCurrentCatalog().getSchemaChangeHandler().getAlterJobsV2(); + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + + tbl.readLock(); + try { + Assertions.assertEquals(9, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + + //process uniq drop val column schema change + String dropValColStmtStr = "alter table test.sc_uniq drop column new_v1"; + AlterTableStmt dropValColStm = (AlterTableStmt) parseAndAnalyzeStmt(dropValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(dropValColStm); + jobSize++; + //check alter job + Assertions.assertEquals(jobSize, alterJobs.size()); + tbl.readLock(); + try { + Assertions.assertEquals(8, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + } + + @Test + public void testDupAddOrDropColumn() throws Exception { + + LOG.info("dbName: {}", Catalog.getCurrentInternalCatalog().getDbNames()); + + Database db = Catalog.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + OlapTable tbl = (OlapTable) db.getTableOrMetaException("sc_dup", Table.TableType.OLAP); + tbl.readLock(); + try { + Assertions.assertNotNull(tbl); + System.out.println(tbl.getName()); + Assertions.assertEquals("Doris", tbl.getEngine()); + Assertions.assertEquals(6, tbl.getBaseSchema().size()); + } finally { + tbl.readUnlock(); + } + + //process uniq add value column schema change + String addValColStmtStr = "alter table test.sc_dup add column new_v1 int default '0'"; + AlterTableStmt addValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(addValColStmt); + jobSize++; + //check alter job, do not create job + Map alterJobs = Catalog.getCurrentCatalog().getSchemaChangeHandler().getAlterJobsV2(); + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + + tbl.readLock(); + try { + Assertions.assertEquals(7, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + + //process uniq drop val column schema change + String dropValColStmtStr = "alter table test.sc_dup drop column new_v1"; + AlterTableStmt dropValColStm = (AlterTableStmt) parseAndAnalyzeStmt(dropValColStmtStr); + Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(dropValColStm); + jobSize++; + //check alter job + Assertions.assertEquals(jobSize, alterJobs.size()); + tbl.readLock(); + try { + Assertions.assertEquals(6, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + tbl.readUnlock(); + } + } @Test public void testAddValueColumnOnAggMV(@Injectable OlapTable olapTable, @Injectable Column newColumn, - @Injectable ColumnPosition columnPosition) { + @Injectable ColumnPosition columnPosition) { SchemaChangeHandler schemaChangeHandler = new SchemaChangeHandler(); new Expectations() { { @@ -51,8 +368,7 @@ public class SchemaChangeHandlerTest { try { Deencapsulation.invoke(schemaChangeHandler, "addColumnInternal", olapTable, newColumn, columnPosition, - new Long(2), new Long(1), - Maps.newHashMap(), Sets.newHashSet()); + new Long(2), new Long(1), Maps.newHashMap(), Sets.newHashSet(), false); Assert.fail(); } catch (Exception e) { System.out.println(e.getMessage()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java new file mode 100644 index 0000000000..a97dd27bf9 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.io.Text; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.common.collect.Lists; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class TableAddOrDropColumnsInfoTest { + private static String fileName = "./TableAddOrDropColumnsInfoTest"; + + @Test + public void testSerialization() throws IOException { + // 1. Write objects to file + File file = new File(fileName); + file.createNewFile(); + + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + + long dbId = 12345678; + long tableId = 87654321; + long jobId = 23456781; + LinkedList fullSchema = new LinkedList<>(); + fullSchema.add(new Column("testCol1", ScalarType.createType(PrimitiveType.INT))); + fullSchema.add(new Column("testCol2", ScalarType.createType(PrimitiveType.VARCHAR))); + fullSchema.add(new Column("testCol3", ScalarType.createType(PrimitiveType.DATE))); + fullSchema.add(new Column("testCol4", ScalarType.createType(PrimitiveType.DATETIME))); + + Map> indexSchemaMap = new HashMap<>(); + indexSchemaMap.put(tableId, fullSchema); + + List indexes = Lists.newArrayList( + new Index("index", Lists.newArrayList("testCol1"), IndexDef.IndexType.BITMAP, "xxxxxx")); + + TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo(dbId, tableId, + indexSchemaMap, indexes, jobId); + + String c1Json = GsonUtils.GSON.toJson(tableAddOrDropColumnsInfo1); + Text.writeString(out, c1Json); + out.flush(); + out.close(); + + // 2. Read objects from file + DataInputStream in = new DataInputStream(new FileInputStream(file)); + + String readJson = Text.readString(in); + TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo2 = GsonUtils.GSON.fromJson(readJson, + TableAddOrDropColumnsInfo.class); + + Assert.assertEquals(tableAddOrDropColumnsInfo1.getDbId(), tableAddOrDropColumnsInfo2.getDbId()); + Assert.assertEquals(tableAddOrDropColumnsInfo1.getTableId(), tableAddOrDropColumnsInfo2.getTableId()); + Assert.assertEquals(tableAddOrDropColumnsInfo1.getIndexSchemaMap(), + tableAddOrDropColumnsInfo2.getIndexSchemaMap()); + + } + + @After + public void tearDown() { + File file = new File(fileName); + file.delete(); + } +} diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index 3dab3f0bee..acde58bbfa 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -21,6 +21,7 @@ package doris; option java_package = "org.apache.doris.proto"; import "types.proto"; +import "olap_file.proto"; message PSlotDescriptor { required int32 id = 1; @@ -47,6 +48,7 @@ message POlapTableIndexSchema { required int64 id = 1; repeated string columns = 2; required int32 schema_hash = 3; + repeated ColumnPB columns_desc = 4; }; message POlapTableSchemaParam { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index a9ba857c7e..e7a3893fc1 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -107,6 +107,8 @@ message RowsetMetaPB { // the encoded segment min/max key of segments in this rowset, // only used in unique key data model with primary_key_index support. repeated KeyBoundsPB segments_key_bounds = 27; + // tablet meta pb, for compaction + optional TabletSchemaPB tablet_schema = 28; // spare field id for future use optional AlphaRowsetExtraMetaPB alpha_rowset_extra_meta_pb = 50; // to indicate whether the data between the segments overlap @@ -190,6 +192,7 @@ message ColumnPB { optional bool visible = 16 [default=true]; repeated ColumnPB children_columns = 17; repeated string children_column_names = 18; + optional int32 col_unique_id = 19 [default=-1]; // default is important } enum SortType { @@ -211,6 +214,7 @@ message TabletSchemaPB { optional SortType sort_type = 11; optional int32 sort_col_num = 12; optional segment_v2.CompressionTypePB compression_type = 13 [default=LZ4F]; + optional int32 schema_version = 14; } enum TabletStatePB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 4e420de9df..78c1fe0c3b 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -25,25 +25,12 @@ include "PlanNodes.thrift" include "Descriptors.thrift" include "Exprs.thrift" -struct TColumn { - 1: required string column_name - 2: required Types.TColumnType column_type - 3: optional Types.TAggregationType aggregation_type - 4: optional bool is_key - 5: optional bool is_allow_null - 6: optional string default_value - 7: optional bool is_bloom_filter_column - 8: optional Exprs.TExpr define_expr - 9: optional bool visible = true - 10: optional list children_column -} - struct TTabletSchema { 1: required i16 short_key_column_count 2: required Types.TSchemaHash schema_hash 3: required Types.TKeysType keys_type 4: required Types.TStorageType storage_type - 5: required list columns + 5: required list columns 6: optional double bloom_filter_fpp 7: optional list indexes 8: optional bool is_in_memory @@ -168,6 +155,7 @@ struct TAlterTabletReqV2 { 7: optional list materialized_view_params 8: optional TAlterTabletType alter_tablet_type = TAlterTabletType.SCHEMA_CHANGE 9: optional Descriptors.TDescriptorTable desc_tbl + 10: optional list columns } struct TAlterMaterializedViewParam { @@ -209,6 +197,7 @@ struct TPushReq { // 14 and 15 are used by spark load 14: optional PlanNodes.TBrokerScanRange broker_scan_range 15: optional Descriptors.TDescriptorTable desc_tbl + 16: optional list columns_desc } struct TCloneReq { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 187df7e0b3..787a6e606e 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -21,6 +21,21 @@ namespace java org.apache.doris.thrift include "Types.thrift" include "Exprs.thrift" +struct TColumn { + 1: required string column_name + 2: required Types.TColumnType column_type + 3: optional Types.TAggregationType aggregation_type + 4: optional bool is_key + 5: optional bool is_allow_null + 6: optional string default_value + 7: optional bool is_bloom_filter_column + 8: optional Exprs.TExpr define_expr + 9: optional bool visible = true + 10: optional list children_column + 11: optional i32 col_unique_id = -1 + 12: optional bool has_bitmap_index = false +} + struct TSlotDescriptor { 1: required Types.TSlotId id 2: required Types.TTupleId parent @@ -32,6 +47,7 @@ struct TSlotDescriptor { 8: required string colName; 9: required i32 slotIdx 10: required bool isMaterialized + 11: optional i32 col_unique_id = -1 } struct TTupleDescriptor { @@ -158,6 +174,7 @@ struct TOlapTableIndexSchema { 1: required i64 id 2: required list columns 3: required i32 schema_hash + 4: required list columns_desc } struct TOlapTableSchemaParam { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index dec3ca2f64..2469ebd4e2 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -419,6 +419,7 @@ struct TOlapScanNode { 5: optional string sort_column 6: optional Types.TKeysType keyType 7: optional string table_name + 8: required list columns_desc } struct TEqJoinCondition { diff --git a/regression-test/data/schema_change/test_compaction_schema_change.out b/regression-test/data/schema_change/test_compaction_schema_change.out new file mode 100644 index 0000000000..c7f14cd658 --- /dev/null +++ b/regression-test/data/schema_change/test_compaction_schema_change.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !compaction -- +100 + +-- !compaction -- +100 + +-- !compaction -- +1 +2 + diff --git a/regression-test/data/schema_change/test_delete_schema_change.out b/regression-test/data/schema_change/test_delete_schema_change.out new file mode 100644 index 0000000000..ff8a11901d --- /dev/null +++ b/regression-test/data/schema_change/test_delete_schema_change.out @@ -0,0 +1,44 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_delete_schema_change -- +0 + +-- !test_delete_schema_change_2 -- +0 + +-- !test_delete_schema_change_3 -- +1 + +-- !test_delete_schema_change_4 -- +1 + +-- !test_delete_schema_change_5 -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 + +-- !test_delete_schema_change_6 -- +0 + +-- !test_delete_schema_change_7 -- +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1 +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 1 + +-- !test_delete_schema_change_8 -- +1 + +-- !test_delete_schema_change_9 -- +1 + +-- !test_delete_schema_change_10 -- +1 + +-- !test_delete_schema_change_11 -- +1 + +-- !test_delete_schema_change_12 -- +0 + +-- !test_delete_schema_change_13 -- +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 32 20 2 +2 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 32 20 2 +1 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 19 2 + diff --git a/regression-test/data/schema_change/test_partition_schema_change.out b/regression-test/data/schema_change/test_partition_schema_change.out new file mode 100644 index 0000000000..5c568d5ecd --- /dev/null +++ b/regression-test/data/schema_change/test_partition_schema_change.out @@ -0,0 +1,33 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_partition_schema_change -- +0 + +-- !test_partition_schema_change_2 -- +0 + +-- !test_partition_schema_change_3 -- +1 + +-- !test_partition_schema_change_4 -- +1 + +-- !test_partition_schema_change_5 -- +1 + +-- !test_partition_schema_change_6 -- +1 2017-01-02 Beijing 10 1 2017-01-02T00:00 1 30 20 +1 2017-02-02 Beijing 10 1 2017-02-02T00:00 1 30 20 +1 2017-03-02 Beijing 10 1 2017-03-02T00:00 1 30 20 + +-- !test_partition_schema_change_7 -- +0 + +-- !test_partition_schema_change_8 -- +1 + +-- !test_partition_schema_change_9 -- +1 2017-01-02 Beijing 10 1 2017-01-02T00:00 1 30 20 1 +1 2017-02-02 Beijing 10 1 2017-02-02T00:00 1 30 20 1 +2 2017-02-03 Beijing 10 1 2017-02-02T00:00 1 30 20 2 +1 2017-03-02 Beijing 10 1 2017-03-02T00:00 1 30 20 1 + diff --git a/regression-test/data/schema_change/test_update_schema_change.out b/regression-test/data/schema_change/test_update_schema_change.out new file mode 100644 index 0000000000..aeacfd28c6 --- /dev/null +++ b/regression-test/data/schema_change/test_update_schema_change.out @@ -0,0 +1,64 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_update_schema_change -- +0 + +-- !test_update_schema_change_2 -- +0 + +-- !test_update_schema_change_3 -- +1 + +-- !test_update_schema_change_4 -- +1 + +-- !test_update_schema_change_5 -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 + +-- !test_update_schema_change_6 -- +0 + +-- !test_update_schema_change_7 -- +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1 +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 1 + +-- !test_update_schema_change_8 -- +1 + +-- !test_update_schema_change_9 -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 2 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1 + +-- !test_update_schema_change_10 -- +1 + +-- !test_update_schema_change_11 -- +1 + +-- !test_update_schema_change_12 -- +2 + +-- !test_update_schema_change_13 -- +5 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 20 +3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1 +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 20 + +-- !test_update_schema_change_14 -- +0 + +-- !test_update_schema_change_15 -- +5 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 +3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 + +-- !test_update_schema_change_16 -- +1 + +-- !test_update_schema_change_17 -- +5 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 20 31 21 +3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 + diff --git a/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy b/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy new file mode 100644 index 0000000000..13594474f7 --- /dev/null +++ b/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_agg_keys_schema_change") { + def tableName = "schema_change_agg_keys_regression_test" + + try { + + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", + `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", + `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") + AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, 1, 30, 20, hll_hash(1), to_bitmap(1)) + """ + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, 1, 31, 19, hll_hash(2), to_bitmap(2)) + """ + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, 1, 31, 21, hll_hash(2), to_bitmap(2)) + """ + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(3), to_bitmap(3)) + """ + + def result = "null"; + result = sql """ select * from ${tableName} order by user_id""" + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 10) + assertTrue(result[0][5] == 2, "user id 1 cost should be 2") + assertTrue(result[1][5] == 2, "user id 2 cost should be 2") + + // add key column case 1, not light schema change + sql """ + ALTER table ${tableName} ADD COLUMN new_key_column INT default "2" + """ + + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`cost`,`max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, 100, 32, 20, hll_hash(4), to_bitmap(4)) + """ + result = "null" + result = sql """SELECT * FROM ${tableName} WHERE user_id = 3""" + + assertTrue(result.size() == 1) + assertTrue(result[0][5] == 2, "new key column default value should be 2") + assertTrue(result[0].size() == 11) + + // add key column case 2 + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, 3, 110, 32, 20, hll_hash(4), to_bitmap(4)) + """ + result = "null" + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + result = "null" + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 4, "total count is 4") + + // drop key column, not light schema change + sql """ + ALTER TABLE ${tableName} DROP COLUMN new_key_column + """ + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + result = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 10) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 4) + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 10) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 8) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} \ No newline at end of file diff --git a/regression-test/suites/schema_change/test_agg_mv_schema_change.groovy b/regression-test/suites/schema_change/test_agg_mv_schema_change.groovy new file mode 100644 index 0000000000..d13c66ebdf --- /dev/null +++ b/regression-test/suites/schema_change/test_agg_mv_schema_change.groovy @@ -0,0 +1,258 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_agg_mv_schema_change") { + def tableName = "schema_change_agg_mv_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", + `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", + `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") + AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + //add materialized view + def result = "null" + def mvName = "mv1" + sql "create materialized view ${mvName} as select user_id, date, city, age, sex, sum(cost) from ${tableName} group by user_id, date, city, age, sex, cost;" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, 1, 30, 20, hll_hash(1), to_bitmap(1)) + """ + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, 1, 31, 19, hll_hash(2), to_bitmap(2)) + """ + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, 1, 31, 21, hll_hash(2), to_bitmap(2)) + """ + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(3), to_bitmap(3)) + """ + + result = "null"; + result = sql """ select * from ${tableName} """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 10) + assertTrue(result[0][5] == 2, "user id 1 cost should be 2") + assertTrue(result[1][5] == 2, "user id 2 cost should be 2") + assertTrue(result[0].size() == 10) + + // drop value column with mv, not light schema change + sql """ + ALTER TABLE ${tableName} DROP COLUMN cost + """ + + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")) { + log.info("rollup job is cancelled, result: ${result}".toString()) + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} (`user_id`, `date`, `city`, `age`, `sex`, `max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(4), to_bitmap(4)) + """ + result = "null" + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 9) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 4) + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 9) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 14) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +} + + \ No newline at end of file diff --git a/regression-test/suites/schema_change/test_agg_rollup_schema_change.groovy b/regression-test/suites/schema_change/test_agg_rollup_schema_change.groovy new file mode 100644 index 0000000000..c77cb5e946 --- /dev/null +++ b/regression-test/suites/schema_change/test_agg_rollup_schema_change.groovy @@ -0,0 +1,256 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_agg_rollup_schema_change") { + def tableName = "schema_change_agg_rollup_regression_test" + + try { + + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", + `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", + `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") + AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + //add rollup + def result = "null" + def rollupName = "rollup_cost" + sql "ALTER TABLE ${tableName} ADD ROLLUP ${rollupName}(`user_id`,`date`,`city`,`age`,`sex`, cost);" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, 1, 30, 20, hll_hash(1), to_bitmap(1)) + """ + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, 1, 31, 19, hll_hash(2), to_bitmap(2)) + """ + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, 1, 31, 21, hll_hash(2), to_bitmap(2)) + """ + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(3), to_bitmap(3)) + """ + + result = "null"; + result = sql """ select * from ${tableName} """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 10) + assertTrue(result[0][5] == 2, "user id 1 cost should be 2") + assertTrue(result[1][5] == 2, "user id 2 cost should be 2") + assertTrue(result[0].size() == 10) + + // drop value column with rollup, not light schema change + sql """ + ALTER TABLE ${tableName} DROP COLUMN cost + """ + + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")) { + log.info("rollup job is cancelled, result: ${result}".toString()) + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} (`user_id`, `date`, `city`, `age`, `sex`, `max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(4), to_bitmap(4)) + """ + result = "null" + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 9) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5)) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 4) + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 9) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 12) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/schema_change/test_agg_vals_schema_change.groovy b/regression-test/suites/schema_change/test_agg_vals_schema_change.groovy new file mode 100644 index 0000000000..304f3e4033 --- /dev/null +++ b/regression-test/suites/schema_change/test_agg_vals_schema_change.groovy @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_agg_vals_schema_change") { + def tableName = "schema_change_agg_vals_regression_test" + + try { + + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME REPLACE NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", + `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", + `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") + AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20, hll_hash(1), to_bitmap(1)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19, hll_hash(2), to_bitmap(2)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21, hll_hash(2), to_bitmap(2)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(3), to_bitmap(3)) + """ + def result1 = sql """ + select * from ${tableName} + """ + assertTrue(result1.size() == 2) + assertTrue(result1[0].size() == 13) + assertTrue(result1[0][8] == 2, "user id 1 cost should be 2") + assertTrue(result1[1][8] == 2, "user id 2 cost should be 2") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT MAX default "1" + """ + + def result2 = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result1[0][8] == 2, "user id 2 cost should be 2") + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) + """ + result2 = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result2[0][8] == 3, "user id 2 cost should be 3") + + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4)) + """ + + result2 = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 14) + assertTrue(result2[0][13] == 1, "new add column default value should be 1") + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) + """ + def result3 = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result3.size() == 1) + assertTrue(result3[0].size() == 14) + assertTrue(result3[0][13] == 2, "new add column value is set to 2") + + def result4 = sql """ select count(*) from ${tableName} """ + logger.info("result4.size:"+result4.size() + " result4[0].size:" + result4[0].size + " " + result4[0][0]) + assertTrue(result4.size() == 1) + assertTrue(result4[0].size() == 1) + assertTrue(result4[0][0] == 3, "total count is 3") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN last_visit_date + """ + def result5 = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result5.size() == 1) + assertTrue(result5[0].size() == 13) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(4), to_bitmap(4), 2) + """ + + def result6 = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result6.size() == 1) + assertTrue(result6[0].size() == 13) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(5), to_bitmap(5), 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + def result7 = sql """ select count(*) from ${tableName} """ + assertTrue(result7.size() == 1) + assertTrue(result7[0][0] == 5) + + def result8 = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result8.size() == 1) + assertTrue(result8[0].size() == 13) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 8) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +} diff --git a/regression-test/suites/schema_change/test_delete_schema_change.sql b/regression-test/suites/schema_change/test_delete_schema_change.sql new file mode 100644 index 0000000000..e3a92a38f5 --- /dev/null +++ b/regression-test/suites/schema_change/test_delete_schema_change.sql @@ -0,0 +1,44 @@ +DROP TABLE IF EXISTS schema_change_delete_regression_test; + +CREATE TABLE schema_change_delete_regression_test ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + +INSERT INTO schema_change_delete_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20); + +INSERT INTO schema_change_delete_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21); + +SELECT * FROM schema_change_delete_regression_test order by user_id ASC, last_visit_date; + +ALTER table schema_change_delete_regression_test ADD COLUMN new_column INT default "1"; + +SELECT * FROM schema_change_delete_regression_test order by user_id DESC, last_visit_date; + +INSERT INTO schema_change_delete_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19, 2); + +INSERT INTO schema_change_delete_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2); + +INSERT INTO schema_change_delete_regression_test VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 1); + +INSERT INTO schema_change_delete_regression_test VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2); + +DELETE FROM schema_change_delete_regression_test where new_column = 1; + +SELECT * FROM schema_change_delete_regression_test order by user_id DESC, last_visit_date; \ No newline at end of file diff --git a/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy new file mode 100644 index 0000000000..6711429d5a --- /dev/null +++ b/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_dup_keys_schema_change") { + def tableName = "schema_change_dup_keys_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + def result = sql """ + select count(*) from ${tableName} + """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 4, "total columns should be 4 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 1, "new add column default value should be 1") + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 12) + assertTrue(result[1][11] == 2, "new add column value is set to 2") + + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 6, "total count is 6") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN sex + """ + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")) { + log.info("rollup job is cancelled, result: ${result}".toString()) + break + } + Thread.sleep(1000) + } + result = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + result = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 13) + + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount < 10) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +} diff --git a/regression-test/suites/schema_change/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change/test_dup_mv_schema_change.groovy new file mode 100644 index 0000000000..810d75d7b8 --- /dev/null +++ b/regression-test/suites/schema_change/test_dup_mv_schema_change.groovy @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_dup_mv_schema_change") { + def tableName = "schema_change_dup_mv_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + //add materialized view + def result = "null" + def mvName = "mv1" + sql "create materialized view ${mvName} as select user_id, date, city, age,sex from ${tableName};" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + result = sql """ + select count(*) from ${tableName} + """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 4, "total columns should be 4 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 1, "new add column default value should be 1") + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 12) + assertTrue(result[1][11] == 2, "new add column value is set to 2") + + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 6, "total count is 6") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN sex + """ + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")) { + log.info("rollup job is cancelled, result: ${result}".toString()) + break + } + Thread.sleep(1000) + } + + result = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + result = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 13) + + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 14) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +} diff --git a/regression-test/suites/schema_change/test_dup_rollup_schema_change.groovy b/regression-test/suites/schema_change/test_dup_rollup_schema_change.groovy new file mode 100644 index 0000000000..6c1e8c2e42 --- /dev/null +++ b/regression-test/suites/schema_change/test_dup_rollup_schema_change.groovy @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_dup_rollup_schema_change") { + def tableName = "schema_change_dup_rollup_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + //add rollup + def result = "null" + def rollupName = "rollup_cost" + sql "ALTER TABLE ${tableName} ADD ROLLUP ${rollupName}(`user_id`,`date`,`city`,`age`,`sex`, cost);" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + result = sql """ + select count(*) from ${tableName} + """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 4, "total columns should be 4 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 1, "new add column default value should be 1") + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 12) + assertTrue(result[1][11] == 2, "new add column value is set to 2") + + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 6, "total count is 6") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN sex + """ + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")) { + log.info("rollup job is cancelled, result: ${result}".toString()) + break + } + Thread.sleep(1000) + } + + result = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + result = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 13) + + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 2) + assertTrue(result[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + StringBuilder sb = new StringBuilder(); + backend_id = tablet[2] + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 14) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +} diff --git a/regression-test/suites/schema_change/test_dup_vals_schema_change.groovy b/regression-test/suites/schema_change/test_dup_vals_schema_change.groovy new file mode 100644 index 0000000000..e10e0724ce --- /dev/null +++ b/regression-test/suites/schema_change/test_dup_vals_schema_change.groovy @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_dup_vals_schema_change") { + def tableName = "schema_change_dup_vals_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + def result1 = sql """ + select count(*) from ${tableName} + """ + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 4, "total columns should be 4 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + def result2 = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 12) + assertTrue(result2[0][11] == 1, "new add column default value should be 1") + + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + def result3 = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result3.size() == 2) + assertTrue(result3[0].size() == 12) + assertTrue(result3[1][11] == 2, "new add column value is set to 2") + + def result4 = sql """ select count(*) from ${tableName} """ + logger.info("result4.size:"+result4.size() + " result4[0].size:" + result4[0].size + " " + result4[0][0]) + assertTrue(result4.size() == 1) + assertTrue(result4[0].size() == 1) + assertTrue(result4[0][0] == 6, "total count is 6") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN last_visit_date + """ + def result5 = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result5.size() == 2) + assertTrue(result5[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + def result6 = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result6.size() == 1) + assertTrue(result6[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + def result7 = sql """ select count(*) from ${tableName} """ + assertTrue(result7.size() == 1) + assertTrue(result7[0][0] == 13) + + def result8 = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result8.size() == 2) + assertTrue(result8[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount < 10) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/schema_change/test_partition_schema_change.sql b/regression-test/suites/schema_change/test_partition_schema_change.sql new file mode 100644 index 0000000000..58925efc3b --- /dev/null +++ b/regression-test/suites/schema_change/test_partition_schema_change.sql @@ -0,0 +1,44 @@ +DROP TABLE IF EXISTS example_range_tbl; +CREATE TABLE example_range_tbl +( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" +) +ENGINE=OLAP +AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) +PARTITION BY RANGE(`date`) +( + PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), + PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), + PARTITION `p201703` VALUES LESS THAN ("2017-04-01") +) +DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 +PROPERTIES +( + "replication_num" = "1" +); + +INSERT INTO example_range_tbl VALUES + (1, '2017-01-02', 'Beijing', 10, 1, "2017-01-02 00:00:00", 1, 30, 20); + +INSERT INTO example_range_tbl VALUES + (1, '2017-02-02', 'Beijing', 10, 1, "2017-02-02 00:00:00", 1, 30, 20); + +INSERT INTO example_range_tbl VALUES + (1, '2017-03-02', 'Beijing', 10, 1, "2017-03-02 00:00:00", 1, 30, 20); + +select * from example_range_tbl order by `date`; + +ALTER table example_range_tbl ADD COLUMN new_column INT MAX default "1"; + +INSERT INTO example_range_tbl VALUES + (2, '2017-02-03', 'Beijing', 10, 1, "2017-02-02 00:00:00", 1, 30, 20, 2); + +select * from example_range_tbl order by `date`; \ No newline at end of file diff --git a/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy new file mode 100644 index 0000000000..568df986f5 --- /dev/null +++ b/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy @@ -0,0 +1,270 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_uniq_keys_schema_change") { + def tableName = "schema_change_uniq_keys_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE schema_change_uniq_keys_regression_test ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO schema_change_uniq_keys_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + def result = sql """ + select count(*) from schema_change_uniq_keys_regression_test + """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 2, "total columns should be 2 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 1, "new add column default value should be 1") + + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 2, "new add column value is set to 2") + + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 3, "total count is 3") + + // drop column will throws exception 'Can not drop key column in Unique data model table' + // sql """ + // ALTER TABLE ${tableName} DROP COLUMN sex + // """ + // result = "null" + // while (!result.contains("FINISHED")){ + // result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + // result = result.toString() + // logger.info("result: ${result}") + // if(result.contains("CANCELLED")) { + // log.info("rollup job is cancelled, result: ${result}".toString()) + // break + // } + // Thread.sleep(1000) + // } + // result = sql """ select * from ${tableName} where user_id = 3 """ + // assertTrue(result.size() == 1) + // assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + result = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 5) + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 10) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/schema_change/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change/test_uniq_mv_schema_change.groovy new file mode 100644 index 0000000000..19070173f3 --- /dev/null +++ b/regression-test/suites/schema_change/test_uniq_mv_schema_change.groovy @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_uniq_mv_schema_change") { + def tableName = "schema_change_uniq_mv_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + //add materialized view + def result = "null" + def mvName = "mv1" + sql "create materialized view ${mvName} as select user_id, date, city, age, sex from ${tableName} group by user_id, date, city, age, sex;" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + result = sql """ + select count(*) from ${tableName} + """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 2, "total columns should be 2 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 1, "new add column default value should be 1") + + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 2, "new add column value is set to 2") + + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 3, "total count is 3") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN cost + """ + + result = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + + result = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 5) + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 14) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/schema_change/test_uniq_rollup_schema_change.groovy b/regression-test/suites/schema_change/test_uniq_rollup_schema_change.groovy new file mode 100644 index 0000000000..317e099b2c --- /dev/null +++ b/regression-test/suites/schema_change/test_uniq_rollup_schema_change.groovy @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_uniq_rollup_schema_change") { + def tableName = "schema_change_uniq_rollup_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + //add rollup + def result = "null" + def rollupName = "rollup_cost" + sql "ALTER TABLE ${tableName} ADD ROLLUP ${rollupName}(`user_id`,`date`,`city`,`age`,`sex`, cost);" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + break + } + Thread.sleep(1000) + } + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + result = sql """ + select count(*) from ${tableName} + """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 2, "total columns should be 2 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 1, "new add column default value should be 1") + + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 12) + assertTrue(result[0][11] == 2, "new add column value is set to 2") + + result = sql """ select count(*) from ${tableName} """ + logger.info("result.size:" + result.size() + " result[0].size:" + result[0].size + " " + result[0][0]) + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 1) + assertTrue(result[0][0] == 3, "total count is 3") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN cost + """ + + result = "null" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")) { + log.info("rollup job is cancelled, result: ${result}".toString()) + break + } + Thread.sleep(1000) + } + + result = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1,'2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + + result = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1,'2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + result = sql """ select count(*) from ${tableName} """ + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 5) + + result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result.size() == 1) + assertTrue(result[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 14) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/schema_change/test_uniq_vals_schema_change.groovy b/regression-test/suites/schema_change/test_uniq_vals_schema_change.groovy new file mode 100644 index 0000000000..3c5ff5b3a7 --- /dev/null +++ b/regression-test/suites/schema_change/test_uniq_vals_schema_change.groovy @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_uniq_vals_schema_change") { + def tableName = "schema_change_uniq_vals_regression_test" + + try { + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + for (String[] backend in backends) { + backendId_to_backendIP.put(backend[0], backend[2]) + backendId_to_backendHttpPort.put(backend[0], backend[5]) + } + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + def result1 = sql """ + select count(*) from ${tableName} + """ + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 2, "total columns should be 2 rows") + + // add column + sql """ + ALTER table ${tableName} ADD COLUMN new_column INT default "1" + """ + + sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + + sql """ INSERT INTO ${tableName} (`user_id`,`date`,`city`,`age`,`sex`,`last_visit_date`,`last_update_date`, + `last_visit_date_not_null`,`cost`,`max_dwell_time`,`min_dwell_time`) + VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + def result2 = sql """ SELECT * FROM ${tableName} WHERE user_id=3 """ + + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 12) + assertTrue(result2[0][11] == 1, "new add column default value should be 1") + + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + def result3 = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """ + + assertTrue(result3.size() == 1) + assertTrue(result3[0].size() == 12) + assertTrue(result3[0][11] == 2, "new add column value is set to 2") + + def result4 = sql """ select count(*) from ${tableName} """ + logger.info("result4.size:"+result4.size() + " result4[0].size:" + result4[0].size + " " + result4[0][0]) + assertTrue(result4.size() == 1) + assertTrue(result4[0].size() == 1) + assertTrue(result4[0][0] == 3, "total count is 3") + + // drop column + sql """ + ALTER TABLE ${tableName} DROP COLUMN last_visit_date + """ + def result5 = sql """ select * from ${tableName} where user_id = 3 """ + assertTrue(result5.size() == 1) + assertTrue(result5[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + def result6 = sql """ select * from ${tableName} where user_id = 4 """ + assertTrue(result6.size() == 1) + assertTrue(result6[0].size() == 11) + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2) + """ + + Thread.sleep(30 * 1000) + // compaction + String[][] tablets = sql """ show tablets from ${tableName}; """ + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + logger.info("run compaction:" + tablet_id) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + //assertEquals(code, 0) + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + def result7 = sql """ select count(*) from ${tableName} """ + assertTrue(result7.size() == 1) + assertTrue(result7[0][0] == 5) + + def result8 = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ + assertTrue(result8.size() == 1) + assertTrue(result8[0].size() == 11) + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + logger.info("size:" + rowCount) + assertTrue(rowCount <= 10) + } finally { + //try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +} diff --git a/regression-test/suites/schema_change/test_update_schema_change.sql b/regression-test/suites/schema_change/test_update_schema_change.sql new file mode 100644 index 0000000000..e2cd8263c2 --- /dev/null +++ b/regression-test/suites/schema_change/test_update_schema_change.sql @@ -0,0 +1,50 @@ +DROP TABLE IF EXISTS schema_change_update_regression_test; + +CREATE TABLE schema_change_update_regression_test ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + +INSERT INTO schema_change_update_regression_test VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20); + +INSERT INTO schema_change_update_regression_test VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21); + +SELECT * FROM schema_change_update_regression_test order by user_id ASC, last_visit_date; + +ALTER table schema_change_update_regression_test ADD COLUMN new_column INT default "1"; + +SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date; + +UPDATE schema_change_update_regression_test set new_column = 2 where user_id = 1; + +SELECT * FROM schema_change_update_regression_test order by user_id ASC, last_visit_date; + +INSERT INTO schema_change_update_regression_test VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20, 2); + +INSERT INTO schema_change_update_regression_test VALUES + (5, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21, 20); + +UPDATE schema_change_update_regression_test set new_column = 20 where new_column = 2; + +SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date; + +ALTER TABLE schema_change_update_regression_test DROP COLUMN new_column; + +SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date; + +UPDATE schema_change_update_regression_test set cost = 20 where user_id = 5; + +SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date; \ No newline at end of file