From ae352997b46036342cd229e4fa57d0f77e6d0f57 Mon Sep 17 00:00:00 2001 From: YueW <45946325+Tanya-W@users.noreply.github.com> Date: Sun, 28 May 2023 11:23:07 +0800 Subject: [PATCH] [Enhancement](alter inverted index) Improve alter inverted index performance with light weight add or drop inverted index (#19063) --- be/src/agent/task_worker_pool.cpp | 141 ++--- be/src/agent/task_worker_pool.h | 4 - be/src/olap/CMakeLists.txt | 2 + be/src/olap/olap_server.cpp | 23 + be/src/olap/rowset/beta_rowset.cpp | 8 +- be/src/olap/rowset/beta_rowset.h | 3 +- be/src/olap/rowset/rowset.h | 3 +- be/src/olap/schema_change.cpp | 533 ------------------ be/src/olap/schema_change.h | 52 -- be/src/olap/storage_engine.h | 4 + be/src/olap/tablet.cpp | 31 + be/src/olap/tablet.h | 5 + be/src/olap/tablet_schema.cpp | 15 + be/src/olap/tablet_schema.h | 2 + be/src/olap/task/engine_alter_tablet_task.cpp | 30 - be/src/olap/task/engine_alter_tablet_task.h | 14 - be/src/olap/task/engine_clone_task.cpp | 1 + be/src/olap/task/engine_index_change_task.cpp | 64 +++ be/src/olap/task/engine_index_change_task.h | 42 ++ be/src/olap/task/index_builder.cpp | 484 ++++++++++++++++ be/src/olap/task/index_builder.h | 85 +++ be/test/testutil/mock_rowset.h | 4 +- .../apache/doris/common/FeMetaVersion.java | 4 +- fe/fe-core/src/main/cup/sql_parser.cup | 17 + .../apache/doris/alter/IndexChangeJob.java | 439 +++++++++++++++ .../doris/alter/SchemaChangeHandler.java | 466 ++++++++++----- .../apache/doris/alter/SchemaChangeJobV2.java | 135 ----- .../doris/analysis/BuildIndexClause.java | 87 +++ .../org/apache/doris/analysis/IndexDef.java | 27 + .../doris/analysis/ShowBuildIndexStmt.java | 229 ++++++++ .../org/apache/doris/catalog/OlapTable.java | 7 + .../apache/doris/catalog/TableIndexes.java | 10 + .../doris/common/proc/BuildIndexProcDir.java | 217 +++++++ .../apache/doris/common/proc/JobsProcDir.java | 3 + .../common/proc/SchemaChangeProcDir.java | 1 - .../apache/doris/journal/JournalEntity.java | 6 + .../org/apache/doris/master/MasterImpl.java | 18 +- .../org/apache/doris/persist/EditLog.java | 12 +- .../apache/doris/persist/OperationType.java | 1 + .../TableAddOrDropInvertedIndicesInfo.java | 25 +- .../org/apache/doris/qe/ShowExecutor.java | 14 + .../doris/service/FrontendServiceImpl.java | 2 +- .../doris/task/AlterInvertedIndexTask.java | 76 ++- gensrc/thrift/AgentService.thrift | 6 +- .../test_index_change_with_compaction.out | 21 + .../test_dytable_complex_data.groovy | 27 + .../test_index_change_with_compaction.groovy | 212 +++++++ ...t_add_drop_index_ignore_case_column.groovy | 27 + .../test_add_drop_index_with_data.groovy | 31 +- ...est_add_drop_index_with_delete_data.groovy | 27 + ..._index_match_term_and_phrase_select.groovy | 37 ++ .../test_index_range_in_select.groovy | 37 ++ .../test_index_range_not_in_select.groovy | 37 ++ 53 files changed, 2726 insertions(+), 1082 deletions(-) create mode 100644 be/src/olap/task/engine_index_change_task.cpp create mode 100644 be/src/olap/task/engine_index_change_task.h create mode 100644 be/src/olap/task/index_builder.cpp create mode 100644 be/src/olap/task/index_builder.h create mode 100644 fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java create mode 100644 regression-test/data/inverted_index_p0/index_change/test_index_change_with_compaction.out create mode 100644 regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ff49f04ffe..61bdec6862 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -65,6 +65,7 @@ #include "olap/task/engine_batch_load_task.h" #include "olap/task/engine_checksum_task.h" #include "olap/task/engine_clone_task.h" +#include "olap/task/engine_index_change_task.h" #include "olap/task/engine_publish_version_task.h" #include "olap/task/engine_storage_migration_task.h" #include "olap/txn_manager.h" @@ -334,11 +335,11 @@ void TaskWorkerPool::_finish_task(const TFinishTaskRequest& finish_task_request) void TaskWorkerPool::_alter_inverted_index_worker_thread_callback() { while (_is_work) { TAgentTaskRequest agent_task_req; + { std::unique_lock worker_thread_lock(_worker_thread_lock); - while (_is_work && _tasks.empty()) { - _worker_thread_condition_variable.wait(worker_thread_lock); - } + _worker_thread_condition_variable.wait( + worker_thread_lock, [this]() { return !_is_work || !_tasks.empty(); }); if (!_is_work) { return; } @@ -346,106 +347,54 @@ void TaskWorkerPool::_alter_inverted_index_worker_thread_callback() { agent_task_req = _tasks.front(); _tasks.pop_front(); } - int64_t signature = agent_task_req.signature; - LOG(INFO) << "get alter inverted index task, signature: " << agent_task_req.signature; - bool is_task_timeout = false; - if (agent_task_req.__isset.recv_time) { - int64_t time_elapsed = time(nullptr) - agent_task_req.recv_time; - if (time_elapsed > config::report_task_interval_seconds * 20) { - LOG(INFO) << "task elapsed " << time_elapsed - << " seconds since it is inserted to queue, it is timeout"; - is_task_timeout = true; - } + + auto& alter_inverted_index_rq = agent_task_req.alter_inverted_index_req; + LOG(INFO) << "get alter inverted index task. signature=" << agent_task_req.signature + << ", tablet_id=" << alter_inverted_index_rq.tablet_id + << ", job_id=" << alter_inverted_index_rq.job_id; + + Status status = Status::OK(); + TabletSharedPtr tablet_ptr = StorageEngine::instance()->tablet_manager()->get_tablet( + alter_inverted_index_rq.tablet_id); + if (tablet_ptr != nullptr) { + EngineIndexChangeTask engine_task(alter_inverted_index_rq); + status = _env->storage_engine()->execute_task(&engine_task); + } else { + status = + Status::NotFound("could not find tablet {}", alter_inverted_index_rq.tablet_id); } - if (!is_task_timeout) { - TFinishTaskRequest finish_task_request; - TTaskType::type task_type = agent_task_req.task_type; - switch (task_type) { - case TTaskType::ALTER_INVERTED_INDEX: - _alter_inverted_index(agent_task_req, signature, task_type, &finish_task_request); - break; - default: - // pass - break; + + // Return result to fe + TFinishTaskRequest finish_task_request; + finish_task_request.__set_backend(_backend); + finish_task_request.__set_task_type(agent_task_req.task_type); + finish_task_request.__set_signature(agent_task_req.signature); + std::vector finish_tablet_infos; + if (!status.ok()) { + LOG(WARNING) << "failed to alter inverted index task, signature=" + << agent_task_req.signature + << ", tablet_id=" << alter_inverted_index_rq.tablet_id + << ", job_id=" << alter_inverted_index_rq.job_id << ", error=" << status; + } else { + LOG(INFO) << "successfully alter inverted index task, signature=" + << agent_task_req.signature + << ", tablet_id=" << alter_inverted_index_rq.tablet_id + << ", job_id=" << alter_inverted_index_rq.job_id; + TTabletInfo tablet_info; + status = _get_tablet_info(alter_inverted_index_rq.tablet_id, + alter_inverted_index_rq.schema_hash, agent_task_req.signature, + &tablet_info); + if (status.ok()) { + finish_tablet_infos.push_back(tablet_info); } - _finish_task(finish_task_request); + finish_task_request.__set_finish_tablet_infos(finish_tablet_infos); } + finish_task_request.__set_task_status(status.to_thrift()); + _finish_task(finish_task_request); _remove_task_info(agent_task_req.task_type, agent_task_req.signature); } } -void TaskWorkerPool::_alter_inverted_index(const TAgentTaskRequest& alter_inverted_index_request, - int64_t signature, const TTaskType::type task_type, - TFinishTaskRequest* finish_task_request) { - Status status = Status::OK(); - string process_name; - switch (task_type) { - case TTaskType::ALTER_INVERTED_INDEX: - process_name = "AlterInvertedIndex"; - break; - default: - std::string task_name; - EnumToString(TTaskType, task_type, task_name); - LOG(WARNING) << "schema change type invalid. type: " << task_name - << ", signature: " << signature; - status = Status::NotSupported("Schema change type invalid"); - break; - } - - TTabletId tablet_id; - TSchemaHash schema_hash = 0; - if (status.ok()) { - tablet_id = alter_inverted_index_request.alter_inverted_index_req.tablet_id; - schema_hash = alter_inverted_index_request.alter_inverted_index_req.schema_hash; - EngineAlterInvertedIndexTask engine_task( - alter_inverted_index_request.alter_inverted_index_req); - Status sc_status = _env->storage_engine()->execute_task(&engine_task); - if (!sc_status.ok()) { - status = Status::DataQualityError("The data quality does not satisfy"); - } else { - status = Status::OK(); - } - } - - if (status.ok()) { - ++_s_report_version; - LOG(INFO) << process_name << " finished. signature: " << signature; - } - - // Return result to fe - finish_task_request->__set_backend(BackendOptions::get_local_backend()); - finish_task_request->__set_report_version(_s_report_version); - finish_task_request->__set_task_type(task_type); - finish_task_request->__set_signature(signature); - - std::vector finish_tablet_infos; - if (status.ok()) { - TTabletInfo tablet_info; - status = _get_tablet_info(tablet_id, schema_hash, signature, &tablet_info); - - if (!status.ok()) { - LOG(WARNING) << process_name << " success, but get tablet info failed." - << "tablet_id: " << tablet_id << ", schema_hash: " << schema_hash - << ", signature: " << signature; - } else { - finish_tablet_infos.push_back(tablet_info); - } - } - - if (status.ok()) { - finish_task_request->__set_finish_tablet_infos(finish_tablet_infos); - LOG_INFO("successfully {}", process_name) - .tag("signature", signature) - .tag("tablet_id", tablet_id); - } else { - LOG_WARNING("failed to {}", process_name) - .tag("signature", signature) - .tag("tablet_id", tablet_id) - .error(status); - } - finish_task_request->__set_task_status(status.to_thrift()); -} - void TaskWorkerPool::_alter_tablet_worker_thread_callback() { while (_is_work) { TAgentTaskRequest agent_task_req; diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h index 7287da3f03..d24ba647cf 100644 --- a/be/src/agent/task_worker_pool.h +++ b/be/src/agent/task_worker_pool.h @@ -201,10 +201,6 @@ protected: void _push_cooldown_conf_worker_thread_callback(); void _push_storage_policy_worker_thread_callback(); - void _alter_inverted_index(const TAgentTaskRequest& alter_inverted_index_request, - int64_t signature, const TTaskType::type task_type, - TFinishTaskRequest* finish_task_request); - void _alter_tablet(const TAgentTaskRequest& alter_tablet_request, int64_t signature, const TTaskType::type task_type, TFinishTaskRequest* finish_task_request); void _handle_report(const TReportRequest& request, ReportType type); diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt index ab270c8247..1380d9df2b 100644 --- a/be/src/olap/CMakeLists.txt +++ b/be/src/olap/CMakeLists.txt @@ -109,6 +109,8 @@ add_library(Olap STATIC task/engine_storage_migration_task.cpp task/engine_publish_version_task.cpp task/engine_alter_tablet_task.cpp + task/engine_index_change_task.cpp + task/index_builder.cpp segment_loader.cpp storage_policy.cpp ) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index c3e94483ff..152a3ebb0d 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -51,11 +51,13 @@ #include "olap/olap_common.h" #include "olap/rowset/beta_rowset_writer.h" #include "olap/rowset/segcompaction.h" +#include "olap/schema_change.h" #include "olap/storage_engine.h" #include "olap/tablet.h" #include "olap/tablet_manager.h" #include "olap/tablet_meta.h" #include "olap/tablet_schema.h" +#include "olap/task/index_builder.h" #include "service/point_query_executor.h" #include "util/countdown_latch.h" #include "util/doris_metrics.h" @@ -738,6 +740,27 @@ Status StorageEngine::submit_seg_compaction_task(BetaRowsetWriter* writer, std::bind(&StorageEngine::_handle_seg_compaction, this, writer, segments)); } +Status StorageEngine::process_index_change_task(const TAlterInvertedIndexReq& request) { + auto tablet_id = request.tablet_id; + TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet == nullptr) { + LOG(WARNING) << "tablet: " << tablet_id << " not exist"; + return Status::InternalError("tablet not exist, tablet_id={}.", tablet_id); + } + + IndexBuilderSharedPtr index_builder = + std::make_shared(tablet, request.columns, request.indexes_desc, + request.alter_inverted_indexes, request.is_drop_op); + RETURN_IF_ERROR(_handle_index_change(index_builder)); + return Status::OK(); +} + +Status StorageEngine::_handle_index_change(IndexBuilderSharedPtr index_builder) { + RETURN_IF_ERROR(index_builder->init()); + RETURN_IF_ERROR(index_builder->do_build_inverted_index()); + return Status::OK(); +} + void StorageEngine::_cooldown_tasks_producer_callback() { int64_t interval = config::generate_cooldown_task_interval_sec; // the cooldown replica may be slow to upload it's meta file, so we should wait diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index ee2faada7c..4ab84dd710 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -220,7 +220,8 @@ void BetaRowset::do_close() { } Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, - size_t new_rowset_start_seg_id) { + size_t new_rowset_start_seg_id, + std::set* without_index_column_uids) { DCHECK(is_local()); auto fs = _rowset_meta->fs(); if (!fs) { @@ -246,7 +247,10 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, return Status::Error(); } for (auto& column : _schema->columns()) { - // if (column.has_inverted_index()) { + if (without_index_column_uids != nullptr && + without_index_column_uids->count(column.unique_id())) { + continue; + } const TabletIndex* index_meta = _schema->get_inverted_index(column.unique_id()); if (index_meta) { std::string inverted_index_src_file_path = diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 02db3a2892..643782c461 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -71,7 +71,8 @@ public: Status remove() override; Status link_files_to(const std::string& dir, RowsetId new_rowset_id, - size_t new_rowset_start_seg_id = 0) override; + size_t new_rowset_start_seg_id = 0, + std::set* without_index_column_uids = nullptr) override; Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index b8a83008a7..2a8ff5a4bb 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -202,7 +202,8 @@ public: // hard link all files in this rowset to `dir` to form a new rowset with id `new_rowset_id`. virtual Status link_files_to(const std::string& dir, RowsetId new_rowset_id, - size_t new_rowset_start_seg_id = 0) = 0; + size_t new_rowset_start_seg_id = 0, + std::set* without_index_column_uids = nullptr) = 0; // copy all files to `dir` virtual Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) = 0; diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 3cf51bf3e5..5952108eab 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -631,245 +631,6 @@ Status VSchemaChangeWithSorting::_external_sorting(vector& src_ return Status::OK(); } -SchemaChangeForInvertedIndex::SchemaChangeForInvertedIndex( - const std::vector& alter_inverted_indexs, - const TabletSchemaSPtr& tablet_schema) - : _alter_inverted_indexs(alter_inverted_indexs), _tablet_schema(tablet_schema) { - _olap_data_convertor = std::make_unique(); -} - -SchemaChangeForInvertedIndex::~SchemaChangeForInvertedIndex() { - VLOG_NOTICE << "~SchemaChangeForInvertedIndex()"; - _inverted_index_builders.clear(); - _index_metas.clear(); - _olap_data_convertor.reset(); -} - -Status SchemaChangeForInvertedIndex::process(RowsetReaderSharedPtr rowset_reader, - RowsetWriter* rowset_writer, - TabletSharedPtr new_tablet, - TabletSharedPtr base_tablet, - TabletSchemaSPtr base_tablet_schema) { - Status res = Status::OK(); - if (rowset_reader->rowset()->empty() || rowset_reader->rowset()->num_rows() == 0) { - return Status::OK(); - } - - // create inverted index writer - auto rowset_meta = rowset_reader->rowset()->rowset_meta(); - std::string segment_dir = base_tablet->tablet_path(); - auto fs = rowset_meta->fs(); - - // load segments - SegmentCacheHandle segment_cache_handle; - RETURN_IF_ERROR(SegmentLoader::instance()->load_segments( - std::static_pointer_cast(rowset_reader->rowset()), &segment_cache_handle, - false)); - - for (auto& seg_ptr : segment_cache_handle.get_segments()) { - std::string segment_filename = - fmt::format("{}_{}.dat", rowset_meta->rowset_id().to_string(), seg_ptr->id()); - std::vector return_columns; - std::vector> inverted_index_writer_signs; - _olap_data_convertor->reserve(_alter_inverted_indexs.size()); - // create inverted index writer - for (auto& inverted_index : _alter_inverted_indexs) { - DCHECK_EQ(inverted_index.columns.size(), 1); - auto index_id = inverted_index.index_id; - auto column_name = inverted_index.columns[0]; - auto column_idx = _tablet_schema->field_index(column_name); - if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << column_idx - << "]"; - return Status::Error(); - } - auto column = _tablet_schema->column(column_idx); - return_columns.emplace_back(column_idx); - _olap_data_convertor->add_column_data_convertor(column); - - std::unique_ptr field(FieldFactory::create(column)); - _index_metas.emplace_back(new TabletIndex()); - _index_metas.back()->init_from_thrift(inverted_index, *_tablet_schema); - std::unique_ptr inverted_index_builder; - try { - RETURN_IF_ERROR(segment_v2::InvertedIndexColumnWriter::create( - field.get(), &inverted_index_builder, segment_filename, segment_dir, - _index_metas.back().get(), fs)); - } catch (const std::exception& e) { - LOG(WARNING) << "CLuceneError occured: " << e.what(); - return Status::Error(); - } - - if (inverted_index_builder) { - auto writer_sign = std::make_pair(seg_ptr->id(), index_id); - _inverted_index_builders.insert( - std::make_pair(writer_sign, std::move(inverted_index_builder))); - inverted_index_writer_signs.push_back(writer_sign); - } - } - - // create iterator for each segment - StorageReadOptions read_options; - OlapReaderStatistics stats; - read_options.stats = &stats; - read_options.tablet_schema = _tablet_schema; - std::unique_ptr schema = - std::make_unique(_tablet_schema->columns(), return_columns); - std::unique_ptr iter; - res = seg_ptr->new_iterator(*schema, read_options, &iter); - if (!res.ok()) { - LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() - << "]: " << res.to_string(); - return Status::Error(); - } - - std::shared_ptr block = - std::make_shared(_tablet_schema->create_block(return_columns)); - while (true) { - res = iter->next_batch(block.get()); - if (!res.ok()) { - if (res.is()) { - break; - } - RETURN_NOT_OK_STATUS_WITH_WARN( - res, "failed to read next block when schema change for inverted index."); - } - - // write inverted index - if (_write_inverted_index(iter->data_id(), block.get()) != Status::OK()) { - res = Status::Error(); - LOG(WARNING) << "failed to write block."; - return res; - } - block->clear_column_data(); - } - - // finish write inverted index, flush data to compound file - for (auto& writer_sign : inverted_index_writer_signs) { - try { - if (_inverted_index_builders[writer_sign]) { - _inverted_index_builders[writer_sign]->finish(); - } - } catch (const std::exception& e) { - LOG(WARNING) << "CLuceneError occured: " << e.what(); - return Status::Error(); - } - } - - _olap_data_convertor->reset(); - } - - _inverted_index_builders.clear(); - _index_metas.clear(); - - LOG(INFO) << "all row nums. source_rows=" << rowset_reader->rowset()->num_rows(); - return res; -} - -Status SchemaChangeForInvertedIndex::_add_nullable( - const std::string& column_name, const std::pair& index_writer_sign, - Field* field, const uint8_t* null_map, const uint8_t** ptr, size_t num_rows) { - size_t offset = 0; - auto next_run_step = [&]() { - size_t step = 1; - for (auto i = offset + 1; i < num_rows; ++i) { - if (null_map[offset] == null_map[i]) { - step++; - } else { - break; - } - } - return step; - }; - - try { - do { - auto step = next_run_step(); - if (null_map[offset]) { - RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_nulls(step)); - } else { - if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - DCHECK(field->get_sub_field_count() == 1); - const auto* col_cursor = reinterpret_cast(*ptr); - RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_array_values( - field->get_sub_field(0)->size(), col_cursor, step)); - } else { - RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_values( - column_name, *ptr, step)); - } - } - *ptr += field->size() * step; - offset += step; - } while (offset < num_rows); - } catch (const std::exception& e) { - LOG(WARNING) << "CLuceneError occured: " << e.what(); - return Status::Error(); - } - - return Status::OK(); -} - -Status SchemaChangeForInvertedIndex::_add_data(const std::string& column_name, - const std::pair& index_writer_sign, - Field* field, const uint8_t** ptr, size_t num_rows) { - try { - if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - DCHECK(field->get_sub_field_count() == 1); - const auto* col_cursor = reinterpret_cast(*ptr); - RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_array_values( - field->get_sub_field(0)->size(), col_cursor, num_rows)); - } else { - RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_values( - column_name, *ptr, num_rows)); - } - } catch (const std::exception& e) { - LOG(WARNING) << "CLuceneError occured: " << e.what(); - return Status::Error(); - } - - return Status::OK(); -} - -Status SchemaChangeForInvertedIndex::_write_inverted_index(int32_t segment_idx, - vectorized::Block* block) { - VLOG_DEBUG << "begin to write inverted index"; - // converter block data - _olap_data_convertor->set_source_content(block, 0, block->rows()); - int idx = 0; - for (auto& inverted_index : _alter_inverted_indexs) { - auto column_name = inverted_index.columns[0]; - auto column_idx = _tablet_schema->field_index(column_name); - if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << column_idx << "]"; - return Status::Error(); - } - auto column = _tablet_schema->column(column_idx); - auto index_id = inverted_index.index_id; - - auto converted_result = _olap_data_convertor->convert_column_data(idx++); - if (converted_result.first != Status::OK()) { - LOG(WARNING) << "failed to convert block, errcode: " << converted_result.first; - return converted_result.first; - } - - auto writer_sign = std::make_pair(segment_idx, index_id); - std::unique_ptr field(FieldFactory::create(column)); - const auto* ptr = (const uint8_t*)converted_result.second->get_data(); - if (converted_result.second->get_nullmap()) { - RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, field.get(), - converted_result.second->get_nullmap(), &ptr, - block->rows())); - } else { - RETURN_IF_ERROR(_add_data(column_name, writer_sign, field.get(), &ptr, block->rows())); - } - } - _olap_data_convertor->clear_source_content(); - - return Status::OK(); -} - Status SchemaChangeHandler::process_alter_tablet_v2(const TAlterTabletReqV2& request) { if (!request.__isset.desc_tbl) { return Status::Error().append( @@ -901,31 +662,6 @@ Status SchemaChangeHandler::process_alter_tablet_v2(const TAlterTabletReqV2& req return res; } -Status SchemaChangeHandler::process_alter_inverted_index(const TAlterInvertedIndexReq& request) { - LOG(INFO) << "begin to do request alter inverted index: tablet_id=" << request.tablet_id - << ", schema_hash=" << request.schema_hash - << ", alter_version=" << request.alter_version; - TabletSharedPtr tablet = - StorageEngine::instance()->tablet_manager()->get_tablet(request.tablet_id); - if (tablet == nullptr) { - LOG(WARNING) << "fail to find tablet. tablet=" << request.tablet_id; - return Status::Error(); - } - - // // Lock schema_change_lock util schema change info is stored in tablet header - // std::unique_lock schema_change_lock(tablet->get_schema_change_lock(), - // std::try_to_lock); - // if (!schema_change_lock.owns_lock()) { - // LOG(WARNING) << "failed to obtain schema change lock. " - // << "tablet=" << request.tablet_id; - // return Status::Error(); - // } - - Status res = _do_process_alter_inverted_index(tablet, request); - LOG(INFO) << "finished alter inverted index process, res=" << res; - return res; -} - std::shared_mutex SchemaChangeHandler::_mutex; std::unordered_set SchemaChangeHandler::_tablet_ids_in_converting; @@ -1265,275 +1001,6 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& return res; } -Status SchemaChangeHandler::_do_process_alter_inverted_index( - TabletSharedPtr tablet, const TAlterInvertedIndexReq& request) { - Status res = Status::OK(); - // TODO(wy): check whether the tablet's max continuous version == request.version - if (tablet->tablet_state() == TABLET_TOMBSTONED || tablet->tablet_state() == TABLET_STOPPED || - tablet->tablet_state() == TABLET_SHUTDOWN) { - LOG(WARNING) << "tablet's state=" << tablet->tablet_state() - << " cannot alter inverted index"; - return Status::Error(); - } - - std::shared_lock base_migration_rlock(tablet->get_migration_lock(), std::try_to_lock); - if (!base_migration_rlock.owns_lock()) { - return Status::Error(); - } - - TabletSchemaSPtr tablet_schema = std::make_shared(); - tablet_schema->copy_from(*tablet->tablet_schema()); - if (!request.columns.empty() && request.columns[0].col_unique_id >= 0) { - tablet_schema->clear_columns(); - for (const auto& column : request.columns) { - tablet_schema->append_column(TabletColumn(column)); - } - } - - // get rowset reader - std::vector rs_readers; - RETURN_IF_ERROR(_get_rowset_readers(tablet, tablet_schema, request, &rs_readers)); - if (request.__isset.is_drop_op && request.is_drop_op) { - // drop index - res = _drop_inverted_index(rs_readers, tablet_schema, tablet, request); - } else { - // add index - res = _add_inverted_index(rs_readers, tablet_schema, tablet, request); - } - - if (!res.ok()) { - LOG(WARNING) << "failed to alter tablet. tablet=" << tablet->full_name(); - return res; - } - - return Status::OK(); -} - -Status SchemaChangeHandler::_get_rowset_readers(TabletSharedPtr tablet, - const TabletSchemaSPtr& tablet_schema, - const TAlterInvertedIndexReq& request, - std::vector* rs_readers) { - Status res = Status::OK(); - std::vector versions_to_be_changed; - std::vector return_columns; - std::vector alter_inverted_indexs; - - if (request.__isset.alter_inverted_indexes) { - alter_inverted_indexs = request.alter_inverted_indexes; - } - - for (auto& inverted_index : alter_inverted_indexs) { - DCHECK_EQ(inverted_index.columns.size(), 1); - auto column_name = inverted_index.columns[0]; - auto idx = tablet_schema->field_index(column_name); - if (idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << idx << "]"; - return Status::Error(); - } - return_columns.emplace_back(idx); - } - - // obtain base tablet's push lock and header write lock to prevent loading data - { - std::lock_guard tablet_lock(tablet->get_push_lock()); - std::lock_guard tablet_wlock(tablet->get_header_lock()); - - do { - RowsetSharedPtr max_rowset; - // get history data to rebuild inverted index and it will check if there is hold in base tablet - res = _get_versions_to_be_changed(tablet, &versions_to_be_changed, &max_rowset); - if (!res.ok()) { - LOG(WARNING) << "fail to get version to be rebuild inverted index. res=" << res; - break; - } - - // should check the max_version >= request.alter_version, if not the rebuild index is useless - if (max_rowset == nullptr || max_rowset->end_version() < request.alter_version) { - LOG(WARNING) << "base tablet's max version=" - << (max_rowset == nullptr ? 0 : max_rowset->end_version()) - << " is less than request version=" << request.alter_version; - res = Status::InternalError( - "base tablet's max version={} is less than request version={}", - (max_rowset == nullptr ? 0 : max_rowset->end_version()), - request.alter_version); - break; - } - - // init one delete handler - int64_t end_version = -1; - for (auto& version : versions_to_be_changed) { - end_version = std::max(end_version, version.second); - } - - auto& all_del_preds = tablet->delete_predicates(); - for (auto& delete_pred : all_del_preds) { - if (delete_pred->version().first > end_version) { - continue; - } - tablet_schema->merge_dropped_columns(tablet->tablet_schema(delete_pred->version())); - } - DeleteHandler delete_handler; - res = delete_handler.init(tablet_schema, all_del_preds, end_version); - if (!res) { - LOG(WARNING) << "init delete handler failed. tablet=" << tablet->full_name() - << ", end_version=" << end_version; - break; - } - - // acquire data sources correspond to history versions - tablet->capture_rs_readers(versions_to_be_changed, rs_readers); - if (rs_readers->size() < 1) { - LOG(WARNING) << "fail to acquire all data sources. " - << "version_num=" << versions_to_be_changed.size() - << ", data_source_num=" << rs_readers->size(); - res = Status::Error(); - break; - } - - // reader_context is stack variables, it's lifetime should keep the same with rs_readers - RowsetReaderContext reader_context; - reader_context.reader_type = ReaderType::READER_ALTER_TABLE; - reader_context.tablet_schema = tablet_schema; - reader_context.need_ordered_result = false; - reader_context.delete_handler = &delete_handler; - reader_context.return_columns = &return_columns; - reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx(); - reader_context.is_unique = tablet->keys_type() == UNIQUE_KEYS; - reader_context.batch_size = ALTER_TABLE_BATCH_SIZE; - reader_context.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); - reader_context.version = Version(0, end_version); - - for (auto& rs_reader : *rs_readers) { - res = rs_reader->init(&reader_context); - if (!res.ok()) { - LOG(WARNING) << "failed to init rowset reader: " << tablet->full_name(); - break; - } - } - } while (false); - } - - return res; -} - -Status SchemaChangeHandler::_drop_inverted_index(std::vector rs_readers, - const TabletSchemaSPtr& tablet_schema, - TabletSharedPtr tablet, - const TAlterInvertedIndexReq& request) { - LOG(INFO) << "begin to drop inverted index"; - Status res = Status::OK(); - - std::vector alter_inverted_indexs; - if (request.__isset.alter_inverted_indexes) { - alter_inverted_indexs = request.alter_inverted_indexes; - } - - for (auto& rs_reader : rs_readers) { - auto rowset_meta = rs_reader->rowset()->rowset_meta(); - auto fs = rowset_meta->fs(); - for (auto i = 0; i < rowset_meta->num_segments(); ++i) { - std::string segment_path = BetaRowset::segment_file_path(tablet->tablet_path(), - rowset_meta->rowset_id(), i); - for (auto& inverted_index : alter_inverted_indexs) { - auto column_name = inverted_index.columns[0]; - auto column_idx = tablet_schema->field_index(column_name); - if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << column_idx - << "]"; - return Status::Error(); - } - auto column = tablet_schema->column(column_idx); - auto index_id = inverted_index.index_id; - - std::string inverted_index_file = - InvertedIndexDescriptor::get_index_file_name(segment_path, index_id); - bool file_exist = false; - fs->exists(inverted_index_file, &file_exist); - if (!file_exist) { - return Status::OK(); - } - LOG(INFO) << "will drop inverted index cid: " << index_id - << ", column_name: " << column_name - << ", inverted_index_file: " << inverted_index_file; - res = fs->delete_file(inverted_index_file); - if (!res.ok()) { - LOG(WARNING) << "failed to delete file: " << inverted_index_file - << ", res: " << res.to_string(); - return res; - } - } - } - } - - return Status::OK(); -} - -Status SchemaChangeHandler::_add_inverted_index(std::vector rs_readers, - const TabletSchemaSPtr& tablet_schema, - TabletSharedPtr tablet, - const TAlterInvertedIndexReq& request) { - LOG(INFO) << "begin to add inverted index, tablet=" << tablet->full_name(); - Status res = Status::OK(); - std::vector alter_inverted_indexs; - if (request.__isset.alter_inverted_indexes) { - alter_inverted_indexs = request.alter_inverted_indexes; - } - - { - std::lock_guard wrlock(_mutex); - _tablet_ids_in_converting.insert(tablet->tablet_id()); - } - - res = _rebuild_inverted_index(rs_readers, tablet_schema, tablet, alter_inverted_indexs); - - { - std::lock_guard wrlock(_mutex); - _tablet_ids_in_converting.erase(tablet->tablet_id()); - } - - // if (res.ok()) { - // // _validate_alter_result should be outside the above while loop. - // // to avoid requiring the header lock twice. - // res = _validate_alter_result(tablet, request); - // } - - if (!res.ok()) { - LOG(WARNING) << "failed to alter add inverte index. tablet=" << tablet->full_name(); - } - return res; -} - -Status SchemaChangeHandler::_rebuild_inverted_index( - const std::vector& rs_readers, const TabletSchemaSPtr& tablet_schema, - TabletSharedPtr tablet, const std::vector& alter_inverted_indexs) { - LOG(INFO) << "begin to rebuild inverted index" - << ", tablet=" << tablet->full_name(); - Status res = Status::OK(); - auto sc_procedure = - std::make_unique(alter_inverted_indexs, tablet_schema); - // read tablet data and write inverted index - for (auto& rs_reader : rs_readers) { - VLOG_TRACE << "begin to read a history rowset. version=" << rs_reader->version().first - << "-" << rs_reader->version().second; - res = sc_procedure->process(rs_reader, nullptr, nullptr, tablet, nullptr); - if (!res.ok() && res.code() != ErrorCode::END_OF_FILE) { - LOG(WARNING) << "failed to process the version." - << " version=" << rs_reader->version().first << "-" - << rs_reader->version().second; - return res; - } - - VLOG_TRACE << "succeed to write inverted index." - << " version=" << rs_reader->version().first << "-" - << rs_reader->version().second; - } - - LOG(INFO) << "finish to write inverted index to tablet: " << tablet->full_name(); - return Status::OK(); -} - bool SchemaChangeHandler::tablet_in_converting(int64_t tablet_id) { std::shared_lock rdlock(_mutex); return _tablet_ids_in_converting.find(tablet_id) != _tablet_ids_in_converting.end(); diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index b74113c8c7..dc1312171e 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -222,44 +222,11 @@ private: std::unique_ptr _mem_tracker; }; -class SchemaChangeForInvertedIndex : public SchemaChange { -public: - explicit SchemaChangeForInvertedIndex(const std::vector& alter_inverted_indexs, - const TabletSchemaSPtr& tablet_schema); - ~SchemaChangeForInvertedIndex() override; - - Status process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer, - TabletSharedPtr new_tablet, TabletSharedPtr base_tablet, - TabletSchemaSPtr base_tablet_schema) override; - -private: - DISALLOW_COPY_AND_ASSIGN(SchemaChangeForInvertedIndex); - Status _write_inverted_index(int32_t segment_idx, vectorized::Block* block); - Status _add_data(const std::string& column_name, - const std::pair& index_writer_sign, Field* field, - const uint8_t** ptr, size_t num_rows); - Status _add_nullable(const std::string& column_name, - const std::pair& index_writer_sign, Field* field, - const uint8_t* null_map, const uint8_t** ptr, size_t num_rows); - - std::vector _alter_inverted_indexs; - TabletSchemaSPtr _tablet_schema; - - // "" -> InvertedIndexColumnWriter - std::unordered_map, - std::unique_ptr> - _inverted_index_builders; - std::vector> _index_metas; - std::unique_ptr _olap_data_convertor; -}; - class SchemaChangeHandler { public: // schema change v2, it will not set alter task in base tablet static Status process_alter_tablet_v2(const TAlterTabletReqV2& request); - static Status process_alter_inverted_index(const TAlterInvertedIndexReq& request); - static std::unique_ptr get_sc_procedure(const BlockChanger& changer, bool sc_sorting, bool sc_directly) { if (sc_sorting) { @@ -310,25 +277,6 @@ private: static Status _parse_request(const SchemaChangeParams& sc_params, BlockChanger* changer, bool* sc_sorting, bool* sc_directly); - static Status _do_process_alter_inverted_index(TabletSharedPtr tablet, - const TAlterInvertedIndexReq& request); - - static Status _get_rowset_readers(TabletSharedPtr tablet, const TabletSchemaSPtr& tablet_schema, - const TAlterInvertedIndexReq& request, - std::vector* rs_readers); - static Status _add_inverted_index(std::vector rs_readers, - const TabletSchemaSPtr& tablet_schema, TabletSharedPtr tablet, - const TAlterInvertedIndexReq& request); - static Status _drop_inverted_index(std::vector rs_readers, - const TabletSchemaSPtr& tablet_schema, - TabletSharedPtr tablet, - const TAlterInvertedIndexReq& request); - - static Status _rebuild_inverted_index( - const std::vector& rs_readers, - const TabletSchemaSPtr& tablet_schema, TabletSharedPtr tablet, - const std::vector& alter_inverted_indexs); - // Initialization Settings for creating a default value static Status _init_column_mapping(ColumnMapping* column_mapping, const TabletColumn& column_schema, const std::string& value); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index ecd6a62a27..ad24dd4b1a 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -43,6 +43,7 @@ #include "olap/rowset/rowset_id_generator.h" #include "olap/rowset/segment_v2/segment.h" #include "olap/tablet.h" +#include "olap/task/index_builder.h" #include "runtime/heartbeat_flags.h" #include "util/countdown_latch.h" @@ -199,6 +200,8 @@ public: bool stopped() { return _stopped; } ThreadPool* get_bg_multiget_threadpool() { return _bg_multi_get_thread_pool.get(); } + Status process_index_change_task(const TAlterInvertedIndexReq& reqest); + private: // Instance should be inited from `static open()` // MUST NOT be called in other circumstances. @@ -286,6 +289,7 @@ private: Status _handle_seg_compaction(BetaRowsetWriter* writer, SegCompactionCandidatesSharedPtr segments); + Status _handle_index_change(IndexBuilderSharedPtr index_builder); void _gc_binlogs(); private: diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 1dec889c53..0cda3978ae 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1283,6 +1283,37 @@ std::vector Tablet::pick_candidate_rowsets_to_base_compaction() return candidate_rowsets; } +std::vector Tablet::pick_candidate_rowsets_to_build_inverted_index( + const std::set& alter_index_uids, bool is_drop_op) { + std::vector candidate_rowsets; + { + std::shared_lock rlock(_meta_lock); + auto has_alter_inverted_index = [&](RowsetSharedPtr rowset) -> bool { + for (const auto& index_id : alter_index_uids) { + if (rowset->tablet_schema()->has_inverted_index_with_index_id(index_id)) { + return true; + } + } + return false; + }; + + for (const auto& [version, rs] : _rs_version_map) { + if (!has_alter_inverted_index(rs) && is_drop_op) { + continue; + } + if (has_alter_inverted_index(rs) && !is_drop_op) { + continue; + } + + if (rs->is_local()) { + candidate_rowsets.push_back(rs); + } + } + } + std::sort(candidate_rowsets.begin(), candidate_rowsets.end(), Rowset::comparator); + return candidate_rowsets; +} + // For http compaction action void Tablet::get_compaction_status(std::string* json_result) { rapidjson::Document root; diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index f1eb005886..1399b4c253 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -201,6 +201,8 @@ public: std::mutex& get_schema_change_lock() { return _schema_change_lock; } + std::mutex& get_build_inverted_index_lock() { return _build_inverted_index_lock; } + // operation for compaction bool can_do_compaction(size_t path_hash, CompactionType compaction_type); uint32_t calc_compaction_score( @@ -248,6 +250,8 @@ public: std::vector pick_candidate_rowsets_to_cumulative_compaction(); std::vector pick_candidate_rowsets_to_base_compaction(); + std::vector pick_candidate_rowsets_to_build_inverted_index( + const std::set& alter_index_uids, bool is_drop_op); void calculate_cumulative_point(); // TODO(ygl): @@ -564,6 +568,7 @@ private: std::mutex _cumulative_compaction_lock; std::mutex _schema_change_lock; std::shared_mutex _migration_lock; + std::mutex _build_inverted_index_lock; // TODO(lingbin): There is a _meta_lock TabletMeta too, there should be a comment to // explain how these two locks work together. diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 60d117cb29..699d938938 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -32,6 +32,7 @@ #include "common/consts.h" #include "common/status.h" #include "exec/tablet_info.h" +#include "olap/inverted_index_parser.h" #include "olap/olap_define.h" #include "olap/types.h" #include "olap/utils.h" @@ -631,6 +632,10 @@ void TabletSchema::append_column(TabletColumn column, bool is_dropped_column) { _num_columns++; } +void TabletSchema::append_index(TabletIndex index) { + _indexes.push_back(std::move(index)); +} + void TabletSchema::clear_columns() { _field_name_to_index.clear(); _field_id_to_index.clear(); @@ -940,6 +945,16 @@ bool TabletSchema::has_inverted_index(int32_t col_unique_id) const { return false; } +bool TabletSchema::has_inverted_index_with_index_id(int32_t index_id) const { + for (size_t i = 0; i < _indexes.size(); i++) { + if (_indexes[i].index_type() == IndexType::INVERTED && _indexes[i].index_id() == index_id) { + return true; + } + } + + return false; +} + const TabletIndex* TabletSchema::get_inverted_index(int32_t col_unique_id) const { // TODO use more efficient impl for (size_t i = 0; i < _indexes.size(); i++) { diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 85c58c02c8..7a1b34745c 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -193,6 +193,7 @@ public: void init_from_pb(const TabletSchemaPB& schema); void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const; void append_column(TabletColumn column, bool is_dropped_column = false); + void append_index(TabletIndex index); // Must make sure the row column is always the last column void add_row_column(); void copy_from(const TabletSchema& tablet_schema); @@ -239,6 +240,7 @@ public: const std::vector& indexes() const { return _indexes; } std::vector get_indexes_for_column(int32_t col_unique_id) const; bool has_inverted_index(int32_t col_unique_id) const; + bool has_inverted_index_with_index_id(int32_t index_id) const; const TabletIndex* get_inverted_index(int32_t col_unique_id) const; bool has_ngram_bf_index(int32_t col_unique_id) const; const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const; diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp b/be/src/olap/task/engine_alter_tablet_task.cpp index df801a6d1b..f2df074494 100644 --- a/be/src/olap/task/engine_alter_tablet_task.cpp +++ b/be/src/olap/task/engine_alter_tablet_task.cpp @@ -54,34 +54,4 @@ Status EngineAlterTabletTask::execute() { return res; } // execute -EngineAlterInvertedIndexTask::EngineAlterInvertedIndexTask( - const TAlterInvertedIndexReq& alter_inverted_index_request) - : _alter_inverted_index_req(alter_inverted_index_request) { - _mem_tracker = std::make_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, - fmt::format("EngineAlterInvertedIndexTask#tabletId={}", - std::to_string(_alter_inverted_index_req.tablet_id)), - config::memory_limitation_per_thread_for_schema_change_bytes); -} - -Status EngineAlterInvertedIndexTask::execute() { - SCOPED_ATTACH_TASK(_mem_tracker); - DorisMetrics::instance()->alter_inverted_index_requests_total->increment(1); - - Status res = SchemaChangeHandler::process_alter_inverted_index(_alter_inverted_index_req); - - if (!res.ok()) { - LOG(WARNING) << "failed to do alter inverted index task. res=" << res - << " tablet_ud=" << _alter_inverted_index_req.tablet_id - << ", schema_hash=" << _alter_inverted_index_req.schema_hash; - DorisMetrics::instance()->alter_inverted_index_requests_failed->increment(1); - return res; - } - - LOG(INFO) << "success to execute alter inverted index. res=" << res - << " tablet_id=" << _alter_inverted_index_req.tablet_id - << ", schema_hash=" << _alter_inverted_index_req.schema_hash; - return res; -} // execute - } // namespace doris diff --git a/be/src/olap/task/engine_alter_tablet_task.h b/be/src/olap/task/engine_alter_tablet_task.h index 0c7851c98c..7e6ae5be9d 100644 --- a/be/src/olap/task/engine_alter_tablet_task.h +++ b/be/src/olap/task/engine_alter_tablet_task.h @@ -43,18 +43,4 @@ private: std::shared_ptr _mem_tracker; }; // EngineTask -class EngineAlterInvertedIndexTask : public EngineTask { -public: - virtual Status execute(); - -public: - EngineAlterInvertedIndexTask(const TAlterInvertedIndexReq& alter_inverted_index_request); - ~EngineAlterInvertedIndexTask() = default; - -private: - const TAlterInvertedIndexReq& _alter_inverted_index_req; - - std::shared_ptr _mem_tracker; -}; - } // namespace doris diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 0540208020..34a6b34a8e 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -577,6 +577,7 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d std::lock_guard base_compaction_lock(tablet->get_base_compaction_lock()); std::lock_guard cumulative_compaction_lock(tablet->get_cumulative_compaction_lock()); std::lock_guard cold_compaction_lock(tablet->get_cold_compaction_lock()); + std::lock_guard build_inverted_index_lock(tablet->get_build_inverted_index_lock()); tablet->set_clone_occurred(true); std::lock_guard push_lock(tablet->get_push_lock()); std::lock_guard rwlock(tablet->get_rowset_update_lock()); diff --git a/be/src/olap/task/engine_index_change_task.cpp b/be/src/olap/task/engine_index_change_task.cpp new file mode 100644 index 0000000000..9fcdedf0f1 --- /dev/null +++ b/be/src/olap/task/engine_index_change_task.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/task/engine_index_change_task.h" + +#include "runtime/memory/mem_tracker.h" +#include "runtime/thread_context.h" + +namespace doris { + +EngineIndexChangeTask::EngineIndexChangeTask( + const TAlterInvertedIndexReq& alter_inverted_index_request) + : _alter_inverted_index_req(alter_inverted_index_request) { + _mem_tracker = std::make_shared( + MemTrackerLimiter::Type::SCHEMA_CHANGE, + fmt::format("EngineIndexChangeTask#tabletId={}", + std::to_string(_alter_inverted_index_req.tablet_id)), + config::memory_limitation_per_thread_for_schema_change_bytes); +} + +Status EngineIndexChangeTask::execute() { + SCOPED_ATTACH_TASK(_mem_tracker); + DorisMetrics::instance()->alter_inverted_index_requests_total->increment(1); + uint64_t start = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + Status res = StorageEngine::instance()->process_index_change_task(_alter_inverted_index_req); + + if (!res.ok()) { + LOG(WARNING) << "failed to do index change task. res=" << res + << " tablet_id=" << _alter_inverted_index_req.tablet_id + << ", job_id=" << _alter_inverted_index_req.job_id + << ", schema_hash=" << _alter_inverted_index_req.schema_hash; + DorisMetrics::instance()->alter_inverted_index_requests_failed->increment(1); + return res; + } + + uint64_t end = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + LOG(INFO) << "success to execute index change task. res=" << res + << " tablet_id=" << _alter_inverted_index_req.tablet_id + << ", job_id=" << _alter_inverted_index_req.job_id + << ", schema_hash=" << _alter_inverted_index_req.schema_hash + << ", start time=" << start << ", end time=" << end + << ", cost time=" << (end - start); + return res; +} // execute + +} // namespace doris diff --git a/be/src/olap/task/engine_index_change_task.h b/be/src/olap/task/engine_index_change_task.h new file mode 100644 index 0000000000..5021bacc0a --- /dev/null +++ b/be/src/olap/task/engine_index_change_task.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gen_cpp/AgentService_types.h" +#include "olap/olap_define.h" +#include "olap/task/engine_task.h" + +namespace doris { + +// base class for storage engine +// add "Engine" as task prefix to prevent duplicate name with agent task +class EngineIndexChangeTask : public EngineTask { +public: + Status execute() override; + +public: + EngineIndexChangeTask(const TAlterInvertedIndexReq& alter_inverted_index_request); + ~EngineIndexChangeTask() = default; + +private: + const TAlterInvertedIndexReq& _alter_inverted_index_req; + + std::shared_ptr _mem_tracker; +}; // EngineTask + +} // namespace doris diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp new file mode 100644 index 0000000000..c60b2b122c --- /dev/null +++ b/be/src/olap/task/index_builder.cpp @@ -0,0 +1,484 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/task/index_builder.h" + +#include "common/status.h" +#include "olap/rowset/beta_rowset.h" +#include "olap/rowset/rowset_writer.h" +#include "olap/rowset/rowset_writer_context.h" +#include "olap/segment_loader.h" +#include "olap/storage_engine.h" +#include "olap/tablet_schema.h" +#include "runtime/memory/mem_tracker.h" +#include "runtime/thread_context.h" + +namespace doris { + +IndexBuilder::IndexBuilder(const TabletSharedPtr& tablet, const std::vector& columns, + const std::vector exist_indexes, + const std::vector& alter_inverted_indexes, + bool is_drop_op) + : _tablet(tablet), + _columns(columns), + _exist_indexes(exist_indexes), + _alter_inverted_indexes(alter_inverted_indexes), + _is_drop_op(is_drop_op) { + _olap_data_convertor = std::make_unique(); +} + +IndexBuilder::~IndexBuilder() { + _olap_data_convertor.reset(); + _inverted_index_builders.clear(); +} + +Status IndexBuilder::init() { + for (auto inverted_index : _alter_inverted_indexes) { + _alter_index_ids.insert(inverted_index.index_id); + } + return Status::OK(); +} + +Status IndexBuilder::update_inverted_index_info() { + // just do link files + LOG(INFO) << "begin to update_inverted_index_info, tablet=" << _tablet->tablet_id() + << ", is_drop_op=" << _is_drop_op; + for (auto i = 0; i < _input_rowsets.size(); ++i) { + auto input_rowset = _input_rowsets[i]; + TabletSchemaSPtr output_rs_tablet_schema = std::make_shared(); + auto input_rs_tablet_schema = input_rowset->tablet_schema(); + output_rs_tablet_schema->copy_from(*input_rs_tablet_schema); + if (_is_drop_op) { + output_rs_tablet_schema->update_indexes_from_thrift(_exist_indexes); + } else { + for (auto t_inverted_index : _alter_inverted_indexes) { + TabletIndex index; + index.init_from_thrift(t_inverted_index, *input_rs_tablet_schema); + output_rs_tablet_schema->append_index(std::move(index)); + } + } + // construct input rowset reader + RowsetReaderSharedPtr input_rs_reader; + RETURN_IF_ERROR(input_rowset->create_reader(&input_rs_reader)); + + // construct output rowset writer + std::unique_ptr output_rs_writer; + RowsetWriterContext context; + context.version = input_rs_reader->version(); + context.rowset_state = VISIBLE; + context.segments_overlap = input_rs_reader->rowset()->rowset_meta()->segments_overlap(); + context.tablet_schema = output_rs_tablet_schema; + context.newest_write_timestamp = input_rs_reader->newest_write_timestamp(); + context.fs = input_rs_reader->rowset()->rowset_meta()->fs(); + Status status = _tablet->create_rowset_writer(context, &output_rs_writer); + if (!status.ok()) { + return Status::Error(); + } + std::set alter_column_ids_set = + _rowset_alter_index_column_ids[input_rowset->rowset_id().to_string()]; + RETURN_IF_ERROR(input_rowset->link_files_to(_tablet->tablet_path(), + output_rs_writer->rowset_id(), 0, + &alter_column_ids_set)); // build output rowset + + auto input_rowset_meta = input_rowset->rowset_meta(); + RowsetMetaSharedPtr rowset_meta = std::make_shared(); + rowset_meta->set_num_rows(input_rowset_meta->num_rows()); + rowset_meta->set_total_disk_size(input_rowset_meta->total_disk_size()); + rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); + rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size()); + rowset_meta->set_empty(input_rowset_meta->empty()); + rowset_meta->set_num_segments(input_rowset_meta->num_segments()); + rowset_meta->set_segments_overlap(input_rowset_meta->segments_overlap()); + rowset_meta->set_rowset_state(input_rowset_meta->rowset_state()); + std::vector key_bounds; + input_rowset->get_segments_key_bounds(&key_bounds); + rowset_meta->set_segments_key_bounds(key_bounds); + auto output_rowset = output_rs_writer->manual_build(rowset_meta); + if (input_rowset_meta->has_delete_predicate()) { + output_rowset->rowset_meta()->set_delete_predicate( + input_rowset_meta->delete_predicate()); + } + _output_rowsets.push_back(output_rowset); + } + + return Status::OK(); +} + +Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta, + std::vector& segments) { + if (_is_drop_op) { + // delete invertd index file by gc thread when gc input rowset + return Status::OK(); + } else { + // create inverted index writer + std::string segment_dir = _tablet->tablet_path(); + auto fs = output_rowset_meta->fs(); + auto output_rowset_schema = output_rowset_meta->tablet_schema(); + for (auto& seg_ptr : segments) { + std::string segment_filename = fmt::format( + "{}_{}.dat", output_rowset_meta->rowset_id().to_string(), seg_ptr->id()); + std::vector return_columns; + std::vector> inverted_index_writer_signs; + _olap_data_convertor->reserve(_alter_inverted_indexes.size()); + // create inverted index writer + for (auto i = 0; i < _alter_inverted_indexes.size(); ++i) { + auto inverted_index = _alter_inverted_indexes[i]; + DCHECK_EQ(inverted_index.columns.size(), 1); + auto index_id = inverted_index.index_id; + auto column_name = inverted_index.columns[0]; + auto column_idx = output_rowset_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx + << "]"; + continue; + } + auto column = output_rowset_schema->column(column_idx); + DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id)); + _olap_data_convertor->add_column_data_convertor(column); + return_columns.emplace_back(column_idx); + std::unique_ptr field(FieldFactory::create(column)); + auto index_meta = output_rowset_schema->get_inverted_index(column.unique_id()); + std::unique_ptr inverted_index_builder; + try { + RETURN_IF_ERROR(segment_v2::InvertedIndexColumnWriter::create( + field.get(), &inverted_index_builder, segment_filename, segment_dir, + index_meta, fs)); + } catch (const std::exception& e) { + LOG(WARNING) << "CLuceneError occured: " << e.what(); + return Status::Error(); + } + + if (inverted_index_builder) { + auto writer_sign = std::make_pair(seg_ptr->id(), index_id); + _inverted_index_builders.insert( + std::make_pair(writer_sign, std::move(inverted_index_builder))); + inverted_index_writer_signs.push_back(writer_sign); + } + } + + // create iterator for each segment + StorageReadOptions read_options; + OlapReaderStatistics stats; + read_options.stats = &stats; + read_options.tablet_schema = output_rowset_schema; + std::unique_ptr schema = + std::make_unique(output_rowset_schema->columns(), return_columns); + std::unique_ptr iter; + auto res = seg_ptr->new_iterator(*schema, read_options, &iter); + if (!res.ok()) { + LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() + << "]: " << res.to_string(); + return Status::Error(); + } + + std::shared_ptr block = std::make_shared( + output_rowset_schema->create_block(return_columns)); + while (true) { + auto st = iter->next_batch(block.get()); + if (!st.ok()) { + if (st.is()) { + break; + } + LOG(WARNING) + << "failed to read next block when schema change for inverted index." + << ", err=" << st.to_string(); + } + + // write inverted index data + if (_write_inverted_index_data(output_rowset_schema, iter->data_id(), + block.get()) != Status::OK()) { + LOG(WARNING) << "failed to write block."; + return Status::Error(); + } + block->clear_column_data(); + } + + // finish write inverted index, flush data to compound file + for (auto& writer_sign : inverted_index_writer_signs) { + try { + if (_inverted_index_builders[writer_sign]) { + _inverted_index_builders[writer_sign]->finish(); + } + } catch (const std::exception& e) { + LOG(WARNING) << "CLuceneError occured: " << e.what(); + return Status::Error(); + } + } + + _olap_data_convertor->reset(); + } + _inverted_index_builders.clear(); + LOG(INFO) << "all row nums. source_rows=" << output_rowset_meta->num_rows(); + } + + return Status::OK(); +} + +Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, int32_t segment_idx, + vectorized::Block* block) { + VLOG_DEBUG << "begin to write inverted index"; + // converter block data + _olap_data_convertor->set_source_content(block, 0, block->rows()); + for (auto i = 0; i < _alter_inverted_indexes.size(); ++i) { + auto inverted_index = _alter_inverted_indexes[i]; + auto index_id = inverted_index.index_id; + auto converted_result = _olap_data_convertor->convert_column_data(i); + if (converted_result.first != Status::OK()) { + LOG(WARNING) << "failed to convert block, errcode: " << converted_result.first; + return converted_result.first; + } + + auto column_name = inverted_index.columns[0]; + auto column_idx = tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx << "]"; + continue; + } + auto column = tablet_schema->column(column_idx); + auto writer_sign = std::make_pair(segment_idx, index_id); + std::unique_ptr field(FieldFactory::create(column)); + const auto* ptr = (const uint8_t*)converted_result.second->get_data(); + if (converted_result.second->get_nullmap()) { + RETURN_IF_ERROR(_add_nullable(column_name, writer_sign, field.get(), + converted_result.second->get_nullmap(), &ptr, + block->rows())); + } else { + RETURN_IF_ERROR(_add_data(column_name, writer_sign, field.get(), &ptr, block->rows())); + } + } + _olap_data_convertor->clear_source_content(); + + return Status::OK(); +} + +Status IndexBuilder::_add_nullable(const std::string& column_name, + const std::pair& index_writer_sign, + Field* field, const uint8_t* null_map, const uint8_t** ptr, + size_t num_rows) { + size_t offset = 0; + auto next_run_step = [&]() { + size_t step = 1; + for (auto i = offset + 1; i < num_rows; ++i) { + if (null_map[offset] == null_map[i]) { + step++; + } else { + break; + } + } + return step; + }; + + try { + do { + auto step = next_run_step(); + if (null_map[offset]) { + RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_nulls(step)); + } else { + if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + DCHECK(field->get_sub_field_count() == 1); + const auto* col_cursor = reinterpret_cast(*ptr); + RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_array_values( + field->get_sub_field(0)->size(), col_cursor, step)); + } else { + RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_values( + column_name, *ptr, step)); + } + } + *ptr += field->size() * step; + offset += step; + } while (offset < num_rows); + } catch (const std::exception& e) { + LOG(WARNING) << "CLuceneError occured: " << e.what(); + return Status::Error(); + } + + return Status::OK(); +} + +Status IndexBuilder::_add_data(const std::string& column_name, + const std::pair& index_writer_sign, Field* field, + const uint8_t** ptr, size_t num_rows) { + try { + if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { + DCHECK(field->get_sub_field_count() == 1); + const auto* col_cursor = reinterpret_cast(*ptr); + RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_array_values( + field->get_sub_field(0)->size(), col_cursor, num_rows)); + } else { + RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_values( + column_name, *ptr, num_rows)); + } + } catch (const std::exception& e) { + LOG(WARNING) << "CLuceneError occured: " << e.what(); + return Status::Error(); + } + + return Status::OK(); +} + +Status IndexBuilder::handle_inverted_index_data() { + LOG(INFO) << "begint to handle_inverted_index_data"; + DCHECK(_input_rowsets.size() == _output_rowsets.size()); + for (auto i = 0; i < _output_rowsets.size(); ++i) { + SegmentCacheHandle segment_cache_handle; + RETURN_IF_ERROR(SegmentLoader::instance()->load_segments( + std::static_pointer_cast(_output_rowsets[i]), &segment_cache_handle)); + auto output_rowset_meta = _output_rowsets[i]->rowset_meta(); + auto& segments = segment_cache_handle.get_segments(); + RETURN_IF_ERROR(handle_single_rowset(output_rowset_meta, segments)); + } + return Status::OK(); +} + +Status IndexBuilder::do_build_inverted_index() { + LOG(INFO) << "begine to do_build_inverted_index, tablet=" << _tablet->tablet_id() + << ", is_drop_op=" << _is_drop_op; + if (_alter_inverted_indexes.empty()) { + return Status::OK(); + } + + std::unique_lock schema_change_lock(_tablet->get_schema_change_lock(), + std::try_to_lock); + if (!schema_change_lock.owns_lock()) { + return Status::Error("try schema_change_lock failed"); + } + // Check executing serially with compaction task. + std::unique_lock base_compaction_lock(_tablet->get_base_compaction_lock(), + std::try_to_lock); + if (!base_compaction_lock.owns_lock()) { + return Status::Error("try base_compaction_lock failed"); + } + std::unique_lock cumu_compaction_lock(_tablet->get_cumulative_compaction_lock(), + std::try_to_lock); + if (!cumu_compaction_lock.owns_lock()) { + return Status::Error("try cumu_compaction_lock failed"); + } + + std::unique_lock cold_compaction_lock(_tablet->get_cold_compaction_lock(), + std::try_to_lock); + if (!cold_compaction_lock.owns_lock()) { + return Status::Error("try cold_compaction_lock failed"); + } + + std::unique_lock build_inverted_index_lock(_tablet->get_build_inverted_index_lock(), + std::try_to_lock); + if (!build_inverted_index_lock.owns_lock()) { + LOG(WARNING) << "failed to obtain build inverted index lock. " + << "tablet=" << _tablet->tablet_id(); + return Status::Error(); + } + + if (_tablet->get_clone_occurred()) { + _tablet->set_clone_occurred(false); + return Status::Error(); + } + + _input_rowsets = + _tablet->pick_candidate_rowsets_to_build_inverted_index(_alter_index_ids, _is_drop_op); + if (_input_rowsets.empty()) { + LOG(INFO) << "_input_rowsets is empty"; + return Status::OK(); + } + + _calc_alter_column_ids(); + + auto st = update_inverted_index_info(); + if (!st.ok()) { + LOG(WARNING) << "failed to update_inverted_index_info. " + << "tablet=" << _tablet->tablet_id() << ", error=" << st; + gc_output_rowset(); + } + + // create inverted index file for output rowset + st = handle_inverted_index_data(); + if (!st.ok()) { + LOG(WARNING) << "failed to handle_inverted_index_data. " + << "tablet=" << _tablet->tablet_id() << ", error=" << st; + gc_output_rowset(); + } + + // modify rowsets in memory + st = modify_rowsets(); + if (!st.ok()) { + LOG(WARNING) << "failed to modify rowsets in memory. " + << "tablet=" << _tablet->tablet_id() << ", error=" << st; + gc_output_rowset(); + } + return st; +} + +Status IndexBuilder::modify_rowsets(const Merger::Statistics* stats) { + if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write()) { + std::lock_guard rwlock(_tablet->get_rowset_update_lock()); + std::shared_lock wrlock(_tablet->get_header_lock()); + for (auto rowset_ptr : _output_rowsets) { + RETURN_IF_ERROR(_tablet->update_delete_bitmap_without_lock(rowset_ptr)); + } + RETURN_IF_ERROR(_tablet->modify_rowsets(_output_rowsets, _input_rowsets, true)); + } else { + std::lock_guard wrlock(_tablet->get_header_lock()); + RETURN_IF_ERROR(_tablet->modify_rowsets(_output_rowsets, _input_rowsets, true)); + } + + { + std::shared_lock rlock(_tablet->get_header_lock()); + _tablet->save_meta(); + } + return Status::OK(); +} + +void IndexBuilder::gc_output_rowset() { + for (auto output_rowset : _output_rowsets) { + if (!output_rowset->is_local()) { + Tablet::erase_pending_remote_rowset(output_rowset->rowset_id().to_string()); + _tablet->record_unused_remote_rowset(output_rowset->rowset_id(), + output_rowset->rowset_meta()->resource_id(), + output_rowset->num_segments()); + return; + } + StorageEngine::instance()->add_unused_rowset(output_rowset); + } +} + +Status IndexBuilder::_calc_alter_column_ids() { + for (auto& rs : _input_rowsets) { + RowsetId rowset_id = rs->rowset_id(); + auto rs_tablet_schema = rs->tablet_schema(); + std::set alter_column_uids; + for (auto inverted_index : _alter_inverted_indexes) { + auto column_name = inverted_index.columns[0]; + auto column_idx = rs_tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx + << "]"; + continue; + } + auto column = rs_tablet_schema->column(column_idx); + alter_column_uids.insert(column.unique_id()); + } + _rowset_alter_index_column_ids.insert( + std::make_pair(rowset_id.to_string(), alter_column_uids)); + } + + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h new file mode 100644 index 0000000000..562cb1148d --- /dev/null +++ b/be/src/olap/task/index_builder.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "olap/merger.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/inverted_index_writer.h" +#include "olap/tablet.h" +#include "olap/tablet_meta.h" +#include "olap/utils.h" +#include "vec/olap/olap_data_convertor.h" + +namespace doris { + +class RowsetWriter; + +using RowsetWriterUniquePtr = std::unique_ptr; + +class IndexBuilder { +public: + IndexBuilder(const TabletSharedPtr& tablet, const std::vector& columns, + const std::vector exist_indexes, + const std::vector& alter_inverted_indexes, + bool is_drop_op = false); + ~IndexBuilder(); + + Status init(); + Status do_build_inverted_index(); + Status update_inverted_index_info(); + Status handle_inverted_index_data(); + Status handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta, + std::vector& segments); + Status modify_rowsets(const Merger::Statistics* stats = nullptr); + void gc_output_rowset(); + +private: + Status _write_inverted_index_data(TabletSchemaSPtr tablet_schema, int32_t segment_idx, + vectorized::Block* block); + Status _add_data(const std::string& column_name, + const std::pair& index_writer_sign, Field* field, + const uint8_t** ptr, size_t num_rows); + Status _add_nullable(const std::string& column_name, + const std::pair& index_writer_sign, Field* field, + const uint8_t* null_map, const uint8_t** ptr, size_t num_rows); + + Status _calc_alter_column_ids(); + +private: + TabletSharedPtr _tablet; + std::vector _columns; + std::vector _exist_indexes; + std::vector _alter_inverted_indexes; + bool _is_drop_op; + std::unordered_map> _rowset_alter_index_column_ids; + std::set _alter_index_ids; + std::vector _input_rowsets; + std::vector _output_rowsets; + std::vector _input_rs_readers; + std::unique_ptr _olap_data_convertor; + // "" -> InvertedIndexColumnWriter + std::unordered_map, + std::unique_ptr> + _inverted_index_builders; +}; + +using IndexBuilderSharedPtr = std::shared_ptr; + +} // namespace doris diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h index e13008536a..a16a37e3cd 100644 --- a/be/test/testutil/mock_rowset.h +++ b/be/test/testutil/mock_rowset.h @@ -29,8 +29,8 @@ class MockRowset : public Rowset { Status remove() override { return Status::NotSupported("MockRowset not support this method."); } - Status link_files_to(const std::string& dir, RowsetId new_rowset_id, - size_t start_seg_id) override { + Status link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t start_seg_id, + std::set* without_index_column_uids) override { return Status::NotSupported("MockRowset not support this method."); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 6b00e97412..e06de15db0 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -62,9 +62,11 @@ public final class FeMetaVersion { public static final int VERSION_120 = 120; // For BackendHbResponse node type public static final int VERSION_121 = 121; + // For IndexChangeJob + public static final int VERSION_122 = 122; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_121; + public static final int VERSION_CURRENT = VERSION_122; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 0a1dcc5afd..305e033427 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -807,6 +807,7 @@ nonterminal Map key_value_map, opt_key_value_map, opt_key_value_ opt_ext_properties, opt_enable_feature_properties, properties; nonterminal ColumnDef column_definition; nonterminal IndexDef index_definition; +nonterminal IndexDef build_index_definition; nonterminal ArrayList column_definition_list; nonterminal ArrayList index_definition_list; nonterminal AggregateType opt_agg_type; @@ -1959,6 +1960,10 @@ create_stmt ::= {: RESULT = new CreatePolicyStmt(PolicyTypeEnum.STORAGE, ifNotExists, policyName, properties); :} + | KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName opt_partition_names:partitionNames + {: + RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true), false))); + :} ; channel_desc_list ::= @@ -3466,6 +3471,13 @@ index_definition ::= :} ; +build_index_definition ::= + KW_INDEX ident:indexName opt_partition_names:partitionNames + {: + RESULT = new IndexDef(indexName, partitionNames, true); + :} + ; + opt_is_allow_null ::= {: RESULT = true; @@ -4008,6 +4020,11 @@ show_param ::= {: RESULT = new ShowQueryStatsStmt(dbTblName, true, true); :} + /* show build index job */ + | KW_BUILD KW_INDEX opt_db:db opt_wild_where order_by_clause:orderByClause limit_clause:limitClause + {: + RESULT = new ShowBuildIndexStmt(db, parser.where, orderByClause, limitClause); + :} ; opt_tmp ::= diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java new file mode 100644 index 0000000000..0709f6282f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java @@ -0,0 +1,439 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.AlterInvertedIndexTask; +import org.apache.doris.thrift.TColumn; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + + +public class IndexChangeJob implements Writable { + private static final Logger LOG = LogManager.getLogger(IndexChangeJob.class); + + + public enum JobState { + // CHECKSTYLE OFF + WAITING_TXN, // waiting for previous txns to be finished + // CHECKSTYLE ON + RUNNING, // waiting for inverted index tasks finished. + FINISHED, // job is done + CANCELLED; // job is cancelled + + public boolean isFinalState() { + return this == JobState.FINISHED || this == JobState.CANCELLED; + } + } + + @SerializedName(value = "jobId") + private long jobId; + @SerializedName(value = "jobState") + private JobState jobState; + + @SerializedName(value = "dbId") + private long dbId; + @SerializedName(value = "tableId") + private long tableId; + @SerializedName(value = "tableName") + private String tableName; + @SerializedName(value = "partitionId") + private long partitionId; + @SerializedName(value = "partitionName") + private String partitionName; + + @SerializedName(value = "errMsg") + private String errMsg = ""; + @SerializedName(value = "createTimeMs") + private long createTimeMs = -1; + @SerializedName(value = "finishedTimeMs") + private long finishedTimeMs = -1; + @SerializedName(value = "watershedTxnId") + protected long watershedTxnId = -1; + + @SerializedName(value = "isDropOp") + private boolean isDropOp = false; + @SerializedName(value = "alterInvertedIndexes") + private List alterInvertedIndexes = null; + @SerializedName(value = "originIndexId") + private long originIndexId; + @SerializedName(value = "invertedIndexBatchTask") + AgentBatchTask invertedIndexBatchTask = new AgentBatchTask(); + + public IndexChangeJob() { + this.jobId = -1; + this.dbId = -1; + this.tableId = -1; + this.tableName = ""; + + this.createTimeMs = System.currentTimeMillis(); + this.jobState = JobState.WAITING_TXN; + } + + public IndexChangeJob(long jobId, long dbId, long tableId, String tableName) { + this.jobId = jobId; + this.dbId = dbId; + this.tableId = tableId; + this.tableName = tableName; + + this.createTimeMs = System.currentTimeMillis(); + this.jobState = JobState.WAITING_TXN; + } + + public long getJobId() { + return jobId; + } + + public long getOriginIndexId() { + return originIndexId; + } + + public JobState getJobState() { + return jobState; + } + + public void setJobState(JobState jobState) { + this.jobState = jobState; + } + + public void setPartitionId(long partitionId) { + this.partitionId = partitionId; + } + + public void setPartitionName(String partitionName) { + this.partitionName = partitionName; + } + + public void setOriginIndexId(long originIndexId) { + this.originIndexId = originIndexId; + } + + public void setAlterInvertedIndexInfo(boolean isDropOp, List alterInvertedIndexes) { + this.isDropOp = isDropOp; + this.alterInvertedIndexes = alterInvertedIndexes; + } + + public boolean hasSameAlterInvertedIndex(boolean isDropOp, List inputAlterInvertedIndexes) { + if (this.isDropOp == isDropOp) { + for (Index inputIndex : inputAlterInvertedIndexes) { + for (Index existIndex : this.alterInvertedIndexes) { + if (inputIndex.getIndexId() == existIndex.getIndexId()) { + return true; + } + } + } + } + return false; + } + + public long getDbId() { + return dbId; + } + + public long getTableId() { + return tableId; + } + + public String getTableName() { + return tableName; + } + + public String getPartitionName() { + return partitionName; + } + + public boolean isExpire() { + return isDone() && (System.currentTimeMillis() - finishedTimeMs) / 1000 > Config.history_job_keep_max_second; + } + + public boolean isDone() { + return jobState.isFinalState(); + } + + public long getFinishedTimeMs() { + return finishedTimeMs; + } + + public void setFinishedTimeMs(long finishedTimeMs) { + this.finishedTimeMs = finishedTimeMs; + } + + /** + * The keyword 'synchronized' only protects 2 methods: + * run() and cancel() + * Only these 2 methods can be visited by different thread(internal working thread and user connection thread) + * So using 'synchronized' to make sure only one thread can run the job at one time. + * + * lock order: + * synchronized + * db lock + */ + public synchronized void run() { + try { + this.watershedTxnId = Env.getCurrentGlobalTransactionMgr() + .getTransactionIDGenerator().getNextTransactionId(); + switch (jobState) { + case WAITING_TXN: + runWaitingTxnJob(); + break; + case RUNNING: + runRunningJob(); + break; + default: + break; + } + } catch (AlterCancelException e) { + cancelImpl(e.getMessage()); + } + } + + public final synchronized boolean cancel(String errMsg) { + return cancelImpl(errMsg); + } + + // Check whether transactions of the given database which txnId is less than 'watershedTxnId' are finished. + protected boolean isPreviousLoadFinished() throws AnalysisException { + return Env.getCurrentGlobalTransactionMgr().isPreviousTransactionsFinished( + watershedTxnId, dbId, Lists.newArrayList(tableId)); + } + + protected void runWaitingTxnJob() throws AlterCancelException { + Preconditions.checkState(jobState == JobState.WAITING_TXN, jobState); + try { + if (!isPreviousLoadFinished()) { + LOG.info("wait transactions before {} to be finished, inverted index job: {}", watershedTxnId, jobId); + return; + } + } catch (AnalysisException e) { + throw new AlterCancelException(e.getMessage()); + } + + LOG.info("previous transactions are all finished, begin to send build or delete inverted index file tasks." + + "job: {}, is delete: {}", jobId, isDropOp); + Database db = Env.getCurrentInternalCatalog() + .getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist")); + OlapTable olapTable; + try { + olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); + } catch (MetaNotFoundException e) { + throw new AlterCancelException(e.getMessage()); + } + + olapTable.readLock(); + try { + List originSchemaColumns = olapTable.getSchemaByIndexId(originIndexId, true); + for (Column col : originSchemaColumns) { + TColumn tColumn = col.toThrift(); + col.setIndexFlag(tColumn, olapTable); + } + int originSchemaHash = olapTable.getSchemaHashByIndexId(originIndexId); + Partition partition = olapTable.getPartition(partitionId); + MaterializedIndex origIdx = partition.getIndex(originIndexId); + for (Tablet originTablet : origIdx.getTablets()) { + long taskSignature = Env.getCurrentEnv().getNextId(); + long originTabletId = originTablet.getId(); + List originReplicas = originTablet.getReplicas(); + for (Replica originReplica : originReplicas) { + if (originReplica.getBackendId() < 0) { + LOG.warn("replica:{}, backendId: {}", originReplica, originReplica.getBackendId()); + throw new AlterCancelException("originReplica:" + originReplica.getId() + + " backendId < 0"); + } + AlterInvertedIndexTask alterInvertedIndexTask = new AlterInvertedIndexTask( + originReplica.getBackendId(), db.getId(), olapTable.getId(), + partitionId, originIndexId, originTabletId, + originSchemaHash, olapTable.getIndexes(), + alterInvertedIndexes, originSchemaColumns, + isDropOp, taskSignature); + invertedIndexBatchTask.addTask(alterInvertedIndexTask); + } + } // end for tablet + + LOG.info("invertedIndexBatchTask:{}", invertedIndexBatchTask); + AgentTaskQueue.addBatchTask(invertedIndexBatchTask); + AgentTaskExecutor.submit(invertedIndexBatchTask); + } finally { + olapTable.readUnlock(); + } + this.jobState = JobState.RUNNING; + LOG.info("transfer inverted index job {} state to {}", jobId, this.jobState); + } + + protected void runRunningJob() throws AlterCancelException { + Preconditions.checkState(jobState == JobState.RUNNING, jobState); + + if (!invertedIndexBatchTask.isFinished()) { + LOG.info("inverted index tasks not finished. job: {}, partitionId: {}", jobId, partitionId); + // TODO: task failed limit + return; + } + + this.jobState = JobState.FINISHED; + this.finishedTimeMs = System.currentTimeMillis(); + + Env.getCurrentEnv().getEditLog().logIndexChangeJob(this); + LOG.info("inverted index job finished: {}", jobId); + } + + protected boolean cancelImpl(String errMsg) { + return true; + } + + public void replay(IndexChangeJob replayedJob) { + try { + IndexChangeJob replayedIndexChangeJob = (IndexChangeJob) replayedJob; + switch (replayedJob.jobState) { + case WAITING_TXN: + replayCreateJob(replayedIndexChangeJob); + break; + case FINISHED: + replayRunningJob(replayedIndexChangeJob); + break; + case CANCELLED: + // TODO: + // replayCancelled(replayedIndexChangeJob); + break; + default: + break; + } + } catch (MetaNotFoundException e) { + LOG.warn("[INCONSISTENT META] replay inverted index job failed {}", replayedJob.getJobId(), e); + } + } + + private void replayCreateJob(IndexChangeJob replayedJob) throws MetaNotFoundException { + // do nothing, resend inverted index task to be + this.watershedTxnId = replayedJob.watershedTxnId; + this.jobState = JobState.WAITING_TXN; + LOG.info("replay waiting_txn inverted index job: {}, table id: {}", jobId, tableId); + } + + private void replayRunningJob(IndexChangeJob replayedJob) { + // do nothing, finish inverted index task + this.jobState = JobState.FINISHED; + this.finishedTimeMs = replayedJob.finishedTimeMs; + LOG.info("replay finished inverted index job: {} table id: {}", jobId, tableId); + } + + public static IndexChangeJob read(DataInput in) throws IOException { + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_122) { + IndexChangeJob job = new IndexChangeJob(); + job.readFields(in); + return job; + } else { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, IndexChangeJob.class); + } + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this, IndexChangeJob.class); + Text.writeString(out, json); + } + + protected void readFields(DataInput in) throws IOException { + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_122) { + jobId = in.readLong(); + jobState = JobState.valueOf(Text.readString(in)); + dbId = in.readLong(); + tableId = in.readLong(); + tableName = Text.readString(in); + partitionId = in.readLong(); + partitionName = Text.readString(in); + errMsg = Text.readString(in); + createTimeMs = in.readLong(); + finishedTimeMs = in.readLong(); + watershedTxnId = in.readLong(); + isDropOp = in.readBoolean(); + alterInvertedIndexes = Lists.newArrayList(); + int alterInvertedIndexesSize = in.readInt(); + for (int i = 0; i < alterInvertedIndexesSize; ++i) { + Index alterIndex = Index.read(in); + alterInvertedIndexes.add(alterIndex); + } + originIndexId = in.readLong(); + invertedIndexBatchTask = new AgentBatchTask(); + } + } + + public String getAlterInvertedIndexesInfo() { + String info = null; + List infoList = Lists.newArrayList(); + String invertedIndexChangeInfo = ""; + for (Index invertedIndex : alterInvertedIndexes) { + invertedIndexChangeInfo += "[" + (isDropOp ? "DROP " : "ADD ") + invertedIndex.toString() + "], "; + } + infoList.add(invertedIndexChangeInfo); + info = Joiner.on(", ").join(infoList.subList(0, infoList.size())); + return info; + } + + public void getInfo(List> infos) { + // calc progress first. all index share the same process + String progress = FeConstants.null_string; + if (jobState == JobState.RUNNING && invertedIndexBatchTask.getTaskNum() > 0) { + progress = invertedIndexBatchTask.getFinishedTaskNum() + "/" + invertedIndexBatchTask.getTaskNum(); + } + + List info = Lists.newArrayList(); + info.add(jobId); + info.add(tableName); + info.add(partitionName); + info.add(getAlterInvertedIndexesInfo()); + info.add(TimeUtils.longToTimeStringWithms(createTimeMs)); + info.add(TimeUtils.longToTimeStringWithms(finishedTimeMs)); + info.add(watershedTxnId); + info.add(jobState.name()); + info.add(errMsg); + info.add(progress); + + infos.add(info); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 91c8d13150..f16f45c968 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -20,6 +20,7 @@ package org.apache.doris.alter; import org.apache.doris.analysis.AddColumnClause; import org.apache.doris.analysis.AddColumnsClause; import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.BuildIndexClause; import org.apache.doris.analysis.CancelAlterTableStmt; import org.apache.doris.analysis.CancelStmt; import org.apache.doris.analysis.ColumnPosition; @@ -27,6 +28,7 @@ import org.apache.doris.analysis.CreateIndexClause; import org.apache.doris.analysis.DropColumnClause; import org.apache.doris.analysis.DropIndexClause; import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; import org.apache.doris.analysis.ModifyColumnClause; import org.apache.doris.analysis.ModifyTablePropertiesClause; import org.apache.doris.analysis.ReorderColumnsClause; @@ -72,6 +74,7 @@ import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.IdGeneratorUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.AlterLightSchemaChangeInfo; @@ -112,6 +115,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -133,6 +137,13 @@ public class SchemaChangeHandler extends AlterHandler { public final Map runnableSchemaChangeJobV2 = Maps.newConcurrentMap(); + // queue of inverted index job + public ConcurrentMap indexChangeJobs = Maps.newConcurrentMap(); + + public final Map activeIndexChangeJob = Maps.newConcurrentMap(); + + public final Map runnableIndexChangeJob = Maps.newConcurrentMap(); + public int cycleCount = 0; public SchemaChangeHandler() { @@ -1582,10 +1593,12 @@ public class SchemaChangeHandler extends AlterHandler { protected void runAfterCatalogReady() { if (cycleCount >= CYCLE_COUNT_TO_CHECK_EXPIRE_SCHEMA_CHANGE_JOB) { clearFinishedOrCancelledSchemaChangeJobV2(); + clearExpireFinishedOrCancelledIndexChangeJobs(); super.runAfterCatalogReady(); cycleCount = 0; } runAlterJobV2(); + runIndexChangeJob(); cycleCount++; } @@ -1610,6 +1623,81 @@ public class SchemaChangeHandler extends AlterHandler { }); } + private void runIndexChangeJob() { + runnableIndexChangeJob.values().forEach(indexChangeJob -> { + if (!indexChangeJob.isDone() && !activeIndexChangeJob.containsKey(indexChangeJob.getJobId()) + && activeIndexChangeJob.size() < MAX_ACTIVE_SCHEMA_CHANGE_JOB_V2_SIZE) { + if (FeConstants.runningUnitTest) { + indexChangeJob.run(); + } else { + schemaChangeThreadPool.submit(() -> { + if (activeIndexChangeJob.putIfAbsent( + indexChangeJob.getJobId(), indexChangeJob) == null) { + try { + indexChangeJob.run(); + } finally { + activeIndexChangeJob.remove(indexChangeJob.getJobId()); + } + } + }); + } + } + + if (indexChangeJob.isDone()) { + runnableIndexChangeJob.remove(indexChangeJob.getJobId()); + } + }); + } + + private void changeTableState(long dbId, long tableId, OlapTableState olapTableState) { + try { + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(dbId); + OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, Table.TableType.OLAP); + olapTable.writeLockOrMetaException(); + try { + if (olapTable.getState() == olapTableState) { + return; + } else if (olapTable.getState() == OlapTableState.SCHEMA_CHANGE) { + olapTable.setState(olapTableState); + } + } finally { + olapTable.writeUnlock(); + } + } catch (MetaNotFoundException e) { + LOG.warn("[INCONSISTENT META] changing table status failed after schema change job done", e); + } + } + + public List> getAllIndexChangeJobInfos() { + List> indexChangeJobInfos = new LinkedList<>(); + for (IndexChangeJob indexChangeJob : ImmutableList.copyOf(indexChangeJobs.values())) { + // no need to check priv here. This method is only called in show proc stmt, + // which already check the ADMIN priv. + indexChangeJob.getInfo(indexChangeJobInfos); + } + + // sort by "JobId", "TableName", "PartitionName", "CreateTime", "FinishTime" + ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4); + indexChangeJobInfos.sort(comparator); + return indexChangeJobInfos; + } + + public List> getAllIndexChangeJobInfos(Database db) { + List> indexChangeJobInfos = new LinkedList<>(); + for (IndexChangeJob indexChangeJob : ImmutableList.copyOf(indexChangeJobs.values())) { + if (indexChangeJob.getDbId() != db.getId()) { + continue; + } + indexChangeJob.getInfo(indexChangeJobInfos); + } + + + // sort by "JobId", "TableName", "PartitionName", "CreateTime", "FinishTime" + ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4); + indexChangeJobInfos.sort(comparator); + return indexChangeJobInfos; + } + public List> getAllAlterJobInfos() { List> schemaChangeJobInfos = new LinkedList<>(); for (AlterJobV2 alterJob : ImmutableList.copyOf(alterJobsV2.values())) { @@ -1663,7 +1751,8 @@ public class SchemaChangeHandler extends AlterHandler { try { //alterClauses can or cannot light schema change boolean lightSchemaChange = true; - boolean lightSchemaChangeWithInvertedIndex = false; + boolean lightIndexChange = false; + boolean buildIndexChange = false; // index id -> index schema Map> indexSchemaMap = new HashMap<>(); @@ -1693,10 +1782,10 @@ public class SchemaChangeHandler extends AlterHandler { } LOG.debug("in process indexSchemaMap:{}", indexSchemaMap); - List oriIndexes = olapTable.getCopiedIndexes(); List newIndexes = olapTable.getCopiedIndexes(); - List alterInvertedIndexes = new ArrayList<>(); - boolean isDropInvertedIndex = false; + List alterIndexes = new ArrayList<>(); + Map> invertedIndexOnPartitions = new HashMap<>(); + boolean isDropIndex = false; Map propertyMap = new HashMap<>(); for (AlterClause alterClause : alterClauses) { Map properties = alterClause.getProperties(); @@ -1813,55 +1902,102 @@ public class SchemaChangeHandler extends AlterHandler { // do nothing, properties are already in propertyMap lightSchemaChange = false; } else if (alterClause instanceof CreateIndexClause) { - if (processAddIndex((CreateIndexClause) alterClause, olapTable, newIndexes)) { + CreateIndexClause createIndexClause = (CreateIndexClause) alterClause; + IndexDef indexDef = createIndexClause.getIndexDef(); + Index index = createIndexClause.getIndex(); + if (processAddIndex(createIndexClause, olapTable, newIndexes)) { return; } lightSchemaChange = false; - // CreateIndexClause createIndexClause = (CreateIndexClause) alterClause; - // if (createIndexClause.getIndexDef().isInvertedIndex()) { - // alterInvertedIndexes.add(createIndexClause.getIndex()); - // isDropInvertedIndex = false; - // lightSchemaChangeWithInvertedIndex = true; - // } + + if (indexDef.isInvertedIndex()) { + alterIndexes.add(index); + isDropIndex = false; + // now only support light index change for inverted index + lightIndexChange = true; + } + } else if (alterClause instanceof BuildIndexClause) { + BuildIndexClause buildIndexClause = (BuildIndexClause) alterClause; + IndexDef indexDef = buildIndexClause.getIndexDef(); + Index index = buildIndexClause.getIndex(); + if (!olapTable.isPartitioned()) { + List specifiedPartitions = indexDef.getPartitionNames(); + if (!specifiedPartitions.isEmpty()) { + throw new DdlException("table " + olapTable.getName() + + " is not partitioned, cannot build index with partitions."); + } + } + List existedIndexes = olapTable.getIndexes(); + boolean found = false; + for (Index existedIdx : existedIndexes) { + if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) { + found = true; + index.setIndexId(existedIdx.getIndexId()); + index.setColumns(existedIdx.getColumns()); + index.setProperties(existedIdx.getProperties()); + if (indexDef.getPartitionNames().isEmpty()) { + invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); + } else { + invertedIndexOnPartitions.put( + index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); + } + break; + } + } + if (!found) { + throw new DdlException("index " + indexDef.getIndexName() + + " not exist, cannot build it with defferred."); + } + + if (indexDef.isInvertedIndex()) { + alterIndexes.add(index); + } + buildIndexChange = true; + lightSchemaChange = false; } else if (alterClause instanceof DropIndexClause) { if (processDropIndex((DropIndexClause) alterClause, olapTable, newIndexes)) { return; } lightSchemaChange = false; - // DropIndexClause dropIndexClause = (DropIndexClause) alterClause; - // List existedIndexes = olapTable.getIndexes(); - // Index found = null; - // for (Index existedIdx : existedIndexes) { - // if (existedIdx.getIndexName().equalsIgnoreCase(dropIndexClause.getIndexName())) { - // found = existedIdx; - // break; - // } - // } - // IndexDef.IndexType indexType = found.getIndexType(); - // if (indexType == IndexType.INVERTED) { - // alterInvertedIndexes.add(found); - // isDropInvertedIndex = true; - // lightSchemaChangeWithInvertedIndex = true; - // } + + DropIndexClause dropIndexClause = (DropIndexClause) alterClause; + List existedIndexes = olapTable.getIndexes(); + Index found = null; + for (Index existedIdx : existedIndexes) { + if (existedIdx.getIndexName().equalsIgnoreCase(dropIndexClause.getIndexName())) { + found = existedIdx; + break; + } + } + IndexDef.IndexType indexType = found.getIndexType(); + if (indexType == IndexType.INVERTED) { + alterIndexes.add(found); + isDropIndex = true; + lightIndexChange = true; + } } else { Preconditions.checkState(false); } } // end for alter clauses - LOG.debug("table: {}({}), lightSchemaChange: {}, lightSchemaChangeWithInvertedIndex: {}, indexSchemaMap:{}", - olapTable.getName(), olapTable.getId(), lightSchemaChange, lightSchemaChangeWithInvertedIndex, - indexSchemaMap); + LOG.debug("table: {}({}), lightSchemaChange: {}, lightIndexChange: {}," + + " buildIndexChange: {}, indexSchemaMap:{}", + olapTable.getName(), olapTable.getId(), lightSchemaChange, + lightIndexChange, buildIndexChange, indexSchemaMap); if (lightSchemaChange) { long jobId = Env.getCurrentEnv().getNextId(); //for schema change add/drop value column optimize, direct modify table meta. - modifyTableLightSchemaChange(db, olapTable, indexSchemaMap, newIndexes, jobId, false); - return; - } else if (lightSchemaChangeWithInvertedIndex) { + modifyTableLightSchemaChange(db, olapTable, indexSchemaMap, newIndexes, + null, isDropIndex, jobId, false); + } else if (lightIndexChange) { long jobId = Env.getCurrentEnv().getNextId(); //for schema change add/drop inverted index optimize, direct modify table meta firstly. - modifyTableAddOrDropInvertedIndices(db, olapTable, indexSchemaMap, propertyMap, newIndexes, - alterInvertedIndexes, isDropInvertedIndex, oriIndexes, jobId, false); + modifyTableLightSchemaChange(db, olapTable, indexSchemaMap, newIndexes, + alterIndexes, isDropIndex, jobId, false); + } else if (buildIndexChange) { + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, invertedIndexOnPartitions, false); } else { createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); } @@ -2273,6 +2409,12 @@ public class SchemaChangeHandler extends AlterHandler { runnableSchemaChangeJobV2.put(alterJob.getJobId(), alterJob); } + public void addIndexChangeJob(IndexChangeJob indexChangeJob) { + indexChangeJobs.put(indexChangeJob.getJobId(), indexChangeJob); + runnableIndexChangeJob.put(indexChangeJob.getJobId(), indexChangeJob); + LOG.info("add inverted index job {}", indexChangeJob.getJobId()); + } + private void clearFinishedOrCancelledSchemaChangeJobV2() { Iterator> iterator = runnableSchemaChangeJobV2.entrySet().iterator(); while (iterator.hasNext()) { @@ -2283,6 +2425,26 @@ public class SchemaChangeHandler extends AlterHandler { } } + private void clearExpireFinishedOrCancelledIndexChangeJobs() { + Iterator> iterator = indexChangeJobs.entrySet().iterator(); + while (iterator.hasNext()) { + IndexChangeJob indexChangeJob = iterator.next().getValue(); + if (indexChangeJob.isExpire()) { + iterator.remove(); + LOG.info("remove expired inverted index job {}. finish at {}", + indexChangeJob.getJobId(), TimeUtils.longToTimeString(indexChangeJob.getFinishedTimeMs())); + } + } + + Iterator> runnableIterator = runnableIndexChangeJob.entrySet().iterator(); + while (runnableIterator.hasNext()) { + IndexChangeJob indexChangeJob = runnableIterator.next().getValue(); + if (indexChangeJob.isDone()) { + runnableIterator.remove(); + } + } + } + @Override public void replayRemoveAlterJobV2(RemoveAlterJobV2OperationLog log) { if (runnableSchemaChangeJobV2.containsKey(log.getJobId())) { @@ -2302,6 +2464,7 @@ public class SchemaChangeHandler extends AlterHandler { // the invoker should keep table's write lock public void modifyTableLightSchemaChange(Database db, OlapTable olapTable, Map> indexSchemaMap, List indexes, + List alterIndexes, boolean isDropIndex, long jobId, boolean isReplay) throws DdlException { @@ -2357,20 +2520,46 @@ public class SchemaChangeHandler extends AlterHandler { throw new DdlException(e.getMessage()); } - if (!isReplay) { - TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(db.getId(), olapTable.getId(), - indexSchemaMap, indexes, jobId); - LOG.debug("logModifyTableAddOrDropColumns info:{}", info); - Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropColumns(info); - } - // set Job state then add job schemaChangeJob.setJobState(AlterJobV2.JobState.FINISHED); schemaChangeJob.setFinishedTimeMs(System.currentTimeMillis()); this.addAlterJobV2(schemaChangeJob); - LOG.info("finished modify table's add or drop or modify columns. table: {}, is replay: {}", olapTable.getName(), - isReplay); + if (alterIndexes != null) { + if (!isReplay) { + TableAddOrDropInvertedIndicesInfo info = new TableAddOrDropInvertedIndicesInfo( + db.getId(), olapTable.getId(), indexSchemaMap, indexes, + alterIndexes, isDropIndex, jobId); + LOG.debug("logModifyTableAddOrDropInvertedIndices info:{}", info); + Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropInvertedIndices(info); + + if (isDropIndex) { + // send drop rpc to be + Map> invertedIndexOnPartitions = new HashMap<>(); + for (Index index : alterIndexes) { + invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); + } + try { + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, invertedIndexOnPartitions, true); + } catch (Exception e) { + throw new DdlException(e.getMessage()); + } + } + } + + LOG.info("finished modify table's meta for add or drop inverted index. table: {}, job: {}, is replay: {}", + olapTable.getName(), jobId, isReplay); + } else { + if (!isReplay) { + TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(db.getId(), olapTable.getId(), + indexSchemaMap, indexes, jobId); + LOG.debug("logModifyTableAddOrDropColumns info:{}", info); + Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropColumns(info); + } + LOG.info("finished modify table's add or drop or modify columns. table: {}, job: {}, is replay: {}", + olapTable.getName(), jobId, isReplay); + } } public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info) throws MetaNotFoundException { @@ -2385,7 +2574,7 @@ public class SchemaChangeHandler extends AlterHandler { OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); olapTable.writeLock(); try { - modifyTableLightSchemaChange(db, olapTable, indexSchemaMap, indexes, jobId, true); + modifyTableLightSchemaChange(db, olapTable, indexSchemaMap, indexes, null, false, jobId, true); } catch (DdlException e) { // should not happen LOG.warn("failed to replay modify table add or drop or modify columns", e); @@ -2501,112 +2690,23 @@ public class SchemaChangeHandler extends AlterHandler { olapTable.rebuildFullSchema(); } - // the invoker should keep table's write lock - public void modifyTableAddOrDropInvertedIndices(Database db, OlapTable olapTable, - Map> indexSchemaMap, - Map propertyMap, List indexes, - List alterInvertedIndexes, boolean isDropInvertedIndex, - List oriIndexes, long jobId, - boolean isReplay) throws UserException { - LOG.info("begin to modify table's meta for add or drop inverted index. table: {}, job: {}", olapTable.getName(), - jobId); - LOG.info("indexSchemaMap:{}, indexes:{}, alterInvertedIndexes:{}, isDropInvertedIndex: {}", indexSchemaMap, - indexes, alterInvertedIndexes, isDropInvertedIndex); - if (olapTable.getState() == OlapTableState.ROLLUP) { - throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); - } - - // for now table's state can only be NORMAL - Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); - - boolean hasInvertedIndexChange = false; - if (!alterInvertedIndexes.isEmpty()) { - hasInvertedIndexChange = true; - } - - // begin checking each table - Map> changedIndexIdToSchema = Maps.newHashMap(); - try { - changedIndexIdToSchema = checkTable(db, olapTable, indexSchemaMap); - } catch (DdlException e) { - throw new DdlException("Table " + db.getFullName() + "." + olapTable.getName() + " check failed"); - } - if (changedIndexIdToSchema.isEmpty() && !hasInvertedIndexChange) { - throw new DdlException("Nothing is changed. please check your alter stmt."); - } - - // update base index schema - try { - updateBaseIndexSchema(olapTable, indexSchemaMap, indexes); - } catch (Exception e) { - throw new UserException(e.getMessage()); - } - - if (!isReplay) { - TableAddOrDropInvertedIndicesInfo info = new TableAddOrDropInvertedIndicesInfo(db.getId(), - olapTable.getId(), indexSchemaMap, propertyMap, indexes, alterInvertedIndexes, isDropInvertedIndex, - oriIndexes, jobId); - LOG.debug("logModifyTableAddOrDropInvertedIndices info:{}", info); - Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropInvertedIndices(info); - - // after modify tablet meta, we will create a WAITING_TXN state schema change job v2 - // to handle alter inverted index task - long timeoutSecond = PropertyAnalyzer.analyzeTimeout(propertyMap, Config.alter_table_timeout_second); - SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, db.getId(), olapTable.getId(), - olapTable.getName(), timeoutSecond * 1000); - schemaChangeJob.setAlterInvertedIndexInfo(hasInvertedIndexChange, isDropInvertedIndex, - alterInvertedIndexes); - schemaChangeJob.setOriIndexInfo(oriIndexes); - // only V2 support index, so if there is index changed, storage format must be V2 - schemaChangeJob.setStorageFormat(TStorageFormat.V2); - - for (Map.Entry> entry : changedIndexIdToSchema.entrySet()) { - long originIndexId = entry.getKey(); - for (Partition partition : olapTable.getPartitions()) { - long partitionId = partition.getId(); - schemaChangeJob.addPartitionOriginIndexIdMap(partitionId, originIndexId); - } // end for partition - String newIndexName = SHADOW_NAME_PREFIX + olapTable.getIndexNameById(originIndexId); - MaterializedIndexMeta currentIndexMeta = olapTable.getIndexMetaByIndexId(originIndexId); - // 1. get new schema version/schema version hash, short key column count - int currentSchemaVersion = currentIndexMeta.getSchemaVersion(); - int newSchemaVersion = currentSchemaVersion + 1; - schemaChangeJob.addIndexSchema(originIndexId, originIndexId, newIndexName, newSchemaVersion, - currentIndexMeta.getSchemaHash(), currentIndexMeta.getShortKeyColumnCount(), entry.getValue()); - } // end for index - - // set table state - olapTable.setState(OlapTableState.SCHEMA_CHANGE); - - // set Job state then add job - schemaChangeJob.setJobState(AlterJobV2.JobState.WAITING_TXN); - this.addAlterJobV2(schemaChangeJob); - LOG.debug("logAlterJob schemaChangeJob:{}", schemaChangeJob); - Env.getCurrentEnv().getEditLog().logAlterJob(schemaChangeJob); - } - LOG.info("finished modify table's meta for add or drop inverted index. table: {}, job: {}, is replay: {}", - olapTable.getName(), jobId, isReplay); - } - - public void replaymodifyTableAddOrDropInvertedIndices(TableAddOrDropInvertedIndicesInfo info) + public void replayModifyTableAddOrDropInvertedIndices(TableAddOrDropInvertedIndicesInfo info) throws MetaNotFoundException { LOG.debug("info:{}", info); long dbId = info.getDbId(); long tableId = info.getTableId(); Map> indexSchemaMap = info.getIndexSchemaMap(); - Map propertyMap = info.getPropertyMap(); List newIndexes = info.getIndexes(); - List alterInvertedIndexes = info.getAlterInvertedIndexes(); - boolean isDropInvertedIndex = info.getIsDropInvertedIndex(); - List oriIndexes = info.getOriIndexes(); + List alterIndexes = info.getAlterInvertedIndexes(); + boolean isDropIndex = info.getIsDropInvertedIndex(); long jobId = info.getJobId(); Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrMetaException(dbId); OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP); olapTable.writeLock(); try { - modifyTableAddOrDropInvertedIndices(db, olapTable, indexSchemaMap, propertyMap, newIndexes, - alterInvertedIndexes, isDropInvertedIndex, oriIndexes, jobId, true); + modifyTableLightSchemaChange(db, olapTable, indexSchemaMap, newIndexes, + alterIndexes, isDropIndex, jobId, true); } catch (UserException e) { // should not happen LOG.warn("failed to replay modify table add or drop indexes", e); @@ -2615,6 +2715,96 @@ public class SchemaChangeHandler extends AlterHandler { } } + public void buildOrDeleteTableInvertedIndices(Database db, OlapTable olapTable, + Map> indexSchemaMap, List alterIndexes, + Map> invertedIndexOnPartitions, boolean isDropOp) throws UserException { + LOG.info("begin to build table's inverted index. table: {}", olapTable.getName()); + + // for now table's state can only be NORMAL + Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); + + // begin checking each table + Map> changedIndexIdToSchema = Maps.newHashMap(); + try { + changedIndexIdToSchema = checkTable(db, olapTable, indexSchemaMap); + } catch (DdlException e) { + throw new DdlException("Table " + db.getFullName() + "." + olapTable.getName() + + " check failed"); + } + if (changedIndexIdToSchema.isEmpty() && alterIndexes.isEmpty()) { + throw new DdlException("Nothing is changed. please check your alter stmt."); + } + + for (Map.Entry> entry : changedIndexIdToSchema.entrySet()) { + long originIndexId = entry.getKey(); + for (Partition partition : olapTable.getPartitions()) { + // create job + long jobId = Env.getCurrentEnv().getNextId(); + IndexChangeJob indexChangeJob = new IndexChangeJob( + jobId, db.getId(), olapTable.getId(), olapTable.getName()); + indexChangeJob.setOriginIndexId(originIndexId); + indexChangeJob.setAlterInvertedIndexInfo(isDropOp, alterIndexes); + long partitionId = partition.getId(); + String partitionName = partition.getName(); + boolean found = false; + for (Set partitions : invertedIndexOnPartitions.values()) { + if (partitions.contains(partitionName)) { + found = true; + break; + } + } + if (!found) { + continue; + } + + if (hasIndexChangeJobOnPartition(originIndexId, db.getId(), olapTable.getId(), + partitionName, alterIndexes, isDropOp)) { + throw new DdlException("partition " + partitionName + " has been built specified index." + + " please check your build stmt."); + } + + indexChangeJob.setPartitionId(partitionId); + indexChangeJob.setPartitionName(partitionName); + + addIndexChangeJob(indexChangeJob); + + // write edit log + Env.getCurrentEnv().getEditLog().logIndexChangeJob(indexChangeJob); + LOG.info("finish create table's inverted index job. table: {}, partition: {}, job: {}", + olapTable.getName(), partitionName, jobId); + } // end for partition + } // end for index + } + + public boolean hasIndexChangeJobOnPartition( + long originIndexId, long dbId, long tableId, String partitionName, + List alterIndexes, boolean isDrop) { + // TODO: this is temporary methods + for (IndexChangeJob indexChangeJob : ImmutableList.copyOf(indexChangeJobs.values())) { + if (indexChangeJob.getOriginIndexId() == originIndexId + && indexChangeJob.getDbId() == dbId + && indexChangeJob.getTableId() == tableId + && indexChangeJob.getPartitionName().equals(partitionName) + && indexChangeJob.hasSameAlterInvertedIndex(isDrop, alterIndexes) + && indexChangeJob.getJobState() != IndexChangeJob.JobState.CANCELLED) { + // if JobState is CANCELLED, also allow user to create job again + return true; + } + } + return false; + } + + public void replayIndexChangeJob(IndexChangeJob indexChangeJob) throws MetaNotFoundException { + if (!indexChangeJob.isDone() && !runnableSchemaChangeJobV2.containsKey(indexChangeJob.getJobId())) { + runnableIndexChangeJob.put(indexChangeJob.getJobId(), indexChangeJob); + } + indexChangeJobs.put(indexChangeJob.getJobId(), indexChangeJob); + indexChangeJob.replay(indexChangeJob); + if (indexChangeJob.isDone()) { + runnableIndexChangeJob.remove(indexChangeJob.getJobId()); + } + } + public boolean updateBinlogConfig(Database db, OlapTable olapTable, List alterClauses) throws DdlException, UserException { // TODO(Drogon): check olapTable read binlog thread safety diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 988781e5bc..968001d902 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -51,10 +51,8 @@ import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; -import org.apache.doris.task.AlterInvertedIndexTask; import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.task.CreateReplicaTask; -import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; @@ -123,16 +121,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 { // alter index info @SerializedName(value = "indexChange") private boolean indexChange = false; - @SerializedName(value = "invertedIndexChange") - private boolean invertedIndexChange = false; - @SerializedName(value = "isDropOp") - private boolean isDropOp = false; @SerializedName(value = "indexes") private List indexes = null; - @SerializedName(value = "alterInvertedIndexes") - private List alterInvertedIndexes = null; - @SerializedName(value = "oriIndexes") - private List oriIndexes = null; // The schema change job will wait all transactions before this txn id finished, then send the schema change tasks. @SerializedName(value = "watershedTxnId") @@ -191,17 +181,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { this.indexes = indexes; } - public void setAlterInvertedIndexInfo(boolean invertedIndexChange, - boolean isDropOp, List alterInvertedIndexes) { - this.invertedIndexChange = invertedIndexChange; - this.isDropOp = isDropOp; - this.alterInvertedIndexes = alterInvertedIndexes; - } - - public void setOriIndexInfo(List oriIndexes) { - this.oriIndexes = oriIndexes; - } - public void setStorageFormat(TStorageFormat storageFormat) { this.storageFormat = storageFormat; } @@ -367,11 +346,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { } private void addShadowIndexToCatalog(OlapTable tbl) { - if (invertedIndexChange) { - // change inverted index no need create shadow index, it modifies on origin base tablet. - return; - } - for (long partitionId : partitionIndexMap.rowKeySet()) { Partition partition = tbl.getPartition(partitionId); if (partition == null) { @@ -404,12 +378,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { @Override protected void runWaitingTxnJob() throws AlterCancelException { Preconditions.checkState(jobState == JobState.WAITING_TXN, jobState); - if (invertedIndexChange) { - // generate a watershedTxnId - this.watershedTxnId = Env.getCurrentGlobalTransactionMgr() - .getTransactionIDGenerator().getNextTransactionId(); - } - try { if (!isPreviousLoadFinished()) { LOG.info("wait transactions before {} to be finished, schema change job: {}", watershedTxnId, jobId); @@ -431,58 +399,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { } tbl.readLock(); - if (invertedIndexChange) { - try { - long expiration = (createTimeMs + timeoutMs) / 1000; - Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); - - for (Map.Entry entry : partitionOriginIndexIdMap.entrySet()) { - long partitionId = entry.getKey(); - long originIdxId = entry.getValue(); - Partition partition = tbl.getPartition(partitionId); - Preconditions.checkNotNull(partition, partitionId); - // the schema change task will transform the data before visible - // version(included). - long visibleVersion = partition.getVisibleVersion(); - List originSchemaColumns = tbl.getSchemaByIndexId(originIdxId, true); - for (Column col : originSchemaColumns) { - TColumn tColumn = col.toThrift(); - col.setIndexFlag(tColumn, tbl); - } - int originSchemaHash = tbl.getSchemaHashByIndexId(originIdxId); - MaterializedIndex origIdx = partition.getIndex(originIdxId); - for (Tablet originTablet : origIdx.getTablets()) { - long originTabletId = originTablet.getId(); - List originReplicas = originTablet.getReplicas(); - for (Replica originReplica : originReplicas) { - if (originReplica.getBackendId() < 0) { - LOG.warn("replica:{}, backendId: {}", originReplica, originReplica.getBackendId()); - throw new AlterCancelException("originReplica:" + originReplica.getId() - + " backendId < 0"); - } - AlterInvertedIndexTask alterInvertedIndexTask = new AlterInvertedIndexTask( - originReplica.getBackendId(), dbId, tableId, - partitionId, originIdxId, visibleVersion, - originTabletId, originSchemaHash, - jobId, JobType.SCHEMA_CHANGE, - isDropOp, alterInvertedIndexes, indexes, originSchemaColumns, expiration); - schemaChangeBatchTask.addTask(alterInvertedIndexTask); - } - } - } - } finally { - tbl.readUnlock(); - } - LOG.debug("schemaChangeBatchTask:{}", schemaChangeBatchTask); - AgentTaskQueue.addBatchTask(schemaChangeBatchTask); - AgentTaskExecutor.submit(schemaChangeBatchTask); - - this.jobState = JobState.RUNNING; - - // DO NOT write edit log here, tasks will be send again if FE restart or master changed. - LOG.info("transfer schema change job {} state to {}", jobId, this.jobState); - return; - } try { Map indexColumnMap = Maps.newHashMap(); @@ -592,13 +508,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { LOG.info("schema change tasks not finished. job: {}", jobId); List tasks = schemaChangeBatchTask.getUnfinishedTasks(2000); for (AgentTask task : tasks) { - if (invertedIndexChange) { - // TODO(wy): more elegant implementation - // keep alterInvertedIndex task in agent task queue, and try again after failed - LOG.debug("alter inverted index task failed after try {} times, err msg: {}", - task.getFailedTimes(), task.getErrorMsg()); - continue; - } if (task.getFailedTimes() >= 3) { task.setFinished(true); AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); @@ -624,21 +533,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { * we just check whether all new replicas are healthy. */ tbl.writeLockOrAlterCancelException(); - if (invertedIndexChange) { - // do nothing - try { - Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); - onFinished(tbl); - } finally { - tbl.writeUnlock(); - } - pruneMeta(); - this.jobState = JobState.FINISHED; - this.finishedTimeMs = System.currentTimeMillis(); - Env.getCurrentEnv().getEditLog().logAlterJob(this); - LOG.info("schema change job finished: {}", jobId); - return; - } try { Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); @@ -698,10 +592,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { } private void onFinished(OlapTable tbl) { - if (invertedIndexChange) { - tbl.setStorageFormat(storageFormat); - return; - } // replace the origin index with shadow index, set index state as NORMAL for (Partition partition : tbl.getPartitions()) { // drop the origin index from partitions @@ -834,10 +724,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { for (String shadowIndexName : indexIdToName.values()) { tbl.deleteIndexInfo(shadowIndexName); } - if (invertedIndexChange) { - // update index to oriIndexes, because inverted index updated before WAIT_TXN state. - tbl.setIndexes(oriIndexes); - } } finally { tbl.writeUnlock(); } @@ -902,11 +788,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { olapTable.writeLock(); try { addShadowIndexToCatalog(olapTable); - if (invertedIndexChange) { - // invertedIndexChange job begin with WAITING_TXN, no PENDING state, - // here need set table state to SCHEMA_CHANGE when replay job begin at WAITING_TXN - olapTable.setState(OlapTableState.SCHEMA_CHANGE); - } } finally { olapTable.writeUnlock(); } @@ -1006,7 +887,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { info.add(errMsg); info.add(progress); info.add(timeoutMs / 1000); - info.add(getOtherInfo()); infos.add(info); } } @@ -1046,21 +926,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 { } } - public String getOtherInfo() { - String info = null; - // can add info as needed - List infoList = Lists.newArrayList(); - if (invertedIndexChange) { - String invertedIndexChangeInfo = ""; - for (Index invertedIndex : alterInvertedIndexes) { - invertedIndexChangeInfo += "[" + (isDropOp ? "DROP " : "ADD ") + invertedIndex.toString() + "], "; - } - infoList.add(invertedIndexChangeInfo); - } - info = Joiner.on(", ").join(infoList.subList(0, infoList.size())); - return info; - } - @Override public void write(DataOutput out) throws IOException { String json = GsonUtils.GSON.toJson(this, AlterJobV2.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java new file mode 100644 index 0000000000..99aee373b1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.alter.AlterOpType; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Index; +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.Maps; + +import java.util.Map; + +public class BuildIndexClause extends AlterTableClause { + // in which table the index on, only used when alter = false + private TableName tableName; + // index definition class + private IndexDef indexDef; + // when alter = true, clause like: alter table add index xxxx + // when alter = false, clause like: create index xx on table xxxx + private boolean alter; + // index internal class + private Index index; + + public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean alter) { + super(AlterOpType.SCHEMA_CHANGE); + this.tableName = tableName; + this.indexDef = indexDef; + this.alter = alter; + } + + @Override + public Map getProperties() { + return Maps.newHashMap(); + } + + public Index getIndex() { + return index; + } + + public IndexDef getIndexDef() { + return indexDef; + } + + public boolean isAlter() { + return alter; + } + + public TableName getTableName() { + return tableName; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (indexDef == null) { + throw new AnalysisException("index definition expected."); + } + indexDef.analyze(); + this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), + indexDef.getColumns(), indexDef.getIndexType(), + indexDef.getProperties(), indexDef.getComment()); + } + + @Override + public String toSql() { + if (alter) { + return indexDef.toSql(); + } else { + return "BUILD " + indexDef.toSql(tableName.toSql()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index a5eb735b1d..9aa0a4685a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -41,6 +41,8 @@ public class IndexDef { private IndexType indexType; private String comment; private Map properties; + private boolean isBuildDeferred = false; + private PartitionNames partitionNames; public static final String NGRAM_SIZE_KEY = "gram_size"; public static final String NGRAM_BF_SIZE_KEY = "bf_size"; @@ -73,7 +75,24 @@ public class IndexDef { } } + public IndexDef(String indexName, PartitionNames partitionNames, boolean isBuildDeferred) { + this.indexName = indexName; + this.indexType = IndexType.INVERTED; + this.partitionNames = partitionNames; + this.isBuildDeferred = isBuildDeferred; + } + public void analyze() throws AnalysisException { + if (isBuildDeferred && indexType == IndexDef.IndexType.INVERTED) { + if (Strings.isNullOrEmpty(indexName)) { + throw new AnalysisException("index name cannot be blank."); + } + if (indexName.length() > 128) { + throw new AnalysisException("index name too long, the index name length at most is 128."); + } + return; + } + if (indexType == IndexDef.IndexType.BITMAP || indexType == IndexDef.IndexType.INVERTED) { if (columns == null || columns.size() != 1) { @@ -168,6 +187,14 @@ public class IndexDef { return ifNotExists; } + public boolean isBuildDeferred() { + return isBuildDeferred; + } + + public List getPartitionNames() { + return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames(); + } + public enum IndexType { BITMAP, INVERTED, diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java new file mode 100644 index 0000000000..525fd49204 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.common.proc.BuildIndexProcDir; +import org.apache.doris.common.proc.ProcNodeInterface; +import org.apache.doris.common.proc.ProcService; +import org.apache.doris.common.util.OrderByPair; +import org.apache.doris.qe.ShowResultSetMetaData; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +// SHOW LOAD STATUS statement used to get status of load job. +// +// syntax: +// SHOW LOAD [FROM db] [LIKE mask] +public class ShowBuildIndexStmt extends ShowStmt { + private static final Logger LOG = LogManager.getLogger(ShowBuildIndexStmt.class); + + private String dbName; + private Expr whereClause; + private LimitElement limitElement; + private List orderByElements; + private HashMap filterMap; + private ArrayList orderByPairs; + + private ProcNodeInterface node; + + public ShowBuildIndexStmt(String dbName, Expr whereClause, + List orderByElements, LimitElement limitElement) { + this.dbName = dbName; + this.whereClause = whereClause; + this.orderByElements = orderByElements; + this.limitElement = limitElement; + this.filterMap = new HashMap(); + } + + public String getDbName() { + return dbName; + } + + public HashMap getFilterMap() { + return filterMap; + } + + public LimitElement getLimitElement() { + return limitElement; + } + + public ArrayList getOrderPairs() { + return orderByPairs; + } + + public ProcNodeInterface getNode() { + return this.node; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, UserException { + super.analyze(analyzer); + if (Strings.isNullOrEmpty(dbName)) { + dbName = analyzer.getDefaultDb(); + if (Strings.isNullOrEmpty(dbName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); + } + } else { + dbName = ClusterNamespace.getFullName(getClusterName(), dbName); + } + + // analyze where clause if not null + if (whereClause != null) { + analyzeSubPredicate(whereClause); + } + + // order by + if (orderByElements != null && !orderByElements.isEmpty()) { + orderByPairs = new ArrayList(); + for (OrderByElement orderByElement : orderByElements) { + if (!(orderByElement.getExpr() instanceof SlotRef)) { + throw new AnalysisException("Should order by column"); + } + SlotRef slotRef = (SlotRef) orderByElement.getExpr(); + int index = BuildIndexProcDir.analyzeColumn(slotRef.getColumnName()); + OrderByPair orderByPair = new OrderByPair(index, !orderByElement.getIsAsc()); + orderByPairs.add(orderByPair); + } + } + + if (limitElement != null) { + limitElement.analyze(analyzer); + } + + DatabaseIf db = analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(dbName); + // build proc path + StringBuilder sb = new StringBuilder(); + sb.append("/jobs/"); + sb.append(db.getId()); + sb.append("/build_index"); + + LOG.debug("process SHOW PROC '{}';", sb.toString()); + // create show proc stmt + // '/jobs/db_name/build_index/ + node = ProcService.getInstance().open(sb.toString()); + if (node == null) { + throw new AnalysisException("Failed to show build index"); + } + } + + private void analyzeSubPredicate(Expr subExpr) throws AnalysisException { + if (subExpr == null) { + return; + } + if (subExpr instanceof CompoundPredicate) { + CompoundPredicate cp = (CompoundPredicate) subExpr; + if (cp.getOp() != org.apache.doris.analysis.CompoundPredicate.Operator.AND) { + throw new AnalysisException("Only allow compound predicate with operator AND"); + } + analyzeSubPredicate(cp.getChild(0)); + analyzeSubPredicate(cp.getChild(1)); + return; + } + getPredicateValue(subExpr); + } + + private void getPredicateValue(Expr subExpr) throws AnalysisException { + if (!(subExpr instanceof BinaryPredicate)) { + throw new AnalysisException("The operator =|>=|<=|>|<|!= are supported."); + } + + BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr; + if (!(subExpr.getChild(0) instanceof SlotRef)) { + throw new AnalysisException("Only support column = xxx syntax."); + } + String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName().toLowerCase(); + if (leftKey.equalsIgnoreCase("tablename") + || leftKey.equalsIgnoreCase("state") + || leftKey.equalsIgnoreCase("partitionname")) { + if (!(subExpr.getChild(1) instanceof StringLiteral) + || binaryPredicate.getOp() != BinaryPredicate.Operator.EQ) { + throw new AnalysisException("Where clause : TableName = \"table1\" or " + + "State = \"FINISHED|CANCELLED|RUNNING|PENDING|WAITING_TXN\""); + } + } else if (leftKey.equalsIgnoreCase("createtime") || leftKey.equalsIgnoreCase("finishtime")) { + if (!(subExpr.getChild(1) instanceof StringLiteral)) { + throw new AnalysisException("Where clause : CreateTime/FinishTime =|>=|<=|>|<|!= " + + "\"2019-12-02|2019-12-02 14:54:00\""); + } + subExpr.setChild(1, (subExpr.getChild(1)).castTo( + ScalarType.getDefaultDateType(Type.DATETIME))); + } else { + throw new AnalysisException( + "The columns of TableName/PartitionName/CreateTime/FinishTime/State are supported."); + } + filterMap.put(leftKey, subExpr); + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("SHOW BUILD INDEX "); + if (!Strings.isNullOrEmpty(dbName)) { + sb.append("FROM `").append(dbName).append("`"); + } + if (whereClause != null) { + sb.append(" WHERE ").append(whereClause.toSql()); + } + // Order By clause + if (orderByElements != null) { + sb.append(" ORDER BY "); + for (int i = 0; i < orderByElements.size(); ++i) { + sb.append(orderByElements.get(i).toSql()); + sb.append((i + 1 != orderByElements.size()) ? ", " : ""); + } + } + + if (limitElement != null) { + sb.append(limitElement.toSql()); + } + return sb.toString(); + } + + @Override + public String toString() { + return toSql(); + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + ImmutableList titleNames = BuildIndexProcDir.TITLE_NAMES; + + for (String title : titleNames) { + builder.addColumn(new Column(title, ScalarType.createVarchar(30))); + } + + return builder.build(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 48d2368ad1..72af863b1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -251,6 +251,13 @@ public class OlapTable extends Table { return indexes.getIndexes(); } + public List getIndexIds() { + if (indexes == null) { + return Lists.newArrayList(); + } + return indexes.getIndexIds(); + } + public TableIndexes getTableIndexes() { return indexes; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIndexes.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIndexes.java index 52ed6f011f..ddc76bb7c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIndexes.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIndexes.java @@ -63,6 +63,16 @@ public class TableIndexes implements Writable { return indexes; } + public List getIndexIds() { + List indexIds = Lists.newArrayList(); + if (indexes != null) { + for (Index index : indexes) { + indexIds.add(index.getIndexId()); + } + } + return indexIds; + } + public List getCopiedIndexes() { if (indexes == null || indexes.size() == 0) { return Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java new file mode 100644 index 0000000000..9c381fe567 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.proc; + +import org.apache.doris.alter.SchemaChangeHandler; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.DateLiteral; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.LimitElement; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.ListComparator; +import org.apache.doris.common.util.OrderByPair; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class BuildIndexProcDir implements ProcDirInterface { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("JobId").add("TableName") + .add("PartitionName").add("AlterInvertedIndexes") + .add("CreateTime").add("FinishTime") + .add("TransactionId").add("State") + .add("Msg").add("Progress") + .build(); + + private static final Logger LOG = LogManager.getLogger(BuildIndexProcDir.class); + + private SchemaChangeHandler schemaChangeHandler; + private Database db; + + public BuildIndexProcDir(SchemaChangeHandler schemaChangeHandler, Database db) { + this.schemaChangeHandler = schemaChangeHandler; + this.db = db; + } + + boolean filterResult(String columnName, Comparable element, HashMap filter) throws AnalysisException { + if (filter == null) { + return true; + } + Expr subExpr = filter.get(columnName.toLowerCase()); + if (subExpr == null) { + return true; + } + BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr; + if (subExpr.getChild(1) instanceof StringLiteral && binaryPredicate.getOp() == BinaryPredicate.Operator.EQ) { + return ((StringLiteral) subExpr.getChild(1)).getValue().equals(element); + } + if (subExpr.getChild(1) instanceof DateLiteral) { + Type type; + switch (subExpr.getChild(1).getType().getPrimitiveType()) { + case DATE: + case DATETIME: + type = Type.DATETIME; + break; + case DATEV2: + type = Type.DATETIMEV2; + break; + case DATETIMEV2: + type = subExpr.getChild(1).getType(); + break; + default: + throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); + } + Long leftVal = (new DateLiteral((String) element, type)).getLongValue(); + Long rightVal = ((DateLiteral) subExpr.getChild(1)).getLongValue(); + switch (binaryPredicate.getOp()) { + case EQ: + case EQ_FOR_NULL: + return leftVal.equals(rightVal); + case GE: + return leftVal >= rightVal; + case GT: + return leftVal > rightVal; + case LE: + return leftVal <= rightVal; + case LT: + return leftVal < rightVal; + case NE: + return !leftVal.equals(rightVal); + default: + Preconditions.checkState(false, "No defined binary operator."); + } + } + return true; + } + + public ProcResult fetchResultByFilter(HashMap filter, ArrayList orderByPairs, + LimitElement limitElement) throws AnalysisException { + Preconditions.checkNotNull(db); + Preconditions.checkNotNull(schemaChangeHandler); + + List> indexChangeJobInfos = schemaChangeHandler.getAllIndexChangeJobInfos(db); + + //where + List> jobInfos; + if (filter == null || filter.size() == 0) { + jobInfos = indexChangeJobInfos; + } else { + jobInfos = Lists.newArrayList(); + for (List infoStr : indexChangeJobInfos) { + if (infoStr.size() != TITLE_NAMES.size()) { + LOG.warn("indexChangeJobInfos.size() " + indexChangeJobInfos.size() + + " not equal TITLE_NAMES.size() " + TITLE_NAMES.size()); + continue; + } + boolean isNeed = true; + for (int i = 0; i < infoStr.size(); i++) { + isNeed = filterResult(TITLE_NAMES.get(i), infoStr.get(i), filter); + if (!isNeed) { + break; + } + } + if (isNeed) { + jobInfos.add(infoStr); + } + } + } + + // order by + if (orderByPairs != null) { + ListComparator> comparator = null; + OrderByPair[] orderByPairArr = new OrderByPair[orderByPairs.size()]; + comparator = new ListComparator>(orderByPairs.toArray(orderByPairArr)); + Collections.sort(jobInfos, comparator); + } + + //limit + if (limitElement != null && limitElement.hasLimit()) { + int beginIndex = (int) limitElement.getOffset(); + int endIndex = (int) (beginIndex + limitElement.getLimit()); + if (endIndex > jobInfos.size()) { + endIndex = jobInfos.size(); + } + jobInfos = jobInfos.subList(beginIndex, endIndex); + } + + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + for (List jobInfo : jobInfos) { + List oneResult = new ArrayList(jobInfos.size()); + for (Comparable column : jobInfo) { + oneResult.add(column.toString()); + } + result.addRow(oneResult); + } + return result; + } + + @Override + public ProcResult fetchResult() throws AnalysisException { + Preconditions.checkNotNull(schemaChangeHandler); + + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + List> indexChangeJobInfos; + // db is null means need total result of all databases + if (db == null) { + indexChangeJobInfos = schemaChangeHandler.getAllIndexChangeJobInfos(); + } else { + indexChangeJobInfos = schemaChangeHandler.getAllIndexChangeJobInfos(db); + } + for (List infoStr : indexChangeJobInfos) { + List oneInfo = new ArrayList(TITLE_NAMES.size()); + for (Comparable element : infoStr) { + oneInfo.add(element.toString()); + } + result.addRow(oneInfo); + } + return result; + } + + public static int analyzeColumn(String columnName) throws AnalysisException { + for (int i = 0; i < TITLE_NAMES.size(); ++i) { + if (TITLE_NAMES.get(i).equalsIgnoreCase(columnName)) { + return i; + } + } + throw new AnalysisException("Title name[" + columnName + "] does not exist"); + } + + @Override + public boolean register(String name, ProcNodeInterface node) { + return false; + } + + @Override + public ProcNodeInterface lookup(String name) { + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/JobsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/JobsProcDir.java index 597066536b..b81561399d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/JobsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/JobsProcDir.java @@ -46,6 +46,7 @@ public class JobsProcDir implements ProcDirInterface { private static final String ROLLUP = "rollup"; private static final String SCHEMA_CHANGE = "schema_change"; private static final String EXPORT = "export"; + private static final String BUILD_INDEX = "build_index"; private Env env; private Database db; @@ -77,6 +78,8 @@ public class JobsProcDir implements ProcDirInterface { return new SchemaChangeProcDir(env.getSchemaChangeHandler(), db); } else if (jobTypeName.equals(EXPORT)) { return new ExportProcNode(env.getExportMgr(), db); + } else if (jobTypeName.equals(BUILD_INDEX)) { + return new BuildIndexProcDir(env.getSchemaChangeHandler(), db); } else { throw new AnalysisException("Invalid job type: " + jobTypeName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java index d89792ddf5..03ccfaa3ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java @@ -48,7 +48,6 @@ public class SchemaChangeProcDir implements ProcDirInterface { .add("JobId").add("TableName").add("CreateTime").add("FinishTime") .add("IndexName").add("IndexId").add("OriginIndexId").add("SchemaVersion") .add("TransactionId").add("State").add("Msg").add("Progress").add("Timeout") - .add("OtherInfos") .build(); private static final Logger LOG = LogManager.getLogger(SchemaChangeProcDir.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 313dcea247..006930dbea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -19,6 +19,7 @@ package org.apache.doris.journal; import org.apache.doris.alter.AlterJobV2; import org.apache.doris.alter.BatchAlterJobPersistInfo; +import org.apache.doris.alter.IndexChangeJob; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; @@ -723,6 +724,11 @@ public class JournalEntity implements Writable { isRead = true; break; } + case OperationType.OP_INVERTED_INDEX_JOB: { + data = IndexChangeJob.read(in); + isRead = true; + break; + } case OperationType.OP_CLEAN_LABEL: { data = CleanLabelOperationLog.read(in); isRead = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java index 4e89cd56ae..4e031c0dac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java @@ -194,7 +194,7 @@ public class MasterImpl { finishAlterTask(task); break; case ALTER_INVERTED_INDEX: - finishAlterInvertedIndexTask(task); + finishAlterInvertedIndexTask(task, request); break; case UPDATE_TABLET_META_INFO: finishUpdateTabletMeta(task, request); @@ -581,11 +581,19 @@ public class MasterImpl { AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); } - private void finishAlterInvertedIndexTask(AgentTask task) { + private void finishAlterInvertedIndexTask(AgentTask task, TFinishTaskRequest request) { + TStatus taskStatus = request.getTaskStatus(); + if (taskStatus.getStatusCode() != TStatusCode.OK) { + LOG.warn("AlterInvertedIndexTask: {} failed, failed times: {}, remaining it in agent task queue", + task.getSignature(), task.getFailedTimes()); + return; + } + AlterInvertedIndexTask alterInvertedIndexTask = (AlterInvertedIndexTask) task; - LOG.info("beigin finish AlterInvertedIndexTask: {}, JobId: {}, Jobtype: {}", - alterInvertedIndexTask.getSignature(), alterInvertedIndexTask.getJobId(), - alterInvertedIndexTask.getJobType()); + LOG.info("beigin finish AlterInvertedIndexTask: {}, tablet: {}, toString: {}", + alterInvertedIndexTask.getSignature(), + alterInvertedIndexTask.getTabletId(), + alterInvertedIndexTask.toString()); // TODO: more check alterInvertedIndexTask.setFinished(true); AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER_INVERTED_INDEX, task.getSignature()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 3fb6d9001b..d72c2f8f6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -19,6 +19,7 @@ package org.apache.doris.persist; import org.apache.doris.alter.AlterJobV2; import org.apache.doris.alter.BatchAlterJobPersistInfo; +import org.apache.doris.alter.IndexChangeJob; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; @@ -881,7 +882,12 @@ public class EditLog { case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES: { final TableAddOrDropInvertedIndicesInfo info = (TableAddOrDropInvertedIndicesInfo) journal.getData(); - env.getSchemaChangeHandler().replaymodifyTableAddOrDropInvertedIndices(info); + env.getSchemaChangeHandler().replayModifyTableAddOrDropInvertedIndices(info); + break; + } + case OperationType.OP_INVERTED_INDEX_JOB: { + IndexChangeJob indexChangeJob = (IndexChangeJob) journal.getData(); + env.getSchemaChangeHandler().replayIndexChangeJob(indexChangeJob); break; } case OperationType.OP_CLEAN_LABEL: { @@ -1735,6 +1741,10 @@ public class EditLog { logEdit(OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES, info); } + public void logIndexChangeJob(IndexChangeJob indexChangeJob) { + logEdit(OperationType.OP_INVERTED_INDEX_JOB, indexChangeJob); + } + public void logCleanLabel(CleanLabelOperationLog log) { logEdit(OperationType.OP_CLEAN_LABEL, log); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 2e63824786..8ffa0b609a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -79,6 +79,7 @@ public class OperationType { //schema change for add and drop inverted indices public static final short OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES = 220; + public static final short OP_INVERTED_INDEX_JOB = 221; // 30~39 130~139 230~239 ... // load job for only hadoop load diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java index e094aa1e3e..57c7c5ede0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java @@ -42,31 +42,25 @@ public class TableAddOrDropInvertedIndicesInfo implements Writable { private long tableId; @SerializedName(value = "indexSchemaMap") private Map> indexSchemaMap; - @SerializedName(value = "propertyMap") - private Map propertyMap; @SerializedName(value = "indexes") private List indexes; @SerializedName(value = "alterInvertedIndexes") private List alterInvertedIndexes; @SerializedName(value = "isDropInvertedIndex") private boolean isDropInvertedIndex; - @SerializedName(value = "oriIndexes") - private List oriIndexes; @SerializedName(value = "jobId") private long jobId; public TableAddOrDropInvertedIndicesInfo(long dbId, long tableId, - Map> indexSchemaMap, Map propertyMap, - List indexes, List alterInvertedIndexes, boolean isDropInvertedIndex, - List oriIndexes, long jobId) { + Map> indexSchemaMap, List indexes, + List alterInvertedIndexes, boolean isDropInvertedIndex, + long jobId) { this.dbId = dbId; this.tableId = tableId; this.indexSchemaMap = indexSchemaMap; - this.propertyMap = propertyMap; this.indexes = indexes; this.alterInvertedIndexes = alterInvertedIndexes; this.isDropInvertedIndex = isDropInvertedIndex; - this.oriIndexes = oriIndexes; this.jobId = jobId; } @@ -82,10 +76,6 @@ public class TableAddOrDropInvertedIndicesInfo implements Writable { return indexSchemaMap; } - public Map getPropertyMap() { - return propertyMap; - } - public List getIndexes() { return indexes; } @@ -98,10 +88,6 @@ public class TableAddOrDropInvertedIndicesInfo implements Writable { return isDropInvertedIndex; } - public List getOriIndexes() { - return oriIndexes; - } - public long getJobId() { return jobId; } @@ -129,11 +115,10 @@ public class TableAddOrDropInvertedIndicesInfo implements Writable { return (dbId == info.dbId && tableId == tableId && indexSchemaMap.equals(info.indexSchemaMap) - && propertyMap.equals(info.propertyMap) && indexes.equals(info.indexes) && alterInvertedIndexes.equals(info.alterInvertedIndexes) && isDropInvertedIndex == info.isDropInvertedIndex - && oriIndexes.equals(info.oriIndexes) && jobId == info.jobId); + && jobId == info.jobId); } @Override @@ -142,11 +127,9 @@ public class TableAddOrDropInvertedIndicesInfo implements Writable { sb.append(" dbId: ").append(dbId); sb.append(" tableId: ").append(tableId); sb.append(" indexSchemaMap: ").append(indexSchemaMap); - sb.append(" propertyMap: ").append(propertyMap); sb.append(" indexes: ").append(indexes); sb.append(" alterInvertedIndexes: ").append(alterInvertedIndexes); sb.append(" isDropInvertedIndex: ").append(isDropInvertedIndex); - sb.append(" oriIndexes: ").append(oriIndexes); sb.append(" jobId: ").append(jobId); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8f440cb750..42b7d52b2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -32,6 +32,7 @@ import org.apache.doris.analysis.ShowAuthorStmt; import org.apache.doris.analysis.ShowBackendsStmt; import org.apache.doris.analysis.ShowBackupStmt; import org.apache.doris.analysis.ShowBrokerStmt; +import org.apache.doris.analysis.ShowBuildIndexStmt; import org.apache.doris.analysis.ShowCatalogRecycleBinStmt; import org.apache.doris.analysis.ShowCatalogStmt; import org.apache.doris.analysis.ShowCollationStmt; @@ -151,6 +152,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.proc.BackendsProcDir; +import org.apache.doris.common.proc.BuildIndexProcDir; import org.apache.doris.common.proc.FrontendsProcNode; import org.apache.doris.common.proc.LoadProcDir; import org.apache.doris.common.proc.PartitionsProcDir; @@ -412,6 +414,8 @@ public class ShowExecutor { handleMTMVTasks(); } else if (stmt instanceof ShowTypeCastStmt) { handleShowTypeCastStmt(); + } else if (stmt instanceof ShowBuildIndexStmt) { + handleShowBuildIndexStmt(); } else { handleEmtpy(); } @@ -2705,5 +2709,15 @@ public class ShowExecutor { ShowResultSetMetaData showMetaData = showStmt.getMetaData(); resultSet = new ShowResultSet(showMetaData, resultRowSet); } + + private void handleShowBuildIndexStmt() throws AnalysisException { + ShowBuildIndexStmt showStmt = (ShowBuildIndexStmt) stmt; + ProcNodeInterface procNodeI = showStmt.getNode(); + Preconditions.checkNotNull(procNodeI); + // List> rows = ((BuildIndexProcDir) procNodeI).fetchResult().getRows(); + List> rows = ((BuildIndexProcDir) procNodeI).fetchResultByFilter(showStmt.getFilterMap(), + showStmt.getOrderPairs(), showStmt.getLimitElement()).getRows(); + resultSet = new ShowResultSet(showStmt.getMetaData(), rows); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index ace979d759..7bd366e909 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -454,7 +454,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { List newIndexes = olapTable.getCopiedIndexes(); long jobId = Env.getCurrentEnv().getNextId(); Env.getCurrentEnv().getSchemaChangeHandler().modifyTableLightSchemaChange( - db, olapTable, indexSchemaMap, newIndexes, jobId, false); + db, olapTable, indexSchemaMap, newIndexes, null, false, jobId, false); } else { throw new MetaNotFoundException("table_id " + request.getTableId() + " cannot light schema change through rpc."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java index 2dea8e7ff7..c199d1d482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java @@ -17,7 +17,6 @@ package org.apache.doris.task; -import org.apache.doris.alter.AlterJobV2; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Index; import org.apache.doris.thrift.TAlterInvertedIndexReq; @@ -25,6 +24,9 @@ import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TOlapTableIndex; import org.apache.doris.thrift.TTaskType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; @@ -35,68 +37,63 @@ import java.util.List; * The new replica can be a rollup replica, or a shadow replica of schema change. */ public class AlterInvertedIndexTask extends AgentTask { + private static final Logger LOG = LogManager.getLogger(AlterInvertedIndexTask.class); private long tabletId; - private long version; - private long jobId; - private AlterJobV2.JobType jobType; private int schemaHash; - private boolean isDropOp = false; private List alterInvertedIndexes; - List indexes; private List schemaColumns; - private long expiration; + private List existIndexes; + private boolean isDropOp = false; public AlterInvertedIndexTask(long backendId, long dbId, long tableId, - long partitionId, long indexId, long version, - long tabletId, int schemaHash, - long jobId, AlterJobV2.JobType jobType, - boolean isDropOp, List alterInvertedIndexes, - List indexes, List schemaColumns, long expiration) { - super(null, backendId, TTaskType.ALTER_INVERTED_INDEX, dbId, tableId, partitionId, indexId, tabletId); + long partitionId, long indexId, long tabletId, int schemaHash, + List existIndexes, List alterInvertedIndexes, + List schemaColumns, boolean isDropOp, long taskSignature) { + super(null, backendId, TTaskType.ALTER_INVERTED_INDEX, dbId, tableId, + partitionId, indexId, tabletId, taskSignature); this.tabletId = tabletId; - this.version = version; - this.jobId = jobId; - this.jobType = jobType; this.schemaHash = schemaHash; - this.isDropOp = isDropOp; + this.existIndexes = existIndexes; this.alterInvertedIndexes = alterInvertedIndexes; - this.indexes = indexes; this.schemaColumns = schemaColumns; - this.expiration = expiration; + this.isDropOp = isDropOp; } public long getTabletId() { return tabletId; } - public long getVersion() { - return version; - } - - public long getJobId() { - return jobId; - } - public int getSchemaHash() { return schemaHash; } - public AlterJobV2.JobType getJobType() { - return jobType; - } - public List getAlterInvertedIndexes() { return alterInvertedIndexes; } + public String toString() { + StringBuilder sb = new StringBuilder(""); + if (isDropOp) { + sb.append("DROP"); + } else { + sb.append("ADD"); + } + sb.append(" ("); + for (Index alterIndex : alterInvertedIndexes) { + sb.append(alterIndex.getIndexId()); + sb.append(": "); + sb.append(alterIndex.toString()); + sb.append(", "); + } + sb.append(" )"); + return sb.toString(); + } + public TAlterInvertedIndexReq toThrift() { TAlterInvertedIndexReq req = new TAlterInvertedIndexReq(); req.setTabletId(tabletId); - req.setAlterVersion(version); req.setSchemaHash(schemaHash); req.setIsDropOp(isDropOp); - req.setJobId(jobId); - req.setExpiration(expiration); if (!alterInvertedIndexes.isEmpty()) { List tIndexes = new ArrayList<>(); @@ -106,12 +103,13 @@ public class AlterInvertedIndexTask extends AgentTask { req.setAlterInvertedIndexes(tIndexes); } - if (indexes != null && !indexes.isEmpty()) { - List tIndexes = new ArrayList<>(); - for (Index index : indexes) { - tIndexes.add(index.toThrift()); + if (existIndexes != null) { + List indexDesc = new ArrayList(); + for (Index index : existIndexes) { + TOlapTableIndex tIndex = index.toThrift(); + indexDesc.add(tIndex); } - req.setIndexes(tIndexes); + req.setIndexesDesc(indexDesc); } if (schemaColumns != null) { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index d5241a45ac..a16022b116 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -186,11 +186,11 @@ struct TAlterTabletReqV2 { struct TAlterInvertedIndexReq { 1: required Types.TTabletId tablet_id 2: required Types.TSchemaHash schema_hash - 3: optional Types.TVersion alter_version - 4: optional TAlterTabletType alter_tablet_type = TAlterTabletType.SCHEMA_CHANGE + 3: optional Types.TVersion alter_version // Deprecated + 4: optional TAlterTabletType alter_tablet_type = TAlterTabletType.SCHEMA_CHANGE // Deprecated 5: optional bool is_drop_op= false 6: optional list alter_inverted_indexes - 7: optional list indexes + 7: optional list indexes_desc 8: optional list columns 9: optional i64 job_id 10: optional i64 expiration diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_with_compaction.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_with_compaction.out new file mode 100644 index 0000000000..1e15e2d8b0 --- /dev/null +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_with_compaction.out @@ -0,0 +1,21 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.150 2017-10-01T11:11:11.130111 2020-01-02T00:00 1 31 21 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.100 2017-10-01T11:11:11.140111 2020-01-03T00:00 1 32 22 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-04T00:00 2020-01-04T00:00 2017-10-01T11:11:11.110 2017-10-01T11:11:11.150111 2020-01-04T00:00 1 33 21 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + +-- !select_default2 -- +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.150 2017-10-01T11:11:11.130111 2020-01-02T00:00 1 31 21 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.100 2017-10-01T11:11:11.140111 2020-01-03T00:00 1 32 22 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-04T00:00 2020-01-04T00:00 2017-10-01T11:11:11.110 2017-10-01T11:11:11.150111 2020-01-04T00:00 1 33 21 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + diff --git a/regression-test/suites/dynamic_table_p0/test_dytable_complex_data.groovy b/regression-test/suites/dynamic_table_p0/test_dytable_complex_data.groovy index 2e2028bb60..245b228f5e 100644 --- a/regression-test/suites/dynamic_table_p0/test_dytable_complex_data.groovy +++ b/regression-test/suites/dynamic_table_p0/test_dytable_complex_data.groovy @@ -129,6 +129,29 @@ suite("test_dynamic_table", "dynamic_table"){ assertTrue(useTime <= OpTimeout) } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + def index_res = "" def create_index = { table_name, colume_name, index_name, index_type, expect_success -> // create index @@ -146,6 +169,10 @@ suite("test_dynamic_table", "dynamic_table"){ logger.info("create index res: ${index_res} \n".toString()) wait_for_latest_op_on_table_finish(table_name, timeout) + index_res = sql """ build index ${index_name} on ${table_name} """ + logger.info("build index res: ${index_res} \n".toString()) + wait_for_build_index_on_partition_finish(table_name, timeout) + }catch(Exception ex){ logger.info("create create index ${index_name} on ${table_name}(`${colume_name}`) using inverted(${index_type}) fail, catch exception: ${ex} \n".toString()) real_res = "false" diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy new file mode 100644 index 0000000000..c37f9aa0cd --- /dev/null +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_compaction.groovy @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_index_change_with_compaction") { + def tableName = "index_change_with_compaction_dup_keys" + + try { + //BackendId,Cluster,IP,HeartbeatPort,BePort,HttpPort,BrpcPort,LastStartTime,LastHeartbeat,Alive,SystemDecommissioned,ClusterDecommissioned,TabletNum,DataUsedCapacity,AvailCapacity,TotalCapacity,UsedPct,MaxDiskUsedPct,Tag,ErrMsg,Version,Status + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + backend_id = backendId_to_backendIP.keySet()[0] + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(backendId_to_backendIP.get(backend_id)) + showConfigCommand.append(":") + showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) + showConfigCommand.append("/api/show_config") + logger.info(showConfigCommand.toString()) + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `datev2` DATEV2 NOT NULL COMMENT "数据灌入日期时间", + `datetimev2_1` DATETIMEV2(3) NOT NULL COMMENT "数据灌入日期时间", + `datetimev2_2` DATETIMEV2(6) NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `datetime_val1` DATETIMEV2(3) DEFAULT "1970-01-01 00:00:00.111" COMMENT "用户最后一次访问时间", + `datetime_val2` DATETIME(6) DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `datev2`, `datetimev2_1`, `datetimev2_2`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2017-10-01 11:11:11.170000', '2017-10-01 11:11:11.110111', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.160000', '2017-10-01 11:11:11.100111', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.150000', '2017-10-01 11:11:11.130111', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.140000', '2017-10-01 11:11:11.120111', '2020-01-03', 1, 32, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.100000', '2017-10-01 11:11:11.140111', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-04', '2020-01-04', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.150111', '2020-01-04', 1, 33, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ + + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus + String[][] tablets = sql """ show tablets from ${tableName}; """ + + // create inverted index + sql """ CREATE INDEX idx_user_id ON ${tableName}(`user_id`) USING INVERTED """ + sql """ CREATE INDEX idx_date ON ${tableName}(`date`) USING INVERTED """ + sql """ CREATE INDEX idx_city ON ${tableName}(`city`) USING INVERTED """ + + // trigger compactions for all tablets in ${tableName} + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + if (compactJson.status.toLowerCase() == "fail") { + assertEquals(disableAutoCompaction, false) + logger.info("Compaction was done automatically!") + } + if (disableAutoCompaction) { + assertEquals("success", compactJson.status.toLowerCase()) + } + } + + // build index + sql "build index idx_user_id on ${tableName}" + sql "build index idx_date on ${tableName}" + sql "build index idx_city on ${tableName}" + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(backendId_to_backendIP.get(backend_id)) + sb.append(":") + sb.append(backendId_to_backendHttpPort.get(backend_id)) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + StringBuilder sb = new StringBuilder(); + def compactionStatusUrlIndex = 18 + sb.append("curl -X GET ") + sb.append(tablet[compactionStatusUrlIndex]) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + assert (rowCount <= 8) + qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ + } finally { + // try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy index a68cd2c59e..cd04866dcf 100644 --- a/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy +++ b/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy @@ -37,6 +37,29 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + def indexTbName1 = "test_add_drop_inverted_index4" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -103,6 +126,8 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ // add index on column description sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql "build index idx_desc on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // show index after add index show_result = sql "show index from ${indexTbName1}" @@ -167,6 +192,8 @@ suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){ // add index on column description sql "create index idx_desc on ${indexTbName1}(DESCRIPTION) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql "build index idx_desc on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // query rows where description match 'desc' select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index_with_data.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index_with_data.groovy index ec267a3f27..2f5d8f7e43 100644 --- a/regression-test/suites/inverted_index_p0/test_add_drop_index_with_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_add_drop_index_with_data.groovy @@ -37,6 +37,29 @@ suite("test_add_drop_index_with_data", "inverted_index"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + def indexTbName1 = "test_add_drop_inverted_index2" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -103,7 +126,8 @@ suite("test_add_drop_index_with_data", "inverted_index"){ // add index on column description sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_latest_op_on_table_finish(indexTbName1, timeout) - + sql "build index idx_desc on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // show index after add index show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) @@ -210,7 +234,8 @@ suite("test_add_drop_index_with_data", "inverted_index"){ // add index on column description sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_latest_op_on_table_finish(indexTbName1, timeout) - + sql "build index idx_desc on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // query rows where description match 'desc' select_result = sql "select * from ${indexTbName1} where description match 'desc' order by id" assertEquals(select_result.size(), 2) @@ -245,6 +270,8 @@ suite("test_add_drop_index_with_data", "inverted_index"){ show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) assertEquals(show_result.size(), 3) + sql "build index idx_name on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // query rows where name match 'name1' select_result = sql "select * from ${indexTbName1} where name match 'name1'" diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index_with_delete_data.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index_with_delete_data.groovy index b2c7b3c7ae..927a31dfc9 100644 --- a/regression-test/suites/inverted_index_p0/test_add_drop_index_with_delete_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_add_drop_index_with_delete_data.groovy @@ -37,6 +37,29 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + def indexTbName1 = "test_add_drop_inverted_index3" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -104,6 +127,8 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ // add index on column description sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql "build index idx_desc on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // show index after add index show_result = sql "show index from ${indexTbName1}" @@ -202,6 +227,8 @@ suite("test_add_drop_index_with_delete_data", "inverted_index"){ // add index on column description sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED PROPERTIES(\"parser\"=\"standard\");" wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql "build index idx_desc on ${indexTbName1}" + wait_for_build_index_on_partition_finish(indexTbName1, timeout) // show index after add index show_result = sql "show index from ${indexTbName1}" diff --git a/regression-test/suites/inverted_index_p0/test_index_match_term_and_phrase_select.groovy b/regression-test/suites/inverted_index_p0/test_index_match_term_and_phrase_select.groovy index d857e6b27f..1d6ba2f751 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_term_and_phrase_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_term_and_phrase_select.groovy @@ -47,6 +47,29 @@ suite("test_index_match_term_and_phrase_select", "inverted_index_select"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + sql "DROP TABLE IF EXISTS ${indexTbName1}" // create table with different index @@ -116,6 +139,20 @@ suite("test_index_match_term_and_phrase_select", "inverted_index_select"){ add index ${text_colume1}_idx(`${text_colume1}`) USING INVERTED PROPERTIES("parser"="standard") COMMENT '${text_colume1} index'; """ wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql """ build index ${varchar_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${varchar_colume2}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${varchar_colume3}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${int_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${string_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${char_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${text_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) } // case1: match term diff --git a/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy index 49ce39e26c..bb17d0f76b 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy @@ -85,6 +85,29 @@ suite("test_index_range_in_select", "inverted_index_select"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + for (int i = 0; i < 2; i++) { logger.info("select table with index times " + i) // case 1 @@ -115,6 +138,20 @@ suite("test_index_range_in_select", "inverted_index_select"){ add index ${text_colume1}_idx(`${text_colume1}`) USING INVERTED PROPERTIES("parser"="standard") COMMENT '${text_colume1} index'; """ wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql """ build index ${varchar_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${varchar_colume2}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${varchar_colume3}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${int_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${string_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${char_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) + sql """ build index ${text_colume1}_idx on ${indexTbName1} """ + wait_for_build_index_on_partition_finish(indexTbName1, timeout) } // case1: select in diff --git a/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy index 319e1b5d15..04a2df58a6 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy @@ -91,6 +91,29 @@ suite("test_index_range_not_in_select", "inverted_index_select"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + expected_finished_num = alter_res.size(); + finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } else { + finished_num = 0; + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + for (int i = 0; i < 2; i++) { logger.info("select table with index times " + i) // case 1 @@ -121,6 +144,20 @@ suite("test_index_range_not_in_select", "inverted_index_select"){ add index ${text_colume1}_idx(`${text_colume1}`) USING INVERTED PROPERTIES("parser"="standard") COMMENT '${text_colume1} index'; """ wait_for_latest_op_on_table_finish(Tb_name, timeout) + sql """ build index ${varchar_colume1}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) + sql """ build index ${varchar_colume2}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) + sql """ build index ${varchar_colume3}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) + sql """ build index ${int_colume1}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) + sql """ build index ${string_colume1}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) + sql """ build index ${char_colume1}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) + sql """ build index ${text_colume1}_idx on ${Tb_name} """ + wait_for_build_index_on_partition_finish(Tb_name, timeout) } // case1: select in