From ca23b7a511ebfb57110db9a145fcf05f8c110b56 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 9 Sep 2019 13:59:19 +0800 Subject: [PATCH] Should create init rowset for alter task v2 (#1767) --- be/src/olap/schema_change.cpp | 14 ++++++++------ be/src/olap/tablet.cpp | 3 +-- be/src/olap/tablet_manager.cpp | 34 ++++++++++++++++------------------ 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 488cc52720..f3af6d6d10 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1058,7 +1058,7 @@ SORTING_PROCESS_ERR: // remove the intermediate rowsets generated by internal sorting for (vector::iterator it = src_rowsets.begin(); it != src_rowsets.end(); ++it) { - (*it)->remove(); + StorageEngine::instance()->add_unused_rowset(*it); } for (vector::iterator it = row_block_arr.begin(); @@ -1261,7 +1261,8 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe << new_tablet->full_name(); } for (auto& rowset : rowsets_to_delete) { - rowset->remove(); + // do not call rowset.remove directly, using gc thread to delete it + StorageEngine::instance()->add_unused_rowset(rowset); } // init one delete handler @@ -1339,7 +1340,8 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe if (res != OLAP_SUCCESS) { LOG(WARNING) << "failed to alter tablet. base_tablet=" << base_tablet->full_name() << ", drop new_tablet=" << new_tablet->full_name(); - StorageEngine::instance()->tablet_manager()->drop_tablet(new_tablet->tablet_id(), new_tablet->schema_hash()); + // do not drop the new tablet and its data. GC thread will + // StorageEngine::instance()->tablet_manager()->drop_tablet(new_tablet->tablet_id(), new_tablet->schema_hash()); } return res; @@ -1715,7 +1717,7 @@ OLAPStatus SchemaChangeHandler::schema_version_convert( SCHEMA_VERSION_CONVERT_ERR: if (*new_rowset != nullptr) { - (*new_rowset)->remove(); + StorageEngine::instance()->add_unused_rowset(*new_rowset); } SAFE_DELETE(sc_procedure); @@ -1938,14 +1940,14 @@ OLAPStatus SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangePa << "tablet=" << sc_params.new_tablet->full_name() << ", version='" << rs_reader->version().first << "-" << rs_reader->version().second; - new_rowset->remove(); + StorageEngine::instance()->add_unused_rowset(new_rowset); res = OLAP_SUCCESS; } else if (res != OLAP_SUCCESS) { LOG(WARNING) << "failed to register new version. " << " tablet=" << sc_params.new_tablet->full_name() << ", version=" << rs_reader->version().first << "-" << rs_reader->version().second; - new_rowset->remove(); + StorageEngine::instance()->add_unused_rowset(new_rowset); sc_params.new_tablet->release_push_lock(); goto PROCESS_ALTER_EXIT; } else { diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 76ce76fe80..f4fc08ddd5 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -954,8 +954,7 @@ TabletInfo Tablet::get_tablet_info() { void Tablet::pick_candicate_rowsets_to_cumulative_compaction(std::vector* candidate_rowsets) { ReadLock rdlock(&_meta_lock); for (auto& it : _rs_version_map) { - if (it.first.first >= _cumulative_point - && it.first.first == it.first.second) { + if (it.first.first >= _cumulative_point) { candidate_rowsets->push_back(it.second); } } diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index c0110a3a51..cdc4f2b694 100755 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -353,6 +353,9 @@ TabletSharedPtr TabletManager::_internal_create_tablet(const AlterTabletType alt return nullptr; } + // TODO(yiguolei) + // the following code is very difficult to understand because it mixed alter tablet v2 and alter tablet v1 + // should remove alter tablet v1 code after v0.12 OLAPStatus res = OLAP_SUCCESS; do { res = tablet->init(); @@ -360,7 +363,7 @@ TabletSharedPtr TabletManager::_internal_create_tablet(const AlterTabletType alt LOG(WARNING) << "tablet init failed. tablet:" << tablet->full_name(); break; } - if (!is_schema_change_tablet) { + if (!is_schema_change_tablet || (request.__isset.base_tablet_id && request.base_tablet_id > 0)) { // Create init version if this is not a restore mode replica and request.version is set // bool in_restore_mode = request.__isset.in_restore_mode && request.in_restore_mode; // if (!in_restore_mode && request.__isset.version) { @@ -370,9 +373,14 @@ TabletSharedPtr TabletManager::_internal_create_tablet(const AlterTabletType alt LOG(WARNING) << "fail to create initial version for tablet. res=" << res; break; } - } else { + } + if (is_schema_change_tablet) { if (request.__isset.base_tablet_id && request.base_tablet_id > 0) { LOG(INFO) << "this request is for alter tablet request v2, so that not add alter task to tablet"; + // if this is a new alter tablet, has to set its state to not ready + // because schema change hanlder depends on it to check whether history data + // convert finished + tablet->set_tablet_state(TabletState::TABLET_NOTREADY); } else { // add alter task to new tablet if it is a new tablet during schema change tablet->add_alter_task(ref_tablet->tablet_id(), ref_tablet->schema_hash(), @@ -393,16 +401,6 @@ TabletSharedPtr TabletManager::_internal_create_tablet(const AlterTabletType alt tablet->set_creation_time(new_creation_time); } } - if (request.__isset.base_tablet_id) { - if (request.base_tablet_id < 1) { - LOG(FATAL) << "base tablet is set but it value=" << request.base_tablet_id; - - } - // if this is a new alter tablet, has to set its state to not ready - // because schema change hanlder depends on it to check whether history data - // convert finished - tablet->set_tablet_state(TabletState::TABLET_NOTREADY); - } // Add tablet to StorageEngine will make it visiable to user res = _add_tablet_unlock(request.tablet_id, request.tablet_schema.schema_hash, tablet, true, false); if (res != OLAP_SUCCESS) { @@ -849,12 +847,12 @@ OLAPStatus TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tab _shutdown_tablets.push_back(tablet); return OLAP_ERR_TABLE_ALREADY_DELETED_ERROR; } - - if (tablet->max_version().first == -1 && tablet->alter_task() == nullptr) { - LOG(WARNING) << "tablet not in schema change state without delta is invalid." - << "tablet=" << tablet->full_name(); - // tablet state is invalid, drop tablet - return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR; + // not check tablet init version because when be restarts during alter task the new tablet may be empty + if (tablet->max_version().first == -1 && tablet->tablet_state() == TABLET_RUNNING) { + LOG(WARNING) << "tablet is in running state without delta is invalid." + << "tablet=" << tablet->full_name(); + // tablet state is invalid, drop tablet + return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR; } OLAPStatus res = tablet->init();