From fbee3c7722302f2b352bb65bee2477154e9a482d Mon Sep 17 00:00:00 2001 From: lichaoyong Date: Wed, 4 Dec 2019 20:09:03 +0800 Subject: [PATCH] Remove VersionHash used to comparison in BE (#2358) --- be/src/olap/schema_change.cpp | 6 ++-- be/src/olap/tablet.cpp | 11 +----- be/src/olap/task/engine_checksum_task.h | 2 +- be/src/olap/task/engine_clone_task.cpp | 36 ++++++------------- be/test/exec/new_olap_scan_node_test.cpp | 4 +-- be/test/exec/olap_common_test.cpp | 2 +- be/test/exec/olap_scan_node_test.cpp | 2 +- be/test/exec/olap_scanner_test.cpp | 2 +- be/test/olap/mock_command_executor.h | 1 - be/test/olap/olap_reader_test.cpp | 2 +- be/test/olap/olap_snapshot_converter_test.cpp | 7 ++-- be/test/olap/rowset/alpha_rowset_test.cpp | 1 - be/test/olap/rowset/beta_rowset_test.cpp | 1 - be/test/olap/rowset/rowset_meta_test.cpp | 2 -- be/test/olap/vectorized_olap_reader_test.cpp | 2 +- gensrc/proto/AgentService.thrift | 0 gensrc/thrift/AgentService.thrift | 18 +++++----- gensrc/thrift/PlanNodes.thrift | 2 +- gensrc/thrift/QueryPlanExtra.thrift | 2 +- 19 files changed, 34 insertions(+), 69 deletions(-) create mode 100644 gensrc/proto/AgentService.thrift diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 281d2ceb08..475a0bac93 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1932,10 +1932,8 @@ OLAPStatus SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_table new_tablet->max_continuous_version_from_begining(&max_continuous_version, &max_continuous_version_hash); LOG(INFO) << "find max continuous version of tablet=" << new_tablet->full_name() << ", start_version=" << max_continuous_version.first - << ", end_version=" << max_continuous_version.second - << ", version_hash=" << max_continuous_version_hash; - if (max_continuous_version.second > request.alter_version - || (max_continuous_version.second == request.alter_version && max_continuous_version_hash == request.alter_version_hash)) { + << ", end_version=" << max_continuous_version.second; + if (max_continuous_version.second >= request.alter_version) { return OLAP_SUCCESS; } else { return OLAP_ERR_VERSION_NOT_EXIST; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 998b61d3d6..0ae9171534 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -982,19 +982,10 @@ void Tablet::_print_missed_versions(const std::vector& missed_versions) LOG(WARNING) << ss.str(); } - OLAPStatus Tablet::_check_added_rowset(const RowsetSharedPtr& rowset) { +OLAPStatus Tablet::_check_added_rowset(const RowsetSharedPtr& rowset) { if (rowset == nullptr) { return OLAP_ERR_ROWSET_INVALID; } - Version version = {rowset->start_version(), rowset->end_version()}; - RowsetSharedPtr exist_rs = get_rowset_by_version(version); - // if there exist a rowset with version_hash == 0, should delete it - if (exist_rs != nullptr && exist_rs->version_hash() == 0) { - vector to_add; - vector to_delete; - to_delete.push_back(exist_rs); - RETURN_NOT_OK(modify_rowsets(to_add, to_delete)); - } // check if there exist a rowset contains the added rowset for (auto& it : _rs_version_map) { diff --git a/be/src/olap/task/engine_checksum_task.h b/be/src/olap/task/engine_checksum_task.h index 9a114435ef..3e6bc32481 100644 --- a/be/src/olap/task/engine_checksum_task.h +++ b/be/src/olap/task/engine_checksum_task.h @@ -51,4 +51,4 @@ private: }; // EngineTask } // doris -#endif //DORIS_BE_SRC_OLAP_TASK_ENGINE_CHECKSUM_TASK_H \ No newline at end of file +#endif //DORIS_BE_SRC_OLAP_TASK_ENGINE_CHECKSUM_TASK_H diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index e029b47afa..23c31b2cc6 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -88,7 +88,7 @@ OLAPStatus EngineCloneTask::execute() { // version 2 may be an invalid rowset Version clone_version = {_clone_req.committed_version, _clone_req.committed_version}; RowsetSharedPtr clone_rowset = tablet->get_rowset_by_version(clone_version); - if (clone_rowset == nullptr || clone_rowset->version_hash() == _clone_req.committed_version_hash) { + if (clone_rowset == nullptr) { // try to incremental clone vector missed_versions; tablet->calc_missed_versions(_clone_req.committed_version, &missed_versions); @@ -110,8 +110,6 @@ OLAPStatus EngineCloneTask::execute() { } else { LOG(INFO) << "current tablet has invalid rowset that's version == commit_version but version hash not equal" << " clone req commit_version=" << _clone_req.committed_version - << " clone req commit_version_hash=" << _clone_req.committed_version_hash - << " cur rowset version=" << clone_rowset->version_hash() << " tablet info = " << tablet->full_name(); } if (status == DORIS_SUCCESS && allow_incremental_clone) { @@ -245,19 +243,13 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status, bool is_new_tablet) { << " signature: " << _signature; _error_msgs->push_back("clone success, but get tablet info failed."); status = DORIS_ERROR; - } else if ( - (_clone_req.__isset.committed_version - && _clone_req.__isset.committed_version_hash) - && (tablet_info.version < _clone_req.committed_version || - (tablet_info.version == _clone_req.committed_version - && tablet_info.version_hash != _clone_req.committed_version_hash))) { + } else if (_clone_req.__isset.committed_version + && tablet_info.version < _clone_req.committed_version) { LOG(WARNING) << "failed to clone tablet. tablet_id:" << _clone_req.tablet_id << ", schema_hash:" << _clone_req.schema_hash << ", signature:" << _signature << ", version:" << tablet_info.version - << ", version_hash:" << tablet_info.version_hash - << ", expected_version: " << _clone_req.committed_version - << ", version_hash:" << _clone_req.committed_version_hash; + << ", expected_version: " << _clone_req.committed_version; // if it is a new tablet and clone failed, then remove the tablet // if it is incremental clone, then must not drop the tablet if (is_new_tablet) { @@ -268,9 +260,7 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status, bool is_new_tablet) { << ", schema_hash:" << _clone_req.schema_hash << ", signature:" << _signature << ", version:" << tablet_info.version - << ", version_hash:" << tablet_info.version_hash - << ", expected_version: " << _clone_req.committed_version - << ", version_hash:" << _clone_req.committed_version_hash; + << ", expected_version: " << _clone_req.committed_version; OLAPStatus drop_status = StorageEngine::instance()->tablet_manager()->drop_tablet(_clone_req.tablet_id, _clone_req.schema_hash); if (drop_status != OLAP_SUCCESS && drop_status != OLAP_ERR_TABLE_NOT_FOUND) { @@ -283,8 +273,7 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status, bool is_new_tablet) { LOG(INFO) << "clone get tablet info success. tablet_id:" << _clone_req.tablet_id << ", schema_hash:" << _clone_req.schema_hash << ", signature:" << _signature - << ", version:" << tablet_info.version - << ", version_hash:" << tablet_info.version_hash; + << ", version:" << tablet_info.version; _tablet_infos->push_back(tablet_info); } } @@ -787,7 +776,6 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_ // check local versions for (auto& rs_meta : tablet->tablet_meta()->all_rs_metas()) { Version local_version(rs_meta->start_version(), rs_meta->end_version()); - VersionHash local_version_hash = rs_meta->version_hash(); LOG(INFO) << "check local delta when full clone." << "tablet=" << tablet->full_name() << ", local_version=" << local_version.first << "-" << local_version.second; @@ -813,8 +801,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_ // there is no necessity to clone it. for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) { if (rs_meta->version().first == local_version.first - && rs_meta->version().second == local_version.second - && rs_meta->version_hash() == local_version_hash) { + && rs_meta->version().second == local_version.second) { existed_in_src = true; break; } @@ -830,8 +817,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_ } else { LOG(INFO) << "Delta has already existed in local header, no need to clone." << "tablet=" << tablet->full_name() - << ", version='" << local_version.first<< "-" << local_version.second - << ", version_hash=" << local_version_hash; + << ", version='" << local_version.first<< "-" << local_version.second; } } else { // Delta labeled in local_version is not existed in clone header, @@ -840,8 +826,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_ versions_to_delete.push_back(local_version); LOG(INFO) << "Delete delta not included by the clone header, should delete it from local header." << "tablet=" << tablet->full_name() << "," - << ", version=" << local_version.first<< "-" << local_version.second - << ", version_hash=" << local_version_hash; + << ", version=" << local_version.first<< "-" << local_version.second; } } } @@ -851,8 +836,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_ LOG(INFO) << "Delta to clone." << "tablet=" << tablet->full_name() << ", version=" << rs_meta->version().first << "-" - << rs_meta->version().second - << ", version_hash=" << rs_meta->version_hash(); + << rs_meta->version().second; } // clone_data to tablet diff --git a/be/test/exec/new_olap_scan_node_test.cpp b/be/test/exec/new_olap_scan_node_test.cpp index b59bcc15d0..a9e3982d1c 100644 --- a/be/test/exec/new_olap_scan_node_test.cpp +++ b/be/test/exec/new_olap_scan_node_test.cpp @@ -228,7 +228,7 @@ public: doris_scan_range.hosts.push_back(host); doris_scan_range.__set_schema_hash("1709394"); doris_scan_range.__set_version("0"); - doris_scan_range.__set_version_hash("2709394"); + doris_scan_range.__set_version_hash("0"); config::olap_index_name = "userid_type_planid_unitid_winfoid"; doris_scan_range.engine_table_name.push_back("clickuserid_online"); doris_scan_range.__set_db_name("fc"); @@ -306,7 +306,7 @@ TEST_F(TestOlapScanNode, SimpleTest) { TEST_F(TestOlapScanNode, MultiColumnSingleVersionTest) { _scan_ranges[0].scan_range.doris_scan_range.__set_version("0"); - _scan_ranges[0].scan_range.doris_scan_range.__set_version_hash("2709394"); + _scan_ranges[0].scan_range.doris_scan_range.__set_version_hash("0"); vector data; read_data(0, &data); diff --git a/be/test/exec/olap_common_test.cpp b/be/test/exec/olap_common_test.cpp index f550267680..6c85d2a1b2 100644 --- a/be/test/exec/olap_common_test.cpp +++ b/be/test/exec/olap_common_test.cpp @@ -40,7 +40,7 @@ void construct_scan_range(TPaloScanRange* doris_scan_range) { doris_scan_range->hosts.push_back(host); doris_scan_range->__set_schema_hash("216424022"); doris_scan_range->__set_version("0"); - doris_scan_range->__set_version_hash("3997217299075720338"); + doris_scan_range->__set_version_hash("0"); doris_scan_range->engine_table_name.push_back("ShowQStats"); doris_scan_range->__set_db_name("olap"); TKeyRange key_range; diff --git a/be/test/exec/olap_scan_node_test.cpp b/be/test/exec/olap_scan_node_test.cpp index 96948191a5..4531cebf4c 100644 --- a/be/test/exec/olap_scan_node_test.cpp +++ b/be/test/exec/olap_scan_node_test.cpp @@ -177,7 +177,7 @@ public: doris_scan_range.hosts.push_back(host); doris_scan_range.__set_schema_hash("462300563"); doris_scan_range.__set_version("94"); - doris_scan_range.__set_version_hash("422202811388534102"); + doris_scan_range.__set_version_hash("0"); doris_scan_range.engine_table_name.push_back("DorisTestStats"); doris_scan_range.__set_db_name("olap"); //TKeyRange key_range; diff --git a/be/test/exec/olap_scanner_test.cpp b/be/test/exec/olap_scanner_test.cpp index 05a9dd820c..b3f99b0f5d 100644 --- a/be/test/exec/olap_scanner_test.cpp +++ b/be/test/exec/olap_scanner_test.cpp @@ -42,7 +42,7 @@ boost::shared_ptr construct_scan_ranges() { doris_scan_range.hosts.push_back(host); doris_scan_range.__set_schema_hash("462300563"); doris_scan_range.__set_version("94"); - doris_scan_range.__set_version_hash("422202811388534102"); + doris_scan_range.__set_version_hash("0"); doris_scan_range.engine_table_name.push_back("DorisTestStats"); doris_scan_range.__set_db_name("olap"); TKeyRange key_range; diff --git a/be/test/olap/mock_command_executor.h b/be/test/olap/mock_command_executor.h index e2f402139b..2632b5ef61 100644 --- a/be/test/olap/mock_command_executor.h +++ b/be/test/olap/mock_command_executor.h @@ -81,7 +81,6 @@ public: TTabletId tablet_id, TSchemaHash schema_hash, TVersion version, - TVersionHash version_hash, uint32_t* checksum)); MOCK_METHOD1(reload_root_path, OLAPStatus(const std::string& root_paths)); MOCK_METHOD2(check_tablet_exist, bool(TTabletId tablet_id, TSchemaHash schema_hash)); diff --git a/be/test/olap/olap_reader_test.cpp b/be/test/olap/olap_reader_test.cpp index f866bd2feb..be793f4d90 100644 --- a/be/test/olap/olap_reader_test.cpp +++ b/be/test/olap/olap_reader_test.cpp @@ -128,7 +128,7 @@ void set_default_push_request(TPushReq* request) { request->tablet_id = 10003; request->schema_hash = 1508825676; request->__set_version(2); - request->__set_version_hash(1); + request->__set_version_hash(0); request->timeout = 86400; request->push_type = TPushType::LOAD; request->__set_http_file_path("./be/test/olap/test_data/all_types_1000"); diff --git a/be/test/olap/olap_snapshot_converter_test.cpp b/be/test/olap/olap_snapshot_converter_test.cpp index 75348eb7b4..45d7ee2904 100644 --- a/be/test/olap/olap_snapshot_converter_test.cpp +++ b/be/test/olap/olap_snapshot_converter_test.cpp @@ -143,12 +143,10 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) { for (auto& pdelta : header_msg.delta()) { int64_t start_version = pdelta.start_version(); int64_t end_version = pdelta.end_version(); - int64_t version_hash = pdelta.version_hash(); bool found = false; for (auto& visible_rowset : tablet_meta_pb.rs_metas()) { if (visible_rowset.start_version() == start_version - && visible_rowset.end_version() == end_version - && visible_rowset.version_hash() == version_hash) { + && visible_rowset.end_version() == end_version) { found = true; } } @@ -226,8 +224,7 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) { bool found = false; for (auto& converted_pdelta : old_header_msg.delta()) { if (converted_pdelta.start_version() == pdelta.start_version() - && converted_pdelta.end_version() == pdelta.end_version() - && converted_pdelta.version_hash() == pdelta.version_hash()) { + && converted_pdelta.end_version() == pdelta.end_version()) { found = true; } } diff --git a/be/test/olap/rowset/alpha_rowset_test.cpp b/be/test/olap/rowset/alpha_rowset_test.cpp index 7cdef70d4e..c1ea0092c7 100644 --- a/be/test/olap/rowset/alpha_rowset_test.cpp +++ b/be/test/olap/rowset/alpha_rowset_test.cpp @@ -85,7 +85,6 @@ void create_rowset_writer_context(TabletSchema* tablet_schema, rowset_writer_context->tablet_schema = tablet_schema; rowset_writer_context->version.first = 0; rowset_writer_context->version.second = 1; - rowset_writer_context->version_hash = 110; } void create_rowset_reader_context(TabletSchema* tablet_schema, const std::vector* return_columns, diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp index 9cc3eb63e8..e88f638b0f 100644 --- a/be/test/olap/rowset/beta_rowset_test.cpp +++ b/be/test/olap/rowset/beta_rowset_test.cpp @@ -114,7 +114,6 @@ protected: rowset_writer_context->tablet_schema = tablet_schema; rowset_writer_context->version.first = 10; rowset_writer_context->version.second = 10; - rowset_writer_context->version_hash = 110; } void create_and_init_rowset_reader(Rowset* rowset, RowsetReaderContext& context, RowsetReaderSharedPtr* result) { diff --git a/be/test/olap/rowset/rowset_meta_test.cpp b/be/test/olap/rowset/rowset_meta_test.cpp index 7320ec7332..95323dbb0b 100644 --- a/be/test/olap/rowset/rowset_meta_test.cpp +++ b/be/test/olap/rowset/rowset_meta_test.cpp @@ -82,7 +82,6 @@ void do_check(RowsetMeta rowset_meta) { ASSERT_EQ(VISIBLE, rowset_meta.rowset_state()); ASSERT_EQ(2, rowset_meta.start_version()); ASSERT_EQ(2, rowset_meta.end_version()); - ASSERT_EQ(8391828013814912580, rowset_meta.version_hash()); ASSERT_EQ(3929, rowset_meta.num_rows()); ASSERT_EQ(84699, rowset_meta.total_disk_size()); ASSERT_EQ(84464, rowset_meta.data_disk_size()); @@ -124,7 +123,6 @@ void do_check_for_alpha(AlphaRowsetMeta alpha_rowset_meta) { ASSERT_EQ(VISIBLE, alpha_rowset_meta.rowset_state()); ASSERT_EQ(2, alpha_rowset_meta.start_version()); ASSERT_EQ(2, alpha_rowset_meta.end_version()); - ASSERT_EQ(8391828013814912580, alpha_rowset_meta.version_hash()); ASSERT_EQ(3929, alpha_rowset_meta.num_rows()); ASSERT_EQ(84699, alpha_rowset_meta.total_disk_size()); ASSERT_EQ(84464, alpha_rowset_meta.data_disk_size()); diff --git a/be/test/olap/vectorized_olap_reader_test.cpp b/be/test/olap/vectorized_olap_reader_test.cpp index c07ff68cf5..650da3033c 100644 --- a/be/test/olap/vectorized_olap_reader_test.cpp +++ b/be/test/olap/vectorized_olap_reader_test.cpp @@ -131,7 +131,7 @@ void set_default_push_request(TPushReq* request) { request->tablet_id = 10003; request->schema_hash = 1508825676; request->__set_version(2); - request->__set_version_hash(1); + request->__set_version_hash(0); request->timeout = 86400; request->push_type = TPushType::LOAD; request->__set_http_file_path("./be/test/olap/test_data/all_types_1000"); diff --git a/gensrc/proto/AgentService.thrift b/gensrc/proto/AgentService.thrift new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 3f6c906d90..ad4e15f8c4 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -45,7 +45,7 @@ struct TCreateTabletReq { 1: required Types.TTabletId tablet_id 2: required TTabletSchema tablet_schema 3: optional Types.TVersion version - 4: optional Types.TVersionHash version_hash + 4: optional Types.TVersionHash version_hash // Deprecated 5: optional Types.TStorageMedium storage_medium 6: optional bool in_restore_mode // this new tablet should be colocate with base tablet @@ -80,7 +80,7 @@ struct TAlterTabletReqV2 { 4: required Types.TSchemaHash new_schema_hash // version of data which this alter task should transform 5: optional Types.TVersion alter_version - 6: optional Types.TVersionHash alter_version_hash + 6: optional Types.TVersionHash alter_version_hash // Deprecated } struct TClusterInfo { @@ -92,7 +92,7 @@ struct TPushReq { 1: required Types.TTabletId tablet_id 2: required Types.TSchemaHash schema_hash 3: required Types.TVersion version - 4: required Types.TVersionHash version_hash + 4: required Types.TVersionHash version_hash // Deprecated 5: required i64 timeout 6: required Types.TPushType push_type 7: optional string http_file_path @@ -114,7 +114,7 @@ struct TCloneReq { 4: optional Types.TStorageMedium storage_medium // these are visible version(hash) actually 5: optional Types.TVersion committed_version - 6: optional Types.TVersionHash committed_version_hash + 6: optional Types.TVersionHash committed_version_hash // Deprecated 7: optional i32 task_version; 8: optional i64 src_path_hash; 9: optional i64 dest_path_hash; @@ -132,14 +132,14 @@ struct TCancelDeleteDataReq { 1: required Types.TTabletId tablet_id 2: required Types.TSchemaHash schema_hash 3: required Types.TVersion version - 4: required Types.TVersionHash version_hash + 4: required Types.TVersionHash version_hash // Deprecated } struct TCheckConsistencyReq { 1: required Types.TTabletId tablet_id 2: required Types.TSchemaHash schema_hash 3: required Types.TVersion version - 4: required Types.TVersionHash version_hash + 4: required Types.TVersionHash version_hash // Deprecated } struct TUploadReq { @@ -160,7 +160,7 @@ struct TSnapshotRequest { 1: required Types.TTabletId tablet_id 2: required Types.TSchemaHash schema_hash 3: optional Types.TVersion version - 4: optional Types.TVersionHash version_hash + 4: optional Types.TVersionHash version_hash // Deprecated 5: optional i64 timeout 6: optional list missing_version 7: optional bool list_files @@ -181,7 +181,7 @@ struct TClearRemoteFileReq { struct TPartitionVersionInfo { 1: required Types.TPartitionId partition_id 2: required Types.TVersion version - 3: required Types.TVersionHash version_hash + 3: required Types.TVersionHash version_hash // Deprecated } struct TMoveDirReq { @@ -217,7 +217,7 @@ struct TRecoverTabletReq { 1: optional Types.TTabletId tablet_id 2: optional Types.TSchemaHash schema_hash 3: optional Types.TVersion version - 4: optional Types.TVersionHash version_hash + 4: optional Types.TVersionHash version_hash // Deprecated } struct TTabletMetaInfo { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 9b15b30a48..b13c70d3d2 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -81,7 +81,7 @@ struct TPaloScanRange { 1: required list hosts 2: required string schema_hash 3: required string version - 4: required string version_hash + 4: required string version_hash // Deprecated 5: required Types.TTabletId tablet_id 6: required string db_name 7: optional list partition_column_ranges diff --git a/gensrc/thrift/QueryPlanExtra.thrift b/gensrc/thrift/QueryPlanExtra.thrift index e952680766..bdd0ae33b9 100644 --- a/gensrc/thrift/QueryPlanExtra.thrift +++ b/gensrc/thrift/QueryPlanExtra.thrift @@ -38,4 +38,4 @@ struct TQueryPlanInfo { 3: required Descriptors.TDescriptorTable desc_tbl // all tablet scan should share one query_id 4: required Types.TUniqueId query_id -} \ No newline at end of file +}