database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Remove VersionHash used to comparison in BE (#2358)

Browse Source
This commit is contained in:
lichaoyong
2019-12-04 20:09:03 +08:00
committed by Mingyu Chen
parent 92536272d3
commit fbee3c7722
19 changed files with 34 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
be/src/olap/schema_change.cpp
View File

@ -1932,10 +1932,8 @@ OLAPStatus SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_table
new_tablet->max_continuous_version_from_begining(&max_continuous_version, &max_continuous_version_hash);
LOG(INFO) << "find max continuous version of tablet=" << new_tablet->full_name()
<< ", start_version=" << max_continuous_version.first
<< ", end_version=" << max_continuous_version.second
<< ", version_hash=" << max_continuous_version_hash;
if (max_continuous_version.second > request.alter_version
|| (max_continuous_version.second == request.alter_version && max_continuous_version_hash == request.alter_version_hash)) {
<< ", end_version=" << max_continuous_version.second;
if (max_continuous_version.second >= request.alter_version) {
return OLAP_SUCCESS;
} else {
return OLAP_ERR_VERSION_NOT_EXIST;

11
be/src/olap/tablet.cpp
View File

@ -982,19 +982,10 @@ void Tablet::_print_missed_versions(const std::vector<Version>& missed_versions)
LOG(WARNING) << ss.str();
}
OLAPStatus Tablet::_check_added_rowset(const RowsetSharedPtr& rowset) {
OLAPStatus Tablet::_check_added_rowset(const RowsetSharedPtr& rowset) {
if (rowset == nullptr) {
return OLAP_ERR_ROWSET_INVALID;
}
Version version = {rowset->start_version(), rowset->end_version()};
RowsetSharedPtr exist_rs = get_rowset_by_version(version);
// if there exist a rowset with version_hash == 0, should delete it
if (exist_rs != nullptr && exist_rs->version_hash() == 0) {
vector<RowsetSharedPtr> to_add;
vector<RowsetSharedPtr> to_delete;
to_delete.push_back(exist_rs);
RETURN_NOT_OK(modify_rowsets(to_add, to_delete));
}
// check if there exist a rowset contains the added rowset
for (auto& it : _rs_version_map) {

2
be/src/olap/task/engine_checksum_task.h
View File

@ -51,4 +51,4 @@ private:
}; // EngineTask
} // doris
#endif //DORIS_BE_SRC_OLAP_TASK_ENGINE_CHECKSUM_TASK_H
#endif //DORIS_BE_SRC_OLAP_TASK_ENGINE_CHECKSUM_TASK_H

36
be/src/olap/task/engine_clone_task.cpp
View File

@ -88,7 +88,7 @@ OLAPStatus EngineCloneTask::execute() {
// version 2 may be an invalid rowset
Version clone_version = {_clone_req.committed_version, _clone_req.committed_version};
RowsetSharedPtr clone_rowset = tablet->get_rowset_by_version(clone_version);
if (clone_rowset == nullptr || clone_rowset->version_hash() == _clone_req.committed_version_hash) {
if (clone_rowset == nullptr) {
// try to incremental clone
vector<Version> missed_versions;
tablet->calc_missed_versions(_clone_req.committed_version, &missed_versions);
@ -110,8 +110,6 @@ OLAPStatus EngineCloneTask::execute() {
} else {
LOG(INFO) << "current tablet has invalid rowset that's version == commit_version but version hash not equal"
<< " clone req commit_version=" << _clone_req.committed_version
<< " clone req commit_version_hash=" << _clone_req.committed_version_hash
<< " cur rowset version=" << clone_rowset->version_hash()
<< " tablet info = " << tablet->full_name();
}
if (status == DORIS_SUCCESS && allow_incremental_clone) {
@ -245,19 +243,13 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status, bool is_new_tablet) {
<< " signature: " << _signature;
_error_msgs->push_back("clone success, but get tablet info failed.");
status = DORIS_ERROR;
} else if (
(_clone_req.__isset.committed_version
&& _clone_req.__isset.committed_version_hash)
&& (tablet_info.version < _clone_req.committed_version ||
(tablet_info.version == _clone_req.committed_version
&& tablet_info.version_hash != _clone_req.committed_version_hash))) {
} else if (_clone_req.__isset.committed_version
&& tablet_info.version < _clone_req.committed_version) {
LOG(WARNING) << "failed to clone tablet. tablet_id:" << _clone_req.tablet_id
<< ", schema_hash:" << _clone_req.schema_hash
<< ", signature:" << _signature
<< ", version:" << tablet_info.version
<< ", version_hash:" << tablet_info.version_hash
<< ", expected_version: " << _clone_req.committed_version
<< ", version_hash:" << _clone_req.committed_version_hash;
<< ", expected_version: " << _clone_req.committed_version;
// if it is a new tablet and clone failed, then remove the tablet
// if it is incremental clone, then must not drop the tablet
if (is_new_tablet) {
@ -268,9 +260,7 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status, bool is_new_tablet) {
<< ", schema_hash:" << _clone_req.schema_hash
<< ", signature:" << _signature
<< ", version:" << tablet_info.version
<< ", version_hash:" << tablet_info.version_hash
<< ", expected_version: " << _clone_req.committed_version
<< ", version_hash:" << _clone_req.committed_version_hash;
<< ", expected_version: " << _clone_req.committed_version;
OLAPStatus drop_status = StorageEngine::instance()->tablet_manager()->drop_tablet(_clone_req.tablet_id,
_clone_req.schema_hash);
if (drop_status != OLAP_SUCCESS && drop_status != OLAP_ERR_TABLE_NOT_FOUND) {
@ -283,8 +273,7 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status, bool is_new_tablet) {
LOG(INFO) << "clone get tablet info success. tablet_id:" << _clone_req.tablet_id
<< ", schema_hash:" << _clone_req.schema_hash
<< ", signature:" << _signature
<< ", version:" << tablet_info.version
<< ", version_hash:" << tablet_info.version_hash;
<< ", version:" << tablet_info.version;
_tablet_infos->push_back(tablet_info);
}
}
@ -787,7 +776,6 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_
// check local versions
for (auto& rs_meta : tablet->tablet_meta()->all_rs_metas()) {
Version local_version(rs_meta->start_version(), rs_meta->end_version());
VersionHash local_version_hash = rs_meta->version_hash();
LOG(INFO) << "check local delta when full clone."
<< "tablet=" << tablet->full_name()
<< ", local_version=" << local_version.first << "-" << local_version.second;
@ -813,8 +801,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_
// there is no necessity to clone it.
for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
if (rs_meta->version().first == local_version.first
&& rs_meta->version().second == local_version.second
&& rs_meta->version_hash() == local_version_hash) {
&& rs_meta->version().second == local_version.second) {
existed_in_src = true;
break;
}
@ -830,8 +817,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_
} else {
LOG(INFO) << "Delta has already existed in local header, no need to clone."
<< "tablet=" << tablet->full_name()
<< ", version='" << local_version.first<< "-" << local_version.second
<< ", version_hash=" << local_version_hash;
<< ", version='" << local_version.first<< "-" << local_version.second;
}
} else {
// Delta labeled in local_version is not existed in clone header,
@ -840,8 +826,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_
versions_to_delete.push_back(local_version);
LOG(INFO) << "Delete delta not included by the clone header, should delete it from local header."
<< "tablet=" << tablet->full_name() << ","
<< ", version=" << local_version.first<< "-" << local_version.second
<< ", version_hash=" << local_version_hash;
<< ", version=" << local_version.first<< "-" << local_version.second;
}
}
}
@ -851,8 +836,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_
LOG(INFO) << "Delta to clone."
<< "tablet=" << tablet->full_name()
<< ", version=" << rs_meta->version().first << "-"
<< rs_meta->version().second
<< ", version_hash=" << rs_meta->version_hash();
<< rs_meta->version().second;
}
// clone_data to tablet

4
be/test/exec/new_olap_scan_node_test.cpp
View File

@ -228,7 +228,7 @@ public:
doris_scan_range.hosts.push_back(host);
doris_scan_range.__set_schema_hash("1709394");
doris_scan_range.__set_version("0");
doris_scan_range.__set_version_hash("2709394");
doris_scan_range.__set_version_hash("0");
config::olap_index_name = "userid_type_planid_unitid_winfoid";
doris_scan_range.engine_table_name.push_back("clickuserid_online");
doris_scan_range.__set_db_name("fc");
@ -306,7 +306,7 @@ TEST_F(TestOlapScanNode, SimpleTest) {
TEST_F(TestOlapScanNode, MultiColumnSingleVersionTest) {
_scan_ranges[0].scan_range.doris_scan_range.__set_version("0");
_scan_ranges[0].scan_range.doris_scan_range.__set_version_hash("2709394");
_scan_ranges[0].scan_range.doris_scan_range.__set_version_hash("0");
vector<string> data;
read_data(0, &data);

2
be/test/exec/olap_common_test.cpp
View File

@ -40,7 +40,7 @@ void construct_scan_range(TPaloScanRange* doris_scan_range) {
doris_scan_range->hosts.push_back(host);
doris_scan_range->__set_schema_hash("216424022");
doris_scan_range->__set_version("0");
doris_scan_range->__set_version_hash("3997217299075720338");
doris_scan_range->__set_version_hash("0");
doris_scan_range->engine_table_name.push_back("ShowQStats");
doris_scan_range->__set_db_name("olap");
TKeyRange key_range;

2
be/test/exec/olap_scan_node_test.cpp
View File

@ -177,7 +177,7 @@ public:
doris_scan_range.hosts.push_back(host);
doris_scan_range.__set_schema_hash("462300563");
doris_scan_range.__set_version("94");
doris_scan_range.__set_version_hash("422202811388534102");
doris_scan_range.__set_version_hash("0");
doris_scan_range.engine_table_name.push_back("DorisTestStats");
doris_scan_range.__set_db_name("olap");
//TKeyRange key_range;

2
be/test/exec/olap_scanner_test.cpp
View File

@ -42,7 +42,7 @@ boost::shared_ptr<DorisScanRange> construct_scan_ranges() {
doris_scan_range.hosts.push_back(host);
doris_scan_range.__set_schema_hash("462300563");
doris_scan_range.__set_version("94");
doris_scan_range.__set_version_hash("422202811388534102");
doris_scan_range.__set_version_hash("0");
doris_scan_range.engine_table_name.push_back("DorisTestStats");
doris_scan_range.__set_db_name("olap");
TKeyRange key_range;

1
be/test/olap/mock_command_executor.h
View File

@ -81,7 +81,6 @@ public:
TTabletId tablet_id,
TSchemaHash schema_hash,
TVersion version,
TVersionHash version_hash,
uint32_t* checksum));
MOCK_METHOD1(reload_root_path, OLAPStatus(const std::string& root_paths));
MOCK_METHOD2(check_tablet_exist, bool(TTabletId tablet_id, TSchemaHash schema_hash));

2
be/test/olap/olap_reader_test.cpp
View File

@ -128,7 +128,7 @@ void set_default_push_request(TPushReq* request) {
request->tablet_id = 10003;
request->schema_hash = 1508825676;
request->__set_version(2);
request->__set_version_hash(1);
request->__set_version_hash(0);
request->timeout = 86400;
request->push_type = TPushType::LOAD;
request->__set_http_file_path("./be/test/olap/test_data/all_types_1000");

7
be/test/olap/olap_snapshot_converter_test.cpp
View File

@ -143,12 +143,10 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
for (auto& pdelta : header_msg.delta()) {
int64_t start_version = pdelta.start_version();
int64_t end_version = pdelta.end_version();
int64_t version_hash = pdelta.version_hash();
bool found = false;
for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
if (visible_rowset.start_version() == start_version
&& visible_rowset.end_version() == end_version
&& visible_rowset.version_hash() == version_hash) {
&& visible_rowset.end_version() == end_version) {
found = true;
}
}
@ -226,8 +224,7 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
bool found = false;
for (auto& converted_pdelta : old_header_msg.delta()) {
if (converted_pdelta.start_version() == pdelta.start_version()
&& converted_pdelta.end_version() == pdelta.end_version()
&& converted_pdelta.version_hash() == pdelta.version_hash()) {
&& converted_pdelta.end_version() == pdelta.end_version()) {
found = true;
}
}

1
be/test/olap/rowset/alpha_rowset_test.cpp
View File

@ -85,7 +85,6 @@ void create_rowset_writer_context(TabletSchema* tablet_schema,
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 0;
rowset_writer_context->version.second = 1;
rowset_writer_context->version_hash = 110;
}
void create_rowset_reader_context(TabletSchema* tablet_schema, const std::vector<uint32_t>* return_columns,

1
be/test/olap/rowset/beta_rowset_test.cpp
View File

@ -114,7 +114,6 @@ protected:
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
rowset_writer_context->version_hash = 110;
}
void create_and_init_rowset_reader(Rowset* rowset, RowsetReaderContext& context, RowsetReaderSharedPtr* result) {

2
be/test/olap/rowset/rowset_meta_test.cpp
View File

@ -82,7 +82,6 @@ void do_check(RowsetMeta rowset_meta) {
ASSERT_EQ(VISIBLE, rowset_meta.rowset_state());
ASSERT_EQ(2, rowset_meta.start_version());
ASSERT_EQ(2, rowset_meta.end_version());
ASSERT_EQ(8391828013814912580, rowset_meta.version_hash());
ASSERT_EQ(3929, rowset_meta.num_rows());
ASSERT_EQ(84699, rowset_meta.total_disk_size());
ASSERT_EQ(84464, rowset_meta.data_disk_size());
@ -124,7 +123,6 @@ void do_check_for_alpha(AlphaRowsetMeta alpha_rowset_meta) {
ASSERT_EQ(VISIBLE, alpha_rowset_meta.rowset_state());
ASSERT_EQ(2, alpha_rowset_meta.start_version());
ASSERT_EQ(2, alpha_rowset_meta.end_version());
ASSERT_EQ(8391828013814912580, alpha_rowset_meta.version_hash());
ASSERT_EQ(3929, alpha_rowset_meta.num_rows());
ASSERT_EQ(84699, alpha_rowset_meta.total_disk_size());
ASSERT_EQ(84464, alpha_rowset_meta.data_disk_size());

2
be/test/olap/vectorized_olap_reader_test.cpp
View File

@ -131,7 +131,7 @@ void set_default_push_request(TPushReq* request) {
request->tablet_id = 10003;
request->schema_hash = 1508825676;
request->__set_version(2);
request->__set_version_hash(1);
request->__set_version_hash(0);
request->timeout = 86400;
request->push_type = TPushType::LOAD;
request->__set_http_file_path("./be/test/olap/test_data/all_types_1000");

0
gensrc/proto/AgentService.thrift Normal file
View File

18
gensrc/thrift/AgentService.thrift
View File

@ -45,7 +45,7 @@ struct TCreateTabletReq {
1: required Types.TTabletId tablet_id
2: required TTabletSchema tablet_schema
3: optional Types.TVersion version
4: optional Types.TVersionHash version_hash
4: optional Types.TVersionHash version_hash // Deprecated
5: optional Types.TStorageMedium storage_medium
6: optional bool in_restore_mode
// this new tablet should be colocate with base tablet
@ -80,7 +80,7 @@ struct TAlterTabletReqV2 {
4: required Types.TSchemaHash new_schema_hash
// version of data which this alter task should transform
5: optional Types.TVersion alter_version
6: optional Types.TVersionHash alter_version_hash
6: optional Types.TVersionHash alter_version_hash // Deprecated
}
struct TClusterInfo {
@ -92,7 +92,7 @@ struct TPushReq {
1: required Types.TTabletId tablet_id
2: required Types.TSchemaHash schema_hash
3: required Types.TVersion version
4: required Types.TVersionHash version_hash
4: required Types.TVersionHash version_hash // Deprecated
5: required i64 timeout
6: required Types.TPushType push_type
7: optional string http_file_path
@ -114,7 +114,7 @@ struct TCloneReq {
4: optional Types.TStorageMedium storage_medium
// these are visible version(hash) actually
5: optional Types.TVersion committed_version
6: optional Types.TVersionHash committed_version_hash
6: optional Types.TVersionHash committed_version_hash // Deprecated
7: optional i32 task_version;
8: optional i64 src_path_hash;
9: optional i64 dest_path_hash;
@ -132,14 +132,14 @@ struct TCancelDeleteDataReq {
1: required Types.TTabletId tablet_id
2: required Types.TSchemaHash schema_hash
3: required Types.TVersion version
4: required Types.TVersionHash version_hash
4: required Types.TVersionHash version_hash // Deprecated
}
struct TCheckConsistencyReq {
1: required Types.TTabletId tablet_id
2: required Types.TSchemaHash schema_hash
3: required Types.TVersion version
4: required Types.TVersionHash version_hash
4: required Types.TVersionHash version_hash // Deprecated
}
struct TUploadReq {
@ -160,7 +160,7 @@ struct TSnapshotRequest {
1: required Types.TTabletId tablet_id
2: required Types.TSchemaHash schema_hash
3: optional Types.TVersion version
4: optional Types.TVersionHash version_hash
4: optional Types.TVersionHash version_hash // Deprecated
5: optional i64 timeout
6: optional list<Types.TVersion> missing_version
7: optional bool list_files
@ -181,7 +181,7 @@ struct TClearRemoteFileReq {
struct TPartitionVersionInfo {
1: required Types.TPartitionId partition_id
2: required Types.TVersion version
3: required Types.TVersionHash version_hash
3: required Types.TVersionHash version_hash // Deprecated
}
struct TMoveDirReq {
@ -217,7 +217,7 @@ struct TRecoverTabletReq {
1: optional Types.TTabletId tablet_id
2: optional Types.TSchemaHash schema_hash
3: optional Types.TVersion version
4: optional Types.TVersionHash version_hash
4: optional Types.TVersionHash version_hash // Deprecated
}
struct TTabletMetaInfo {

2
gensrc/thrift/PlanNodes.thrift
View File

@ -81,7 +81,7 @@ struct TPaloScanRange {
1: required list<Types.TNetworkAddress> hosts
2: required string schema_hash
3: required string version
4: required string version_hash
4: required string version_hash // Deprecated
5: required Types.TTabletId tablet_id
6: required string db_name
7: optional list<TKeyRange> partition_column_ranges

2
gensrc/thrift/QueryPlanExtra.thrift
View File

@ -38,4 +38,4 @@ struct TQueryPlanInfo {
3: required Descriptors.TDescriptorTable desc_tbl
// all tablet scan should share one query_id
4: required Types.TUniqueId query_id
}
}