database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Write delete predicate into RowsetMeta upon upgrade from Doris-0.10 to Doris-0.11 (#3044)

Browse Source 
If delete predicate exists in meta in Doris-0.10, all of this predicates should
be remained. There is an confused place in Doris-0.10. The delete predicate
only exists in OLAPHeaderMessage and PPendingDelta, not in PDelta.
This trick results this bug.
This commit is contained in:
lichaoyong
2020-03-07 11:16:48 +08:00
committed by GitHub
parent 1d296e907d
commit c83729435f
3 changed files with 36 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
be/src/olap/olap_snapshot_converter.cpp
View File

@ -66,8 +66,13 @@ OLAPStatus OlapSnapshotConverter::to_olap_header(const TabletMetaPB& tablet_meta
}
for (auto& rs_meta : tablet_meta_pb.rs_metas()) {
// Add delete predicate OLAPHeaderMessage from PDelta.
PDelta* pdelta = olap_header->add_delta();
convert_to_pdelta(rs_meta, pdelta);
if (pdelta->has_delete_condition()) {
DeletePredicatePB* delete_condition = olap_header->add_delete_data_conditions();
*delete_condition = pdelta->delete_condition();
}
}
// not add pending delta, it is usedless in clone or backup restore
for (auto& inc_rs_meta : tablet_meta_pb.inc_rs_metas()) {
@ -131,11 +136,24 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola
}
std::unordered_map<Version, RowsetMetaPB*, HashOfVersion> _rs_version_map;
const DelPredicateArray& delete_conditions = olap_header.delete_data_conditions();
for (auto& delta : olap_header.delta()) {
RowsetId next_id = StorageEngine::instance()->next_rowset_id();
RowsetMetaPB* rowset_meta = tablet_meta_pb->add_rs_metas();
convert_to_rowset_meta(delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta);
Version rowset_version = { delta.start_version(), delta.end_version() };
PDelta temp_delta = delta;
// PDelta is not corresponding with RowsetMeta in DeletePredicate
// Add delete predicate to PDelta from OLAPHeaderMessage.
// Only after this, convert from PDelta to RowsetMeta is valid.
if (temp_delta.start_version() == temp_delta.end_version()) {
for (auto& del_pred : delete_conditions) {
if (temp_delta.start_version() == del_pred.version()) {
DeletePredicatePB* delete_condition = temp_delta.mutable_delete_condition();
*delete_condition = del_pred;
}
}
}
convert_to_rowset_meta(temp_delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta);
Version rowset_version = { temp_delta.start_version(), temp_delta.end_version() };
_rs_version_map[rowset_version] = rowset_meta;
}
@ -150,7 +168,16 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola
}
RowsetId next_id = StorageEngine::instance()->next_rowset_id();
RowsetMetaPB* rowset_meta = tablet_meta_pb->add_inc_rs_metas();
convert_to_rowset_meta(inc_delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta);
PDelta temp_inc_delta = inc_delta;
if (temp_inc_delta.start_version() == temp_inc_delta.end_version()) {
for (auto& del_pred : delete_conditions) {
if (temp_inc_delta.start_version() == del_pred.version()) {
DeletePredicatePB* delete_condition = temp_inc_delta.mutable_delete_condition();
*delete_condition = del_pred;
}
}
}
convert_to_rowset_meta(temp_inc_delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta);
}
for (auto& pending_delta : olap_header.pending_delta()) {
@ -414,7 +441,7 @@ OLAPStatus OlapSnapshotConverter::to_alter_tablet_pb(const SchemaChangeStatusMes
// from olap header to tablet meta
OLAPStatus OlapSnapshotConverter::to_new_snapshot(const OLAPHeaderMessage& olap_header, const string& old_data_path_prefix,
const string& new_data_path_prefix, TabletMetaPB* tablet_meta_pb,
const string& new_data_path_prefix, TabletMetaPB* tablet_meta_pb,
vector<RowsetMetaPB>* pending_rowsets, bool is_startup) {
RETURN_NOT_OK(to_tablet_meta_pb(olap_header, tablet_meta_pb, pending_rowsets));

6
be/src/olap/olap_snapshot_converter.h
View File

@ -47,7 +47,7 @@ public:
// convert olap header to tablet meta pb, convert delta to rowsetmetapb
// pending delta is not in tablet meta any more, so that convert pending delta to rowset and add it to pending rowsets
// as a return value
OLAPStatus to_tablet_meta_pb(const OLAPHeaderMessage& olap_header, TabletMetaPB* tablet_meta_pb,
OLAPStatus to_tablet_meta_pb(const OLAPHeaderMessage& olap_header, TabletMetaPB* tablet_meta_pb,
vector<RowsetMetaPB>* pending_rowsets);
OLAPStatus convert_to_pdelta(const RowsetMetaPB& rowset_meta_pb, PDelta* delta);
@ -67,8 +67,8 @@ public:
OLAPStatus to_alter_tablet_pb(const SchemaChangeStatusMessage& schema_change_msg, AlterTabletPB* alter_tablet_pb);
// from olap header to tablet meta
OLAPStatus to_new_snapshot(const OLAPHeaderMessage& olap_header, const string& old_data_path_prefix,
const string& new_data_path_prefix, TabletMetaPB* tablet_meta_pb,
OLAPStatus to_new_snapshot(const OLAPHeaderMessage& olap_header, const string& old_data_path_prefix,
const string& new_data_path_prefix, TabletMetaPB* tablet_meta_pb,
vector<RowsetMetaPB>* pending_rowsets, bool is_startup);
// from tablet meta to olap header

4
be/src/olap/task/engine_clone_task.cpp
View File

@ -602,8 +602,8 @@ OLAPStatus EngineCloneTask::_convert_to_new_snapshot(const string& clone_dir, in
OlapSnapshotConverter converter;
TabletMetaPB tablet_meta_pb;
vector<RowsetMetaPB> pending_rowsets;
res = converter.to_new_snapshot(olap_header_msg, clone_dir, clone_dir, &tablet_meta_pb,
&pending_rowsets, false);
res = converter.to_new_snapshot(olap_header_msg, clone_dir, clone_dir,
&tablet_meta_pb, &pending_rowsets, false);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to convert snapshot to new format. dir='" << clone_dir;
return res;