From 4571b09dd68fa041b85370c22ad274dbcebb8d70 Mon Sep 17 00:00:00 2001 From: ZhangYu0123 <67053339+ZhangYu0123@users.noreply.github.com> Date: Sun, 13 Sep 2020 11:58:22 +0800 Subject: [PATCH] [storage][compatibility] Add meta format detection to prevent data loss. (#4539) After 0.12 version, doris remove the format convert functiion which can convert from hdr_ format to tabletmeta_ format when loading metas, the commit link: 3bca253 When we update doris version and there are old format meta in storage, BE will not read the old format tablet. It can lead to data loss. So we add meta format detection function to prevent data loss. When there are old format meta in olap_meta, BE can find and print log or exit. --- be/src/common/config.h | 6 ++++ be/src/olap/data_dir.cpp | 34 +++++++++++++++++++ be/src/olap/data_dir.h | 4 +++ be/src/olap/tablet_meta_manager.cpp | 3 +- .../administrator-guide/config/be_config.md | 9 +++++ .../administrator-guide/config/be_config.md | 9 +++++ 6 files changed, 64 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 08151adb0e..b074c85cbd 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -223,6 +223,12 @@ namespace config { CONF_mInt32(disk_stat_monitor_interval, "5"); CONF_mInt32(unused_rowset_monitor_interval, "30"); CONF_String(storage_root_path, "${DORIS_HOME}/storage"); + + // Config is used to check incompatible old format hdr_ format + // whether doris uses strict way. When config is true, process will log fatal + // and exit. When config is false, process will only log warning. + CONF_Bool(storage_strict_check_incompatible_old_format, "true"); + // BE process will exit if the percentage of error disk reach this value. CONF_mInt32(max_percentage_of_error_disk, "0"); // CONF_Int32(default_num_rows_per_data_block, "1024"); diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 82613eb646..42afdd8232 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -602,6 +602,36 @@ OLAPStatus DataDir::set_convert_finished() { return OLAP_SUCCESS; } +OLAPStatus DataDir::_check_incompatible_old_format_tablet() { + + auto check_incompatible_old_func = [this](int64_t tablet_id, int32_t schema_hash, + const std::string& value) -> bool { + + // if strict check incompatible old format, then log fatal + if (config::storage_strict_check_incompatible_old_format) { + LOG(FATAL) << "There are incompatible old format metas, current version does not support " + << "and it may lead to data missing!!! " + << "talbet_id = " << tablet_id << " schema_hash = " << schema_hash; + } else { + LOG(WARNING) + << "There are incompatible old format metas, current version does not support " + << "and it may lead to data missing!!! " + << "talbet_id = " << tablet_id << " schema_hash = " << schema_hash; + } + return false; + }; + + // seek old header prefix. when check_incompatible_old_func is called, it has old format in olap_meta + OLAPStatus check_incompatible_old_status = TabletMetaManager::traverse_headers( + _meta, check_incompatible_old_func, OLD_HEADER_PREFIX); + if (check_incompatible_old_status != OLAP_SUCCESS) { + LOG(WARNING) << "check incompatible old format meta fails, it may lead to data missing!!! " << _path; + } else { + LOG(INFO) << "successfully check incompatible old format meta " << _path; + } + return check_incompatible_old_status; +} + // TODO(ygl): deal with rowsets and tablets when load failed OLAPStatus DataDir::load() { LOG(INFO) << "start to load tablets from " << _path; @@ -609,6 +639,10 @@ OLAPStatus DataDir::load() { // COMMITTED: add to txn manager // VISIBLE: add to tablet // if one rowset load failed, then the total data dir will not be loaded + + // necessarily check incompatible old format. when there are old metas, it may load to data missing + _check_incompatible_old_format_tablet(); + std::vector dir_rowset_metas; LOG(INFO) << "begin loading rowset from meta"; auto load_rowset_func = [&dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id, diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index be434de3df..0fc8b4a096 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -141,6 +141,10 @@ private: Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id); OLAPStatus _clean_unfinished_converting_data(); OLAPStatus _convert_old_tablet(); + // Check whether has old format (hdr_ start) in olap. When doris updating to current version, + // it may lead to data missing. When conf::storage_strict_check_incompatible_old_format is true, + // process will log fatal. + OLAPStatus _check_incompatible_old_format_tablet(); void _process_garbage_path(const std::string& path); diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index abd89d8805..74fdc1653c 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -137,7 +137,8 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, std::function const& func, const string& header_prefix) { auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool { std::vector parts; - // key format: "hdr_" + tablet_id + "_" + schema_hash + // old format key format: "hdr_" + tablet_id + "_" + schema_hash 0.11 + // new format key format: "tabletmata_" + tablet_id + "_" + schema_hash 0.10 split_string(key, '_', &parts); if (parts.size() != 3) { LOG(WARNING) << "invalid tablet_meta key:" << key << ", splitted size:" << parts.size(); diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md index 1beaa46c67..33a9ef369a 100644 --- a/docs/en/administrator-guide/config/be_config.md +++ b/docs/en/administrator-guide/config/be_config.md @@ -483,6 +483,15 @@ Indicates how many tablets in this data directory failed to load. At the same ti ### `storage_root_path` +### `storage_strict_check_incompatible_old_format` +* Type: bool +* Description: Used to check incompatible old format strictly +* Default value: true +* Dynamically modify: false + +This config is used to check incompatible old format hdr_ format whether doris uses strict way. When config is true, +process will log fatal and exit. When config is false, process will only log warning. + ### `streaming_load_max_mb` * Type: int64 diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md index a5dd8a926b..bde36f44b6 100644 --- a/docs/zh-CN/administrator-guide/config/be_config.md +++ b/docs/zh-CN/administrator-guide/config/be_config.md @@ -482,6 +482,15 @@ load tablets from header failed, failed tablets size: xxx, path=xxx ### `storage_root_path` +### `storage_strict_check_incompatible_old_format` +* 类型：bool +* 描述：用来检查不兼容的旧版本格式时是否使用严格的验证方式 +* 默认值： true +* 可动态修改：否 + +配置用来检查不兼容的旧版本格式时是否使用严格的验证方式，当含有旧版本的 hdr 格式时，使用严谨的方式时，程序会 +打出 fatal log 并且退出运行；否则，程序仅打印 warn log. + ### `streaming_load_max_mb` * 类型：int64