diff --git a/be/src/common/config.h b/be/src/common/config.h index 08151adb0e..b074c85cbd 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -223,6 +223,12 @@ namespace config { CONF_mInt32(disk_stat_monitor_interval, "5"); CONF_mInt32(unused_rowset_monitor_interval, "30"); CONF_String(storage_root_path, "${DORIS_HOME}/storage"); + + // Config is used to check incompatible old format hdr_ format + // whether doris uses strict way. When config is true, process will log fatal + // and exit. When config is false, process will only log warning. + CONF_Bool(storage_strict_check_incompatible_old_format, "true"); + // BE process will exit if the percentage of error disk reach this value. CONF_mInt32(max_percentage_of_error_disk, "0"); // CONF_Int32(default_num_rows_per_data_block, "1024"); diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 82613eb646..42afdd8232 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -602,6 +602,36 @@ OLAPStatus DataDir::set_convert_finished() { return OLAP_SUCCESS; } +OLAPStatus DataDir::_check_incompatible_old_format_tablet() { + + auto check_incompatible_old_func = [this](int64_t tablet_id, int32_t schema_hash, + const std::string& value) -> bool { + + // if strict check incompatible old format, then log fatal + if (config::storage_strict_check_incompatible_old_format) { + LOG(FATAL) << "There are incompatible old format metas, current version does not support " + << "and it may lead to data missing!!! " + << "talbet_id = " << tablet_id << " schema_hash = " << schema_hash; + } else { + LOG(WARNING) + << "There are incompatible old format metas, current version does not support " + << "and it may lead to data missing!!! " + << "talbet_id = " << tablet_id << " schema_hash = " << schema_hash; + } + return false; + }; + + // seek old header prefix. when check_incompatible_old_func is called, it has old format in olap_meta + OLAPStatus check_incompatible_old_status = TabletMetaManager::traverse_headers( + _meta, check_incompatible_old_func, OLD_HEADER_PREFIX); + if (check_incompatible_old_status != OLAP_SUCCESS) { + LOG(WARNING) << "check incompatible old format meta fails, it may lead to data missing!!! " << _path; + } else { + LOG(INFO) << "successfully check incompatible old format meta " << _path; + } + return check_incompatible_old_status; +} + // TODO(ygl): deal with rowsets and tablets when load failed OLAPStatus DataDir::load() { LOG(INFO) << "start to load tablets from " << _path; @@ -609,6 +639,10 @@ OLAPStatus DataDir::load() { // COMMITTED: add to txn manager // VISIBLE: add to tablet // if one rowset load failed, then the total data dir will not be loaded + + // necessarily check incompatible old format. when there are old metas, it may load to data missing + _check_incompatible_old_format_tablet(); + std::vector dir_rowset_metas; LOG(INFO) << "begin loading rowset from meta"; auto load_rowset_func = [&dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id, diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index be434de3df..0fc8b4a096 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -141,6 +141,10 @@ private: Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id); OLAPStatus _clean_unfinished_converting_data(); OLAPStatus _convert_old_tablet(); + // Check whether has old format (hdr_ start) in olap. When doris updating to current version, + // it may lead to data missing. When conf::storage_strict_check_incompatible_old_format is true, + // process will log fatal. + OLAPStatus _check_incompatible_old_format_tablet(); void _process_garbage_path(const std::string& path); diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index abd89d8805..74fdc1653c 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -137,7 +137,8 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, std::function const& func, const string& header_prefix) { auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool { std::vector parts; - // key format: "hdr_" + tablet_id + "_" + schema_hash + // old format key format: "hdr_" + tablet_id + "_" + schema_hash 0.11 + // new format key format: "tabletmata_" + tablet_id + "_" + schema_hash 0.10 split_string(key, '_', &parts); if (parts.size() != 3) { LOG(WARNING) << "invalid tablet_meta key:" << key << ", splitted size:" << parts.size(); diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md index 1beaa46c67..33a9ef369a 100644 --- a/docs/en/administrator-guide/config/be_config.md +++ b/docs/en/administrator-guide/config/be_config.md @@ -483,6 +483,15 @@ Indicates how many tablets in this data directory failed to load. At the same ti ### `storage_root_path` +### `storage_strict_check_incompatible_old_format` +* Type: bool +* Description: Used to check incompatible old format strictly +* Default value: true +* Dynamically modify: false + +This config is used to check incompatible old format hdr_ format whether doris uses strict way. When config is true, +process will log fatal and exit. When config is false, process will only log warning. + ### `streaming_load_max_mb` * Type: int64 diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md index a5dd8a926b..bde36f44b6 100644 --- a/docs/zh-CN/administrator-guide/config/be_config.md +++ b/docs/zh-CN/administrator-guide/config/be_config.md @@ -482,6 +482,15 @@ load tablets from header failed, failed tablets size: xxx, path=xxx ### `storage_root_path` +### `storage_strict_check_incompatible_old_format` +* 类型：bool +* 描述：用来检查不兼容的旧版本格式时是否使用严格的验证方式 +* 默认值： true +* 可动态修改：否 + +配置用来检查不兼容的旧版本格式时是否使用严格的验证方式，当含有旧版本的 hdr 格式时，使用严谨的方式时，程序会 +打出 fatal log 并且退出运行；否则，程序仅打印 warn log. + ### `streaming_load_max_mb` * 类型：int64