[storage][compatibility] Add meta format detection to prevent data loss. (#4539)

After 0.12 version, doris remove the format convert functiion which can convert from hdr_ format
to tabletmeta_ format when loading metas, the commit link: 3bca253

When we update doris version and there are old format meta in storage,
BE will not read the old format tablet. It can lead to data loss.

So we add meta format detection function to prevent data loss.
When there are old format meta in olap_meta, BE can find and print log or exit.

be/src/common/config.h

@@ -223,6 +223,12 @@ namespace config {
     CONF_mInt32(disk_stat_monitor_interval, "5");
     CONF_mInt32(unused_rowset_monitor_interval, "30");
     CONF_String(storage_root_path, "${DORIS_HOME}/storage");
+
+    // Config is used to check incompatible old format hdr_ format
+    // whether doris uses strict way. When config is true, process will log fatal
+    // and exit. When config is false, process will only log warning.
+    CONF_Bool(storage_strict_check_incompatible_old_format, "true");
+
     // BE process will exit if the percentage of error disk reach this value.
     CONF_mInt32(max_percentage_of_error_disk, "0");
     // CONF_Int32(default_num_rows_per_data_block, "1024");

be/src/olap/data_dir.cpp

@@ -602,6 +602,36 @@ OLAPStatus DataDir::set_convert_finished() {
     return OLAP_SUCCESS;
 }
 
+OLAPStatus DataDir::_check_incompatible_old_format_tablet() {
+
+    auto check_incompatible_old_func = [this](int64_t tablet_id, int32_t schema_hash,
+                                              const std::string& value) -> bool {
+                                                  
+        // if strict check incompatible old format, then log fatal
+        if (config::storage_strict_check_incompatible_old_format) {
+            LOG(FATAL) << "There are incompatible old format metas, current version does not support "
+                       << "and it may lead to data missing!!! "
+                       << "talbet_id = " << tablet_id << " schema_hash = " << schema_hash;
+        } else {
+            LOG(WARNING)
+                    << "There are incompatible old format metas, current version does not support "
+                    << "and it may lead to data missing!!! "
+                    << "talbet_id = " << tablet_id << " schema_hash = " << schema_hash;
+        }
+        return false;
+    };
+
+    // seek old header prefix. when check_incompatible_old_func is called, it has old format in olap_meta
+    OLAPStatus check_incompatible_old_status = TabletMetaManager::traverse_headers(
+            _meta, check_incompatible_old_func, OLD_HEADER_PREFIX);
+    if (check_incompatible_old_status != OLAP_SUCCESS) {
+        LOG(WARNING) << "check incompatible old format meta fails, it may lead to data missing!!! " << _path;
+    } else {
+        LOG(INFO) << "successfully check incompatible old format meta " << _path;
+    }
+    return check_incompatible_old_status;
+}
+
 // TODO(ygl): deal with rowsets and tablets when load failed
 OLAPStatus DataDir::load() {
     LOG(INFO) << "start to load tablets from " << _path;
@@ -609,6 +639,10 @@ OLAPStatus DataDir::load() {
     // COMMITTED: add to txn manager
     // VISIBLE: add to tablet
     // if one rowset load failed, then the total data dir will not be loaded
+    
+    // necessarily check incompatible old format. when there are old metas, it may load to data missing
+    _check_incompatible_old_format_tablet();
+
     std::vector<RowsetMetaSharedPtr> dir_rowset_metas;
     LOG(INFO) << "begin loading rowset from meta";
     auto load_rowset_func = [&dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,

be/src/olap/data_dir.h

@@ -141,6 +141,10 @@ private:
     Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id);
     OLAPStatus _clean_unfinished_converting_data();
     OLAPStatus _convert_old_tablet();
+    // Check whether has old format (hdr_ start) in olap. When doris updating to current version, 
+    // it may lead to data missing. When conf::storage_strict_check_incompatible_old_format is true, 
+    // process will log fatal.
+    OLAPStatus _check_incompatible_old_format_tablet();
 
     void _process_garbage_path(const std::string& path);
 

be/src/olap/tablet_meta_manager.cpp

@@ -137,7 +137,8 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta,
         std::function<bool(long, long, const std::string&)> const& func, const string& header_prefix) {
     auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool {
         std::vector<std::string> parts;
-        // key format: "hdr_" + tablet_id + "_" + schema_hash
+        // old format key format: "hdr_" + tablet_id + "_" + schema_hash  0.11
+        // new format key format: "tabletmata_" + tablet_id + "_" + schema_hash  0.10
         split_string<char>(key, '_', &parts);
         if (parts.size() != 3) {
             LOG(WARNING) << "invalid tablet_meta key:" << key << ", splitted size:" << parts.size();

docs/en/administrator-guide/config/be_config.md

@@ -483,6 +483,15 @@ Indicates how many tablets in this data directory failed to load. At the same ti
 
 ### `storage_root_path`
 
+### `storage_strict_check_incompatible_old_format`
+* Type: bool
+* Description: Used to check incompatible old format strictly
+* Default value: true
+* Dynamically modify: false
+
+This config is used to check incompatible old format hdr_ format whether doris uses strict way. When config is true, 
+process will log fatal and exit. When config is false, process will only log warning.
+
 ### `streaming_load_max_mb`
 
 * Type: int64

docs/zh-CN/administrator-guide/config/be_config.md

@@ -482,6 +482,15 @@ load tablets from header failed, failed tablets size: xxx, path=xxx
 
 ### `storage_root_path`
 
+### `storage_strict_check_incompatible_old_format`
+* 类型：bool
+* 描述：用来检查不兼容的旧版本格式时是否使用严格的验证方式
+* 默认值： true
+* 可动态修改：否
+
+配置用来检查不兼容的旧版本格式时是否使用严格的验证方式，当含有旧版本的 hdr 格式时，使用严谨的方式时，程序会
+打出 fatal log 并且退出运行；否则，程序仅打印 warn log.
+
 ### `streaming_load_max_mb`
 
 * 类型：int64