[feature](BE)pad missed version with empty rowset (#15030)

If all replicas of one tablet are broken, user can use this http api to pad the missed version with empty rowset.

be/src/http/CMakeLists.txt

@@ -35,6 +35,7 @@ add_library(Webserver STATIC
   http_client.cpp
   action/download_action.cpp
   action/monitor_action.cpp
+  action/pad_rowset_action.cpp
   action/health_action.cpp
   action/tablet_migration_action.cpp
   action/tablets_info_action.cpp

be/src/http/action/pad_rowset_action.cpp

@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "http/action/pad_rowset_action.h"
+
+#include <memory>
+#include <mutex>
+
+#include "http/http_channel.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/beta_rowset_writer.h"
+#include "olap/rowset/rowset.h"
+#include "olap/storage_engine.h"
+
+namespace doris {
+
+const std::string TABLET_ID = "tablet_id";
+const std::string START_VERSION = "start_version";
+const std::string END_VERSION = "end_version";
+
+Status check_one_param(const std::string& param_val, const std::string& param_name) {
+    if (param_val.empty()) {
+        return Status::InternalError("paramater {} not specified in url", param_name);
+    }
+    return Status::OK();
+}
+
+void PadRowsetAction::handle(HttpRequest* req) {
+    LOG(INFO) << "accept one request " << req->debug_string();
+    Status status = _handle(req);
+    std::string result = status.to_json();
+    LOG(INFO) << "handle request result:" << result;
+    if (status.ok()) {
+        HttpChannel::send_reply(req, HttpStatus::OK, result);
+    } else {
+        HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, result);
+    }
+}
+
+Status PadRowsetAction::check_param(HttpRequest* req) {
+    RETURN_IF_ERROR(check_one_param(req->param(TABLET_ID), TABLET_ID));
+    RETURN_IF_ERROR(check_one_param(req->param(START_VERSION), START_VERSION));
+    RETURN_IF_ERROR(check_one_param(req->param(END_VERSION), END_VERSION));
+    return Status::OK();
+}
+
+Status PadRowsetAction::_handle(HttpRequest* req) {
+    RETURN_IF_ERROR(check_param(req));
+
+    const std::string& tablet_id_str = req->param(TABLET_ID);
+    const std::string& start_version_str = req->param(START_VERSION);
+    const std::string& end_version_str = req->param(END_VERSION);
+
+    // valid str format
+    int64_t tablet_id = std::atol(tablet_id_str.c_str());
+    int32_t start_version = std::atoi(start_version_str.c_str());
+    int32_t end_version = std::atoi(end_version_str.c_str());
+    if (start_version < 0 || end_version < 0 || end_version < start_version) {
+        return Status::InternalError("Invalid input version");
+    }
+
+    auto tablet = StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id);
+    if (nullptr == tablet) {
+        return Status::InternalError("Unknown tablet id {}", tablet_id);
+    }
+    return _pad_rowset(tablet, Version(start_version, end_version));
+}
+
+Status PadRowsetAction::_pad_rowset(TabletSharedPtr tablet, const Version& version) {
+    if (tablet->check_version_exist(version)) {
+        return Status::InternalError("Input version {} exists", version.to_string());
+    }
+
+    std::unique_ptr<RowsetWriter> writer;
+    RETURN_IF_ERROR(tablet->create_rowset_writer(version, VISIBLE, NONOVERLAPPING,
+                                                 tablet->tablet_schema(), -1, -1, &writer));
+    auto rowset = writer->build();
+    rowset->make_visible(version);
+
+    std::vector<RowsetSharedPtr> to_add {rowset};
+    std::vector<RowsetSharedPtr> to_delete;
+    {
+        std::unique_lock wlock(tablet->get_header_lock());
+        tablet->modify_rowsets(to_add, to_delete);
+        tablet->save_meta();
+    }
+
+    return Status::OK();
+}
+
+} // namespace doris

be/src/http/action/pad_rowset_action.h

@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "common/status.h"
+#include "http/http_handler.h"
+#include "http/http_request.h"
+#include "olap/tablet.h"
+
+namespace doris {
+
+class PadRowsetAction : public HttpHandler {
+public:
+    PadRowsetAction() = default;
+
+    ~PadRowsetAction() override = default;
+
+    void handle(HttpRequest* req) override;
+
+private:
+    Status _handle(HttpRequest* req);
+    Status check_param(HttpRequest* req);
+
+#ifdef BE_TEST
+public:
+#endif
+    Status _pad_rowset(TabletSharedPtr tablet, const Version& version);
+};
+} // end namespace doris

be/src/service/http_service.cpp

@@ -27,6 +27,7 @@
 #include "http/action/jeprofile_actions.h"
 #include "http/action/meta_action.h"
 #include "http/action/metrics_action.h"
+#include "http/action/pad_rowset_action.h"
 #include "http/action/pprof_actions.h"
 #include "http/action/reload_tablet_action.h"
 #include "http/action/reset_rpc_channel_action.h"
@@ -184,6 +185,9 @@ Status HttpService::start() {
     _ev_http_server->register_handler(HttpMethod::POST, "/api/check_tablet_segment_lost",
                                       check_tablet_segment_action);
 
+    PadRowsetAction* pad_rowset_action = _pool.add(new PadRowsetAction());
+    _ev_http_server->register_handler(HttpMethod::POST, "api/pad_rowset", pad_rowset_action);
+
     _ev_http_server->start();
     return Status::OK();
 }

be/test/olap/tablet_test.cpp

@@ -21,6 +21,7 @@
 
 #include <sstream>
 
+#include "http/action/pad_rowset_action.h"
 #include "olap/olap_define.h"
 #include "olap/rowset/beta_rowset.h"
 #include "olap/storage_engine.h"
@@ -28,6 +29,7 @@
 #include "olap/tablet_meta.h"
 #include "olap/tablet_schema_cache.h"
 #include "testutil/mock_rowset.h"
+#include "util/file_utils.h"
 #include "util/time.h"
 
 using namespace std;
@@ -38,6 +40,8 @@ using namespace ErrorCode;
 using RowsetMetaSharedContainerPtr = std::shared_ptr<std::vector<RowsetMetaSharedPtr>>;
 
 static StorageEngine* k_engine = nullptr;
+static const std::string kTestDir = "/data_test/data/tablet_test";
+static const uint32_t MAX_PATH_LEN = 1024;
 
 class TestTablet : public testing::Test {
 public:
@@ -65,6 +69,17 @@ public:
             },
             "creation_time": 1553765670
         })";
+        char buffer[MAX_PATH_LEN];
+        EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
+        absolute_dir = std::string(buffer) + kTestDir;
+
+        if (FileUtils::check_exist(absolute_dir)) {
+            EXPECT_TRUE(FileUtils::remove_all(absolute_dir).ok());
+        }
+        EXPECT_TRUE(FileUtils::create_dir(absolute_dir).ok());
+        EXPECT_TRUE(FileUtils::create_dir(absolute_dir + "/tablet_path").ok());
+        _data_dir = std::make_unique<DataDir>(absolute_dir);
+        _data_dir->update_capacity();
 
         doris::EngineOptions options;
         k_engine = new StorageEngine(options);
@@ -72,6 +87,9 @@ public:
     }
 
     void TearDown() override {
+        if (FileUtils::check_exist(absolute_dir)) {
+            EXPECT_TRUE(FileUtils::remove_all(absolute_dir).ok());
+        }
         if (k_engine != nullptr) {
             k_engine->stop();
             delete k_engine;
@@ -197,6 +215,8 @@ public:
 protected:
     std::string _json_rowset_meta;
     TabletMetaSharedPtr _tablet_meta;
+    string absolute_dir;
+    std::unique_ptr<DataDir> _data_dir;
 };
 
 TEST_F(TestTablet, delete_expired_stale_rowset) {
@@ -225,6 +245,41 @@ TEST_F(TestTablet, delete_expired_stale_rowset) {
     _tablet.reset();
 }
 
+TEST_F(TestTablet, pad_rowset) {
+    std::vector<RowsetMetaSharedPtr> rs_metas;
+    auto ptr1 = std::make_shared<RowsetMeta>();
+    init_rs_meta(ptr1, 1, 2);
+    rs_metas.push_back(ptr1);
+    RowsetSharedPtr rowset1 = make_shared<BetaRowset>(nullptr, "", ptr1);
+
+    auto ptr2 = std::make_shared<RowsetMeta>();
+    init_rs_meta(ptr2, 3, 4);
+    rs_metas.push_back(ptr2);
+    RowsetSharedPtr rowset2 = make_shared<BetaRowset>(nullptr, "", ptr2);
+
+    auto ptr3 = std::make_shared<RowsetMeta>();
+    init_rs_meta(ptr3, 6, 7);
+    rs_metas.push_back(ptr3);
+    RowsetSharedPtr rowset3 = make_shared<BetaRowset>(nullptr, "", ptr3);
+
+    for (auto& rowset : rs_metas) {
+        _tablet_meta->add_rs_meta(rowset);
+    }
+
+    _data_dir->init();
+    TabletSharedPtr _tablet(new Tablet(_tablet_meta, _data_dir.get()));
+    _tablet->init();
+
+    Version version(5, 5);
+    std::vector<RowsetReaderSharedPtr> readers;
+    ASSERT_FALSE(_tablet->capture_rs_readers(version, &readers).ok());
+    readers.clear();
+
+    PadRowsetAction action;
+    action._pad_rowset(_tablet, version);
+    ASSERT_TRUE(_tablet->capture_rs_readers(version, &readers).ok());
+}
+
 TEST_F(TestTablet, cooldown_policy) {
     std::vector<RowsetMetaSharedPtr> rs_metas;
     RowsetMetaSharedPtr ptr1(new RowsetMeta());

docs/en/docs/admin-manual/http-actions/pad-rowset.md

@@ -0,0 +1,41 @@
+---
+{
+    "title": "PAD ROWSET",
+    "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# PAD ROWSET
+## description
+   
+    Pad one empty rowset as one substitute for error replica.
+
+    METHOD: POST
+    URI: http://be_host:be_http_port/api/pad_rowset?tablet_id=xxx&start_version=xxx&end_version=xxx
+
+## example
+
+    curl -X POST "http://hostname:8088/api/pad_rowset?tablet_id=123456\&start_version=1111111\$end_version=1111112"
+
+## keyword
+
+    ROWSET,TABLET,ROWSET,TABLET

docs/sidebars.json

@@ -1119,6 +1119,7 @@
                             ]
                         },
                         "admin-manual/http-actions/restore-tablet",
+                        "admin-manual/http-actions/pad-rowset",
                         "admin-manual/http-actions/get-load-state",
                         "admin-manual/http-actions/tablet-migration-action",
                         "admin-manual/http-actions/cancel-label",

docs/zh-CN/docs/admin-manual/http-actions/pad_rowset.md

@@ -0,0 +1,43 @@
+---
+{
+    "title": "PAD ROWSET",
+    "language": "zh-CN"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# PAD ROWSET
+## description
+   
+    该功能用于使用一个空的rowset填充损坏的副本。
+
+    说明：这个功能暂时只在be服务中提供一个http接口。如果要使用，
+    需要向要进行数据恢复的那台be机器的http端口发送pad rowset api请求。api格式如下：
+    METHOD: POST
+    URI: http://be_host:be_http_port/api/pad_rowset?tablet_id=xxx&start_version=xxx&end_version=xxx
+
+## example
+
+    curl -X POST "http://hostname:8088/api/pad_rowset?tablet_id=123456\&start_version=1111111\&end_version=1111112"
+
+## keyword
+
+    PAD,ROWSET,PAD,ROWSET