database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity
Files
b31acee856cec52e45eb8a9550efb2fd39bf4cfc
doris/be/test/olap/rowset/beta_rowset_test.cpp
plat1ko 25b6e4deb2 [fix](daemon) Fix incorrect initialization order of daemon services (#23578) 
Current initialization dependency:

      Daemon ───┬──► StorageEngine ──► ExecEnv ──► Disk/Mem/CpuInfo
                │
                │
BackendService ─┘
However, original code incorrectly initialize Daemon before StorageEngine.
This PR also stop and join threads of daemon services in their dtor, to ensure Daemon services release resources in reverse order of initialization via RAII.
2023-08-31 19:46:38 +08:00

300 lines
10 KiB
C++
Raw Blame History

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "olap/rowset/beta_rowset.h"
#include <aws/core/auth/AWSAuthSigner.h>
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/utils/Outcome.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/S3Errors.h>
#include <aws/s3/model/GetObjectResult.h>
#include <aws/s3/model/HeadObjectResult.h>
#include <gen_cpp/olap_common.pb.h>
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
#include <stdint.h>
#include <unistd.h>
#include <algorithm>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "common/config.h"
#include "common/status.h"
#include "gen_cpp/olap_file.pb.h"
#include "gtest/gtest_pred_impl.h"
#include "io/fs/local_file_system.h"
#include "io/fs/s3_file_system.h"
#include "olap/data_dir.h"
#include "olap/olap_common.h"
#include "olap/options.h"
#include "olap/rowset/rowset.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/rowset/rowset_reader.h"
#include "olap/rowset/rowset_writer_context.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/storage_engine.h"
#include "olap/tablet_schema.h"
#include "runtime/exec_env.h"
#include "util/s3_util.h"
namespace Aws {
namespace S3 {
namespace Model {
class GetObjectRequest;
class HeadObjectRequest;
} // namespace Model
} // namespace S3
} // namespace Aws
namespace doris {
struct RowsetReaderContext;
} // namespace doris
using std::string;
namespace doris {
using namespace ErrorCode;
static const uint32_t MAX_PATH_LEN = 1024;
static std::unique_ptr<StorageEngine> k_engine;
static const std::string kTestDir = "./data_test/data/beta_rowset_test";
class BetaRowsetTest : public testing::Test {
public:
BetaRowsetTest() : _data_dir(std::make_unique<DataDir>(kTestDir)) {
_data_dir->update_capacity();
}
static void SetUpTestSuite() {
config::tablet_map_shard_size = 1;
config::txn_map_shard_size = 1;
config::txn_shard_size = 1;
char buffer[MAX_PATH_LEN];
EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
config::storage_root_path = std::string(buffer) + "/data_test";
EXPECT_TRUE(io::global_local_filesystem()
->delete_and_create_directory(config::storage_root_path)
.ok());
std::vector<StorePath> paths;
paths.emplace_back(config::storage_root_path, -1);
doris::EngineOptions options;
options.store_paths = paths;
Status s = doris::StorageEngine::open(options, &k_engine);
EXPECT_TRUE(s.ok()) << s.to_string();
EXPECT_TRUE(io::global_local_filesystem()->create_directory(kTestDir).ok());
}
static void TearDownTestSuite() { k_engine.reset(); }
protected:
OlapReaderStatistics _stats;
// (k1 int, k2 varchar(20), k3 int) duplicated key (k1, k2)
void create_tablet_schema(TabletSchemaSPtr tablet_schema) {
TabletSchemaPB tablet_schema_pb;
tablet_schema_pb.set_keys_type(DUP_KEYS);
tablet_schema_pb.set_num_short_key_columns(2);
tablet_schema_pb.set_num_rows_per_row_block(1024);
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
tablet_schema_pb.set_next_column_unique_id(4);
ColumnPB* column_1 = tablet_schema_pb.add_column();
column_1->set_unique_id(1);
column_1->set_name("k1");
column_1->set_type("INT");
column_1->set_is_key(true);
column_1->set_length(4);
column_1->set_index_length(4);
column_1->set_is_nullable(true);
column_1->set_is_bf_column(false);
ColumnPB* column_2 = tablet_schema_pb.add_column();
column_2->set_unique_id(2);
column_2->set_name("k2");
column_2->set_type(
"INT"); // TODO change to varchar(20) when dict encoding for string is supported
column_2->set_length(4);
column_2->set_index_length(4);
column_2->set_is_nullable(true);
column_2->set_is_key(true);
column_2->set_is_nullable(true);
column_2->set_is_bf_column(false);
ColumnPB* column_3 = tablet_schema_pb.add_column();
column_3->set_unique_id(3);
column_3->set_name("v1");
column_3->set_type("INT");
column_3->set_length(4);
column_3->set_is_key(false);
column_3->set_is_nullable(false);
column_3->set_is_bf_column(false);
column_3->set_aggregation("SUM");
tablet_schema->init_from_pb(tablet_schema_pb);
}
void create_rowset_writer_context(TabletSchemaSPtr tablet_schema,
RowsetWriterContext* rowset_writer_context) {
RowsetId rowset_id;
rowset_id.init(10000);
// rowset_writer_context->data_dir = _data_dir.get();
rowset_writer_context->rowset_id = rowset_id;
rowset_writer_context->tablet_id = 12345;
rowset_writer_context->tablet_schema_hash = 1111;
rowset_writer_context->partition_id = 10;
rowset_writer_context->rowset_type = BETA_ROWSET;
rowset_writer_context->rowset_dir = kTestDir;
rowset_writer_context->rowset_state = VISIBLE;
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
}
void create_and_init_rowset_reader(Rowset* rowset, RowsetReaderContext& context,
RowsetReaderSharedPtr* result) {
auto s = rowset->create_reader(result);
EXPECT_EQ(Status::OK(), s);
EXPECT_TRUE(*result != nullptr);
s = (*result)->init(&context);
EXPECT_EQ(Status::OK(), s);
}
private:
std::unique_ptr<DataDir> _data_dir;
};
class S3ClientMock : public Aws::S3::S3Client {
S3ClientMock() {}
S3ClientMock(const Aws::Auth::AWSCredentials& credentials,
const Aws::Client::ClientConfiguration& clientConfiguration,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy signPayloads,
bool use_virtual_addressing)
: Aws::S3::S3Client(credentials, clientConfiguration, signPayloads,
use_virtual_addressing) {}
Aws::S3::Model::HeadObjectOutcome HeadObject(
const Aws::S3::Model::HeadObjectRequest& request) const override {
Aws::S3::Model::HeadObjectOutcome response;
response.success = false;
return response;
}
Aws::S3::Model::GetObjectOutcome GetObject(
const Aws::S3::Model::GetObjectRequest& request) const override {
Aws::S3::Model::GetObjectOutcome response;
response.success = false;
return response;
}
};
class S3ClientMockGetError : public S3ClientMock {
Aws::S3::Model::HeadObjectOutcome HeadObject(
const Aws::S3::Model::HeadObjectRequest& request) const override {
Aws::S3::Model::HeadObjectOutcome response;
response.GetResult().SetContentLength(20);
response.success = true;
return response;
}
};
class S3ClientMockGetErrorData : public S3ClientMock {
Aws::S3::Model::HeadObjectOutcome HeadObject(
const Aws::S3::Model::HeadObjectRequest& request) const override {
Aws::S3::Model::HeadObjectOutcome response;
response.GetResult().SetContentLength(20);
response.success = true;
return response;
}
Aws::S3::Model::GetObjectOutcome GetObject(
const Aws::S3::Model::GetObjectRequest& request) const override {
Aws::S3::Model::GetObjectOutcome response;
response.GetResult().SetContentLength(4);
response.success = true;
return response;
}
};
TEST_F(BetaRowsetTest, ReadTest) {
RowsetMetaSharedPtr rowset_meta = std::make_shared<RowsetMeta>();
BetaRowset rowset(nullptr, "", rowset_meta);
S3Conf s3_conf;
s3_conf.ak = "ak";
s3_conf.sk = "sk";
s3_conf.endpoint = "endpoint";
s3_conf.region = "region";
s3_conf.bucket = "bucket";
s3_conf.prefix = "prefix";
std::string resource_id = "10000";
std::shared_ptr<io::S3FileSystem> fs;
ASSERT_TRUE(io::S3FileSystem::create(std::move(s3_conf), resource_id, &fs).ok());
// failed to head object
{
Aws::Auth::AWSCredentials aws_cred("ak", "sk");
Aws::Client::ClientConfiguration aws_config;
fs->_client.reset(
new S3ClientMock(aws_cred, aws_config,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true));
rowset.rowset_meta()->set_num_segments(1);
rowset.rowset_meta()->set_fs(fs);
std::vector<segment_v2::SegmentSharedPtr> segments;
Status st = rowset.load_segments(&segments);
ASSERT_FALSE(st.ok());
}
// failed to get object
{
Aws::Auth::AWSCredentials aws_cred("ak", "sk");
Aws::Client::ClientConfiguration aws_config;
fs->_client.reset(new S3ClientMockGetError());
rowset.rowset_meta()->set_num_segments(1);
rowset.rowset_meta()->set_fs(fs);
std::vector<segment_v2::SegmentSharedPtr> segments;
Status st = rowset.load_segments(&segments);
ASSERT_FALSE(st.ok());
}
// get error data
{
Aws::Auth::AWSCredentials aws_cred("ak", "sk");
Aws::Client::ClientConfiguration aws_config;
fs->_client.reset(new S3ClientMockGetErrorData());
rowset.rowset_meta()->set_num_segments(1);
rowset.rowset_meta()->set_fs(fs);
std::vector<segment_v2::SegmentSharedPtr> segments;
Status st = rowset.load_segments(&segments);
ASSERT_FALSE(st.ok());
}
}
} // namespace doris
Reference in New Issue View Git Blame Copy Permalink