database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Improve unique id performance (#1911)

Browse Source 
Remove the default constructor for UniqueID
Add a gen_uid method in UniqueId. If need to generate a new uid, users should call this api explicitly.
Reuse boost random generator not generate a new one every time.
This commit is contained in:
yiguolei
2019-09-29 18:20:02 +08:00
committed by ZHAO Chun
parent 8f016d3ab2
commit f852f50acb
18 changed files with 83 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/olap/olap_snapshot_converter.cpp
View File

@ -157,7 +157,7 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola
tablet_meta_pb->set_in_restore_mode(olap_header.in_restore_mode());
}
tablet_meta_pb->set_tablet_state(TabletStatePB::PB_RUNNING);
*(tablet_meta_pb->mutable_tablet_uid()) = TabletUid().to_proto();
*(tablet_meta_pb->mutable_tablet_uid()) = TabletUid::gen_uid().to_proto();
VLOG(3) << "convert tablet meta tablet id = " << olap_header.tablet_id()
<< " schema hash = " << olap_header.schema_hash() << " successfully.";
return OLAP_SUCCESS;

2
be/src/olap/options.h
View File

@ -38,7 +38,7 @@ OLAPStatus parse_conf_store_paths(const std::string& config_path, std::vector<St
struct EngineOptions {
// list paths that tablet will be put into.
std::vector<StorePath> store_paths;
UniqueId backend_uid;
UniqueId backend_uid {0, 0};
};
}

5
be/src/olap/rowset/rowset_writer_context.h
View File

@ -39,11 +39,10 @@ struct RowsetWriterContext {
data_dir(nullptr),
version(Version(0, 0)),
version_hash(0),
txn_id(0) {
txn_id(0),
tablet_uid(0, 0) {
load_id.set_hi(0);
load_id.set_lo(0);
tablet_uid.hi = 0;
tablet_uid.lo = 0;
}
RowsetId rowset_id;
int64_t tablet_id;

2
be/src/olap/tablet_manager.cpp
View File

@ -1336,7 +1336,7 @@ OLAPStatus TabletManager::_create_tablet_meta(
LOG(INFO) << "next_unique_id:" << next_unique_id;
// it is a new tablet meta obviously, should generate a new tablet id
TabletUid tablet_uid;
TabletUid tablet_uid = TabletUid::gen_uid();
res = TabletMeta::create(request.table_id, request.partition_id,
request.tablet_id, request.tablet_schema.schema_hash,
shard_id, request.tablet_schema,

2
be/src/olap/tablet_meta.cpp
View File

@ -202,7 +202,7 @@ OLAPStatus TabletMeta::reset_tablet_uid(const std::string& file_path) {
<< " , meta_file=" << file_path;
return res;
}
*(tmp_tablet_meta_pb.mutable_tablet_uid()) = TabletUid().to_proto();
*(tmp_tablet_meta_pb.mutable_tablet_uid()) = TabletUid::gen_uid().to_proto();
res = save(file_path, tmp_tablet_meta_pb);
if (res != OLAP_SUCCESS) {
LOG(FATAL) << "fail to save tablet meta pb to "

2
be/src/runtime/routine_load/data_consumer.cpp
View File

@ -48,7 +48,7 @@ Status KafkaDataConsumer::init(StreamLoadContext* ctx) {
std::stringstream ss;
ss << BackendOptions::get_localhost() << "_";
std::string group_id = ss.str() + UniqueId().to_string();
std::string group_id = ss.str() + UniqueId::gen_uid().to_string();
LOG(INFO) << "init kafka consumer with group id: " << group_id;
std::string errstr;

2
be/src/runtime/routine_load/data_consumer.h
View File

@ -36,6 +36,8 @@ class StreamLoadPipe;
class DataConsumer {
public:
DataConsumer(StreamLoadContext* ctx):
_id(UniqueId::gen_uid()),
_grp_id(UniqueId::gen_uid()),
_has_grp(false),
_init(false),
_cancelled(false),

4
be/src/runtime/routine_load/data_consumer_group.h
View File

@ -31,8 +31,10 @@ public:
typedef std::function<void (const Status&)> ConsumeFinishCallback;
DataConsumerGroup():
_grp_id(UniqueId::gen_uid()),
_thread_pool(3, 10),
_counter(0) {}
_counter(0){
}
virtual ~DataConsumerGroup() {
_consumers.clear();

1
be/src/runtime/stream_load/stream_load_context.h
View File

@ -81,6 +81,7 @@ class MessageBodySink;
class StreamLoadContext {
public:
StreamLoadContext(ExecEnv* exec_env) :
id(UniqueId::gen_uid()),
_exec_env(exec_env),
_refs(0) {
start_nanos = MonotonicNanos();

2
be/src/service/doris_main.cpp
View File

@ -154,7 +154,7 @@ int main(int argc, char** argv) {
// options
doris::EngineOptions options;
options.store_paths = paths;
options.backend_uid = doris::UniqueId();
options.backend_uid = doris::UniqueId::gen_uid();
doris::StorageEngine* engine = nullptr;
auto st = doris::StorageEngine::open(options, &engine);
if (!st.ok()) {

21
be/src/util/uid_util.h
View File

@ -22,14 +22,12 @@
#include <string>
#include <boost/functional/hash.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "gen_cpp/Types_types.h" // for TUniqueId
#include "gen_cpp/types.pb.h" // for PUniqueId
// #include "util/debug_util.h"
#include "util/hash_util.hpp"
#include "util/uuid_generator.h"
namespace doris {
@ -63,13 +61,6 @@ struct UniqueId {
int64_t hi;
int64_t lo;
// !!!! Not modify this method, it is very important. it will generate a random uid
// it need modify it contact yiguolei
UniqueId() {
auto uuid = boost::uuids::basic_random_generator<boost::mt19937>()();
memcpy(&hi, uuid.data, sizeof(int64_t));
memcpy(&lo, uuid.data + sizeof(int64_t), sizeof(int64_t));
}
UniqueId(int64_t hi_, int64_t lo_) : hi(hi_), lo(lo_) { }
UniqueId(const TUniqueId& tuid) : hi(tuid.hi), lo(tuid.lo) { }
UniqueId(const PUniqueId& puid) : hi(puid.hi()), lo(puid.lo()) { }
@ -77,6 +68,16 @@ struct UniqueId {
from_hex(&hi, hi_str);
from_hex(&lo, lo_str);
}
// currently, the implementation is uuid, but it may change in the future
static UniqueId gen_uid() {
UniqueId uid(0, 0);
auto uuid = UUIDGenerator::instance()->next_uuid();
memcpy(&uid.hi, uuid.data, sizeof(int64_t));
memcpy(&uid.lo, uuid.data + sizeof(int64_t), sizeof(int64_t));
return uid;
}
~UniqueId() noexcept { }
std::string to_string() const {

49
be/src/util/uuid_generator.h Normal file
View File

@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <mutex>
#include <ostream>
#include <string>
#include <boost/functional/hash.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "util/spinlock.h"
namespace doris {
class UUIDGenerator {
public:
boost::uuids::uuid next_uuid() {
std::lock_guard<SpinLock> lock(_uuid_gen_lock);
return _boost_uuid_generator();
}
static UUIDGenerator* instance() {
static UUIDGenerator generator;
return &generator;
}
private:
boost::uuids::basic_random_generator<boost::mt19937> _boost_uuid_generator;
SpinLock _uuid_gen_lock;
};
}

2
be/test/olap/olap_snapshot_converter_test.cpp
View File

@ -54,7 +54,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
options.backend_uid = doris::UniqueId();
options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}

4
be/test/olap/rowset/rowset_meta_manager_test.cpp
View File

@ -51,7 +51,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
options.backend_uid = doris::UniqueId();
options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}
@ -83,7 +83,7 @@ public:
private:
OlapMeta* _meta;
std::string _json_rowset_meta;
TabletUid _tablet_uid;
TabletUid _tablet_uid {0, 0};
};
TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) {

4
be/test/olap/rowset/unique_rowset_id_generator_test.cpp
View File

@ -61,8 +61,8 @@ TEST_F(UniqueRowsetIdGeneratorTest, RowsetIdFormatTest) {
TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) {
UniqueId backend_uid;
UniqueId backend_uid2;
UniqueId backend_uid = UniqueId::gen_uid();
UniqueId backend_uid2 = UniqueId::gen_uid();
ASSERT_TRUE(backend_uid != backend_uid2);
UniqueRowsetIdGenerator id_generator(backend_uid);
UniqueRowsetIdGenerator id_generator2(backend_uid2);

2
be/test/olap/tablet_mgr_test.cpp
View File

@ -59,7 +59,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
options.backend_uid = doris::UniqueId();
options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}

4
be/test/olap/txn_manager_test.cpp
View File

@ -98,7 +98,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
options.backend_uid = doris::UniqueId();
options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}
@ -166,7 +166,7 @@ private:
TTransactionId transaction_id = 111;
TTabletId tablet_id = 222;
SchemaHash schema_hash = 333;
TabletUid _tablet_uid;
TabletUid _tablet_uid {0, 0};
PUniqueId load_id;
std::unique_ptr<TabletSchema> _schema;
RowsetSharedPtr _alpha_rowset;

2
be/test/util/uid_util_test.cpp
View File

@ -30,7 +30,7 @@ public:
TEST_F(UidUtilTest, UniqueId) {
{
UniqueId id;
UniqueId id = UniqueId::gen_uid();
std::string hex_str = id.to_string();
ASSERT_STRNE("0000000000000000-0000000000000000", hex_str.c_str());
}