From 47ba4aaf30b857d4b55f67c86f1a3dfcd154f66a Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Wed, 8 Nov 2023 11:22:40 +0800 Subject: [PATCH] [Enhancement](load) add timer and partitions number limit (#26549) add timer and partitions number limit --- be/src/runtime/tablets_channel.cpp | 8 ++++---- be/src/runtime/tablets_channel.h | 1 + docs/en/docs/admin-manual/config/fe-config.md | 8 +++++++- docs/zh-CN/docs/admin-manual/config/fe-config.md | 10 ++++++++-- .../main/java/org/apache/doris/common/Config.java | 7 +++++++ .../java/org/apache/doris/catalog/OlapTable.java | 4 ++++ .../apache/doris/service/FrontendServiceImpl.java | 13 +++++++++++++ 7 files changed, 44 insertions(+), 7 deletions(-) diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 91294135a0..68a35ccc10 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -84,6 +84,7 @@ void TabletsChannel::_init_profile(RuntimeProfile* profile) { _slave_replica_timer = ADD_TIMER(_profile, "SlaveReplicaTime"); _add_batch_timer = ADD_TIMER(_profile, "AddBatchTime"); _write_block_timer = ADD_TIMER(_profile, "WriteBlockTime"); + _incremental_open_timer = ADD_TIMER(_profile, "IncrementalOpenTabletTime"); _memory_usage_counter = memory_usage->AddHighWaterMarkCounter("Total", TUnit::BYTES); _write_memory_usage_counter = memory_usage->AddHighWaterMarkCounter("Write", TUnit::BYTES); _flush_memory_usage_counter = memory_usage->AddHighWaterMarkCounter("Flush", TUnit::BYTES); @@ -120,13 +121,14 @@ Status TabletsChannel::open(const PTabletWriterOpenRequest& request) { } Status TabletsChannel::incremental_open(const PTabletWriterOpenRequest& params) { + SCOPED_TIMER(_incremental_open_timer); if (_state == kInitialized) { // haven't opened return open(params); } std::lock_guard l(_lock); std::vector* index_slots = nullptr; int32_t schema_hash = 0; - for (auto& index : _schema->indexes()) { + for (const auto& index : _schema->indexes()) { if (index->index_id == _index_id) { index_slots = &index->slots; schema_hash = index->schema_hash; @@ -137,14 +139,12 @@ Status TabletsChannel::incremental_open(const PTabletWriterOpenRequest& params) return Status::InternalError("unknown index id, key={}", _key.to_string()); } // update tablets - std::vector tablet_ids; - tablet_ids.reserve(params.tablets_size()); size_t incremental_tablet_num = 0; std::stringstream ss; ss << "LocalTabletsChannel txn_id: " << _txn_id << " load_id: " << print_id(params.id()) << " incremental open delta writer: "; - for (auto& tablet : params.tablets()) { + for (const auto& tablet : params.tablets()) { if (_tablet_writers.find(tablet.tablet_id()) != _tablet_writers.end()) { continue; } diff --git a/be/src/runtime/tablets_channel.h b/be/src/runtime/tablets_channel.h index fe9c226829..4dca905033 100644 --- a/be/src/runtime/tablets_channel.h +++ b/be/src/runtime/tablets_channel.h @@ -196,6 +196,7 @@ private: RuntimeProfile::Counter* _slave_replica_timer = nullptr; RuntimeProfile::Counter* _add_batch_timer = nullptr; RuntimeProfile::Counter* _write_block_timer = nullptr; + RuntimeProfile::Counter* _incremental_open_timer = nullptr; }; template diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index bb54a4fe69..11eab19422 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -167,7 +167,7 @@ Default：100 the max txn number which bdbje can rollback when trying to rejoin the group -### `grpc_threadmgr_threads_nums` +#### `grpc_threadmgr_threads_nums` Default: 4096 @@ -2763,3 +2763,9 @@ Forbid LocalDeployManager drop nodes to prevent errors in the cluster.info file Default: mysql To ensure compatibility with the MySQL ecosystem, Doris includes a built-in database called mysql. If this database conflicts with a user's own database, please modify this field to replace the name of the Doris built-in MySQL database with a different name. + +#### `max_auto_partition_num` + +Default value: 2000 + +For auto-partitioned tables to prevent users from accidentally creating a large number of partitions, the number of partitions allowed per OLAP table is `max_auto_partition_num`. Default 2000. diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 82c718a9a7..9da440ada3 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -173,7 +173,7 @@ Doris 元数据将保存在这里。 强烈建议将此目录的存储为： 元数据会同步写入到多个 Follower FE，这个参数用于控制 Master FE 等待 Follower FE 发送 ack 的超时时间。当写入的数据较大时，可能 ack 时间较长，如果超时，会导致写元数据失败，FE 进程退出。此时可以适当调大这个参数。 -### `grpc_threadmgr_threads_nums` +#### `grpc_threadmgr_threads_nums` 默认值: 4096 @@ -2759,6 +2759,12 @@ show data （其他用法：HELP SHOW DATA） #### `mysqldb_replace_name` -Default: mysql +默认值：mysql Doris 为了兼用 mysql 周边工具生态，会内置一个名为 mysql 的数据库，如果该数据库与用户自建数据库冲突，请修改这个字段，为 doris 内置的 mysql database 更换一个名字 + +#### `max_auto_partition_num` + +默认值：2000 + +对于自动分区表，防止用户意外创建大量分区，每个OLAP表允许的分区数量为`max_auto_partition_num`。默认2000。 diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c6970c7d17..f11ddd1faf 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2249,4 +2249,11 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int publish_topic_info_interval_ms = 30000; // 30s + + @ConfField(mutable = true, masterOnly = true, description = { + "对于自动分区表，防止用户意外创建大量分区，每个OLAP表允许的分区数量为`max_auto_partition_num`。默认2000。", + "For auto-partitioned tables to prevent users from accidentally creating a large number of partitions, " + + "the number of partitions allowed per OLAP table is `max_auto_partition_num`. Default 2000." + }) + public static int max_auto_partition_num = 2000; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d9501f0f1d..2e0df9cb3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -964,6 +964,10 @@ public class OlapTable extends Table { return partition; } + public int getPartitionNum() { + return idToPartition.size(); + } + // get all partitions except temp partitions public Collection getPartitions() { return idToPartition.values(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 6ef0956808..4cea18424f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3295,6 +3295,19 @@ public class FrontendServiceImpl implements FrontendService.Iface { return result; } + // check partition's number limit. + int partitionNum = olapTable.getPartitionNum() + addPartitionClauseMap.size(); + if (partitionNum > Config.max_auto_partition_num) { + olapTable.writeUnlock(); + String errorMessage = String.format( + "create partition failed. partition numbers %d will exceed limit variable max_auto_partition_num%d", + partitionNum, Config.max_auto_partition_num); + LOG.warn(errorMessage); + errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage)); + result.setStatus(errorStatus); + return result; + } + for (AddPartitionClause addPartitionClause : addPartitionClauseMap.values()) { try { // here maybe check and limit created partitions num