[Fix](multi-catalog) delete hdfs hedged configs at BE side. (#25094)
Issue Number: close #25093 We can set hdfs hedged configs when creating catalog, just like this: ``` CREATE CATALOG `test_ctl` PROPERTIES ( ... "dfs.client.hedged.read.threadpool.size" = "128", "dfs.client.hedged.read.threshold.millis" = "500", ... ); ``` It is redundant to set these configs at BE side, and it will brings an occasional bug at #25093 .
This commit is contained in:
@ -1071,10 +1071,6 @@ DEFINE_mInt64(lookup_connection_cache_bytes_limit, "4294967296");
|
||||
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
|
||||
DEFINE_mInt64(LZ4_HC_compression_level, "9");
|
||||
|
||||
DEFINE_Bool(enable_hdfs_hedged_read, "false");
|
||||
DEFINE_Int32(hdfs_hedged_read_thread_num, "128");
|
||||
DEFINE_Int32(hdfs_hedged_read_threshold_time, "500");
|
||||
|
||||
DEFINE_mBool(enable_merge_on_write_correctness_check, "true");
|
||||
|
||||
// The secure path with user files, used in the `local` table function.
|
||||
|
||||
@ -1127,16 +1127,6 @@ DECLARE_mInt64(lookup_connection_cache_bytes_limit);
|
||||
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
|
||||
DECLARE_mInt64(LZ4_HC_compression_level);
|
||||
|
||||
// whether to enable hdfs hedged read.
|
||||
// If set to true, it will be enabled even if user not enable it when creating catalog
|
||||
DECLARE_Bool(enable_hdfs_hedged_read);
|
||||
// hdfs hedged read thread pool size, for "dfs.client.hedged.read.threadpool.size"
|
||||
// Maybe overwritten by the value specified when creating catalog
|
||||
DECLARE_Int32(hdfs_hedged_read_thread_num);
|
||||
// the threshold of doing hedged read, for "dfs.client.hedged.read.threshold.millis"
|
||||
// Maybe overwritten by the value specified when creating catalog
|
||||
DECLARE_Int32(hdfs_hedged_read_threshold_time);
|
||||
|
||||
DECLARE_mBool(enable_merge_on_write_correctness_check);
|
||||
|
||||
// The secure path with user files, used in the `local` table function.
|
||||
|
||||
@ -146,17 +146,6 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_HADOOP_HDFS
|
||||
if (config::enable_hdfs_hedged_read) {
|
||||
hdfsBuilderConfSetStr(builder->get(), "dfs.client.hedged.read.threadpool.size",
|
||||
std::to_string(config::hdfs_hedged_read_thread_num).c_str());
|
||||
hdfsBuilderConfSetStr(builder->get(), "dfs.client.hedged.read.threshold.millis",
|
||||
std::to_string(config::hdfs_hedged_read_threshold_time).c_str());
|
||||
LOG(INFO) << "set hdfs hedged read config: " << config::hdfs_hedged_read_thread_num << ", "
|
||||
<< config::hdfs_hedged_read_threshold_time;
|
||||
}
|
||||
#endif
|
||||
|
||||
hdfsBuilderConfSetStr(builder->get(), "ipc.client.fallback-to-simple-auth-allowed", "true");
|
||||
|
||||
if (builder->is_need_kinit()) {
|
||||
|
||||
@ -259,16 +259,6 @@ under the License.
|
||||
|
||||
`dfs.client.hedged.read.threshold.millis` is the read threshold in milliseconds. When a read request exceeds this threshold and is not returned, Hedged Read will be triggered.
|
||||
|
||||
- Configure parameters in be.conf
|
||||
|
||||
```
|
||||
enable_hdfs_hedged_read = true
|
||||
hdfs_hedged_read_thread_num = 128
|
||||
hdfs_hedged_read_threshold_time = 500
|
||||
```
|
||||
|
||||
This method will enable Hedged Read globally on BE nodes (not enabled by default). And ignore the Hedged Read property set when creating the Catalog.
|
||||
|
||||
After enabling it, you can see related parameters in Query Profile:
|
||||
|
||||
`TotalHedgedRead`: The number of Hedged Reads initiated.
|
||||
|
||||
@ -253,16 +253,7 @@ under the License.
|
||||
`dfs.client.hedged.read.threadpool.size` 表示用于 Hedged Read 的线程数,这些线程由一个 HDFS Client 共享。通常情况下,针对一个 HDFS 集群,BE 节点会共享一个 HDFS Client。
|
||||
|
||||
`dfs.client.hedged.read.threshold.millis` 是读取阈值,单位毫秒。当一个读请求超过这个阈值未返回时,会触发 Hedged Read。
|
||||
|
||||
- 在 be.conf 中配置参数
|
||||
|
||||
```
|
||||
enable_hdfs_hedged_read = true
|
||||
hdfs_hedged_read_thread_num = 128
|
||||
hdfs_hedged_read_threshold_time = 500
|
||||
```
|
||||
|
||||
这种方式会在BE节点全局开启 Hedged Read(默认不开启)。并忽略在创建 Catalog 时设置的 Hedged Read 属性。
|
||||
|
||||
|
||||
开启后,可以在 Query Profile 中看到相关参数:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user