a233349ea2
bp #45148 ### What problem does this PR solve? Problem Summary: Optimize reading of maxcompute partition tables: 1. Introduce batch mode to generate splits for Maxcompute partition tables to optimize scenarios with a large number of partitions. Control it through the variable `num_partitions_in_batch_mode`. 2. Introduce catalog parameter `mc.split_cross_partition`. The parameter is true, which is more friendly to reading partition tables, and false, which is more friendly to debug. 3. Add `-Darrow.enable_null_check_for_get=false` to be jvm to improve the efficiency of mc arrow data conversion.
99 lines
4.4 KiB
Plaintext
99 lines
4.4 KiB
Plaintext
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
CUR_DATE=`date +%Y%m%d-%H%M%S`
|
|
|
|
# Log dir
|
|
LOG_DIR="${DORIS_HOME}/log/"
|
|
|
|
# For jdk 8
|
|
JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -Darrow.enable_null_check_for_get=false"
|
|
|
|
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
|
|
JAVA_OPTS_FOR_JDK_9="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.nio=ALL-UNNAMED -Darrow.enable_null_check_for_get=false"
|
|
|
|
# For jdk 17+, this JAVA_OPTS will be used as default JVM options
|
|
JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED -Darrow.enable_null_check_for_get=false"
|
|
|
|
# since 1.2, the JAVA_HOME need to be set to run BE process.
|
|
# JAVA_HOME=/path/to/jdk/
|
|
|
|
# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
|
|
# https://jemalloc.net/jemalloc.3.html
|
|
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:true,prof_active:false,lg_prof_interval:-1"
|
|
JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_"
|
|
|
|
# ports for admin, web, heartbeat service
|
|
be_port = 9060
|
|
webserver_port = 8040
|
|
heartbeat_service_port = 9050
|
|
brpc_port = 8060
|
|
arrow_flight_sql_port = -1
|
|
|
|
# HTTPS configures
|
|
enable_https = false
|
|
# path of certificate in PEM format.
|
|
ssl_certificate_path = "$DORIS_HOME/conf/cert.pem"
|
|
# path of private key in PEM format.
|
|
ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
|
|
|
|
|
|
# Choose one if there are more than one ip except loopback address.
|
|
# Note that there should at most one ip match this list.
|
|
# If no ip match this rule, will choose one randomly.
|
|
# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
|
|
# Default value is empty.
|
|
# priority_networks = 10.10.10.0/24;192.168.0.0/16
|
|
|
|
# data root path, separate by ';'
|
|
# You can specify the storage type for each root path, HDD (cold data) or SSD (hot data)
|
|
# eg:
|
|
# storage_root_path = /home/disk1/doris;/home/disk2/doris;/home/disk2/doris
|
|
# storage_root_path = /home/disk1/doris,medium:SSD;/home/disk2/doris,medium:SSD;/home/disk2/doris,medium:HDD
|
|
# /home/disk2/doris,medium:HDD(default)
|
|
#
|
|
# you also can specify the properties by setting '<property>:<value>', separate by ','
|
|
# property 'medium' has a higher priority than the extension of path
|
|
#
|
|
# Default value is ${DORIS_HOME}/storage, you should create it by hand.
|
|
# storage_root_path = ${DORIS_HOME}/storage
|
|
|
|
# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
|
|
# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
|
|
|
|
# Advanced configurations
|
|
# INFO, WARNING, ERROR, FATAL
|
|
sys_log_level = INFO
|
|
# sys_log_roll_mode = SIZE-MB-1024
|
|
# sys_log_roll_num = 10
|
|
# sys_log_verbose_modules = *
|
|
# log_buffer_level = -1
|
|
# palo_cgroups
|
|
|
|
# aws sdk log level
|
|
# Off = 0,
|
|
# Fatal = 1,
|
|
# Error = 2,
|
|
# Warn = 3,
|
|
# Info = 4,
|
|
# Debug = 5,
|
|
# Trace = 6
|
|
# Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs
|
|
aws_log_level=0
|
|
## If you are not running in aws cloud, you can disable EC2 metadata
|
|
AWS_EC2_METADATA_DISABLED=true
|