318 lines
13 KiB
Python
318 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding:utf-8 -*-
|
|
#############################################################################
|
|
# Copyright (c) 2023 Huawei Technologies Co.,Ltd.
|
|
#
|
|
# openGauss is licensed under Mulan PSL v2.
|
|
# You can use this software according to the terms
|
|
# and conditions of the Mulan PSL v2.
|
|
# You may obtain a copy of Mulan PSL v2 at:
|
|
#
|
|
# http://license.coscl.org.cn/MulanPSL2
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OF ANY KIND,
|
|
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
# See the Mulan PSL v2 for more details.
|
|
# ----------------------------------------------------------------------------
|
|
# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss
|
|
#############################################################################
|
|
|
|
|
|
import math
|
|
|
|
from base_utils.os.cpu_util import CpuArchitecture, CpuUtil
|
|
|
|
from impl.perf_config.basic.project import Project
|
|
from impl.perf_config.basic.tuner import Tuner, TunerGroup
|
|
from impl.perf_config.basic.guc import GucMap, GUCTuneGroup
|
|
from impl.perf_config.probes.business import BsScenario, TblKind
|
|
from impl.perf_config.tuners.os import CPUTuner
|
|
|
|
class CommonGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(CommonGUC, self).__init__()
|
|
self.pgxc_node_name = self.bind('pgxc_node_name')
|
|
self.application_name = self.bind('application_name')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class FileLocationGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(FileLocationGUC, self).__init__()
|
|
self.data_directory = self.bind('data_directory')
|
|
self.config_file = self.bind('config_file')
|
|
self.hba_file = self.bind('hba_file')
|
|
self.ident_file = self.bind('ident_file')
|
|
self.external_pid_file = self.bind('external_pid_file')
|
|
self.enable_default_cfunc_libpath = self.bind('enable_default_cfunc_libpath')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class KernelResourceGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(KernelResourceGUC, self).__init__()
|
|
self.max_files_per_process = self.bind('max_files_per_process')
|
|
self.shared_preload_libraries = self.bind('shared_preload_libraries')
|
|
self.sql_use_spacelimit = self.bind('sql_use_spacelimit')
|
|
self.temp_file_limit = self.bind('temp_file_limit')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class MemoryGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(MemoryGUC, self).__init__()
|
|
# memory pool
|
|
self.memorypool_enable = self.bind('memorypool_enable')
|
|
self.memorypool_size = self.bind('memorypool_size')
|
|
# memory seciruty
|
|
self.enable_memory_limit = self.bind('enable_memory_limit')
|
|
self.enable_memory_context_control = self.bind('enable_memory_context_control')
|
|
self.uncontrolled_memory_context = self.bind('uncontrolled_memory_context')
|
|
# size, buffer and work mem
|
|
self.max_process_memory = self.bind('max_process_memory')
|
|
self.shared_buffers = self.bind('shared_buffers')
|
|
self.cstore_buffers = self.bind('cstore_buffers')
|
|
self.segment_buffers = self.bind('segment_buffers')
|
|
self.temp_buffers = self.bind('temp_buffers')
|
|
self.work_mem = self.bind('work_mem')
|
|
self.query_mem = self.bind('query_mem')
|
|
self.query_max_mem = self.bind('query_max_mem')
|
|
self.psort_work_mem = self.bind('psort_work_mem')
|
|
self.maintenance_work_mem = self.bind('maintenance_work_mem')
|
|
# huge pages
|
|
self.enable_huge_pages = self.bind('enable_huge_pages')
|
|
self.huge_page_size = self.bind('huge_page_size')
|
|
# others
|
|
self.bulk_write_ring_size = self.bind('bulk_write_ring_size')
|
|
self.max_loaded_cudesc = self.bind('max_loaded_cudesc')
|
|
self.max_stack_depth = self.bind('max_stack_depth')
|
|
self.bulk_read_ring_size = self.bind('bulk_read_ring_size')
|
|
self.enable_early_free = self.bind('enable_early_free')
|
|
self.resilience_memory_reject_percent = self.bind('resilience_memory_reject_percent')
|
|
self.pca_shared_buffers = self.bind('pca_shared_buffers')
|
|
|
|
def calculate(self):
|
|
infos = Project.getGlobalPerfProbe()
|
|
self._calc_memorypool(infos)
|
|
self._calc_memory_security(infos)
|
|
self._calc_memory_size(infos)
|
|
self._calc_memory_others(infos)
|
|
|
|
def _calc_memorypool(self, infos):
|
|
self.memorypool_enable.turn_off()
|
|
|
|
def _calc_memory_security(self, infos):
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
self.enable_memory_limit.turn_off()
|
|
self.enable_memory_context_control.turn_off()
|
|
|
|
def _calc_memory_size(self, infos):
|
|
max_process_memory_rate = 0.55
|
|
shared_buffers_rate = 0.4
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
max_process_memory_rate = 0.6
|
|
shared_buffers_rate = 0.45
|
|
elif infos.business.scenario == BsScenario.AP:
|
|
max_process_memory_rate = 0.65
|
|
shared_buffers_rate = 0.5
|
|
|
|
max_process_memory = int(infos.memory.total_size() * max_process_memory_rate / 1024)
|
|
if max_process_memory == 0:
|
|
Project.fatat('Your machine memory is too small to support configuration.')
|
|
self.max_process_memory.set('{}MB'.format(max_process_memory))
|
|
|
|
shared_buffers = int(infos.memory.total_size() * shared_buffers_rate / 1024)
|
|
if self.shared_buffers == 0:
|
|
Project.fatat('Your machine memory is too small to support configuration.')
|
|
self.shared_buffers.set('{}MB'.format(shared_buffers))
|
|
|
|
cstore_buffers = 16
|
|
if TblKind.haveColumnTbl(infos.business.rel_kind):
|
|
cstore_buffers = 512
|
|
self.cstore_buffers.set('{}MB'.format(cstore_buffers))
|
|
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
self.work_mem.set('1MB')
|
|
self.maintenance_work_mem.set('2GB')
|
|
|
|
def _calc_memory_others(self, infos):
|
|
if infos.business.scenario == BsScenario.AP:
|
|
self.enable_early_free.turn_on()
|
|
|
|
|
|
class SysCacheGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(SysCacheGUC, self).__init__()
|
|
self.enable_global_syscache = self.bind('enable_global_syscache')
|
|
self.local_syscache_threshold = self.bind('local_syscache_threshold')
|
|
self.global_syscache_threshold = self.bind('global_syscache_threshold')
|
|
|
|
def calculate(self):
|
|
infos = Project.getGlobalPerfProbe()
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
self.enable_global_syscache.turn_off()
|
|
self.local_syscache_threshold.set('16MB')
|
|
|
|
|
|
class AlarmGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(AlarmGUC, self).__init__()
|
|
self.enable_alarm = self.bind('enable_alarm')
|
|
self.connection_alarm_rate = self.bind('connection_alarm_rate')
|
|
self.alarm_report_interval = self.bind('alarm_report_interval')
|
|
self.alarm_component = self.bind('alarm_component')
|
|
self.table_skewness_warning_threshold = self.bind('table_skewness_warning_threshold')
|
|
self.table_skewness_warning_rows = self.bind('table_skewness_warning_rows')
|
|
|
|
def calculate(self):
|
|
infos = Project.getGlobalPerfProbe()
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
self.enable_alarm.turn_off()
|
|
|
|
|
|
class RegionFormatGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(RegionFormatGUC, self).__init__()
|
|
self.DateStyle = self.bind('DateStyle')
|
|
self.IntervalStyle = self.bind('IntervalStyle')
|
|
self.TimeZone = self.bind('TimeZone')
|
|
self.timezone_abbreviations = self.bind('timezone_abbreviations')
|
|
self.extra_float_digits = self.bind('extra_float_digits')
|
|
self.client_encoding = self.bind('client_encoding')
|
|
self.lc_messages = self.bind('lc_messages')
|
|
self.lc_monetary = self.bind('lc_monetary')
|
|
self.lc_numeric = self.bind('lc_numeric')
|
|
self.lc_time = self.bind('lc_time')
|
|
self.lc_collate = self.bind('lc_collate')
|
|
self.lc_ctype = self.bind('lc_ctype')
|
|
self.default_text_search_config = self.bind('default_text_search_config')
|
|
|
|
def calculate(self):
|
|
self.lc_messages.set('C')
|
|
self.lc_monetary.set('C')
|
|
self.lc_numeric.set('C')
|
|
self.lc_time.set('C')
|
|
|
|
|
|
class ThreadPoolGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(ThreadPoolGUC, self).__init__()
|
|
self.enable_thread_pool = self.bind('enable_thread_pool')
|
|
self.thread_pool_attr = self.bind('thread_pool_attr')
|
|
self.thread_pool_stream_attr = self.bind('thread_pool_stream_attr')
|
|
self.resilience_threadpool_reject_cond = self.bind('resilience_threadpool_reject_cond')
|
|
self.numa_distribute_mode = self.bind('numa_distribute_mode')
|
|
|
|
def calculate(self):
|
|
infos = Project.getGlobalPerfProbe()
|
|
numa_bind_info = infos.cpu.notebook.read('numa_bind_info')
|
|
if numa_bind_info is None:
|
|
numa_bind_info = CPUTuner.calculate_numa_bind()
|
|
infos.cpu.notebook.write('numa_bind_info', numa_bind_info)
|
|
if not numa_bind_info['use']:
|
|
self.enable_thread_pool.turn_off()
|
|
return
|
|
|
|
self.enable_thread_pool.turn_on()
|
|
self.numa_distribute_mode.set('all')
|
|
|
|
for sug in numa_bind_info['suggestions']:
|
|
Project.report.suggest(sug)
|
|
|
|
numa_group_count = len(infos.cpu.numa())
|
|
thread_count = self._calc_thread_count(infos, numa_bind_info)
|
|
cpubind = 'cpubind:{}'.format(CpuUtil.cpuListToCpuRangeStr(numa_bind_info['threadpool'])) \
|
|
if len(numa_bind_info['threadpool']) != infos.cpu.count() else 'allbind'
|
|
|
|
thread_pool_attr = '{0},{1},({2})'.format(thread_count, numa_group_count, cpubind)
|
|
self.thread_pool_attr.set(thread_pool_attr)
|
|
|
|
def _calc_thread_count(self, infos, numa_bind_info):
|
|
max_count = len(numa_bind_info['threadpool']) * 7.25
|
|
min_count = len(numa_bind_info['threadpool'])
|
|
|
|
ratio = 0.5
|
|
if infos.db.is_single_node:
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
ratio = 0.83
|
|
elif infos.business.scenario == BsScenario.TP_PRODUCE:
|
|
ratio = 0.6
|
|
else:
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
ratio = 0.9
|
|
elif infos.business.scenario == BsScenario.TP_PRODUCE:
|
|
ratio = 0.7
|
|
|
|
Project.log(f'ratio (thread count / parallel) is {ratio}.')
|
|
thread_count = infos.business.parallel * ratio
|
|
|
|
return math.floor(max(min(max_count, thread_count), min_count))
|
|
|
|
|
|
class UpgradeGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(UpgradeGUC, self).__init__()
|
|
self.IsInplaceUpgrade = self.bind('IsInplaceUpgrade')
|
|
self.inplace_upgrade_next_system_object_oids = self.bind('inplace_upgrade_next_system_object_oids')
|
|
self.upgrade_mode = self.bind('upgrade_mode')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class MotGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(MotGUC, self).__init__()
|
|
self.enable_codegen_mot = self.bind('enable_codegen_mot')
|
|
self.force_pseudo_codegen_mot = self.bind('force_pseudo_codegen_mot')
|
|
self.enable_codegen_mot_print = self.bind('enable_codegen_mot_print')
|
|
self.codegen_mot_limit = self.bind('codegen_mot_limit')
|
|
self.mot_allow_index_on_nullable_column = self.bind('mot_allow_index_on_nullable_column')
|
|
self.mot_config_file = self.bind('mot_config_file')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class GlobalTempTableGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(GlobalTempTableGUC, self).__init__()
|
|
self.max_active_global_temporary_table = self.bind('max_active_global_temporary_table')
|
|
self.vacuum_gtt_defer_check_age = self.bind('vacuum_gtt_defer_check_age')
|
|
self.enable_gtt_concurrent_truncate = self.bind('enable_gtt_concurrent_truncate')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class UserDefineFuncGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(UserDefineFuncGUC, self).__init__()
|
|
self.udf_memory_limit = self.bind('udf_memory_limit')
|
|
self.FencedUDFMemoryLimit = self.bind('FencedUDFMemoryLimit')
|
|
self.UDFWorkerMemHardLimit = self.bind('UDFWorkerMemHardLimit')
|
|
self.pljava_vmoptions = self.bind('pljava_vmoptions')
|
|
|
|
def calculate(self):
|
|
pass
|
|
|
|
|
|
class JobScheduleGUC(GUCTuneGroup):
|
|
def __init__(self):
|
|
super(JobScheduleGUC, self).__init__()
|
|
self.job_queue_processes = self.bind('job_queue_processes')
|
|
self.enable_prevent_job_task_startup = self.bind('enable_prevent_job_task_startup')
|
|
|
|
def calculate(self):
|
|
infos = Project.getGlobalPerfProbe()
|
|
if infos.business.scenario == BsScenario.TP_PERFORMANCE:
|
|
self.job_queue_processes.set('0')
|