#!/usr/bin/env python3 # -*- coding:utf-8 -*- ############################################################################# # Copyright (c) 2023 Huawei Technologies Co.,Ltd. # # openGauss is licensed under Mulan PSL v2. # You can use this software according to the terms # and conditions of the Mulan PSL v2. # You may obtain a copy of Mulan PSL v2 at: # # http://license.coscl.org.cn/MulanPSL2 # # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, # WITHOUT WARRANTIES OF ANY KIND, # EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. # ---------------------------------------------------------------------------- # Description : gs_perfconfg is a utility to optimize system and database configure about openGauss ############################################################################# import os import sys import getopt import logging from base_utils.common.dialog import DialogUtil from impl.perf_config.basic.project import Project, PorjectError, ProjectLogLevel from impl.perf_config.basic.anti import AntiLog from impl.perf_config.preset.preset import Preset from impl.perf_config.perf_probe import PerfProbe from impl.perf_config.perf_tuner import PerfTuneTarget, PerfTuner def usage(): Project.msg(""" Usage of gs_perfconfig: gs_perfconfig [ACTION [ ACTION-PARAM ] ] Action, action-params and functions are as follows: help (No params) : Display help document of gs_perfconfig. '--help', '-h', '-?' are also effective. tune [ -t [ all,os,setup,guc,suggest ] ] [ --apply ] [--env envfile] [ -y ]: '-t' include four tuning target modules: os : (only root) Including os parameters, software, hardware configurations. setup : Including database setup configuration. guc : Including database GUC configuration. suggest : Give some additional suggestions. 'all' indicates all modules, but 'os' depend on the executor. '--apply' indicates that the actual application is adjusted, and the database may need to be restarted multiple times during this period. Otherwise, only one report is generated. '--env' specifies the environment variables file. '-y' Accept the requirements for configuring operating system parameters, GUC parameters, and restarting the database during adjustment. When the root user is used to execute the process, the process is completely executed from the beginning and certain information is generated. When the user executes the command, the system first attempts to load the information left by the root user, and then skips certain processes. preset [ --help | (No params) | preset-name | ] Print the preset addition guide, all preset list or specify the contents of the preset. recover [ -y ] : Recover the content of the last tuning. '-y' Accept the requirements for configuring operating system parameters, GUC parameters, and restarting the database during adjustment. The environment variable GS_PERFCONFIG_OPTIONS sets some behavior within the tool. export GS_PERFCONFIG_OPTIONS='param1=value1:param2=value2:param3=value3' params and values: lowest_print_log_level = log/notice/warning/error/fatal. The lowest level of print logs, default is notice. You can also set 'msg', but it will not take effect. """) class TuneTask(object): """ This is the tune task. Responsible for controlling the flow of the entire adjustment task. The operations include environment initialization, performance data detection, adjustment process, and rollback process. """ def __init__(self, argv): self.accept_risk = False target, apply, env = self._parse_arg(argv) self.tune_target = PerfTuneTarget(target, apply) if self.tune_target.noTarget(): return # environment initialization Project.initEnviron(env, True) Project.initRole() Project.initProjectLog(Project.environ.run_log) Project.prepareReport(Project.environ.report) Project.log(Project.environ.__str__()) if self.tune_target.apply(): AntiLog.initAntiLog(Project.environ.anti_log) Project.setTask(self) def _parse_arg(self, argv): short_options = 't:y' long_options = ['apply', 'env='] opts, args = getopt.getopt(argv, short_options, long_options) if len(args) > 1: Project.fatal('unknown param ' + args[0]) target = 'all' apply = False env = None for opt, arg in opts: if opt == '-t': target = arg elif opt == '--apply': apply = True elif opt == '--env': env = arg elif opt == '-y': self.accept_risk = True else: Project.fatal('unknown param ' + opt) return target, apply, env def run(self): """ Task details. Control the entire tune process. """ if self.tune_target.noTarget(): Project.notice('nothing to tune.') return do_apply = self.tune_target.apply() # 1, Operation content and risk tips. self.risk_disclosure() # 2, start probe Project.notice('Start probe detect.') infos = PerfProbe() Project.setGlobalPerfProbe(infos) infos.detect() # 3, shutdown openGauss if necessary og_alive_at_first = Project.isOpenGaussAlive() if og_alive_at_first and do_apply: Project.stopOpenGauss() # 4, tune and apply, and rollback apply when errors. error_occurred = False Project.notice('Prepare tune plan.') tuner = PerfTuner() Project.setGlobalPerfTuner(tuner) tuner.calculate() try: Project.notice('execute tune plan({0})...'.format( 'report and apply' if do_apply else 'just report')) tuner.explain(do_apply) Project.report.dump() Project.notice('Tune finish.') except PorjectError: error_occurred = True except Exception as e: Project.notice('Some errors have occurred.') Project.notice(e.__str__()) logging.exception(e) error_occurred = True if error_occurred and do_apply: Project.notice('start rollback.') PerfTuner.rollback(None) # 5, start og if necessary. if og_alive_at_first and do_apply: Project.startOpenGauss() def risk_disclosure(self): question = ('Certainly, we will perform some stress tests, make adjustments to the \n' 'configuration of operating system parameters, database parameters, and \n' 'also perform a database restart during the optimization process. \n' 'Are you accepting of the aforementioned circumstances?') Project.log('risk disclosure: ' + question) if self.accept_risk: Project.notice(f'user choose yes by "-y" on the question:\n ({question}).') else: ok = DialogUtil.yesOrNot(question) Project.log('user choose: ' + ('y' if ok else 'n')) if not ok: Project.fatal('The user has chosen to cancel the operation.') class PresetTask(object): """ This is the preset task. Responsible for showing how many presets there are, the details of the presets, and showing how to edit the presets. """ def __init__(self, argv): Project.set_lowest_print_level(ProjectLogLevel.WARNING) Project.initEnviron(None, False) Project.reset_lowest_print_level() self.target_preset = None if (argv is None or len(argv) == 0) else argv[0] if len(argv) > 1: Project.fatal('unknown param {}'.format(argv[1])) def run(self): if self.target_preset is None: self._show_all_presets() elif self.target_preset.lower() in ['help', '--help', '-h', '-?']: self._show_preset_usage() else: self._show_one_preset(self.target_preset) def _show_all_presets(self): builtins, usersets = Preset.get_all_presets() msg = 'Builtin Presets:\n' for preset in builtins: msg += f' {preset}\n' Project.msg(msg) msg = 'User Presets:\n' for preset in usersets: msg += f' {preset}\n' Project.msg(msg) def _show_one_preset(self, name): preset = Preset(name) Project.msg(preset.__str__()) def _show_preset_usage(self): preset_usage = Preset.usage() Project.msg(preset_usage) class RecoverTask(object): """ This is a recovery mission Read the anti log, undo the last adjustment, and restore the environment to the way it was before the tune. Only the most recent adjustment can be undone. """ def __init__(self, argv): self.accept_risk = False if len(argv) > 0: if len(argv) == 1 and argv[0] == '-y': self.accept_risk = True else: Project.fatal('invalid param {}'.format(' '.join(argv))) Project.initEnviron() Project.initRole() Project.initProjectLog(Project.environ.run_log) Project.prepareReport(Project.environ.report) Project.log(Project.environ.__str__()) AntiLog.initAntiLog(Project.environ.anti_log, reload=True) def run(self): Project.notice('start rollback.') self.risk_disclosure() og_is_alive_first = Project.isOpenGaussAlive() if og_is_alive_first: Project.stopOpenGauss() PerfTuner.rollback(None) Project.notice('Destroy anti log.') AntiLog.destroyAntiLog() if og_is_alive_first: Project.startOpenGauss() Project.notice('rollback finish.') def risk_disclosure(self): question = ('Certainly, we will make adjustments to the configuration of operating \n' 'system parameters, database parameters, and also perform a database \n' 'restart during the recover process.\n' 'Are you accepting of the aforementioned circumstances?') Project.log('risk disclosure: ' + question) if self.accept_risk: ok = True Project.notice(f'user choose yes by "-y" on the question:\n ({question}).') else: ok = DialogUtil.yesOrNot(question) Project.log('user choose: ' + ('y' if ok else 'n')) if not ok: Project.fatal('The user has chosen to cancel the recover.') if __name__ == '__main__': task = None if len(sys.argv) == 1 or sys.argv[1].lower() in ['help', '--help', '-h', '-?']: usage() exit(0) elif sys.argv[1] == 'tune': task = TuneTask(sys.argv[2:]) elif sys.argv[1] == 'preset': task = PresetTask(sys.argv[2:]) elif sys.argv[1] == 'recover': task = RecoverTask(sys.argv[2:]) else: Project.fatal('Unknown task: ' + sys.argv[1]) task.run()