#!/usr/bin/env python3 # -*- coding:utf-8 -*- ############################################################################# # Copyright (c) 2023 Huawei Technologies Co.,Ltd. # # openGauss is licensed under Mulan PSL v2. # You can use this software according to the terms # and conditions of the Mulan PSL v2. # You may obtain a copy of Mulan PSL v2 at: # # http://license.coscl.org.cn/MulanPSL2 # # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, # WITHOUT WARRANTIES OF ANY KIND, # EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. # ---------------------------------------------------------------------------- # Description : preset management. Including reading and resolving preset. ############################################################################# import os import re import json from enum import Enum from impl.perf_config.basic.project import Project """ preset is a set of information about some business attributes. By writing a preset file, the preset configuration can be directly read during the tool running, thus skipping the stage of business investigation. preset contains Preset information about the service survey, and the configuration parameters are related to the service survey. The preset file is actually a json file. This section describes how to set configuration items in key-value pairs. In json, you can write in strings. However, different parameters have different parsing schemes, such as enumeration, ordinary string, etc., and some parsing rules are needed to parse and determine whether the preset file is valid. The built-in preset is stored in the tool code directory. Also, preset written by the user can be stored in the $GAUSSLOG/om/pg_perfconfig/preset. The difference between the two is that built-in files are automatically replaced when the database is upgrade. The preset name is a file name and needs to end with '.json'. If files with the same name exist in two directories, the internal directory takes precedence. """ class PsOptionType(Enum): INT = 'integer' STR = 'string' ENUM_VALUE = 'enum value' ENUM_LIST = 'list of enum' PATH = 'path' class PsOptionRule(object): @staticmethod def transformIntValue(value, check_range): """ transform an int value. :param value: origin value. :param check_range: function to check range. return T/F. :return: NA """ if type(value) != int: value = int(value) if check_range is not None and not check_range(value): raise ValueError(f'value {value} not in range.') return value @staticmethod def transformStrValue(value, check_range): """ transform a string value. :param value: origin value. :param check_range: function to check range. return T/F. :return: NA """ if type(value) != str: value = str(value) if check_range is not None and not check_range(value): raise ValueError(f'value {value} not in range.') return value @staticmethod def transformEnumValue(value, value_list): """ transform an enum value. :param value: origin value. :param value_list: enumrate list. :return: NA """ if type(value) != str: value = str(value) value = value.strip() if value not in value_list: raise ValueError(f'value {value} not in range.') return value @staticmethod def transformEnumValueList(value, value_list): """ transform an enum value list. :param value: origin value. The value can be a string separated by commas or Spaces. It also could be a list. :param value_list: enumrate list. :return: NA """ opts = value if type(value) == list else re.split(r'[, ]+', str(value)) res = set() for opt in opts: if opt not in value_list: raise ValueError(f'option {opt} not in range.') res.add(opt) return list(res) @staticmethod def transformPath(value, check_exist): """ transform a path. :param value: origin value. :param check_exist: Checks whether the path exists and is accessible. :return: NA """ path = value if type(value) == str else str(value) if check_exist and not os.access(path, os.F_OK): raise ValueError(f'Could not access path Option path {path}.') return path def __init__(self, desc, option_type, default, range_info): self.desc = desc self.option_type = option_type self.default = default self.range_info = range_info def regulate(self, value): if value is None: return self.default try: if self.option_type == PsOptionType.INT: return PsOptionRule.transformIntValue(value, self.range_info) if self.option_type == PsOptionType.STR: return PsOptionRule.transformStrValue(value, self.range_info) elif self.option_type == PsOptionType.ENUM_VALUE: return PsOptionRule.transformEnumValue(value, self.range_info) elif self.option_type == PsOptionType.ENUM_LIST: return PsOptionRule.transformEnumValueList(value, self.range_info) elif self.option_type == PsOptionType.PATH: return PsOptionRule.transformPath(value, self.range_info) else: assert False except ValueError as e: Project.fatal('Preset error, ' + str(e)) class Preset(object): # options list. options = { 'desc': PsOptionRule( 'The description of preset', PsOptionType.STR, 'no description', None ), 'scenario': PsOptionRule( 'Business scenario.', PsOptionType.ENUM_VALUE, 'OLTP-performance', ['OLTP-produce', 'OLTP-performance'] ), 'rel_count': PsOptionRule( 'How many tables do you have?', PsOptionType.INT, 300, lambda x:x > 0 ), 'index_count': PsOptionRule( 'How many indexs do you have?', PsOptionType.INT, 600, lambda x:x > 0 ), 'rel_kind': PsOptionRule( 'What kind of table do you used?', PsOptionType.ENUM_LIST, ['heap-table', 'partition-table'], ['heap-table', 'partition-table', 'column-table', 'column-partition-table'] ), 'part_count': PsOptionRule( 'How many partitions do you have?', PsOptionType.INT, 200, lambda x:x > 0 ), 'data_size': PsOptionRule( 'How much data is there? unit is MB.', PsOptionType.INT, 8192, lambda x:x > 0 ), 'parallel': PsOptionRule( 'How much concurrency is there?', PsOptionType.INT, 400, lambda x:x > 0 ), 'isolated_xlog': PsOptionRule( 'Storing wal on a separate disk.', PsOptionType.PATH, None, True ) } @staticmethod def usage(): """ how to write preset. """ res = 'The preset configure is a file in JSON format that contains the following parameters:\n\n' for opt in Preset.options: res += f'Name: {opt}\n' res += f' Description: {Preset.options[opt].desc}\n' res += f' Type: {Preset.options[opt].option_type.value}\n' if Preset.options[opt].option_type in [PsOptionType.ENUM_VALUE, PsOptionType.ENUM_LIST]: res += f' Range: {str(Preset.options[opt].range_info)}\n' res += '\n' return res @staticmethod def get_preset_dir(): dir1 = os.path.join(Project.environ.workspace1, 'preset') dir2 = os.path.join(Project.environ.workspace2, 'preset') return dir1, dir2 @staticmethod def get_all_presets(): def _read_preset_dir(_dir): """ read all '.json' file in _dir. """ if _dir is None: return [] if not os.access(_dir, os.F_OK): Project.warning('Could not access ' + _dir) return [] files = os.listdir(_dir) res = [] for file in files: if file.endswith('.json'): res.append(file[:-5]) return res dir1, dir2 = Preset.get_preset_dir() builtins = _read_preset_dir(dir1) tmp_usersets = _read_preset_dir(dir2) # when duplicate name, builtin first. usersets = [x for x in tmp_usersets if x not in builtins] return builtins, usersets def __init__(self, preset_name): builtins, usersets = Preset.get_all_presets() dir1, dir2 = Preset.get_preset_dir() file = '' if preset_name in builtins: file = os.path.join(dir1, f'{preset_name}.json') elif preset_name in usersets: file = os.path.join(dir2, f'{preset_name}.json') else: Project.fatal('Could not find preset: ' + preset_name) self.name = preset_name with open(file, 'r') as f: config = json.load(f) for key in config: if not self.options.__contains__(key): Project.warning(f'skip unknown option {key} in preset.') self.content = {} for key in self.options: val = config.get(key) self.content[key] = self.options[key].regulate(val) def __str__(self): res = f'Preset name: {self.name}\n' desc = self.content.get('desc') res += ' {}\n'.format(desc if desc is not None else 'no description') res += 'Detail:\n' for k in self.content: if k == 'desc': continue res += ' {0}: {1}\n'.format(k, self.content[k]) return res def __getitem__(self, item): return self.content.get(item)