#!/usr/bin/env python3 # -*- coding:utf-8 -*- ############################################################################# # Copyright (c) 2023 Huawei Technologies Co.,Ltd. # # openGauss is licensed under Mulan PSL v2. # You can use this software according to the terms # and conditions of the Mulan PSL v2. # You may obtain a copy of Mulan PSL v2 at: # # http://license.coscl.org.cn/MulanPSL2 # # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, # WITHOUT WARRANTIES OF ANY KIND, # EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. # ---------------------------------------------------------------------------- # Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. ############################################################################# """ 标题: 报告总结 数据库信息：版本、节点名等 标准元数据地图：xxxxxxxxx 校验结果：通过 or 失败 内容摘要： 共校验xx个数据库。 累计xx个系统元数据表，其中xx通过，xx个失败。 校验规则共xx条，xx条通过，xx条失败。 Category一共xxx个,其中xx个完全通过，xx个存在失败内容 。。。 失败项摘要： - 规则编号：1 数据库：postgres 校验规则： select ... 规则解读： 使用函数pg_get_ruledef校验重写规则的定义 规则结果： 行号 状态 内容 1 Failed 规则定义def:pg_catalog.gs_session_memory_detail(_RETURN) 缺失 2 ... - 规则编号：8 数据库：postgres 校验规则： select ... 规则解读： ... 规则结果： ... 警告： 存在未分类的系统元数据表: 建议排查来源，并修改本工具，添加至分类列表中。 TIPS:详细校验原理与说明详见本工具的"README.md"。 详细内容： 数据库postgres 元数据系统表结构 共xx个系统表，校验项目xx条，通过xx条，失败xx条 - 规则编号：1 数据库：postgres 校验规则： select ... 规则解读： 使用函数pg_get_ruledef校验重写规则的定义 规则结果： 行号 状态 内容 1 Failed 规则定义def:pg_catalog.gs_session_memory_detail(_RETURN) 缺失 2 ... 分类内容 共xx个系统表，校验项目xx条，通过xx条，失败xx条 数据库对象 数据库user1db 元数据系统表结构 .... """ from enum import Enum from upgrade_checker.utils.param import ReportFormat, ReportMode from upgrade_checker.log import logger from upgrade_checker.rules.rule import StructRule, ContentRule, CommonRule, ConclusionState from upgrade_checker.style.markdown import MarkDown class ConclusionCardType(Enum): SUMMARY = 0 STRUCTURE = 1 CONTENT = 2 COMMON = 3 class ConclusionCard(object): error_num = 0 def __init__(self, id, conclusion, dbname=None, tbname=None, category=None): self.id = id self.dbname = dbname self.tbname = tbname self.category = category self.conclusion = conclusion self.sql = conclusion.rule.sql self.sql_desc = conclusion.rule.sql_desc self.results = conclusion.details def serialize(self, style, cc_type): res = '' res += style.unordered_list([f'规则编号 {self.id}']) if cc_type == ConclusionCardType.SUMMARY: res += style.text('数据库：' + self.dbname) elif cc_type == ConclusionCardType.STRUCTURE: res += style.text('表名：' + self.conclusion.rule.rel_key) elif cc_type == ConclusionCardType.CONTENT: res += style.text('表名：' + self.conclusion.rule.rel_key) res += style.text('类别：' + self.conclusion.rule.category.name) elif cc_type == ConclusionCardType.CONTENT: res += style.text('类别：' + self.conclusion.rule.category.name) res += style.text('校验规则：' + self.sql) res += style.text('规则解读：' + self.sql_desc) res += style.text('规则结果：') rows = [] for i, row in enumerate(self.results): if cc_type == ConclusionCardType.SUMMARY: if row['state'] == ConclusionState.SUCCESS: continue result_state = 'SUCCESS' if row['state'] == ConclusionState.SUCCESS else 'Falied' rows.append(( i, result_state, row['summary'] )) res += style.table(["行号", "状态", "内容"], rows) return res class Report(object): """ 报告 """ def __init__(self): self.db_info = '' self.vmap_info = '' self.pci_suc_count = 0 self.pci_err_count = 0 self.rule_suc_count = 0 self.rule_err_count = 0 self.detail_suc_count = 0 self.detail_err_count = 0 self.err_rules_res = [] self.warnings = [] self.content = {} # { "dbname": [struct rules, content rules, common rules], ... } def serialize(self, style): rule_count = self.rule_suc_count + self.rule_err_count detail_count = self.detail_suc_count + self.detail_err_count verify_result = '成功' if self.rule_err_count == 0 else '失败' db_summary = "共校验 %d 个数据库。" % len(self.content) rul_summary = "校验规则累计执行%d条，%d条通过，%d条失败。" % (rule_count, self.rule_suc_count, self.rule_err_count) detail_summary = "所有校验规则累计生成%d条内容小项，%d条通过，%d条失败。" % (detail_count, self.detail_suc_count, self.detail_err_count) warning_summary = "警告信息%d条。" % len(self.warnings) report = style.title(1, "校验报告") report += style.title_paragraph("数据库信息", self.db_info) report += style.title_paragraph("基准元数据地图", self.vmap_info) report += style.title_paragraph("校验结果", verify_result) report += style.emphasize("内容摘要：") report += style.unordered_list([db_summary, rul_summary, detail_summary, warning_summary]) report += style.title(2, "失败项摘要：") for conclusion_card in self.err_rules_res: report += conclusion_card.serialize(style, ConclusionCardType.SUMMARY) report += style.title(2, "警告：") report += style.unordered_list(self.warnings) report += style.title_paragraph("TIPS", "详细校验原理与说明参考本工具的《README.md》") report += style.title(1, "详细数据") for dbname, contents in self.content.items(): report += style.title(2, "数据库%s" % dbname) report += style.title(3, "元数据系统表结构") for conclusion_card in contents[0]: report += conclusion_card.serialize(style, ConclusionCardType.STRUCTURE) report += style.title(3, "元数据系统表内容") for conclusion_card in contents[1]: report += conclusion_card.serialize(style, ConclusionCardType.CONTENT) report += style.title(3, "常规通用校验") for conclusion_card in contents[2]: report += conclusion_card.serialize(style, ConclusionCardType.COMMON) return report class StyleFactory(object): """ 风格工厂 """ @staticmethod def produce(fmt): if fmt == ReportFormat.MARKDOWN: return MarkDown() else: return MarkDown() class Reporter(object): """ 报告生成器 """ def __init__(self, file, fmt, granularity): self._file_path = file try: self._file = open(file, 'a') # 结果文件 except FileNotFoundError: logger.err('无法打开文件:' + file) self._current_dbname = '' # 当前记录的数据库名称 self._report = Report() # 结果 self._style = StyleFactory.produce(fmt) # 风格 self._granularity = granularity # 粒度 def __del__(self): if self._file is not None: self._file.close() def __str__(self): return 'Reporter: granularity{0}, file({1})'.format(self._granularity, self._file_path) def record_info(self, db_info='', vmap_info=''): self._report.db_info = db_info self._report.vmap_info = vmap_info def _collect_statistic(self, old_errno, new_errno, dbname, conclusion): """ 整理记录校验过程中的统计信息。 :param old_errno: :param new_errno: :param dbname: :param conclusion: :return: """ suc = True if old_errno == new_errno else False if suc: self._report.rule_suc_count += 1 else: self._report.rule_err_count += 1 detail_err_count = new_errno - old_errno detail_suc_count = len(conclusion.details) - detail_err_count self._report.detail_suc_count += detail_suc_count self._report.detail_err_count += detail_err_count if self._granularity == ReportMode.DETAIL: data = self._report.content.get(dbname) assert data is not None logger.debug('当前数据库{0}记录结论小项分别为{1},{2},{3}'.format( dbname, len(data[0]), len(data[1]), len(data[2])) ) def _transform_conclusion(self, dbname, conclusion): """ :param dbname: :param conclusion: :return: """ row_num = 0 for row in conclusion.details: row_num += 1 result_state = 'SUCCESS' if row['state'] == ConclusionState.SUCCESS else \ 'ERROR(err-num-%d)' % self._current_errno row_res_base = [ conclusion.rule.sql, conclusion.rule.sql_desc, row_num, result_state, row['summary'] ] if row['state'] == ConclusionState.FAILED: self._report.err_rules_res.append([dbname] + row_res_base) self._current_errno += 1 def record(self, dbname, conclusion): """ 记录一条结论到report内。并更新统计数据 """ if dbname != self._current_dbname: self._report.content[dbname] = [[], [], []] self._current_dbname = dbname # statistic if conclusion.all_success(): self._report.rule_suc_count += 1 else: self._report.rule_err_count += 1 self._report.detail_suc_count += conclusion.suc_count self._report.detail_err_count += conclusion.err_count logger.debug('reporter 新增记录规则结论，结论成功状态{0}, 结论明细共计{1}条, 成功{2}，失败{3}。'.format( conclusion.all_success(), len(conclusion.details), conclusion.suc_count, conclusion.err_count) ) # record warnings self._report.warnings += conclusion.warnings # record details if conclusion.all_success() and self._granularity == ReportMode.SUMMARY: # if rule success and not need detail, just return return cc = ConclusionCard((self._report.rule_suc_count + self._report.rule_err_count), conclusion, dbname) # generate summary if not conclusion.all_success(): self._report.err_rules_res.append(cc) # generate detail if self._granularity != ReportMode.SUMMARY: data = self._report.content.get(dbname) assert data is not None if isinstance(conclusion.rule, StructRule): data[0].append(cc) elif isinstance(conclusion.rule, ContentRule): data[1].append(cc) elif isinstance(conclusion.rule, CommonRule): data[2].append(cc) else: assert False def report(self): """ :return: """ logger.log('开始整理生成报告......') if self._granularity == ReportMode.DETAIL: logger.info('详细报告整理中，这可能需要较长时间......') self._file.write(self._report.serialize(self._style)) logger.log('报告生成完成 %s。' % self._file_path)