From 75c026c663fbdc8f502ed10a97e33d76c66dbbdc Mon Sep 17 00:00:00 2001 From: hiddenbomb Date: Fri, 12 Jul 2024 03:57:01 +0000 Subject: [PATCH] enhance disk space validation for compat --- tools/upgrade/CMakeLists.txt | 0 tools/upgrade/upgrade_checker.py | 52 ++++++++++++++++++++++++++++++++ tools/upgrade/upgrade_post.py | 52 ++++++++++++++++++++++++++++++++ tools/upgrade/upgrade_pre.py | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+) mode change 100755 => 100644 tools/upgrade/CMakeLists.txt diff --git a/tools/upgrade/CMakeLists.txt b/tools/upgrade/CMakeLists.txt old mode 100755 new mode 100644 diff --git a/tools/upgrade/upgrade_checker.py b/tools/upgrade/upgrade_checker.py index 6ac94546a..3f1f00cea 100755 --- a/tools/upgrade/upgrade_checker.py +++ b/tools/upgrade/upgrade_checker.py @@ -728,6 +728,57 @@ def check_fail_list(): error_msg ="upgrade checker failed with " + str(len(fail_list)) + " reasons: " + ", ".join(['['+x+"] " for x in fail_list]) raise MyError(error_msg) +# 检查升级到4.3.2或更高版本时，剩余的磁盘空间是否足够做多源数据格式转换 +def check_disk_space_for_mds_sstable_compat(query_cur): + need_check_disk_space = False + sql = """select distinct value from GV$OB_PARAMETERS where name='min_observer_version'""" + (desc, results) = query_cur.exec_query(sql) + if len(results) != 1: + fail_list.append('min_observer_version is not sync') + elif len(results[0]) != 1: + fail_list.append('column cnt not match') + else: + min_cluster_version = get_version(results[0][0]) + if min_cluster_version < get_version("4.3.2.0"): + need_check_disk_space = True + logging.info("need check disk space for mds sstable, min observer version: {0}".format(results[0][0])) + else: + logging.info("no need to check disk space, min observer version: {0}".format(results[0][0])) + + if need_check_disk_space: + do_check_disk_space_for_compat(query_cur) + +def do_check_disk_space_for_compat(query_cur): + sql = """select svr_ip, svr_port from __all_server""" + (desc, results) = query_cur.exec_query(sql) + + success = True + for idx in range(len(results)): + svr_ip = results[idx][0] + svr_port = results[idx][1] + + tablet_cnt = get_tablet_cnt(query_cur, svr_ip, svr_port) + disk_free_size = get_disk_free_size(query_cur, svr_ip, svr_port) + needed_size = tablet_cnt * 4096 * 2 + if needed_size > disk_free_size: + fail_list.append("svr_ip: {0}, svr_port: {1}, disk_free_size {2} is not enough for mds sstable, needed_size is {3}, cannot upgrade".format(svr_ip, svr_port, disk_free_size, needed_size)) + success = False + else: + logging.info("svr_ip: {0}, svr_port: {1}, disk_free_size: {2}, needed_size: {3}, can upgrade".format(svr_ip, svr_port, disk_free_size, needed_size)) + + if success: + logging.info("check disk space for mds sstable success") + +def get_tablet_cnt(query_cur, svr_ip, svr_port): + sql = """select /*+ query_timeout(1000000000) */ count(*) from __all_virtual_tablet_pointer_status where svr_ip = '{0}' and svr_port = {1}""".format(svr_ip, svr_port) + (desc, results) = query_cur.exec_query(sql) + return results[0][0] + +def get_disk_free_size(query_cur, svr_ip, svr_port): + sql = """select free_size from __all_virtual_disk_stat where svr_ip = '{0}' and svr_port = {1}""".format(svr_ip, svr_port) + (desc, results) = query_cur.exec_query(sql) + return results[0][0] + def set_query_timeout(query_cur, timeout): if timeout != 0: sql = """set @@session.ob_query_timeout = {0}""".format(timeout * 1000 * 1000) @@ -771,6 +822,7 @@ def do_check(my_host, my_port, my_user, my_passwd, timeout, upgrade_params): check_table_api_transport_compress_func(query_cur) check_variable_binlog_row_image(query_cur) check_oracle_standby_replication_exist(query_cur) + check_disk_space_for_mds_sstable_compat(query_cur) # all check func should execute before check_fail_list check_fail_list() modify_server_permanent_offline_time(cur) diff --git a/tools/upgrade/upgrade_post.py b/tools/upgrade/upgrade_post.py index 6890661a1..3fdc484de 100755 --- a/tools/upgrade/upgrade_post.py +++ b/tools/upgrade/upgrade_post.py @@ -2383,6 +2383,57 @@ # error_msg ="upgrade checker failed with " + str(len(fail_list)) + " reasons: " + ", ".join(['['+x+"] " for x in fail_list]) # raise MyError(error_msg) # +## 检查升级到4.3.2或更高版本时，剩余的磁盘空间是否足够做多源数据格式转换 +#def check_disk_space_for_mds_sstable_compat(query_cur): +# need_check_disk_space = False +# sql = """select distinct value from GV$OB_PARAMETERS where name='min_observer_version'""" +# (desc, results) = query_cur.exec_query(sql) +# if len(results) != 1: +# fail_list.append('min_observer_version is not sync') +# elif len(results[0]) != 1: +# fail_list.append('column cnt not match') +# else: +# min_cluster_version = get_version(results[0][0]) +# if min_cluster_version < get_version("4.3.2.0"): +# need_check_disk_space = True +# logging.info("need check disk space for mds sstable, min observer version: {0}".format(results[0][0])) +# else: +# logging.info("no need to check disk space, min observer version: {0}".format(results[0][0])) +# +# if need_check_disk_space: +# do_check_disk_space_for_compat(query_cur) +# +#def do_check_disk_space_for_compat(query_cur): +# sql = """select svr_ip, svr_port from __all_server""" +# (desc, results) = query_cur.exec_query(sql) +# +# success = True +# for idx in range(len(results)): +# svr_ip = results[idx][0] +# svr_port = results[idx][1] +# +# tablet_cnt = get_tablet_cnt(query_cur, svr_ip, svr_port) +# disk_free_size = get_disk_free_size(query_cur, svr_ip, svr_port) +# needed_size = tablet_cnt * 4096 * 2 +# if needed_size > disk_free_size: +# fail_list.append("svr_ip: {0}, svr_port: {1}, disk_free_size {2} is not enough for mds sstable, needed_size is {3}, cannot upgrade".format(svr_ip, svr_port, disk_free_size, needed_size)) +# success = False +# else: +# logging.info("svr_ip: {0}, svr_port: {1}, disk_free_size: {2}, needed_size: {3}, can upgrade".format(svr_ip, svr_port, disk_free_size, needed_size)) +# +# if success: +# logging.info("check disk space for mds sstable success") +# +#def get_tablet_cnt(query_cur, svr_ip, svr_port): +# sql = """select /*+ query_timeout(1000000000) */ count(*) from __all_virtual_tablet_pointer_status where svr_ip = '{0}' and svr_port = {1}""".format(svr_ip, svr_port) +# (desc, results) = query_cur.exec_query(sql) +# return results[0][0] +# +#def get_disk_free_size(query_cur, svr_ip, svr_port): +# sql = """select free_size from __all_virtual_disk_stat where svr_ip = '{0}' and svr_port = {1}""".format(svr_ip, svr_port) +# (desc, results) = query_cur.exec_query(sql) +# return results[0][0] +# #def set_query_timeout(query_cur, timeout): # if timeout != 0: # sql = """set @@session.ob_query_timeout = {0}""".format(timeout * 1000 * 1000) @@ -2426,6 +2477,7 @@ # check_table_api_transport_compress_func(query_cur) # check_variable_binlog_row_image(query_cur) # check_oracle_standby_replication_exist(query_cur) +# check_disk_space_for_mds_sstable_compat(query_cur) # # all check func should execute before check_fail_list # check_fail_list() # modify_server_permanent_offline_time(cur) diff --git a/tools/upgrade/upgrade_pre.py b/tools/upgrade/upgrade_pre.py index c0ecc5a6c..cb467f24d 100755 --- a/tools/upgrade/upgrade_pre.py +++ b/tools/upgrade/upgrade_pre.py @@ -2383,6 +2383,57 @@ # error_msg ="upgrade checker failed with " + str(len(fail_list)) + " reasons: " + ", ".join(['['+x+"] " for x in fail_list]) # raise MyError(error_msg) # +## 检查升级到4.3.2或更高版本时，剩余的磁盘空间是否足够做多源数据格式转换 +#def check_disk_space_for_mds_sstable_compat(query_cur): +# need_check_disk_space = False +# sql = """select distinct value from GV$OB_PARAMETERS where name='min_observer_version'""" +# (desc, results) = query_cur.exec_query(sql) +# if len(results) != 1: +# fail_list.append('min_observer_version is not sync') +# elif len(results[0]) != 1: +# fail_list.append('column cnt not match') +# else: +# min_cluster_version = get_version(results[0][0]) +# if min_cluster_version < get_version("4.3.2.0"): +# need_check_disk_space = True +# logging.info("need check disk space for mds sstable, min observer version: {0}".format(results[0][0])) +# else: +# logging.info("no need to check disk space, min observer version: {0}".format(results[0][0])) +# +# if need_check_disk_space: +# do_check_disk_space_for_compat(query_cur) +# +#def do_check_disk_space_for_compat(query_cur): +# sql = """select svr_ip, svr_port from __all_server""" +# (desc, results) = query_cur.exec_query(sql) +# +# success = True +# for idx in range(len(results)): +# svr_ip = results[idx][0] +# svr_port = results[idx][1] +# +# tablet_cnt = get_tablet_cnt(query_cur, svr_ip, svr_port) +# disk_free_size = get_disk_free_size(query_cur, svr_ip, svr_port) +# needed_size = tablet_cnt * 4096 * 2 +# if needed_size > disk_free_size: +# fail_list.append("svr_ip: {0}, svr_port: {1}, disk_free_size {2} is not enough for mds sstable, needed_size is {3}, cannot upgrade".format(svr_ip, svr_port, disk_free_size, needed_size)) +# success = False +# else: +# logging.info("svr_ip: {0}, svr_port: {1}, disk_free_size: {2}, needed_size: {3}, can upgrade".format(svr_ip, svr_port, disk_free_size, needed_size)) +# +# if success: +# logging.info("check disk space for mds sstable success") +# +#def get_tablet_cnt(query_cur, svr_ip, svr_port): +# sql = """select /*+ query_timeout(1000000000) */ count(*) from __all_virtual_tablet_pointer_status where svr_ip = '{0}' and svr_port = {1}""".format(svr_ip, svr_port) +# (desc, results) = query_cur.exec_query(sql) +# return results[0][0] +# +#def get_disk_free_size(query_cur, svr_ip, svr_port): +# sql = """select free_size from __all_virtual_disk_stat where svr_ip = '{0}' and svr_port = {1}""".format(svr_ip, svr_port) +# (desc, results) = query_cur.exec_query(sql) +# return results[0][0] +# #def set_query_timeout(query_cur, timeout): # if timeout != 0: # sql = """set @@session.ob_query_timeout = {0}""".format(timeout * 1000 * 1000) @@ -2426,6 +2477,7 @@ # check_table_api_transport_compress_func(query_cur) # check_variable_binlog_row_image(query_cur) # check_oracle_standby_replication_exist(query_cur) +# check_disk_space_for_mds_sstable_compat(query_cur) # # all check func should execute before check_fail_list # check_fail_list() # modify_server_permanent_offline_time(cur)