hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[CP] [UPGRADE] Calculate default timeout according to tenant count

Browse Source
This commit is contained in:
tino247
2023-12-28 05:19:19 +00:00
committed by ob-robot
parent 901bf26411
commit 9093d820a0
6 changed files with 406 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
tools/upgrade/upgrade_post_checker.py
View File

@ -41,8 +41,14 @@ def check_data_version(cur, query_cur, timeout):
# check compatible sync
parameter_count = int(server_count) * int(tenant_count)
current_data_version = actions.get_current_data_version()
query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 2, 60)
actions.set_session_timeout(cur, query_timeout)
sql = """select count(*) as cnt from oceanbase.__all_virtual_tenant_parameter_info where name = 'compatible' and value = '{0}' and tenant_id in ({1})""".format(current_data_version, tenant_ids_str)
times = (timeout if timeout > 0 else 60) / 5
wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 60)
times = wait_timeout / 5
while times >= 0:
logging.info(sql)
cur.execute(sql)
@ -62,6 +68,8 @@ def check_data_version(cur, query_cur, timeout):
raise e
time.sleep(5)
actions.set_session_timeout(cur, 10)
# check target_data_version/current_data_version from __all_core_table
int_current_data_version = actions.get_version(current_data_version)
sql = "select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = {0} and tenant_id in ({1})".format(int_current_data_version, tenant_ids_str)
@ -76,16 +84,20 @@ def check_data_version(cur, query_cur, timeout):
logging.info("all tenant's target_data_version/current_data_version are match with {0}".format(current_data_version))
# 3 检查内部表自检是否成功
def check_root_inspection(query_cur, timeout):
def check_root_inspection(cur, query_cur, timeout):
sql = "select count(*) from oceanbase.__all_virtual_upgrade_inspection where info != 'succeed'"
times = timeout if timeout > 0 else 180
while times > 0 :
wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600)
times = wait_timeout / 10
while times >= 0 :
(desc, results) = query_cur.exec_query(sql)
if results[0][0] == 0:
break
time.sleep(10)
times -= 1
if times == 0:
if times == -1:
logging.warn('check root inspection failed!')
raise e
logging.info('check root inspection success')
@ -113,7 +125,7 @@ def do_check(conn, cur, query_cur, timeout):
try:
check_cluster_version(cur, timeout)
check_data_version(cur, query_cur, timeout)
check_root_inspection(query_cur, timeout)
check_root_inspection(cur, query_cur, timeout)
enable_ddl(cur, timeout)
enable_rebalance(cur, timeout)
enable_rereplication(cur, timeout)