hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] New upgrade framework for 4.x

Browse Source
This commit is contained in:
tino247
2022-12-29 11:40:25 +00:00
committed by ob-robot
parent b39b8c45f0
commit a6098ccbc6
106 changed files with 6140 additions and 3227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
tools/upgrade/do_upgrade_post.py
View File

@ -11,7 +11,9 @@ import config
import opts
import run_modules
import actions
import special_upgrade_action_post
import upgrade_health_checker
import tenant_upgrade_action
import upgrade_post_checker
# 由于用了/*+read_consistency(WEAK) */来查询，因此升级期间不能允许创建或删除租户
@ -37,43 +39,6 @@ def config_logging_module(log_filenamme):
# 将定义好的stdout_handler日志handler添加到root logger
logging.getLogger('').addHandler(stdout_handler)
def dump_sql_to_file(dump_filename, tenant_id_list):
normal_ddls_str = normal_ddl_actions_post.get_normal_ddl_actions_sqls_str()
normal_dmls_str = normal_dml_actions_post.get_normal_dml_actions_sqls_str()
each_tenant_dmls_str = each_tenant_dml_actions_post.get_each_tenant_dml_actions_sqls_str(tenant_id_list)
dump_file = open(dump_filename, 'w')
dump_file.write('# 以下是upgrade_post.py脚本中的步骤\n')
dump_file.write('# 仅供upgrade_post.py脚本运行失败需要人肉的时候参考\n')
dump_file.write('\n\n')
dump_file.write('# normal ddl\n')
dump_file.write(normal_ddls_str + '\n')
dump_file.write('\n\n')
dump_file.write('# normal dml\n')
dump_file.write(normal_dmls_str + '\n')
dump_file.write('\n\n')
dump_file.write('# each tenant dml\n')
dump_file.write(each_tenant_dmls_str + '\n')
dump_file.write('\n\n')
dump_file.write('# do special upgrade actions\n')
dump_file.write('# please run ./upgrade_post.py -h [host] -P [port] -u [user] -p [password] -m special_action\n')
dump_file.write('\n\n')
dump_file.close()
def check_before_upgrade(query_cur, upgrade_params):
check_server_version(query_cur)
return
# 混部阶段执行POST脚本，会导致租户级系统表创建可能多数派处于旧binary而被GC,
# 需要规避OCP升级流程异常导致的混部阶段执行POST脚本的问题
def check_server_version(query_cur):
sql = """select distinct(substring_index(build_version, '_', 1)) from __all_server""";
(desc, results) = query_cur.exec_query(sql);
if len(results) != 1:
raise MyError("servers build_version not match")
else:
logging.info("check server version success")
def print_stats():
logging.info('==================================================================================')
logging.info('============================== STATISTICS BEGIN ==================================')
@ -84,7 +49,7 @@ def print_stats():
logging.info('=============================== STATISTICS END ===================================')
logging.info('==================================================================================')
def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_params):
def do_upgrade(my_host, my_port, my_user, my_passwd, timeout, my_module_set, upgrade_params):
try:
conn = mysql.connector.connect(user = my_user,
password = my_passwd,
@ -95,8 +60,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
cur = conn.cursor(buffered=True)
try:
query_cur = actions.QueryCursor(cur)
# 开始升级前的检查
check_before_upgrade(query_cur, upgrade_params)
actions.check_server_version_by_cluster(cur)
# 获取租户id列表
tenant_id_list = actions.fetch_tenant_ids(query_cur)
if len(tenant_id_list) <= 0:
@ -104,13 +68,45 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
raise MyError('no tenant id')
logging.info('there has %s distinct tenant ids: [%s]', len(tenant_id_list), ','.join(str(tenant_id) for tenant_id in tenant_id_list))
conn.commit()
if run_modules.MODULE_SPECIAL_ACTION in my_module_set:
logging.info('================begin to run special action===============')
if run_modules.MODULE_HEALTH_CHECK in my_module_set:
logging.info('================begin to run health check action ===============')
upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout)
logging.info('================succeed to run health check action ===============')
if run_modules.MODULE_END_ROLLING_UPGRADE in my_module_set:
logging.info('================begin to run end rolling upgrade action ===============')
conn.autocommit = True
special_upgrade_action_post.do_special_upgrade(conn, cur, tenant_id_list, my_user, my_passwd)
actions.do_end_rolling_upgrade(cur, timeout)
conn.autocommit = False
actions.refresh_commit_sql_list()
logging.info('================succeed to commit special action===============')
logging.info('================succeed to run end rolling upgrade action ===============')
if run_modules.MODULE_TENANT_UPRADE in my_module_set:
logging.info('================begin to run tenant upgrade action ===============')
conn.autocommit = True
tenant_upgrade_action.do_upgrade(conn, cur, tenant_id_list, timeout, my_user, my_passwd)
conn.autocommit = False
actions.refresh_commit_sql_list()
logging.info('================succeed to run tenant upgrade action ===============')
if run_modules.MODULE_END_UPRADE in my_module_set:
logging.info('================begin to run end upgrade action ===============')
conn.autocommit = True
actions.do_end_upgrade(cur, timeout)
conn.autocommit = False
actions.refresh_commit_sql_list()
logging.info('================succeed to run end upgrade action ===============')
if run_modules.MODULE_POST_CHECK in my_module_set:
logging.info('================begin to run post check action ===============')
conn.autocommit = True
actions.do_end_upgrade(cur, timeout)
upgrade_post_checker.do_check(conn, cur, query_cur, timeout)
conn.autocommit = False
actions.refresh_commit_sql_list()
logging.info('================succeed to run post check action ===============')
except Exception, e:
logging.exception('run error')
raise e
@ -118,7 +114,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set, upgrade_para
# 打印统计信息
print_stats()
# 将回滚sql写到文件中
actions.dump_rollback_sql_to_file(upgrade_params.rollback_sql_filename)
# actions.dump_rollback_sql_to_file(upgrade_params.rollback_sql_filename)
cur.close()
conn.close()
except mysql.connector.Error, e:
@ -133,7 +129,7 @@ def do_upgrade_by_argv(argv):
opts.change_opt_defult_value('log-file', upgrade_params.log_filename)
opts.parse_options(argv)
if not opts.has_no_local_opts():
opts.deal_with_local_opts()
opts.deal_with_local_opts('upgrade_post')
else:
opts.check_db_client_opts()
log_filename = opts.get_opt_log_file()
@ -145,6 +141,7 @@ def do_upgrade_by_argv(argv):
port = int(opts.get_opt_port())
user = opts.get_opt_user()
password = opts.get_opt_password()
timeout = int(opts.get_opt_timeout())
cmd_module_str = opts.get_opt_module()
module_set = set([])
all_module_set = run_modules.get_all_module_set()
@ -156,9 +153,9 @@ def do_upgrade_by_argv(argv):
module_set.add(cmd_module)
else:
raise MyError('invalid module: {0}'.format(cmd_module))
logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", module=\"%s\", log-file=\"%s\"',\
host, port, user, password, module_set, log_filename)
do_upgrade(host, port, user, password, module_set, upgrade_params)
logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", timeout=\"%s\", module=\"%s\", log-file=\"%s\"',\
host, port, user, password, timeout, module_set, log_filename)
do_upgrade(host, port, user, password, timeout, module_set, upgrade_params)
except mysql.connector.Error, e:
logging.exception('mysql connctor error')
logging.exception('run error, maybe you can reference ' + upgrade_params.rollback_sql_filename + ' to rollback it')