!2816 【共享存储】基于cm模拟文件的HA测试框架以及各个场景用例

Merge pull request !2816 from 董宁/dn_fix1
2023-01-16 07:22:45 +00:00
parent 0b8e1fcf82 d465c8fecf
commit 30b4e5c253
5 changed files with 482 additions and 1 deletions
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@ -441,8 +441,9 @@ const int MAX_COMPACTION_THREAD_NUM = 10;
        (g_instance.attr.attr_storage.dcf_attr.enable_dcf ? \
        g_instance.attr.attr_storage.dcf_attr.dcf_max_workers : 0)

-#define NUM_DMS_REFORM_CALLLBACK_PROCS (3)
+#define NUM_DMS_REFORM_CALLLBACK_PROCS (5)
 #define NUM_DMS_LSNR_CALLBACK_PROC (1)
+#define NUM_DMS_SMON_CALLBACK_PROC (1)
 #define NUM_DMS_RDMA_THREAD_CNT (g_instance.attr.attr_storage.dms_attr.work_thread_count * 2)
 #define NUM_DMS_CALLBACK_PROCS \
        (g_instance.attr.attr_storage.dms_attr.enable_dms ? \
@ -451,6 +452,7 @@ const int MAX_COMPACTION_THREAD_NUM = 10;
        g_instance.attr.attr_storage.dms_attr.work_thread_count :      \
        NUM_DMS_RDMA_THREAD_CNT) + \
        NUM_DMS_LSNR_CALLBACK_PROC + \
+        NUM_DMS_SMON_CALLBACK_PROC + \
        NUM_DMS_REFORM_CALLLBACK_PROCS ) : 0)

 #define GLOBAL_ALL_PROCS \
--- a/src/test/ss/build_ss_database_common.sh
+++ b/src/test/ss/build_ss_database_common.sh
@ -0,0 +1,99 @@
+#!/bin/sh
+CUR_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+echo "CUR_DIR : $CUR_DIR"
+
+echo "check env var"
+if [ ${GAUSSHOME} ] && [ -d ${GAUSSHOME}/bin ];then
+    echo "GAUSSHOME: ${GAUSSHOME}"
+else
+    echo "GAUSSHOME NOT EXIST"
+    exit 1;
+fi
+
+SUPER_PASSWORD=Gauss_234
+
+clean_database_env()
+{
+    SS_DATA=$1
+    if [ -d ${SS_DATA} ]; then
+        echo "${SS_DATA} exists, so need to clean and recreate"
+        rm -rf ${SS_DATA}
+    else
+        echo "${SS_DATA} not exists, so need to recreate"
+    fi
+    mkdir ${SS_DATA}
+}
+
+clear_shm()
+{
+    ipcs -m | grep $USER | awk '{print $2}' | while read shm; do
+        if [ -n ${shm} ]; then
+            ipcrm -m ${shm}
+        fi
+    done
+    ipcs -s | grep $USER | awk '{print $2}' | while read shm; do
+        if [ -n ${shm} ]; then
+            ipcrm -s ${shm}
+        fi
+    done
+}
+
+kill_gaussdb()
+{
+    ps ux | grep gaussdb | grep -v grep | awk '{print $2}' | xargs kill -9 > /dev/null 2>&1
+    ps ux | grep gsql | grep -v grep | awk '{print $2}' | xargs kill -9 > /dev/null 2>&1
+    ps ux | grep dssserver | grep -v grep | awk '{print $2}' | xargs kill -9 > /dev/null 2>&1
+    clear_shm
+    sleep 2
+}
+
+kill_dss()
+{
+    ps ux | grep dssserver | grep -v grep | awk '{print $2}' | xargs kill -9 > /dev/null 2>&1
+}
+
+assign_hatest_parameter()
+{
+    for node in $@
+    do
+        echo -e "\nss_enable_reform = on" >> ${node}/postgresql.conf
+        echo -e "\nlog_min_messages = log" >> ${node}/postgresql.conf
+        echo -e "\nlogging_module = 'on(ALL)'" >> ${node}/postgresql.conf
+        echo -e "\nss_enable_log_level = 'on'" >> ${node}/postgresql.conf
+        echo "${node}:"
+        cat ${node}/postgresql.conf | grep ss_enable_dms
+    done
+}
+
+init_gaussdb()
+{
+    inst_id=$1
+    dss_home=$2
+    SS_DATA=$3
+    nodedata_cfg=$4
+    echo "${GAUSSHOME}/bin/gs_initdb -D ${SS_DATA}/dn${inst_id} --nodename=single_node -w ${SUPER_PASSWORD} --vgname=\"+data,+log${inst_id}\" --enable-dss --dms_url=\"${nodedata_cfg}\" -I ${inst_id} --socketpath=\"UDS:${dss_home}/.dss_unix_d_socket\""
+
+    ${GAUSSHOME}/bin/gs_initdb -D ${SS_DATA}/dn${inst_id} --nodename=single_node -w ${SUPER_PASSWORD} --vgname="+data,+log${inst_id}" --enable-dss --dms_url="${nodedata_cfg}" -I ${inst_id} --socketpath="UDS:${dss_home}/.dss_unix_d_socket"
+}
+
+set_gaussdb_port()
+{
+    data_node=$1
+    pg_port=$2
+    echo "" >> ${data_node}/postgresql.conf
+    echo "port = ${pg_port}" >> ${data_node}/postgresql.conf
+}
+
+start_gaussdb()
+{
+    data_node=$1
+    echo "> starting ${data_node}" && nohup ${GAUSSHOME}/bin/gaussdb -D ${data_node} &
+    sleep 10
+}
+
+stop_gaussdb()
+{
+    data_node=$1
+    echo "> stop ${data_node}" && ${GAUSSHOME}/bin/gs_ctl stop -D ${data_node}
+    sleep 5
+}
--- a/src/test/ss/cm_config.ini
+++ b/src/test/ss/cm_config.ini
@ -0,0 +1,2 @@
+REFORMER_ID = 0
+BITMAP_ONLINE = 3
--- a/src/test/ss/cm_ctl.sh
+++ b/src/test/ss/cm_ctl.sh
@ -0,0 +1,64 @@
+#!/bin/bash
+CMD=$(echo ${1}|tr a-z A-Z)
+KEY=$(echo ${2}|tr a-z A-Z)
+VALUE=${3}
+
+function show_help()
+{
+    echo "Usage: $0 [cmd] [key] [value]"
+    echo "cmd:"
+    echo "    set:          set [key]=[value]"
+    echo "    show:         show key"
+    echo "    help:         show help"
+    echo "key&value:"
+    echo "    REFORMER_ID   [0, 63]"
+    echo "    BITMAP_ONLINE [0, UINT64_MAX]"
+}
+
+function show_param()
+{
+    echo "The parameters are as follows:"
+    cat ${CM_CONFIG_PATH}
+}
+
+function set_param()
+{
+    if [ "$KEY" == "REFORMER_ID" ] || [ "$KEY" == "BITMAP_ONLINE" ]; then
+        PARAMS=$(cat ${CM_CONFIG_PATH}|grep -v ${KEY}|sed '/^$/d')
+        cat /dev/null > ${CM_CONFIG_PATH}
+        echo "${PARAMS}" >> ${CM_CONFIG_PATH}
+        echo "${KEY} = ${VALUE}" >> ${CM_CONFIG_PATH}
+        echo "set ${KEY} = ${VALUE} success"
+    else
+        echo "invalid parameter"
+    fi
+}
+
+function main()
+{
+    if [[ -z "${CM_CONFIG_PATH}" ]]
+    then
+        echo "CM_CONFIG_PATH is NULL"
+        exit 0
+    else
+        echo "CM_CONFIG_PATH=${CM_CONFIG_PATH}"
+    fi
+
+    if [[ ! -e "${CM_CONFIG_PATH}" ]]
+    then
+        touch ${CM_CONFIG_PATH}
+    fi
+
+    if [ "$CMD" == "SET" ];then
+        set_param
+        exit 0
+    elif [ "$CMD" == "SHOW"]; then
+        show_param
+        exit 0
+    else
+        show_help
+        exit 0
+    fi
+}
+
+main
--- a/src/test/ss/ha_test.sh
+++ b/src/test/ss/ha_test.sh
@ -0,0 +1,314 @@
+CURPATH=$(dirname $(readlink -f $0))
+echo "check env var"
+if [ ${GAUSSHOME} ] && [ -d ${GAUSSHOME}/bin ];then
+    echo "GAUSSHOME :${GAUSSHOME}"
+else
+    echo "GAUSSHOME NOT EXIST"
+    exit 1;
+fi
+
+BIN_PATH=${GAUSSHOME}/bin
+LIB_PATH=${GAUSSHOME}/lib
+
+declare -a DB_ROLE
+declare -a DB_STATUS
+declare -a DB_HOME
+
+PGPORT=(6600 6700)
+SS_DATA=${HOME}/ss_hatest
+nodedata_cfg="0:127.0.0.1:6611,1:127.0.0.1:6711"
+export CM_CONFIG_PATH=${CURPATH}/cm_config.ini
+
+function ha_test_init()
+{
+    rm -rf ${CURPATH}/results
+    mkdir -p ${CURPATH}/results
+    touch ${CURPATH}/results/run.log
+
+    echo "CURPATH:$CURPATH"
+    echo "REFORMER_ID = 0" > $CM_CONFIG_PATH
+    echo "BITMAP_ONLINE = 3" >> $CM_CONFIG_PATH
+
+    DB_HOME[0]=$SS_DATA/dn0
+    DB_HOME[1]=$SS_DATA/dn1
+    echo "DB_HOME[0]=${DB_HOME[0]}"
+    echo "DB_HOME[1]=${DB_HOME[1]}"
+}
+
+function print
+{
+    echo `date "+[%Y-%m-%d %H:%M:%S] $1."`
+    echo `date "+[%Y-%m-%d %H:%M:%S] $1."` >> ${CURPATH}/results/run.log
+}
+
+function db_role_get
+{
+    DB_ROLE[$1]=`$BIN_PATH/gsql -d postgres -p ${PGPORT[$1]} -c "select local_role from pg_stat_get_stream_replications();"| grep -E 'Primary|Standby'|sed s/[[:space:]]//g`
+}
+
+function db_role_check
+{
+    role_result="${DB_ROLE[$1]}"
+    role_expect="$2"
+
+    if [ "$role_result" != "$role_expect" ]; then
+        print "node $1 role check fail, should be $role_expect, but $role_result"
+        exit 1
+    fi
+}
+
+function function_enter()
+{
+    echo ""
+    print "===================TestCaseBegin:${FUNCNAME[1]}====================="
+    db_role_get 0
+    db_role_get 1
+    db_role_check 0 Primary
+    db_role_check 1 Standby
+}
+
+function function_leave()
+{
+    db_role_get 0
+    db_role_get 1
+    db_role_check 0 Primary
+    db_role_check 1 Standby
+    print "===================TestCaseEnd:${FUNCNAME[1]}====================="
+}
+
+function db_shutdown
+{
+    print "shutdown node$1"
+    ${BIN_PATH}/gsql -d postgres -p ${PGPORT[$1]} -c "shutdown;"
+
+    pid=`ps ux|grep gaussdb|grep ${DB_HOME[$1]}|grep -v grep`
+    while [[ ! -z $pid ]]; do
+        sleep 1
+        pid=`ps ux|grep gaussdb|grep ${DB_HOME[$1]}|grep -v grep`
+    done
+    sleep 1
+}
+
+function db_kill
+{
+    print "kill node $1"
+    ps ux | grep gaussdb | grep ${DB_HOME[$1]} | grep -v grep |awk '{print $2}' | xargs kill -9 > /dev/null 2>&1
+    pid=`ps ux | grep gaussdb | grep ${DB_HOME[$1]} | grep -v grep`
+    while [[ ! -z $pid ]]; do
+        sleep 1
+        pid=`ps ux | grep gaussdb | grep ${DB_HOME[$1]} | grep -v grep`
+    done
+    sleep 1
+}
+
+function db_start
+{
+    print "node $1 start"
+    ${BIN_PATH}/gaussdb -D ${DB_HOME[$1]} > /dev/null 2>&1 &
+}
+
+function db_start_wait
+{
+    wait_time=0
+    cur_mode=`timeout 5 $BIN_PATH/gsql -d postgres -p ${PGPORT[$1]} -c "select local_role from pg_stat_get_stream_replications();"| grep -E 'Primary|Standby'|sed s/[[:space:]]//g` >/dev/null 2>&1
+    print "wait node $1 start"
+    while [[ $cur_mode != "Primary" ]] && [[ $cur_mode != "Standby" ]]; do
+        sleep 5
+        cur_mode=`timeout 5 $BIN_PATH/gsql -d postgres -p ${PGPORT[$1]} -c "select local_role from pg_stat_get_stream_replications();"| grep -E 'Primary|Standby'|sed s/[[:space:]]//g` >/dev/null 2>&1
+        wait_time=`expr ${wait_time} + 5`
+        temp=`expr ${wait_time} % 30`
+        if [ ${temp} -eq 0 ]; then
+            print "wait node $1 start ${wait_time}s"
+        fi
+        if [ ${wait_time} -gt 600 ]; then
+            print "wait node $1 start ${wait_time}s, db start failed"
+            exit 1
+        fi
+    done
+    sleep 5
+}
+
+function switchover()
+{
+    db_start_wait 0
+    db_start_wait 1
+    $BIN_PATH/gs_ctl switchover -D ${DB_HOME[$1]}
+    sleep 5
+    db_start_wait 0
+    db_start_wait 1
+}
+
+function restart_primary
+{
+    function_enter
+    db_shutdown 0
+    db_start 0
+    db_start_wait 0
+    function_leave
+}
+
+function restart_standby
+{
+    function_enter
+    db_shutdown 1
+    db_start 1
+    db_start_wait 1
+    function_leave
+}
+
+function restart_all
+{
+    function_enter
+    db_shutdown 1
+    db_shutdown 0
+    db_start 0
+    db_start 1
+    db_start_wait 0
+    db_start_wait 1
+    function_leave
+}
+
+function delete_add_standby
+{
+    function_enter
+    db_kill 1
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 1
+    sleep 20
+    db_role_get 0
+    db_role_check 0 Primary
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 3
+    db_start 1
+    db_start_wait 1
+    function_leave
+}
+
+function delete_add_primary
+{
+    function_enter
+    db_kill 0
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 2
+    sleep 5
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 3
+    db_start 0
+    db_start_wait 0
+    function_leave
+}
+
+function failover_alive
+{
+    function_enter
+    db_kill 0
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 2
+    sh ${CURPATH}/cm_ctl.sh set REFORMER_ID 1
+    sleep 30
+    db_start_wait 1
+    db_role_get 1
+    db_role_check 1 Primary
+
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 3
+    db_start 0
+    sleep 10
+    db_start_wait 0
+    db_role_get 0
+    db_role_check 0 Standby
+    switchover 0
+    function_leave
+}
+
+function failover_restart
+{
+    function_enter
+    db_kill 0
+    db_kill 1
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 2
+    sh ${CURPATH}/cm_ctl.sh set REFORMER_ID 1
+    db_start 1
+    sleep 20
+    db_start_wait 1 
+    db_role_get 1
+    db_role_check 1 Primary
+
+    sh ${CURPATH}/cm_ctl.sh set BITMAP_ONLINE 3
+    db_start 0
+    db_start_wait 0
+    db_role_get 0
+    db_role_check 0 Standby
+    switchover 0 
+    function_leave
+}
+
+function switchover_4times
+{
+    function_enter
+    switchover 1
+    db_role_get 0
+    db_role_get 1
+    db_role_check 0 Standby
+    db_role_check 1 Primary
+
+    switchover 0
+    db_role_get 0
+    db_role_get 1
+    db_role_check 0 Primary
+    db_role_check 1 Standby
+
+    switchover 1
+    db_role_get 0
+    db_role_get 1
+    db_role_check 0 Standby
+    db_role_check 1 Primary
+
+    switchover 0
+    db_role_get 0
+    db_role_get 1
+    db_role_check 0 Primary
+    db_role_check 1 Standby
+    function_leave
+}
+
+function deploy_two_inst
+{
+    source ${CURPATH}/build_ss_database_common.sh
+    kill_gaussdb
+    clean_database_env ${SS_DATA}
+    
+    sh ${CURPATH}/conf_start_dss_inst.sh 2 ${SS_DATA} ${SS_DATA}/dss_disk 
+    init_gaussdb 0 ${SS_DATA}/dss_home0 $SS_DATA $nodedata_cfg
+    init_gaussdb 1 ${SS_DATA}/dss_home1 $SS_DATA $nodedata_cfg
+
+    set_gaussdb_port ${SS_DATA}/dn0 ${PGPORT[0]}
+    set_gaussdb_port ${SS_DATA}/dn1 ${PGPORT[1]} 
+
+    assign_hatest_parameter ${SS_DATA}/dn0 ${SS_DATA}/dn1
+
+    export DSS_HOME=${SS_DATA}/dss_home0
+    start_gaussdb ${SS_DATA}/dn0
+    export DSS_HOME=${SS_DATA}/dss_home1
+    start_gaussdb ${SS_DATA}/dn1
+}
+
+function testcase()
+{
+    restart_primary
+    restart_standby
+    restart_all
+    delete_add_standby
+    delete_add_primary
+    failover_alive
+    failover_restart
+    switchover_4times
+}
+
+function check_user()
+{
+    if [[ $USER = "root" ]]; then
+        print "current user is root, do not use root"
+        exit 1
+    fi
+}
+
+check_user
+deploy_two_inst
+ha_test_init
+testcase
+exit 0