From 94c23d74caade7060edab66060b3e1faac961cd4 Mon Sep 17 00:00:00 2001 From: maxz Date: Wed, 31 May 2023 17:10:58 +0800 Subject: [PATCH] add ha testcase for switchover timeout --- src/test/ha/ha_schedule_multi_cascade | 1 + src/test/ha/ha_schedule_multi_single | 1 + .../ha/testcase/cascade/switchover_timeout.sh | 92 +++++++++++++++++++ .../multi_standby_single/switchover.sh | 6 +- .../switchover_timeout.sh | 69 ++++++++++++++ src/test/ha/util.sh | 44 +++++++-- 6 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 src/test/ha/testcase/cascade/switchover_timeout.sh create mode 100644 src/test/ha/testcase/multi_standby_single/switchover_timeout.sh diff --git a/src/test/ha/ha_schedule_multi_cascade b/src/test/ha/ha_schedule_multi_cascade index 9f9d3ee1f..3c890c358 100644 --- a/src/test/ha/ha_schedule_multi_cascade +++ b/src/test/ha/ha_schedule_multi_cascade @@ -1,4 +1,5 @@ cascade/failover cascade/failover_with_data cascade/switchover +cascade/switchover_timeout cascade/inc_build_failover diff --git a/src/test/ha/ha_schedule_multi_single b/src/test/ha/ha_schedule_multi_single index 611b83687..198ffd917 100644 --- a/src/test/ha/ha_schedule_multi_single +++ b/src/test/ha/ha_schedule_multi_single @@ -6,6 +6,7 @@ multi_standby_single/inc_build_reconnect #multi_standby_single/sync_commit #multi_standby_single/quorum multi_standby_single/switchover +multi_standby_single/switchover_timeout multi_standby_single/failover multi_standby_single/params multi_standby_single/xlog_redo_apply_delay diff --git a/src/test/ha/testcase/cascade/switchover_timeout.sh b/src/test/ha/testcase/cascade/switchover_timeout.sh new file mode 100644 index 000000000..71c7a427f --- /dev/null +++ b/src/test/ha/testcase/cascade/switchover_timeout.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# switchover when primary-standby-cascade_standby all ready + +source ./util.sh + +function test_1() +{ + set_cascade_default + check_instance_cascade_standby + check_cascade_detailed_instance + #create table + gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1; CREATE TABLE mpp_test1(id INT,name VARCHAR(15) NOT NULL);" + echo "drop table success" + + #prepare insert sql + cat $scripts_dir'/data/data5_head_100' | python tools.py mpp_test1 '|' > $scripts_dir'/data/data5_head_100_sql' + gsql -d $db -p $dn1_primary_port < $scripts_dir'/data/data5_head_100_sql' &> /dev/null + + #test the insert results primary + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + echo "b=" $b + if [ $(gsql -d $db -p $dn1_primary_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "insert into table success!" + else + echo "insert into table failure $failed_keyword!" + exit 1 + fi + + sleep 10 + + #test the insert results standby + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + if [ $(gsql -d $db -p $dn1_standby_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "test insert result success!" + else + echo "test insert result, $failed_keyword!" + exit 1 + fi + + #test the insert results cascade standby + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + if [ $(gsql -d $db -p $standby2_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "test insert result success!" + else + echo "test insert result, $failed_keyword!" + exit 1 + fi + + echo "begin to switch to cascade standby" + #switchover + switchover_to_cascade_standby 1 + echo "end of switch to cascade standby" + + sleep 60 + + #test the insert results primary + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + if [ $(gsql -d $db -p $dn1_primary_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "test insert result success!" + else + echo "test insert result, $failed_keyword!" + exit 1 + fi + + #test the insert results standby + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + if [ $(gsql -d $db -p $dn1_standby_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "test insert result success!" + else + echo "test insert result, $failed_keyword!" + exit 1 + fi + + #test the insert results cascade standby + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + if [ $(gsql -d $db -p $standby2_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "test insert result success!" + else + echo "test insert result, $failed_keyword!" + exit 1 + fi +} + +function tear_down() +{ + set_cascade_default + sleep 3 + gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1;" +} + +test_1 +tear_down diff --git a/src/test/ha/testcase/multi_standby_single/switchover.sh b/src/test/ha/testcase/multi_standby_single/switchover.sh index a70046aae..39f559e4c 100644 --- a/src/test/ha/testcase/multi_standby_single/switchover.sh +++ b/src/test/ha/testcase/multi_standby_single/switchover.sh @@ -34,21 +34,21 @@ function test_1() echo "begin to switch to standby1" #switchover switchover_to_standby - echo "end of swtich to standby1" + echo "end of switch to standby1" sleep 10 echo "begin to switch to standby2" #switchover switchover_to_standby2 - echo "end of swtich to standby2" + echo "end of switch to standby2" sleep 10 echo "begin to switch to standby3" #switchover switchover_to_standby3 - echo "end of swtich to standby3" + echo "end of switch to standby3" #echo "begin to switch to standby4" #switchover diff --git a/src/test/ha/testcase/multi_standby_single/switchover_timeout.sh b/src/test/ha/testcase/multi_standby_single/switchover_timeout.sh new file mode 100644 index 000000000..43cc35336 --- /dev/null +++ b/src/test/ha/testcase/multi_standby_single/switchover_timeout.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# switchover when primary-standby-dummystandby all ready + +source ./util.sh + +function test_1() +{ + set_default + check_detailed_instance + #create table + gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1; CREATE TABLE mpp_test1(id INT,name VARCHAR(15) NOT NULL);" + echo "drop table success" + + #prepare insert sql + cat $scripts_dir'/data/data5_head_100' | python tools.py mpp_test1 '|' > $scripts_dir'/data/data5_head_100_sql' + gsql -d $db -p $dn1_primary_port < $scripts_dir'/data/data5_head_100_sql' &> /dev/null + + #test the insert results + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + echo "b=" $b + if [ $(gsql -d $db -p $dn1_primary_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "insert into table success!" + else + echo "insert into table failure $failed_keyword!" + exit 1 + fi + + sleep 1 + + echo "begin to switch to standby1 with timeout 1s" + #switchover + switchover_to_standby 1 + echo "end of switch to standby1 with timeout 1s" + + sleep 60 + + echo "begin to switch to standby2 with timeout 2s" + #switchover + switchover_to_standby2 2 + echo "end of switch to standby2" + + sleep 60 + + echo "begin to switch to standby3 with timeout 3s" + #switchover + switchover_to_standby3 3 + echo "end of switch to standby3" + + sleep 60 + + #test the insert results + b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'` + if [ $(gsql -d $db -p $standby3_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then + echo "test insert result success!" + else + echo "test insert result, $failed_keyword!" + exit 1 + fi +} + +function tear_down() +{ + set_default + sleep 3 + gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1;" +} + +test_1 +tear_down diff --git a/src/test/ha/util.sh b/src/test/ha/util.sh index 5911240f2..3f832dbd7 100644 --- a/src/test/ha/util.sh +++ b/src/test/ha/util.sh @@ -301,7 +301,12 @@ function set_most_available_sync_helper() { } function switchover_to_primary() { - gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode1 + if [ $# -eq 0 ]; then + switchover_timeout=$gsctl_wait_time + else + switchover_timeout=$1 + fi + gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode1 if [ $? -eq 0 ]; then echo "switchover to primary success!" else @@ -311,7 +316,12 @@ function switchover_to_primary() { } function switchover_to_standby() { - gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode1_standby + if [ $# -eq 0 ]; then + switchover_timeout=$gsctl_wait_time + else + switchover_timeout=$1 + fi + gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode1_standby if [ $? -eq 0 ]; then echo "switchover to standby1 success!" else @@ -321,7 +331,12 @@ function switchover_to_standby() { } function switchover_to_standby2() { - gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode2_standby + if [ $# -eq 0 ]; then + switchover_timeout=$gsctl_wait_time + else + switchover_timeout=$1 + fi + gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode2_standby if [ $? -eq 0 ]; then echo "switchover to standby2 success!" else @@ -331,7 +346,12 @@ function switchover_to_standby2() { } function switchover_to_standby3() { - gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode3_standby -f + if [ $# -eq 0 ]; then + switchover_timeout=$gsctl_wait_time + else + switchover_timeout=$1 + fi + gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode3_standby -f if [ $? -eq 0 ]; then echo "switchover to standby3 success!" else @@ -341,7 +361,12 @@ function switchover_to_standby3() { } function switchover_to_standby4() { - gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode4_standby -f + if [ $# -eq 0 ]; then + switchover_timeout=$gsctl_wait_time + else + switchover_timeout=$1 + fi + gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode4_standby -f if [ $? -eq 0 ]; then echo "switchover to standby4 success!" else @@ -350,7 +375,6 @@ function switchover_to_standby4() { fi } - function failover_to_primary() { gs_ctl failover -w -t $gsctl_wait_time -D $data_dir/datanode1 if [ $? -eq 0 ]; then @@ -489,7 +513,12 @@ function failover_to_cascade_standby() { fi } function switchover_to_cascade_standby() { - gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode2_standby + if [ $# -eq 0 ]; then + switchover_timeout=$gsctl_wait_time + else + switchover_timeout=$1 + fi + gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode2_standby if [ $? -eq 0 ]; then echo "switchover to cascade standby success!" else @@ -497,6 +526,7 @@ function switchover_to_cascade_standby() { exit 1 fi } + function check_cascade_detailed_instance(){ sleep 2 #date