add ha testcase for switchover timeout

src/test/ha/ha_schedule_multi_cascade

@@ -1,4 +1,5 @@
 cascade/failover
 cascade/failover_with_data
 cascade/switchover
+cascade/switchover_timeout
 cascade/inc_build_failover

src/test/ha/ha_schedule_multi_single

@@ -6,6 +6,7 @@ multi_standby_single/inc_build_reconnect
 #multi_standby_single/sync_commit
 #multi_standby_single/quorum
 multi_standby_single/switchover
+multi_standby_single/switchover_timeout
 multi_standby_single/failover
 multi_standby_single/params
 multi_standby_single/xlog_redo_apply_delay

src/test/ha/testcase/cascade/switchover_timeout.sh

@@ -0,0 +1,92 @@
+#!/bin/sh
+# switchover when primary-standby-cascade_standby all ready
+
+source ./util.sh
+
+function test_1()
+{
+  set_cascade_default
+  check_instance_cascade_standby
+  check_cascade_detailed_instance
+  #create table
+  gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1; CREATE TABLE mpp_test1(id INT,name VARCHAR(15) NOT NULL);"
+  echo "drop table success"
+
+  #prepare insert sql
+  cat $scripts_dir'/data/data5_head_100' | python tools.py mpp_test1 '|' > $scripts_dir'/data/data5_head_100_sql'
+  gsql -d $db -p $dn1_primary_port < $scripts_dir'/data/data5_head_100_sql' &> /dev/null
+
+  #test the insert results primary
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  echo "b=" $b
+  if [ $(gsql -d $db -p $dn1_primary_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "insert into table success!"
+  else
+    echo "insert into table failure $failed_keyword!"
+    exit 1
+  fi
+
+  sleep 10
+
+  #test the insert results standby
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  if [ $(gsql -d $db -p $dn1_standby_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "test insert result success!"
+  else
+    echo "test insert result, $failed_keyword!"
+    exit 1
+  fi
+
+  #test the insert results cascade standby
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  if [ $(gsql -d $db -p $standby2_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "test insert result success!"
+  else
+    echo "test insert result, $failed_keyword!"
+    exit 1
+  fi
+
+  echo "begin to switch to cascade standby"
+  #switchover
+  switchover_to_cascade_standby 1
+  echo "end of switch to cascade standby"
+
+  sleep 60
+
+  #test the insert results primary
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  if [ $(gsql -d $db -p $dn1_primary_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "test insert result success!"
+  else
+    echo "test insert result, $failed_keyword!"
+    exit 1
+  fi
+
+  #test the insert results standby
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  if [ $(gsql -d $db -p $dn1_standby_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "test insert result success!"
+  else
+    echo "test insert result, $failed_keyword!"
+    exit 1
+  fi
+
+  #test the insert results cascade standby
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  if [ $(gsql -d $db -p $standby2_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "test insert result success!"
+  else
+    echo "test insert result, $failed_keyword!"
+    exit 1
+  fi
+}
+
+function tear_down()
+{
+  set_cascade_default
+  sleep 3
+  gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1;"
+}
+
+test_1
+tear_down

src/test/ha/testcase/multi_standby_single/switchover.sh

@@ -34,21 +34,21 @@ function test_1()
   echo "begin to switch to standby1"
   #switchover
   switchover_to_standby
-  echo "end of swtich to standby1"
+  echo "end of switch to standby1"
 
   sleep 10
 
   echo "begin to switch to standby2"
   #switchover
   switchover_to_standby2
-  echo "end of swtich to standby2"
+  echo "end of switch to standby2"
 
   sleep 10
 
   echo "begin to switch to standby3"
   #switchover
   switchover_to_standby3
-  echo "end of swtich to standby3"
+  echo "end of switch to standby3"
 
   #echo "begin to switch to standby4"
   #switchover

src/test/ha/testcase/multi_standby_single/switchover_timeout.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+# switchover when primary-standby-dummystandby all ready
+
+source ./util.sh
+
+function test_1()
+{
+  set_default
+  check_detailed_instance
+  #create table
+  gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1; CREATE TABLE mpp_test1(id INT,name VARCHAR(15) NOT NULL);"
+  echo "drop table success"
+
+  #prepare insert sql
+  cat $scripts_dir'/data/data5_head_100' | python tools.py mpp_test1 '|' > $scripts_dir'/data/data5_head_100_sql'
+  gsql -d $db -p $dn1_primary_port < $scripts_dir'/data/data5_head_100_sql' &> /dev/null
+
+  #test the insert results
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  echo "b=" $b
+  if [ $(gsql -d $db -p $dn1_primary_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "insert into table success!"
+  else
+    echo "insert into table failure $failed_keyword!"
+    exit 1
+  fi
+
+  sleep 1
+
+  echo "begin to switch to standby1 with timeout 1s"
+  #switchover
+  switchover_to_standby 1
+  echo "end of switch to standby1 with timeout 1s"
+
+  sleep 60
+
+  echo "begin to switch to standby2 with timeout 2s"
+  #switchover
+  switchover_to_standby2 2
+  echo "end of switch to standby2"
+
+  sleep 60
+
+  echo "begin to switch to standby3 with timeout 3s"
+  #switchover
+  switchover_to_standby3 3
+  echo "end of switch to standby3"
+
+  sleep 60
+
+  #test the insert results
+  b=`wc $scripts_dir'/data/data5_head_100_sql' | awk '{print $1}'`
+  if [ $(gsql -d $db -p $standby3_port -c "select count(1) from mpp_test1;" | grep $b | wc -l) -eq 1 ]; then
+    echo "test insert result success!"
+  else
+    echo "test insert result, $failed_keyword!"
+    exit 1
+  fi
+}
+
+function tear_down()
+{
+  set_default
+  sleep 3
+  gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_test1;"
+}
+
+test_1
+tear_down

src/test/ha/util.sh

@@ -301,7 +301,12 @@ function set_most_available_sync_helper() {
 }
 
 function switchover_to_primary() {
-  gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode1
+  if [ $# -eq 0 ]; then
+    switchover_timeout=$gsctl_wait_time
+  else
+    switchover_timeout=$1
+  fi
+  gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode1
   if [ $? -eq 0 ]; then
     echo "switchover to primary success!"
   else
@@ -311,7 +316,12 @@ function switchover_to_primary() {
 }
 
 function switchover_to_standby() {
-  gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode1_standby
+  if [ $# -eq 0 ]; then
+    switchover_timeout=$gsctl_wait_time
+  else
+    switchover_timeout=$1
+  fi
+  gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode1_standby
   if [ $? -eq 0 ]; then
     echo "switchover to standby1 success!"
   else
@@ -321,7 +331,12 @@ function switchover_to_standby() {
 }
 
 function switchover_to_standby2() {
-  gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode2_standby
+  if [ $# -eq 0 ]; then
+    switchover_timeout=$gsctl_wait_time
+  else
+    switchover_timeout=$1
+  fi
+  gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode2_standby
   if [ $? -eq 0 ]; then
     echo "switchover to standby2 success!"
   else
@@ -331,7 +346,12 @@ function switchover_to_standby2() {
 }
 
 function switchover_to_standby3() {
-  gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode3_standby -f
+  if [ $# -eq 0 ]; then
+    switchover_timeout=$gsctl_wait_time
+  else
+    switchover_timeout=$1
+  fi
+  gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode3_standby -f
   if [ $? -eq 0 ]; then
     echo "switchover to standby3 success!"
   else
@@ -341,7 +361,12 @@ function switchover_to_standby3() {
 }
 
 function switchover_to_standby4() {
-  gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode4_standby -f
+  if [ $# -eq 0 ]; then
+    switchover_timeout=$gsctl_wait_time
+  else
+    switchover_timeout=$1
+  fi
+  gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode4_standby -f
   if [ $? -eq 0 ]; then
     echo "switchover to standby4 success!"
   else
@@ -350,7 +375,6 @@ function switchover_to_standby4() {
   fi
 }
 
-
 function failover_to_primary() {
   gs_ctl failover -w -t $gsctl_wait_time -D $data_dir/datanode1
   if [ $? -eq 0 ]; then
@@ -489,7 +513,12 @@ function failover_to_cascade_standby() {
   fi
 }
 function switchover_to_cascade_standby() {
-  gs_ctl switchover -w -t $gsctl_wait_time -D $data_dir/datanode2_standby
+  if [ $# -eq 0 ]; then
+    switchover_timeout=$gsctl_wait_time
+  else
+    switchover_timeout=$1
+  fi
+  gs_ctl switchover -w -t $switchover_timeout -D $data_dir/datanode2_standby
   if [ $? -eq 0 ]; then
     echo "switchover to cascade standby success!"
   else
@@ -497,6 +526,7 @@ function switchover_to_cascade_standby() {
     exit 1
   fi
 }
+
 function check_cascade_detailed_instance(){
   sleep 2
   #date