br: fix br integration test (#53836)

close pingcap/tidb#53835

br/pkg/utils/backoff.go

@@ -207,6 +207,11 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 			}
 		}
 	}
+	failpoint.Inject("set-import-attempt-to-one", func(_ failpoint.Value) {
+		if bo.attempt > 1 {
+			bo.attempt = 1
+		}
+	})
 	if bo.delayTime > bo.maxDelayTime {
 		return bo.maxDelayTime
 	}

br/tests/br_file_corruption/run.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Copyright 2024 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+DB="$TEST_NAME"
+TABLE="usertable"
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+run_sql "CREATE DATABASE $DB;"
+go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB"
+
+filename=$(find $TEST_DIR/$DB -regex ".*.sst" | head -n 1)
+filename_temp=$filename"_temp"
+filename_bak=$filename"_bak"
+echo "corruption" > $filename_temp
+cat $filename >> $filename_temp
+
+# file lost
+mv $filename $filename_bak
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'restore success' 
+    exit 1
+fi
+
+# file corruption
+mv $filename_temp $filename
+truncate --size=-11 $filename
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'restore success' 
+    exit 1
+fi

br/tests/br_file_corruption/workload

@@ -0,0 +1,12 @@
+recordcount=10000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+
+requestdistribution=uniform

br/tests/br_full_ddl/run.sh

@@ -23,6 +23,7 @@ RESTORE_LOG=LOG=/$TEST_DIR/restore.log
 BACKUP_STAT=/$TEST_DIR/backup_stat
 RESOTRE_STAT=/$TEST_DIR/restore_stat
 CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 
 run_sql "CREATE DATABASE $DB;"
 go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
@@ -39,6 +40,23 @@ for i in $(seq $DDL_COUNT); do
     fi
 done
 
+# wait until the index creation/drop is done
+retry_cnt=0
+while true; do
+    run_sql "ADMIN SHOW DDL JOBS WHERE DB_NAME = '$DB' AND TABLE_NAME = '$TABLE' AND STATE != 'synced';"
+    if grep -Fq "1. row" $res_file; then
+        cat $res_file
+        retry_cnt=$((retry_cnt+1))
+        if [ "$retry_cnt" -gt 50 ]; then
+            echo 'the wait lag is too large'
+            exit 1
+        fi
+        continue
+    fi
+
+    break
+done
+
 # run analyze to generate stats
 run_sql "analyze table $DB.$TABLE;"
 # record the stats and remove last_update_version

br/tests/br_pitr/run.sh

@@ -166,3 +166,37 @@ if [ $restore_fail -ne 1 ]; then
     echo 'pitr success' 
     exit 1
 fi
+
+# start a new cluster for corruption
+echo "restart a services"
+restart_services
+
+echo "corrupt a log file"
+filename=$(find $TEST_DIR/$PREFIX/log -regex ".*\.log" | grep -v "schema-meta" | tail -n 1)
+filename_temp=$filename"_temp"
+filename_bak=$filename"_bak"
+echo "corruption" > $filename_temp
+cat $filename >> $filename_temp
+
+# file lost
+mv $filename $filename_bak
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi
+
+# file corruption
+mv $filename_temp $filename
+truncate --size=-11 $filename
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi

br/tests/br_txn/run.sh

@@ -97,12 +97,22 @@ run_test() {
     # delete data in range[start-key, end-key)
     clean "hello" "world" 
     # Ensure the data is deleted
-    checksum_new=$(checksum "hello" "world")
+    retry_cnt=0
+    while true; do
+        checksum_new=$(checksum "hello" "world")
 
-    if [ "$checksum_new" != "$checksum_empty" ];then
-        echo "failed to delete data in range after backup"
-        fail_and_exit
-    fi
+        if [ "$checksum_new" != "$checksum_empty" ]; then
+            echo "failed to delete data in range after backup; retry_cnt = $retry_cnt"
+            retry_cnt=$((retry_cnt+1))
+            if [ "$retry_cnt" -gt 50 ]; then
+                fail_and_exit
+            fi
+            sleep 1
+            continue
+        fi
+
+        break
+    done
 
     # restore rawkv
     echo "restore start..."

br/tests/run_group_br_tests.sh

@@ -28,7 +28,7 @@ groups=(
 	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
 	["G06"]='br_tikv_outage br_tikv_outage3'
 	["G07"]='br_pitr'
-	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption'
 )
 
 # Get other cases not in groups, to avoid missing any case