From 58e88e75285f5dfd2f313cfa37b7479645f84acd Mon Sep 17 00:00:00 2001
From: Chunzhu Li <lichunzhu@stu.xjtu.edu.cn>
Date: Mon, 11 Jul 2022 14:55:04 +0800
Subject: [PATCH] dumpling: fix dumpling s3 integration tests (#36054)

close pingcap/tidb#36080
---
 Makefile                 |  1 +
 dumpling/install.sh      |  3 +++
 dumpling/tests/s3/run.sh | 48 ++++++++++++++++++++++++++--------------
 3 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index a4fd0c6394..c670d81f0b 100644
--- a/Makefile
+++ b/Makefile
@@ -412,6 +412,7 @@ dumpling_integration_test: dumpling_bins failpoint-enable build_dumpling
 dumpling_bins:
 	@which bin/tidb-server
 	@which bin/minio
+	@which bin/mc
 	@which bin/tidb-lightning
 	@which bin/sync_diff_inspector
 
diff --git a/dumpling/install.sh b/dumpling/install.sh
index e51ee861b0..65fd56e454 100644
--- a/dumpling/install.sh
+++ b/dumpling/install.sh
@@ -27,3 +27,6 @@ mv tidb/bin/tidb-server bin/
 # download minio
 wget https://dl.min.io/server/minio/release/linux-amd64/minio -O bin/minio
 chmod a+x bin/minio
+
+wget https://dl.minio.io/client/mc/release/linux-amd64/mc -O bin/mc
+chmod a+x bin/mc
diff --git a/dumpling/tests/s3/run.sh b/dumpling/tests/s3/run.sh
index a5011e4707..ede8c31610 100755
--- a/dumpling/tests/s3/run.sh
+++ b/dumpling/tests/s3/run.sh
@@ -47,38 +47,54 @@ run_sql "create database $DB_NAME DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
 (cd "$(dirname "$0")" && GO111MODULE=on go build -o out)
 $DUMPLING_BASE_NAME/out -B $DB_NAME -T $TABLE_NAME -P 3306 -w 16
 
-# run dumpling!
 HOST_DIR=${DUMPLING_OUTPUT_DIR}
+
+# run dumpling local
+export DUMPLING_OUTPUT_DIR=${HOST_DIR}/local
+run_dumpling
+
+# run dumpling s3
 export DUMPLING_OUTPUT_DIR=s3://mybucket/dump
 export DUMPLING_TEST_DATABASE=$DB_NAME
 export AWS_REGION=us-east-1
 export AWS_ACCESS_KEY_ID="$MINIO_ACCESS_KEY"
 export AWS_SECRET_ACCESS_KEY="$MINIO_SECRET_KEY"
 run_dumpling --s3.endpoint="http://$S3_ENDPOINT/"
-ls "${HOST_DIR}"
+ls "${HOST_DIR}" -R
 
-file_should_exist "$DBPATH/mybucket/dump/s3-schema-create.sql"
-file_should_exist "$DBPATH/mybucket/dump/s3.t-schema.sql"
-file_should_exist "$DBPATH/mybucket/dump/s3.t.000000000.sql"
+mkdir -p "${HOST_DIR}/remote"
 
-cnt=`grep -o "('aaaaaaaaaa')" $DBPATH/mybucket/dump/s3.t.000000000.sql|wc -l`
+bin/mc config host add minio http://127.0.0.1:5000 testid testkey8
+bin/mc cp minio/mybucket/dump/s3-schema-create.sql "${HOST_DIR}/remote/s3-schema-create.sql"
+bin/mc cp minio/mybucket/dump/s3.t-schema.sql "${HOST_DIR}/remote/s3.t-schema.sql"
+bin/mc cp minio/mybucket/dump/s3.t.000000000.sql "${HOST_DIR}/remote/s3.t.000000000.sql"
+
+diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/remote/s3-schema-create.sql"
+diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/remote/s3.t-schema.sql"
+diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/remote/s3.t.000000000.sql"
+
+cnt=`grep -o "('aaaaaaaaaa')" ${HOST_DIR}/remote/s3.t.000000000.sql|wc -l`
 echo "1st records count is ${cnt}"
 [ $cnt = 5000000 ]
 
 # run dumpling with compress option
-mv "$DBPATH/mybucket/dump" "$DBPATH/mybucket/expect"
+export DUMPLING_OUTPUT_DIR=s3://mybucket/dump-compress
+
 run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "gzip"
-file_should_exist "$DBPATH/mybucket/dump/s3-schema-create.sql.gz"
-file_should_exist "$DBPATH/mybucket/dump/s3.t-schema.sql.gz"
-file_should_exist "$DBPATH/mybucket/dump/s3.t.000000000.sql.gz"
 
-gzip "$DBPATH/mybucket/dump/s3-schema-create.sql.gz" -d
-diff "$DBPATH/mybucket/expect/s3-schema-create.sql" "$DBPATH/mybucket/dump/s3-schema-create.sql"
+mkdir -p "${HOST_DIR}/compress"
 
-gzip "$DBPATH/mybucket/dump/s3.t-schema.sql.gz" -d
-diff "$DBPATH/mybucket/expect/s3.t-schema.sql" "$DBPATH/mybucket/dump/s3.t-schema.sql"
+bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.gz "${HOST_DIR}/compress/s3-schema-create.sql.gz"
+bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.gz "${HOST_DIR}/compress/s3.t-schema.sql.gz"
+bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.gz "${HOST_DIR}/compress/s3.t.000000000.sql.gz"
 
-gzip "$DBPATH/mybucket/dump/s3.t.000000000.sql.gz" -d
-diff "$DBPATH/mybucket/expect/s3.t.000000000.sql" "$DBPATH/mybucket/dump/s3.t.000000000.sql"
+gzip "${HOST_DIR}/compress/s3-schema-create.sql.gz" -d
+diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
+
+gzip "${HOST_DIR}/compress/s3.t-schema.sql.gz" -d
+diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
+
+gzip "${HOST_DIR}/compress/s3.t.000000000.sql.gz" -d
+diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
 
 run_sql "drop database if exists \`$DB_NAME\`;"