database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dumpling: fix dumpling s3 integration tests (#36054)

Browse Source 
close pingcap/tidb#36080
This commit is contained in:
Chunzhu Li
2022-07-11 14:55:04 +08:00
committed by GitHub
parent c815e5607a
commit 58e88e7528
3 changed files with 36 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -412,6 +412,7 @@ dumpling_integration_test: dumpling_bins failpoint-enable build_dumpling
dumpling_bins:
@which bin/tidb-server
@which bin/minio
@which bin/mc
@which bin/tidb-lightning
@which bin/sync_diff_inspector

3
dumpling/install.sh
View File

@ -27,3 +27,6 @@ mv tidb/bin/tidb-server bin/
# download minio
wget https://dl.min.io/server/minio/release/linux-amd64/minio -O bin/minio
chmod a+x bin/minio
wget https://dl.minio.io/client/mc/release/linux-amd64/mc -O bin/mc
chmod a+x bin/mc

48
dumpling/tests/s3/run.sh
View File

@ -47,38 +47,54 @@ run_sql "create database $DB_NAME DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
(cd "$(dirname "$0")" && GO111MODULE=on go build -o out)
$DUMPLING_BASE_NAME/out -B $DB_NAME -T $TABLE_NAME -P 3306 -w 16
# run dumpling!
HOST_DIR=${DUMPLING_OUTPUT_DIR}
# run dumpling local
export DUMPLING_OUTPUT_DIR=${HOST_DIR}/local
run_dumpling
# run dumpling s3
export DUMPLING_OUTPUT_DIR=s3://mybucket/dump
export DUMPLING_TEST_DATABASE=$DB_NAME
export AWS_REGION=us-east-1
export AWS_ACCESS_KEY_ID="$MINIO_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$MINIO_SECRET_KEY"
run_dumpling --s3.endpoint="http://$S3_ENDPOINT/"
ls "${HOST_DIR}"
ls "${HOST_DIR}" -R
file_should_exist "$DBPATH/mybucket/dump/s3-schema-create.sql"
file_should_exist "$DBPATH/mybucket/dump/s3.t-schema.sql"
file_should_exist "$DBPATH/mybucket/dump/s3.t.000000000.sql"
mkdir -p "${HOST_DIR}/remote"
cnt=`grep -o "('aaaaaaaaaa')" $DBPATH/mybucket/dump/s3.t.000000000.sql|wc -l`
bin/mc config host add minio http://127.0.0.1:5000 testid testkey8
bin/mc cp minio/mybucket/dump/s3-schema-create.sql "${HOST_DIR}/remote/s3-schema-create.sql"
bin/mc cp minio/mybucket/dump/s3.t-schema.sql "${HOST_DIR}/remote/s3.t-schema.sql"
bin/mc cp minio/mybucket/dump/s3.t.000000000.sql "${HOST_DIR}/remote/s3.t.000000000.sql"
diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/remote/s3-schema-create.sql"
diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/remote/s3.t-schema.sql"
diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/remote/s3.t.000000000.sql"
cnt=`grep -o "('aaaaaaaaaa')" ${HOST_DIR}/remote/s3.t.000000000.sql|wc -l`
echo "1st records count is ${cnt}"
[ $cnt = 5000000 ]
# run dumpling with compress option
mv "$DBPATH/mybucket/dump" "$DBPATH/mybucket/expect"
export DUMPLING_OUTPUT_DIR=s3://mybucket/dump-compress
run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "gzip"
file_should_exist "$DBPATH/mybucket/dump/s3-schema-create.sql.gz"
file_should_exist "$DBPATH/mybucket/dump/s3.t-schema.sql.gz"
file_should_exist "$DBPATH/mybucket/dump/s3.t.000000000.sql.gz"
gzip "$DBPATH/mybucket/dump/s3-schema-create.sql.gz" -d
diff "$DBPATH/mybucket/expect/s3-schema-create.sql" "$DBPATH/mybucket/dump/s3-schema-create.sql"
mkdir -p "${HOST_DIR}/compress"
gzip "$DBPATH/mybucket/dump/s3.t-schema.sql.gz" -d
diff "$DBPATH/mybucket/expect/s3.t-schema.sql" "$DBPATH/mybucket/dump/s3.t-schema.sql"
bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.gz "${HOST_DIR}/compress/s3-schema-create.sql.gz"
bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.gz "${HOST_DIR}/compress/s3.t-schema.sql.gz"
bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.gz "${HOST_DIR}/compress/s3.t.000000000.sql.gz"
gzip "$DBPATH/mybucket/dump/s3.t.000000000.sql.gz" -d
diff "$DBPATH/mybucket/expect/s3.t.000000000.sql" "$DBPATH/mybucket/dump/s3.t.000000000.sql"
gzip "${HOST_DIR}/compress/s3-schema-create.sql.gz" -d
diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
gzip "${HOST_DIR}/compress/s3.t-schema.sql.gz" -d
diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
gzip "${HOST_DIR}/compress/s3.t.000000000.sql.gz" -d
diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
run_sql "drop database if exists \`$DB_NAME\`;"