database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhance](regression)docker hive s3 file address is determined based on the configuration (#25905)

Browse Source 
docker hive s3 file address is determined based on the configuration custom_settings.env
This commit is contained in:
zhangguoqiang
2023-10-26 11:58:33 +08:00
committed by GitHub
parent 9aa9894b11
commit e7a3cb079b
3 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/thirdparties/custom_settings.env
View File

@ -21,3 +21,5 @@
# eg: CONTAINER_UID="doris-jack-"
# NOTICE: change this uid will modify the file in docker-compose.
CONTAINER_UID="doris--"
s3Endpoint="cos.ap-hongkong.myqcloud.com"
s3BucketName="doris-build-hk-1308700295"

8
docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
View File

@ -25,7 +25,9 @@ sleep 10s
if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then
echo "/mnt/scripts/tpch1.db does not exist"
cd /mnt/scripts/
wget -P /mnt/scripts https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/load/tpch1_parquet/tpch1.db.tar.gz
#curl -O /mnt/scripts https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/load/tpch1_parquet/tpch1.db.tar.gz
echo "curl -O s3BucketName.s3Endpoint/regression/datalake/pipeline_data/tpch1.db.tar.gz"
curl -O https://s3BucketName.s3Endpoint/regression/datalake/pipeline_data/tpch1.db.tar.gz
tar -zxf tpch1.db.tar.gz
rm -rf tpch1.db.tar.gz
cd -
@ -45,7 +47,9 @@ hadoop fs -put /mnt/scripts/tpch1.db /user/doris/
if [[ ! -d "/mnt/scripts/paimon1" ]]; then
echo "/mnt/scripts/paimon1 does not exist"
cd /mnt/scripts/
curl -O https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/paimon/paimon1.tar.gz
#curl -O https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/paimon/paimon1.tar.gz
echo "curl -O s3BucketName.s3Endpoint/regression/datalake/pipeline_data/paimon1.tar.gz"
curl -O https://s3BucketName.s3Endpoint/regression/datalake/pipeline_data/paimon1.tar.gz
tar -zxf paimon1.tar.gz
rm -rf paimon1.tar.gz
cd -

4
docker/thirdparties/run-thirdparties-docker.sh
View File

@ -309,6 +309,8 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then
sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml
sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
sed -i "s/\${externalEnvIp}/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
sudo bash "${ROOT}"/docker-compose/hive/gen_env.sh
sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
if [[ "${STOP}" -ne 1 ]]; then
@ -334,7 +336,7 @@ if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
sudo docker compose -f "${ROOT}"/docker-compose/iceberg/iceberg.yaml --env-file "${ROOT}"/docker-compose/iceberg/iceberg.env down
sudo rm -rf "${ROOT}"/docker-compose/iceberg/data
if [[ "${STOP}" -ne 1 ]]; then
wget -P ${ROOT}/docker-compose/iceberg https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/iceberg/iceberg_data.zip
wget -P "${ROOT}"/docker-compose/iceberg https://"${s3BucketName}.${s3Endpoint}"/regression/datalake/pipeline_data/iceberg_data.zip
sudo unzip -d "${ROOT}"/docker-compose/iceberg -q ${ROOT}/docker-compose/iceberg/iceberg_data.zip
sudo mv "${ROOT}"/docker-compose/iceberg/iceberg_data "${ROOT}"/docker-compose/iceberg/data
sudo rm -rf ${ROOT}/docker-compose/iceberg/iceberg_data.zip