database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](iceberg)add prefix for endpoint with s3 client for 2.1 (#41336) (#41877)

Browse Source 
bp: #41336
This commit is contained in:
wuwenchi
2024-10-15 19:59:10 +08:00
committed by GitHub
parent b91d8e2327
commit a4b7d93ded
3 changed files with 59 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
View File

@ -67,6 +67,9 @@ public class DLFCatalog extends HiveCompatibleCatalog {
.equalsIgnoreCase("true");
// s3 file io just supports s3-like endpoint
String s3Endpoint = endpoint.replace(region, "s3." + region);
if (!s3Endpoint.contains("://")) {
s3Endpoint = "http://" + s3Endpoint;
}
URI endpointUri = URI.create(s3Endpoint);
FileIO io = new S3FileIO(() -> S3Util.buildS3Client(endpointUri, region, credential, isUsePathStyle));
io.initialize(properties);

5
regression-test/data/external_table_p2/iceberg/test_iceberg_dlf_catalog.out Normal file
View File

@ -0,0 +1,5 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !c1 --
1 a
2 b

51
regression-test/suites/external_table_p2/iceberg/test_iceberg_dlf_catalog.groovy Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_iceberg_dlf_catalog", "p2,external,iceberg,external_remote,external_remote_iceberg") {
String enabled = context.config.otherConfigs.get("enableIcebergTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
return
}
String catalog = "test_iceberg_dlf_catalog"
String uid = context.config.otherConfigs.get("dlf_uid")
String region = context.config.otherConfigs.get("dlf_region")
String catalog_id = context.config.otherConfigs.get("dlf_catalog_id")
String access_key = context.config.otherConfigs.get("dlf_access_key")
String secret_key = context.config.otherConfigs.get("dlf_secret_key")
sql """drop catalog if exists ${catalog};"""
sql """
create catalog if not exists ${catalog} properties (
"type" = "iceberg",
"iceberg.catalog.type" = "dlf",
"warehouse" = "oss://selectdb-qa-datalake-test/p2_regression_case",
"dlf.proxy.mode" = "DLF_ONLY",
"dlf.uid" = "${uid}",
"dlf.region" = "${region}",
"dlf.catalog.id" = "${catalog_id}",
"dlf.access_key" = "${access_key}",
"dlf.secret_key" = "${secret_key}"
);
"""
sql """ use ${catalog}.regression_iceberg """
qt_c1 """ select * from tb_simple order by id """
}