database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](tvf) Pass through user-defined properties (#35515) (#35747)

Browse Source 
bp #35515

Co-authored-by: Tiewei Fang <43782773+BePPPower@users.noreply.github.com>
This commit is contained in:
Mingyu Chen
2024-05-31 22:50:26 +08:00
committed by GitHub
parent 5315df36c0
commit 9468227842
3 changed files with 80 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.URI;
import org.apache.doris.thrift.TFileType;
@ -70,8 +71,10 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction {
locationProperties.put(HdfsResource.HADOOP_FS_NAME, uri.getScheme() + "://" + uri.getAuthority());
}
// 4. parse file
parseFile();
if (!FeConstants.runningUnitTest) {
// 4. parse file
parseFile();
}
}
// =========== implement abstract methods of ExternalFileTableValuedFunction =================

14
fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
View File

@ -73,9 +73,8 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
S3URI s3uri = getS3Uri(uriStr, Boolean.parseBoolean(usePathStyle.toLowerCase()),
Boolean.parseBoolean(forceParsingByStandardUri.toLowerCase()));
String endpoint = otherProps.containsKey(S3Properties.ENDPOINT) ? otherProps.get(S3Properties.ENDPOINT) :
s3uri.getEndpoint().orElseThrow(() ->
new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ENDPOINT)));
String endpoint = getOrDefaultAndRemove(otherProps, S3Properties.ENDPOINT, s3uri.getEndpoint().orElseThrow(() ->
new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ENDPOINT))));
if (!otherProps.containsKey(S3Properties.REGION)) {
String region = s3uri.getRegion().orElseThrow(() ->
new AnalysisException(String.format("Properties '%s' is required.", S3Properties.REGION)));
@ -83,16 +82,17 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
}
checkNecessaryS3Properties(otherProps);
CloudCredentialWithEndpoint credential = new CloudCredentialWithEndpoint(endpoint,
otherProps.get(S3Properties.REGION),
otherProps.get(S3Properties.ACCESS_KEY),
otherProps.get(S3Properties.SECRET_KEY));
getOrDefaultAndRemove(otherProps, S3Properties.REGION, ""),
getOrDefaultAndRemove(otherProps, S3Properties.ACCESS_KEY, ""),
getOrDefaultAndRemove(otherProps, S3Properties.SECRET_KEY, ""));
if (otherProps.containsKey(S3Properties.SESSION_TOKEN)) {
credential.setSessionToken(otherProps.get(S3Properties.SESSION_TOKEN));
credential.setSessionToken(getOrDefaultAndRemove(otherProps, S3Properties.SESSION_TOKEN, ""));
}
locationProperties = S3Properties.credentialToMap(credential);
locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle);
locationProperties.putAll(S3ClientBEProperties.getBeFSProperties(locationProperties));
locationProperties.putAll(otherProps);
filePath = NAME + S3URI.SCHEME_DELIM + s3uri.getBucket() + S3URI.PATH_DELIM + s3uri.getKey();