database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhencement](tvf) select tvf supports using resource (#35139)

Browse Source 
Create an S3/HDFS resource that TVF can use it directly to access the data source.
This commit is contained in:
Tiewei Fang
2024-05-24 12:05:52 +08:00
committed by yiguolei
parent d6e8fb7d77
commit f6beeb1ddd
4 changed files with 259 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
@ -174,9 +175,18 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
//The keys in properties map need to be lowercase.
protected Map<String, String> parseCommonProperties(Map<String, String> properties) throws AnalysisException {
Map<String, String> mergedProperties = Maps.newHashMap();
if (properties.containsKey("resource")) {
Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(properties.get("resource"));
if (resource == null) {
throw new AnalysisException("Can not find resource: " + properties.get("resource"));
}
mergedProperties = resource.getCopiedProperties();
}
mergedProperties.putAll(properties);
// Copy the properties, because we will remove the key from properties.
Map<String, String> copiedProps = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
copiedProps.putAll(properties);
copiedProps.putAll(mergedProperties);
String formatString = getOrDefaultAndRemove(copiedProps, FileFormatConstants.PROP_FORMAT, "").toLowerCase();
String defaultColumnSeparator = FileFormatConstants.DEFAULT_COLUMN_SEPARATOR;