database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](function) support variadic template type in SQL function (#17985)

Browse Source 
Inspired by c++ function `std::vector::emplace_back()`, we can use variadic template for this issue.

e.g.

```
[['struct'], 'STRUCT<TYPES>', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']]
```

`...TYPES` in template_types defines a variadic template `TYPE`.  Then the variadic template will be expanded to multiple normal templates based on actual input arguments at runtime in FE.

But make sure `TYPES...` is placed on the last position in all template type arguments.

BTW, the origin template function logic is not affected.
This commit is contained in:
xy720
2023-03-28 11:08:24 +08:00
committed by GitHub
parent d2839eb41f
commit daeaa91dd6
7 changed files with 263 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java
View File

@ -31,8 +31,12 @@ import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* Describes a STRUCT type. STRUCT types have a list of named struct fields.
@ -58,6 +62,19 @@ public class StructType extends Type {
}
}
public StructType(List<Type> types) {
Preconditions.checkNotNull(types);
ArrayList<StructField> newFields = new ArrayList<>();
for (Type type : types) {
newFields.add(new StructField(type));
}
this.fields = newFields;
}
public StructType(Type... types) {
this(Arrays.asList(types));
}
public StructType() {
this.fields = Lists.newArrayList();
}
@ -171,6 +188,91 @@ public class StructType extends Type {
return true;
}
@Override
public boolean hasTemplateType() {
for (StructField field : fields) {
if (field.type.hasTemplateType()) {
return true;
}
}
return false;
}
@Override
public Type specializeTemplateType(Type specificType, Map<String, Type> specializedTypeMap,
boolean useSpecializedType) throws TypeException {
StructType specificStructType = null;
if (specificType instanceof StructType) {
specificStructType = (StructType) specificType;
} else if (!useSpecializedType) {
throw new TypeException(specificType + " is not StructType");
}
List<Type> newTypes = Lists.newArrayList();
for (int i = 0; i < fields.size(); i++) {
if (fields.get(i).type.hasTemplateType()) {
newTypes.add(fields.get(i).type.specializeTemplateType(
specificStructType != null ? specificStructType.fields.get(i).type : specificType,
specializedTypeMap, useSpecializedType));
}
}
Type newStructType = new StructType(newTypes);
if (Type.canCastTo(specificType, newStructType)
|| (useSpecializedType && !(specificType instanceof StructType))) {
return newStructType;
} else {
throw new TypeException(specificType + " can not cast to specialize type " + newStructType);
}
}
@Override
public boolean needExpandTemplateType() {
Preconditions.checkNotNull(fields);
return fields.get(fields.size() - 1).type.needExpandTemplateType();
}
/**
* A struct variadic template is like `STRUCT<TYPES>` or `STRUCT<T, TYPES>`...
* So that we only need to expand the last field in struct variadic template.
*/
@Override
public void collectTemplateExpandSize(Type[] args, Map<String, Integer> expandSizeMap) throws TypeException {
Preconditions.checkState(needExpandTemplateType());
if (args == null || args.length == 0) {
throw new TypeException("can not expand template type in struct since input args is empty.");
}
if (!(args[0] instanceof StructType)) {
throw new TypeException(args[0] + " is not StructType");
}
StructType structType = (StructType) args[0];
if (structType.fields.size() < fields.size()) {
throw new TypeException("the field size of input struct type " + structType
+ " is less than struct template " + this);
}
Type[] types = structType.fields.subList(fields.size() - 1, structType.fields.size())
.stream().map(field -> field.type).toArray(Type[]::new);
// only the last field is expandable
fields.get(fields.size() - 1).type.collectTemplateExpandSize(types, expandSizeMap);
}
/**
* A struct variadic template is like `STRUCT<TYPES>` or `STRUCT<T, TYPES>`...
* This method is used to expand a variadic template.
* e.g. Expand `STRUCT<T, TYPES>` to `STRUCT<T, TYPES_1, TYPES_2, ..., TYPES_SIZE>`
*/
@Override
public List<Type> expandVariadicTemplateType(Map<String, Integer> expandSizeMap) {
Type type = fields.get(fields.size() - 1).type;
if (type.needExpandTemplateType()) {
List<Type> types = fields.subList(0, fields.size() - 1).stream().map(field -> field.type)
.collect(Collectors.toList());
types.addAll(type.expandVariadicTemplateType(expandSizeMap));
return Lists.newArrayList(new StructType(types));
}
return Lists.newArrayList(this);
}
@Override
public boolean equals(Object other) {
if (!(other instanceof StructType)) {

45
fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java
View File

@ -20,9 +20,12 @@ package org.apache.doris.catalog;
import org.apache.doris.thrift.TColumnType;
import org.apache.doris.thrift.TTypeDesc;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import java.util.List;
import java.util.Map;
/**
@ -34,8 +37,16 @@ public class TemplateType extends Type {
@SerializedName(value = "name")
private final String name;
public TemplateType(String name) {
@SerializedName(value = "isVariadic")
private final boolean isVariadic;
public TemplateType(String name, boolean isVariadic) {
this.name = name;
this.isVariadic = isVariadic;
}
public TemplateType(String name) {
this(name, false);
}
@Override
@ -49,7 +60,7 @@ public class TemplateType extends Type {
return false;
}
TemplateType o = (TemplateType) other;
return o.name.equals(name);
return o.name.equals(name) && o.isVariadic == isVariadic;
}
@Override
@ -63,6 +74,11 @@ public class TemplateType extends Type {
return true;
}
@Override
public boolean needExpandTemplateType() {
return isVariadic;
}
@Override
public Type specializeTemplateType(Type specificType, Map<String, Type> specializedTypeMap,
boolean useSpecializedType) throws TypeException {
@ -94,6 +110,31 @@ public class TemplateType extends Type {
return specializedTypeMap.get(name);
}
@Override
public void collectTemplateExpandSize(Type[] args, Map<String, Integer> expandSizeMap)
throws TypeException {
Preconditions.checkState(isVariadic);
expandSizeMap.computeIfAbsent(name, k -> args.length);
if (expandSizeMap.get(name) != args.length) {
throw new TypeException(
String.format("can not expand variadic template type %s to %s size since it's "
+ "already expand as %s size", name, args.length, expandSizeMap.get(name)));
}
}
@Override
public List<Type> expandVariadicTemplateType(Map<String, Integer> expandSizeMap) {
if (needExpandTemplateType() && expandSizeMap.containsKey(name)) {
List<Type> types = Lists.newArrayList();
int size = expandSizeMap.get(name);
for (int index = 0; index < size; index++) {
types.add(new TemplateType(name + "_" + index));
}
return types;
}
return Lists.newArrayList(this);
}
@Override
public String toSql(int depth) {
return name;

23
fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
View File

@ -529,7 +529,12 @@ public abstract class Type {
return false;
}
// return a new type without template type, by specialize tempalte type in this type
// only used for struct type and variadic template type
public boolean needExpandTemplateType() {
return false;
}
// return a new type without template type, by specialize template type in this type
public Type specializeTemplateType(Type specificType, Map<String, Type> specializedTypeMap,
boolean useSpecializedType) throws TypeException {
if (hasTemplateType()) {
@ -540,6 +545,22 @@ public abstract class Type {
}
}
/**
* Only used for struct type and variadic template type,
* collect variadic template's expand size based on the input arguments
*/
public void collectTemplateExpandSize(Type[] args, Map<String, Integer> expandSizeMap)
throws TypeException {
}
/**
* Only used for struct type and variadic template type,
* Do expand variadic template type
*/
public List<Type> expandVariadicTemplateType(Map<String, Integer> expandSizeMap) {
return Lists.newArrayList(this);
}
/**
* Returns true if Impala supports this type in the metdata. It does not mean we
* can manipulate data of this type. For tables that contain columns with these