database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[FIX](type) fix matchExactType for complex type (#28233)

Browse Source 
fe matchExactType function should call type.matchTypes for its own logic, do not switch case to do special logic otherwise we may meet core in be like this.
 ```
F20231208 18:54:39.359673 680131 block.h:535] Check failed: _data_types[i]->is_nullable()  target type: Struct(l_info:Nullable(Array(Nullable(String)))) src type: Struct(col:Nullable(Array(Nullable(UInt8))))
*** Check failure stack trace: ***
    @     0x5584e952b926  google::LogMessage::SendToLog()
    @     0x5584e9527ef0  google::LogMessage::Flush()
    @     0x5584e952c169  google::LogMessageFatal::~LogMessageFatal()
    @     0x5584cf17201e  doris::vectorized::MutableBlock::merge_impl<>()
    @     0x5584ceac4b1d  doris::vectorized::MutableBlock::merge<>()
    @     0x5584d4dd7de3  doris::vectorized::VUnionNode::get_next_const()
    @     0x5584d4dd9a45  doris::vectorized::VUnionNode::get_next()
    @     0x5584bce469bd  std::__invoke_impl<>()
    @     0x5584bce466d0  std::__invoke<>()
    @     0x5584bce465c7  _ZNSt5_BindIFMN5doris8ExecNodeEFNS0_6StatusEPNS0_12RuntimeStateEPNS0_10vectorized5BlockEPbEPS1_St12_PlaceholderILi1EESC_ILi2EESC_ILi3EEEE6__callIS2_JOS4_OS7_OS8_EJLm0ELm1ELm2ELm3EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
    @     0x5584bce46358  std::_Bind<>::operator()<>()
    @     0x5584bce46208  std::__invoke_impl<>()
    @     0x5584bce46178  _ZSt10__invoke_rIN5doris6StatusERSt5_BindIFMNS0_8ExecNodeEFS1_PNS0_12RuntimeStateEPNS0_10vectorized5BlockEPbEPS3_St12_PlaceholderILi1EESD_ILi2EESD_ILi3EEEEJS5_S8_S9_EENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESL_E4typeEOSM_DpOSN_
    @     0x5584bce45c18  std::_Function_handler<>::_M_invoke()
    @     0x5584bce6412f  std::function<>::operator()()
    @     0x5584bce56382  doris::ExecNode::get_next_after_projects()
    @     0x5584bce26218  doris::PlanFragmentExecutor::get_vectorized_internal()
    @     0x5584bce2431b  doris::PlanFragmentExecutor::open_vectorized_internal()
    @     0x5584bce22a96  doris::PlanFragmentExecutor::open()
    @     0x5584bce27c9d  doris::PlanFragmentExecutor::execute()
    @     0x5584bcbdb3f8  doris::FragmentMgr::_exec_actual()
    @     0x5584bcbf982f  doris::FragmentMgr::exec_plan_fragment()::$_0::operator()()
    @     0x5584bcbf9715  std::__invoke_impl<>()
    @     0x5584bcbf96b5  _ZSt10__invoke_rIvRZN5doris11FragmentMgr18exec_plan_fragmentERKNS0_23TExecPlanFragmentParamsERKSt8functionIFvPNS0_12RuntimeStateEPNS0_6StatusEEEE3$_0JEENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESH_E4typeEOSI_DpOSJ_
    @     0x5584bcbf942d  std::_Function_handler<>::_M_invoke()
    @     0x5584b9dfd883  std::function<>::operator()()
    @     0x5584bd6e3929  doris::FunctionRunnable::run()
    @     0x5584bd6cf8ce  doris::ThreadPool::dispatch_thread()
```
This commit is contained in:
amory
2023-12-21 11:49:05 +08:00
committed by GitHub
parent 12089cda08
commit e9848066c9
12 changed files with 204 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -1875,6 +1875,14 @@ public class FunctionCallExpr extends Expr {
&& fnName.getFunction().equalsIgnoreCase("map")) {
ix = i % 2 == 0 ? 0 : 1;
}
// array_zip varargs special case array_zip(array1, array2, ...)
// we only specialize array_zip with first array type, next type we same with custom type
if (i >= args.length && (fnName.getFunction().equalsIgnoreCase("array_zip"))) {
if (argTypes[i].isNull()) {
uncheckedCastChild(args[i - 1], i);
}
continue;
}
if (i == 0 && (fnName.getFunction().equalsIgnoreCase("char"))) {
continue;

17
fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
View File

@ -47,17 +47,14 @@ public class IsNullPredicate extends Predicate {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(t), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(complexType), Type.BOOLEAN,
NullableMode.ALWAYS_NOT_NULLABLE));
}
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
}

7
fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
View File

@ -23,6 +23,8 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
@ -106,6 +108,11 @@ public class StructLiteral extends LiteralExpr {
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.STRUCT_LITERAL;
((StructType) type).getFields().forEach(v -> msg.setChildType(v.getType().getPrimitiveType().toThrift()));
TTypeDesc container = new TTypeDesc();
container.setTypes(new ArrayList<TTypeNode>());
type.toThrift(container);
msg.setType(container);
}
@Override

42
fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View File

@ -386,16 +386,31 @@ public class FunctionSet<T> {
throw new TypeException(templateFunction
+ " is not support for template since it's not a ScalarFunction");
}
Type[] args = specializedFunction.getArgs();
ArrayList<Type> args = new ArrayList<>();
Collections.addAll(args, specializedFunction.getArgs());
Map<String, Type> specializedTypeMap = Maps.newHashMap();
boolean enableDecimal256 = SessionVariable.getEnableDecimal256();
for (int i = 0; i < args.length; i++) {
if (args[i].hasTemplateType()) {
int i = 0;
for (; i < args.size(); i++) {
if (args.get(i).hasTemplateType()) {
hasTemplateType = true;
args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false,
enableDecimal256);
// if args[i] is template type, and requestFunction.getArgs()[i] NULL_TYPE, we need call function
// deduce to get the specific type
Type deduceType = requestFunction.getArgs()[i];
if (requestFunction.getArgs()[i].isNull()
|| (requestFunction.getArgs()[i] instanceof ArrayType
&& ((ArrayType) requestFunction.getArgs()[i]).getItemType().isNull())
&& FunctionTypeDeducers.DEDUCERS.containsKey(specializedFunction.functionName())) {
deduceType = FunctionTypeDeducers.deduce(specializedFunction.functionName(), i, requestFunction.getArgs());
args.set(i, args.get(i).specializeTemplateType(deduceType == null ? requestFunction.getArgs()[i]
: deduceType, specializedTypeMap, false, enableDecimal256));
} else {
args.set(i, args.get(i).specializeTemplateType(requestFunction.getArgs()[i],
specializedTypeMap, false, enableDecimal256));
}
}
}
specializedFunction.setArgs(args);
if (specializedFunction.getReturnType().hasTemplateType()) {
hasTemplateType = true;
specializedFunction.setReturnType(
@ -426,7 +441,7 @@ public class FunctionSet<T> {
newTypes[i] = inputType;
}
}
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), newTypes);
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), 0, newTypes);
if (newRetType != null && inferenceFunction instanceof ScalarFunction) {
ScalarFunction f = (ScalarFunction) inferenceFunction;
return new ScalarFunction(f.getFunctionName(), Lists.newArrayList(newTypes), newRetType, f.hasVarArgs(),
@ -448,7 +463,20 @@ public class FunctionSet<T> {
final Type[] candicateArgTypes = candicate.getArgs();
if (!(descArgTypes[0] instanceof ScalarType)
|| !(candicateArgTypes[0] instanceof ScalarType)) {
if (candicateArgTypes[0] instanceof ArrayType || candicateArgTypes[0] instanceof MapType) {
if (candicateArgTypes[0] instanceof ArrayType) {
// match is exactly type. but for null type , with in array|map elem can not return true, because for
// be will make null_type to uint8
// so here meet null_type just make true as allowed, descArgTypes[0]).getItemType().isNull() is for
// empty literal like: []|{}
if (descArgTypes[0] instanceof ArrayType && ((ArrayType) descArgTypes[0]).getItemType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
} else if (candicateArgTypes[0] instanceof MapType) {
if (descArgTypes[0] instanceof MapType && ((MapType) descArgTypes[0]).getKeyType().isNull()
&& ((MapType) descArgTypes[0]).getValueType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
}
return false;

34
fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java
View File

@ -23,26 +23,48 @@ import com.google.common.collect.Lists;
import java.util.List;
public class FunctionTypeDeducers {
public interface TypeDeducer {
public Type deduce(Type[] args);
public Type deduce(int argIdx, Type[] args);
}
public static final ImmutableMap<String, TypeDeducer> DEDUCERS = ImmutableMap.<String, TypeDeducer>builder()
.put("named_struct", new NamedStructDeducer())
.put("struct_element", new StructElementDeducer())
.put("array_contains", new ArrayElemFuncDeducer())
.put("array_pushback", new ArrayElemFuncDeducer())
.put("element_at", new ArrayElemFuncDeducer())
.build();
public static Type deduce(String fnName, Type[] args) {
public static Type deduce(String fnName, int argIdx, Type[] args) {
if (DEDUCERS.containsKey(fnName)) {
return DEDUCERS.get(fnName).deduce(args);
return DEDUCERS.get(fnName).deduce(argIdx, args);
}
return null;
}
public static class ArrayElemFuncDeducer implements TypeDeducer {
@Override
public Type deduce(int argIdx, Type[] args) {
if (args.length >= 2) {
if (argIdx == 0) {
// first args should only to be array or null
return args[0] instanceof ArrayType || args[0].isNull() ? new ArrayType(args[1]) : args[0];
} else if (args[0].isNull()) {
// first arg is null, later element is not contains
return args[argIdx];
} else if (Type.isImplicitlyCastable(args[argIdx], ((ArrayType) args[0]).getItemType(), false, true)) {
return args[argIdx];
} else {
return null;
}
}
return null;
}
}
public static class NamedStructDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
List<Type> evenArgs = Lists.newArrayList();
for (int i = 0; i < args.length; i++) {
if ((i & 1) == 1) {
@ -55,7 +77,7 @@ public class FunctionTypeDeducers {
public static class StructElementDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
if (args[0] instanceof StructType) {
return Type.ANY_ELEMENT_TYPE;
}