database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improve](explode) explode function support multi param (#50310)

Browse Source 
### What problem does this PR solve?
backport:https://github.com/apache/doris/pull/48537
Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
This commit is contained in:
Benjaminwei
2025-04-23 23:27:07 +08:00
committed by GitHub
parent e5a9c2552d
commit cf72fa82e2
22 changed files with 1162 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java
View File

@ -41,8 +41,11 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode
import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import java.util.List;
import java.util.Set;
/**
* Builtin table generating functions.
@ -76,6 +79,13 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper {
tableGenerating(PosExplodeOuter.class, "posexplode_outer")
);
public static final ImmutableSet<String> RETURN_MULTI_COLUMNS_FUNCTIONS = new ImmutableSortedSet.Builder<String>(
String.CASE_INSENSITIVE_ORDER).add("explode").add("explode_outer").build();
public Set<String> getReturnManyColumnFunctions() {
return RETURN_MULTI_COLUMNS_FUNCTIONS;
}
public static final BuiltinTableGeneratingFunctions INSTANCE = new BuiltinTableGeneratingFunctions();
// Note: Do not add any code here!

10
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.BuiltinAggregateFunctions;
import org.apache.doris.catalog.BuiltinTableGeneratingFunctions;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.ScalarType;
@ -1206,8 +1207,13 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
String generateName = ctx.tableName.getText();
// if later view explode map type, we need to add a project to convert map to struct
String columnName = ctx.columnNames.get(0).getText();
List<String> expandColumnNames = Lists.newArrayList();
if (ctx.columnNames.size() > 1) {
List<String> expandColumnNames = ImmutableList.of();
// explode can pass multiple columns
// then use struct to return the result of the expansion of multiple columns.
if (ctx.columnNames.size() > 1
|| BuiltinTableGeneratingFunctions.INSTANCE.getReturnManyColumnFunctions()
.contains(ctx.functionName.getText())) {
columnName = ConnectContext.get() != null
? ConnectContext.get().getStatementContext().generateColumnName() : "expand_cols";
expandColumnNames = ctx.columnNames.stream()

61
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java
View File

@ -20,31 +20,34 @@ package org.apache.doris.nereids.trees.expressions.functions.generator;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.nereids.types.coercion.FollowToAnyDataType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
/**
* explode([1, 2, 3]), generate 3 lines include 1, 2 and 3.
* explode([1, 2, 3]), generate three rows include 1, 2 and 3.
* explode([1, 2, 3], [4, 5, 6]) generates two columns and three rows
* where the first column contains 1, 2, 3, and the second column contains 4, 5, 6.
*/
public class Explode extends TableGeneratingFunction implements BinaryExpression, AlwaysNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0)).args(ArrayType.of(new AnyDataType(0)))
);
public class Explode extends TableGeneratingFunction implements CustomSignature, ComputePrecision, AlwaysNullable {
/**
* constructor with 1 argument.
* constructor with one or more argument.
*/
public Explode(Expression arg) {
super("explode", arg);
public Explode(Expression[] args) {
super("explode", args);
}
/**
@ -52,17 +55,43 @@ public class Explode extends TableGeneratingFunction implements BinaryExpression
*/
@Override
public Explode withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 1);
return new Explode(children.get(0));
Preconditions.checkArgument(!children.isEmpty());
return new Explode(children.toArray(new Expression[0]));
}
@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
public FunctionSignature computePrecision(FunctionSignature signature) {
return signature;
}
@Override
public FunctionSignature customSignature() {
List<DataType> arguments = new ArrayList<>();
ImmutableList.Builder<StructField> structFields = ImmutableList.builder();
for (int i = 0; i < children.size(); i++) {
if (children.get(i).getDataType().isNullType()) {
arguments.add(ArrayType.of(NullType.INSTANCE));
structFields.add(
new StructField("col" + (i + 1), NullType.INSTANCE, true, ""));
} else if (children.get(i).getDataType().isArrayType()) {
structFields.add(
new StructField("col" + (i + 1),
((ArrayType) (children.get(i)).getDataType()).getItemType(), true, ""));
arguments.add(children.get(i).getDataType());
} else {
SearchSignature.throwCanNotFoundFunctionException(this.getName(), getArguments());
}
}
return FunctionSignature.of(new StructType(structFields.build()), arguments);
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitExplode(this, context);
}
@Override
public FunctionSignature searchSignature(List<FunctionSignature> signatures) {
return super.searchSignature(signatures);
}
}

61
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java
View File

@ -20,31 +20,34 @@ package org.apache.doris.nereids.trees.expressions.functions.generator;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.nereids.types.coercion.FollowToAnyDataType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
/**
* explode([1, 2, 3]), generate 3 lines include 1, 2 and 3.
* explode_outer([1, 2, 3]), generate three rows include 1, 2 and 3.
* explode_outer([1, 2, 3], [4, 5, 6]) generates two columns and three rows
* where the first column contains 1, 2, 3, and the second column contains 4, 5, 6.
*/
public class ExplodeOuter extends TableGeneratingFunction implements BinaryExpression, AlwaysNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0)).args(ArrayType.of(new AnyDataType(0)))
);
public class ExplodeOuter extends TableGeneratingFunction implements CustomSignature, ComputePrecision, AlwaysNullable {
/**
* constructor with 1 argument.
* constructor with one or more argument.
*/
public ExplodeOuter(Expression arg) {
super("explode_outer", arg);
public ExplodeOuter(Expression[] args) {
super("explode_outer", args);
}
/**
@ -52,17 +55,43 @@ public class ExplodeOuter extends TableGeneratingFunction implements BinaryExpre
*/
@Override
public ExplodeOuter withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 1);
return new ExplodeOuter(children.get(0));
Preconditions.checkArgument(!children.isEmpty());
return new ExplodeOuter(children.toArray(new Expression[0]));
}
@Override
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
public FunctionSignature computePrecision(FunctionSignature signature) {
return signature;
}
@Override
public FunctionSignature customSignature() {
List<DataType> arguments = new ArrayList<>();
ImmutableList.Builder<StructField> structFields = ImmutableList.builder();
for (int i = 0; i < children.size(); i++) {
if (children.get(i).getDataType().isNullType()) {
arguments.add(ArrayType.of(NullType.INSTANCE));
structFields.add(
new StructField("col" + (i + 1), NullType.INSTANCE, true, ""));
} else if (children.get(i).getDataType().isArrayType()) {
structFields.add(
new StructField("col" + (i + 1),
((ArrayType) (children.get(i)).getDataType()).getItemType(), true, ""));
arguments.add(children.get(i).getDataType());
} else {
SearchSignature.throwCanNotFoundFunctionException(this.getName(), getArguments());
}
}
return FunctionSignature.of(new StructType(structFields.build()), arguments);
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitExplodeOuter(this, context);
}
@Override
public FunctionSignature searchSignature(List<FunctionSignature> signatures) {
return super.searchSignature(signatures);
}
}

95
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterTest.java Normal file
View File

@ -0,0 +1,95 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.functions.generator;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.StructType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
public class ExplodeOuterTest {
/////////////////////////////////////////
// GetSignatures
/////////////////////////////////////////
@Test
public void testGetSignatures() {
// build explode_outer(array<int>, array<str>) expression
Expression[] args = {SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE)),
SlotReference.of("str", ArrayType.of(StringType.INSTANCE))};
ExplodeOuter explode = new ExplodeOuter(args);
// check signature
List<FunctionSignature> signatures = explode.getSignatures();
Assertions.assertEquals(1, signatures.size());
FunctionSignature signature = signatures.get(0);
Assertions.assertEquals(2, signature.argumentsTypes.size());
Assertions.assertTrue(signature.argumentsTypes.get(0).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(0)).getItemType().isIntegerType());
Assertions.assertTrue(signature.argumentsTypes.get(1).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(1)).getItemType().isStringType());
Assertions.assertTrue(signature.returnType.isStructType());
StructType returnType = (StructType) signature.returnType;
Assertions.assertEquals(2, returnType.getFields().size());
Assertions.assertEquals(IntegerType.INSTANCE, returnType.getFields().get(0).getDataType());
Assertions.assertEquals(StringType.INSTANCE, returnType.getFields().get(1).getDataType());
}
@Test
public void testGetSignaturesWithNull() {
// build explode(null, array<int>) expression
Expression[] args = { SlotReference.of("null", NullType.INSTANCE), SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE))};
ExplodeOuter explode = new ExplodeOuter(args);
// check signature
List<FunctionSignature> signatures = explode.getSignatures();
Assertions.assertEquals(1, signatures.size());
FunctionSignature signature = signatures.get(0);
Assertions.assertEquals(2, signature.argumentsTypes.size());
Assertions.assertTrue(signature.argumentsTypes.get(0).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(0)).getItemType().isNullType());
Assertions.assertTrue(signature.argumentsTypes.get(1).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(1)).getItemType().isIntegerType());
Assertions.assertTrue(signature.returnType.isStructType());
StructType returnType = (StructType) signature.returnType;
Assertions.assertEquals(2, returnType.getFields().size());
Assertions.assertEquals(NullType.INSTANCE, returnType.getFields().get(0).getDataType());
Assertions.assertEquals(IntegerType.INSTANCE, returnType.getFields().get(1).getDataType());
}
@Test
public void testGetSignaturesWithInvalidArgument() {
// build explode_outer(int)
Expression[] args = { SlotReference.of("int", IntegerType.INSTANCE) };
ExplodeOuter explode = new ExplodeOuter(args);
Assertions.assertThrows(AnalysisException.class, explode::getSignatures);
}
}

95
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeTest.java Normal file
View File

@ -0,0 +1,95 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.functions.generator;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.StructType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
public class ExplodeTest {
/////////////////////////////////////////
// GetSignatures
/////////////////////////////////////////
@Test
public void testGetSignatures() {
// build explode(array<int>, array<str>) expression
Expression[] args = {SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE)),
SlotReference.of("str", ArrayType.of(StringType.INSTANCE))};
Explode explode = new Explode(args);
// check signature
List<FunctionSignature> signatures = explode.getSignatures();
Assertions.assertEquals(1, signatures.size());
FunctionSignature signature = signatures.get(0);
Assertions.assertEquals(2, signature.argumentsTypes.size());
Assertions.assertTrue(signature.argumentsTypes.get(0).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(0)).getItemType().isIntegerType());
Assertions.assertTrue(signature.argumentsTypes.get(1).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(1)).getItemType().isStringType());
Assertions.assertTrue(signature.returnType.isStructType());
StructType returnType = (StructType) signature.returnType;
Assertions.assertEquals(2, returnType.getFields().size());
Assertions.assertEquals(IntegerType.INSTANCE, returnType.getFields().get(0).getDataType());
Assertions.assertEquals(StringType.INSTANCE, returnType.getFields().get(1).getDataType());
}
@Test
public void testGetSignaturesWithNull() {
// build explode(null, array<int>) expression
Expression[] args = { SlotReference.of("null", NullType.INSTANCE), SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE))};
Explode explode = new Explode(args);
// check signature
List<FunctionSignature> signatures = explode.getSignatures();
Assertions.assertEquals(1, signatures.size());
FunctionSignature signature = signatures.get(0);
Assertions.assertEquals(2, signature.argumentsTypes.size());
Assertions.assertTrue(signature.argumentsTypes.get(0).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(0)).getItemType().isNullType());
Assertions.assertTrue(signature.argumentsTypes.get(1).isArrayType());
Assertions.assertTrue(((ArrayType) signature.argumentsTypes.get(1)).getItemType().isIntegerType());
Assertions.assertTrue(signature.returnType.isStructType());
StructType returnType = (StructType) signature.returnType;
Assertions.assertEquals(2, returnType.getFields().size());
Assertions.assertEquals(NullType.INSTANCE, returnType.getFields().get(0).getDataType());
Assertions.assertEquals(IntegerType.INSTANCE, returnType.getFields().get(1).getDataType());
}
@Test
public void testGetSignaturesWithInvalidArgument() {
// build explode(int)
Expression[] args = { SlotReference.of("int", IntegerType.INSTANCE) };
Explode explode = new Explode(args);
Assertions.assertThrows(AnalysisException.class, explode::getSignatures);
}
}