database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](case) fix struct format out files (#37350) (#37499)

Browse Source 
bp #37350
This commit is contained in:
Mingyu Chen
2024-07-09 10:11:50 +08:00
committed by GitHub
parent 3337c1bbe3
commit ca0e44f83f
48 changed files with 1105 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -133,21 +134,21 @@ public class ArrayLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(v.getStringValueForArray()));
children.forEach(v -> list.add(v.getStringValueForArray(options)));
return "[" + StringUtils.join(list, ", ") + "]";
}
@Override
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> {
String stringLiteral;
if (v instanceof NullLiteral) {
stringLiteral = "null";
stringLiteral = options.getNullFormat();
} else {
stringLiteral = getStringLiteralForComplexType(v);
stringLiteral = getStringLiteralForComplexType(v, options);
}
// we should use type to decide we output array is suitable for json format
list.add(stringLiteral);
@ -156,14 +157,14 @@ public class ArrayLiteral extends LiteralExpr {
}
@Override
public String getStringValueForStreamLoad() {
public String getStringValueForStreamLoad(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> {
String stringLiteral;
if (v instanceof NullLiteral) {
stringLiteral = "null";
} else {
stringLiteral = getStringLiteralForStreamLoad(v);
stringLiteral = getStringLiteralForStreamLoad(v, options);
}
// we should use type to decide we output array is suitable for json format
list.add(stringLiteral);

5
fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java
View File

@ -23,6 +23,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TBoolLiteral;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -106,8 +107,8 @@ public class BoolLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.TypeUtils;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.Pair;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExpr;
@ -580,8 +581,8 @@ public class CastExpr extends Expr {
}
@Override
public String getStringValueForArray() {
return children.get(0).getStringValueForArray();
public String getStringValueForArray(FormatOptions options) {
return children.get(0).getStringValueForArray(options);
}
public void setNotFold(boolean notFold) {

5
fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.InvalidFormatException;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.qe.SessionVariable;
@ -727,8 +728,8 @@ public class DateLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
public void roundCeiling(int newScale) {

7
fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.io.Text;
import org.apache.doris.qe.SessionVariable;
@ -258,7 +259,7 @@ public class DecimalLiteral extends NumericLiteralExpr {
}
@Override
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
return value.toPlainString();
}
@ -273,8 +274,8 @@ public class DecimalLiteral extends NumericLiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
@Override

7
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.io.Writable;
import org.apache.doris.nereids.util.Utils;
@ -2200,11 +2201,11 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return "";
}
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
return getStringValue();
}
public String getStringValueForStreamLoad() {
public String getStringValueForStreamLoad(FormatOptions options) {
return getStringValue();
}
@ -2213,7 +2214,7 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
// ["1", "2", "3"]
// ["a", "b", "c"]
// [["1", "2", "3"], ["1"], ["3"]]
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
return null;
}

7
fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -146,7 +147,7 @@ public class FloatLiteral extends NumericLiteralExpr {
}
@Override
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
if (type == Type.TIME || type == Type.TIMEV2) {
// FloatLiteral used to represent TIME type, here we need to remove apostrophe from timeStr
// for example '11:22:33' -> 11:22:33
@ -158,13 +159,13 @@ public class FloatLiteral extends NumericLiteralExpr {
}
@Override
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
String ret = getStringValue();
if (type == Type.TIME || type == Type.TIMEV2) {
// here already wrapped in ''
ret = ret.substring(1, ret.length() - 1);
}
return "\"" + ret + "\"";
return options.getNestedStringWrapper() + ret + options.getNestedStringWrapper();
}
public static Type getDefaultTimeType(Type type) throws AnalysisException {

5
fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TIPv4Literal;
@ -136,7 +137,7 @@ public class IPv4Literal extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TIPv6Literal;
@ -101,7 +102,7 @@ public class IPv6Literal extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.util.ByteBufferUtil;
import org.apache.doris.qe.ConnectContext;
@ -284,8 +285,8 @@ public class IntLiteral extends NumericLiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
@Override

3
fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.io.Text;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -99,7 +100,7 @@ public class JsonLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
return null;
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.io.Text;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -192,8 +193,8 @@ public class LargeIntLiteral extends NumericLiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
@Override

22
fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.mysql.MysqlProto;
import org.apache.doris.thrift.TExprNode;
@ -114,30 +115,29 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
return literalExpr;
}
public static String getStringLiteralForComplexType(Expr v) {
public static String getStringLiteralForComplexType(Expr v, FormatOptions options) {
if (!(v instanceof NullLiteral) && v.getType().isScalarType()
&& (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
|| v.getType() == Type.BOOLEAN)) {
return v.getStringValueInFe();
return v.getStringValueInFe(options);
} else if (v.getType().isComplexType()) {
// these type should also call getStringValueInFe which should handle special case for itself
return v.getStringValueInFe();
return v.getStringValueInFe(options);
} else {
return v.getStringValueForArray();
return v.getStringValueForArray(options);
}
}
public static String getStringLiteralForStreamLoad(Expr v) {
public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options) {
if (!(v instanceof NullLiteral) && v.getType().isScalarType()
&& (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
|| v.getType() == Type.BOOLEAN)) {
return v.getStringValueInFe();
return v.getStringValueInFe(options);
} else if (v.getType().isComplexType()) {
// these type should also call getStringValueInFe which should handle special case for itself
return v.getStringValueForStreamLoad();
return v.getStringValueForStreamLoad(options);
} else {
return v.getStringValueForArray();
return v.getStringValueForArray(options);
}
}
@ -266,12 +266,12 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
@Override
public abstract String getStringValue();
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
return getStringValue();
}
@Override
public abstract String getStringValueForArray();
public abstract String getStringValueForArray(FormatOptions options);
public long getLongValue() {
return 0;

15
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -171,25 +172,27 @@ public class MapLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
list.add(children.get(i).getStringValueForArray() + ":" + children.get(i + 1).getStringValueForArray());
list.add(children.get(i).getStringValueForArray(options)
+ options.getMapKeyDelim()
+ children.get(i + 1).getStringValueForArray(options));
}
return "{" + StringUtils.join(list, ", ") + "}";
}
@Override
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
// we should use type to decide we output array is suitable for json format
if (children.get(i).getType().isComplexType()) {
// map key type do not support complex type
throw new UnsupportedOperationException("Unsupport key type for MAP: " + children.get(i).getType());
throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType());
}
list.add(getStringLiteralForComplexType(children.get(i))
+ ":" + getStringLiteralForComplexType(children.get(i + 1)));
list.add(getStringLiteralForComplexType(children.get(i), options)
+ options.getMapKeyDelim() + getStringLiteralForComplexType(children.get(i + 1), options));
}
return "{" + StringUtils.join(list, ", ") + "}";
}

3
fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.analysis;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TExprNode;
import java.io.DataInput;
@ -73,7 +74,7 @@ public final class MaxLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
return null;
}

7
fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java
View File

@ -23,6 +23,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -102,15 +103,15 @@ public class NullLiteral extends LiteralExpr {
}
@Override
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
return null;
}
// the null value inside an array is represented as "null", for exampe:
// [null, null]. Not same as other primitive type to represent as \N.
@Override
public String getStringValueForArray() {
return "null";
public String getStringValueForArray(FormatOptions options) {
return options.getNullFormat();
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.MysqlColType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.thrift.TExprNode;
@ -196,8 +197,8 @@ public class PlaceHolderExpr extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {

5
fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.io.Text;
import org.apache.doris.qe.VariableVarConverters;
import org.apache.doris.thrift.TExprNode;
@ -149,8 +150,8 @@ public class StringLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
return "\"" + getStringValue() + "\"";
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}
@Override

18
fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TTypeDesc;
@ -107,28 +108,31 @@ public class StructLiteral extends LiteralExpr {
}
@Override
public String getStringValueForArray() {
public String getStringValueForArray(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(v.getStringValueForArray()));
children.forEach(v -> list.add(v.getStringValueForArray(options)));
return "{" + StringUtils.join(list, ", ") + "}";
}
@Override
public String getStringValueInFe() {
public String getStringValueInFe(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
// same with be default field index start with 1
for (int i = 0; i < children.size(); i++) {
Expr child = children.get(i);
list.add("\"" + ((StructType) type).getFields().get(i).getName() + "\": "
+ getStringLiteralForComplexType(child));
list.add(options.getNestedStringWrapper()
+ ((StructType) type).getFields().get(i).getName()
+ options.getNestedStringWrapper()
+ options.getMapKeyDelim()
+ getStringLiteralForComplexType(child, options));
}
return "{" + StringUtils.join(list, ", ") + "}";
}
@Override
public String getStringValueForStreamLoad() {
public String getStringValueForStreamLoad(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(getStringLiteralForComplexType(v)));
children.forEach(v -> list.add(getStringLiteralForComplexType(v, options)));
return "{" + StringUtils.join(list, ", ") + "}";
}

57
fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java Normal file
View File

@ -0,0 +1,57 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common;
/**
* Format options for formatting literals in FE.
* This is mainly for optional compatibility for Presto/Trino.
* User can use different session variable "serde_dialect" to choose different format options.
* This behavior same as in BE. see FormatOptions in be/src/vec/data_types/serde/data_type_serde.h
*/
public class FormatOptions {
private String nestedStringWrapper;
private String mapKeyDelim;
private String nullFormat;
public FormatOptions(String nestedStringWrapper, String mapKeyDelim, String nullFormat) {
this.nestedStringWrapper = nestedStringWrapper;
this.mapKeyDelim = mapKeyDelim;
this.nullFormat = nullFormat;
}
public String getNestedStringWrapper() {
return this.nestedStringWrapper;
}
public String getMapKeyDelim() {
return this.mapKeyDelim;
}
public String getNullFormat() {
return this.nullFormat;
}
public static FormatOptions getDefault() {
return new FormatOptions("\"", ":", "null");
}
public static FormatOptions getForPresto() {
return new FormatOptions("", "=", "NULL");
}
}

20
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.profile.SummaryProfile;
@ -56,6 +57,7 @@ import org.apache.doris.planner.RuntimeFilter;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.qe.SessionVariable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@ -527,6 +529,8 @@ public class NereidsPlanner extends Planner {
if (!(parsedStmt instanceof LogicalPlanAdapter)) {
return Optional.empty();
}
setFormatOptions();
if (physicalPlan instanceof ComputeResultSet) {
Optional<SqlCacheContext> sqlCacheContext = statementContext.getSqlCacheContext();
Optional<ResultSet> resultSet = ((ComputeResultSet) physicalPlan)
@ -539,6 +543,22 @@ public class NereidsPlanner extends Planner {
return Optional.empty();
}
private void setFormatOptions() {
ConnectContext ctx = statementContext.getConnectContext();
SessionVariable sessionVariable = ctx.getSessionVariable();
switch (sessionVariable.serdeDialect) {
case "presto":
case "trino":
statementContext.setFormatOptions(FormatOptions.getForPresto());
break;
case "doris":
statementContext.setFormatOptions(FormatOptions.getDefault());
break;
default:
throw new AnalysisException("Unsupported serde dialect: " + sessionVariable.serdeDialect);
}
}
@VisibleForTesting
public CascadesContext getCascadesContext() {
return cascadesContext;

11
fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.constraint.TableIdentifier;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.Id;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
@ -160,6 +161,8 @@ public class StatementContext implements Closeable {
private ShortCircuitQueryContext shortCircuitQueryContext;
private FormatOptions formatOptions = FormatOptions.getDefault();
public StatementContext() {
this(ConnectContext.get(), null, 0);
}
@ -477,6 +480,14 @@ public class StatementContext implements Closeable {
this.placeholders = placeholders;
}
public void setFormatOptions(FormatOptions options) {
this.formatOptions = options;
}
public FormatOptions getFormatOptions() {
return formatOptions;
}
private static class CloseableResource implements Closeable {
public final String resourceName;
public final String threadName;

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.nereids.analyzer.UnboundAlias;
import org.apache.doris.nereids.analyzer.UnboundHiveTableSink;
@ -99,9 +100,10 @@ public class InsertUtils {
TransactionEntry txnEntry = ctx.getTxnEntry();
int effectRows = 0;
FormatOptions options = FormatOptions.getDefault();
for (List<NamedExpression> row : constantExprsList) {
++effectRows;
InternalService.PDataRow data = getRowStringValue(row);
InternalService.PDataRow data = getRowStringValue(row, options);
if (data == null) {
continue;
}
@ -141,7 +143,7 @@ public class InsertUtils {
ctx.updateReturnRows(effectRows);
}
private static InternalService.PDataRow getRowStringValue(List<NamedExpression> cols) {
private static InternalService.PDataRow getRowStringValue(List<NamedExpression> cols, FormatOptions options) {
if (cols.isEmpty()) {
return null;
}
@ -158,7 +160,7 @@ public class InsertUtils {
row.addColBuilder().setValue(StmtExecutor.NULL_VALUE_FOR_LOAD);
} else if (expr instanceof ArrayLiteral) {
row.addColBuilder().setValue(String.format("\"%s\"",
((ArrayLiteral) expr).toLegacyLiteral().getStringValueForArray()));
((ArrayLiteral) expr).toLegacyLiteral().getStringValueForArray(options)));
} else {
row.addColBuilder().setValue(String.format("\"%s\"",
((Literal) expr).toLegacyLiteral().getStringValue()));

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java
View File

@ -146,7 +146,7 @@ public class PhysicalOneRowRelation extends PhysicalRelation implements OneRowRe
if (expr instanceof Literal) {
LiteralExpr legacyExpr = ((Literal) expr).toLegacyLiteral();
columns.add(new Column(output.getName(), output.getDataType().toCatalogDataType()));
data.add(legacyExpr.getStringValueInFe());
data.add(legacyExpr.getStringValueInFe(cascadesContext.getStatementContext().getFormatOptions()));
} else {
return Optional.empty();
}

8
fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.UserException;
import org.apache.doris.proto.InternalService;
import org.apache.doris.proto.InternalService.PGroupCommitInsertRequest;
@ -192,7 +193,8 @@ public class GroupCommitPlanner {
if (expr instanceof NullLiteral) {
row.addColBuilder().setValue(StmtExecutor.NULL_VALUE_FOR_LOAD);
} else if (expr.getType() instanceof ArrayType) {
row.addColBuilder().setValue(String.format("\"%s\"", expr.getStringValueForArray()));
row.addColBuilder().setValue(String.format("\"%s\"",
expr.getStringValueForArray(FormatOptions.getDefault())));
} else if (!expr.getChildren().isEmpty()) {
expr.getChildren().forEach(child -> processExprVal(child, row));
} else {
@ -213,7 +215,7 @@ public class GroupCommitPlanner {
SelectStmt selectStmt = (SelectStmt) (stmt.getQueryStmt());
if (selectStmt.getValueList() != null) {
for (List<Expr> row : selectStmt.getValueList().getRows()) {
InternalService.PDataRow data = StmtExecutor.getRowStringValue(row);
InternalService.PDataRow data = StmtExecutor.getRowStringValue(row, FormatOptions.getDefault());
if (LOG.isDebugEnabled()) {
LOG.debug("add row: [{}]", data.getColList().stream().map(c -> c.getValue())
.collect(Collectors.joining(",")));
@ -229,7 +231,7 @@ public class GroupCommitPlanner {
exprList.add(resultExpr);
}
}
InternalService.PDataRow data = StmtExecutor.getRowStringValue(exprList);
InternalService.PDataRow data = StmtExecutor.getRowStringValue(exprList, FormatOptions.getDefault());
if (LOG.isDebugEnabled()) {
LOG.debug("add row: [{}]", data.getColList().stream().map(c -> c.getValue())
.collect(Collectors.joining(",")));

4
fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
View File

@ -39,6 +39,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.qe.CommonResultSet;
@ -640,13 +641,14 @@ public class OriginalPlanner extends Planner {
List<Column> columns = new ArrayList<>(selectItems.size());
List<String> columnLabels = parsedSelectStmt.getColLabels();
List<String> data = new ArrayList<>();
FormatOptions options = FormatOptions.getDefault();
for (int i = 0; i < selectItems.size(); i++) {
SelectListItem item = selectItems.get(i);
Expr expr = item.getExpr();
String columnName = columnLabels.get(i);
if (expr instanceof LiteralExpr) {
columns.add(new Column(columnName, expr.getType()));
data.add(((LiteralExpr) expr).getStringValueInFe());
data.add(((LiteralExpr) expr).getStringValueInFe(options));
} else {
return Optional.empty();
}

2
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -3910,7 +3910,7 @@ public class SessionVariable implements Serializable, Writable {
return this.maxMsgSizeOfResultReceiver;
}
private TSerdeDialect getSerdeDialect() {
public TSerdeDialect getSerdeDialect() {
switch (serdeDialect) {
case "doris":
return TSerdeDialect.DORIS;

11
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -98,6 +98,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.NereidsSqlCacheManager;
@ -326,7 +327,8 @@ public class StmtExecutor {
context.getSessionVariable().profileLevel, context.getSessionVariable().getEnablePipelineXEngine());
}
public static InternalService.PDataRow getRowStringValue(List<Expr> cols) throws UserException {
public static InternalService.PDataRow getRowStringValue(List<Expr> cols,
FormatOptions options) throws UserException {
if (cols.isEmpty()) {
return null;
}
@ -339,9 +341,9 @@ public class StmtExecutor {
if (expr instanceof NullLiteral) {
row.addColBuilder().setValue(NULL_VALUE_FOR_LOAD);
} else if (expr instanceof ArrayLiteral) {
row.addColBuilder().setValue(String.format("\"%s\"", expr.getStringValueForStreamLoad()));
row.addColBuilder().setValue(String.format("\"%s\"", expr.getStringValueForStreamLoad(options)));
} else {
String stringValue = expr.getStringValueForStreamLoad();
String stringValue = expr.getStringValueForStreamLoad(options);
if (stringValue.equals(NULL_VALUE_FOR_LOAD) || stringValue.startsWith("\"") || stringValue.endsWith(
"\"")) {
row.addColBuilder().setValue(String.format("\"%s\"", stringValue));
@ -2013,9 +2015,10 @@ public class StmtExecutor {
throw new TException("Column count doesn't match value count");
}
}
FormatOptions options = FormatOptions.getDefault();
for (List<Expr> row : selectStmt.getValueList().getRows()) {
++effectRows;
InternalService.PDataRow data = StmtExecutor.getRowStringValue(row);
InternalService.PDataRow data = StmtExecutor.getRowStringValue(row, options);
if (data == null) {
continue;
}