From a40a4bbc67c9be8a845adb54acf72f4107b68d7d Mon Sep 17 00:00:00 2001 From: LiBinfeng Date: Sat, 22 Mar 2025 07:44:55 +0800 Subject: [PATCH] branch-2.1: [fix](Nereids) fold constant for string function process emoji character by mistake #49087 (#49344) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pick: #49087 Related PR: #40441 Problem Summary: wrong calculation of emoji character length in some String function when do constant folding in FE. For example: select STRLEFT('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘', 2); should return ๐Ÿ˜Š๐Ÿ˜‰, but fe return ๐Ÿ˜Š only when folding constant fixed functions: - left - strleft - right - strright - locate - character_length - split_by_string - overlay - replace_empty --- .../doris/catalog/BuiltinScalarFunctions.java | 8 +- .../executable/StringArithmetic.java | 139 ++++++++++-------- .../expressions/functions/scalar/StrLeft.java | 70 --------- .../functions/scalar/StrRight.java | 70 --------- .../visitor/ScalarFunctionVisitor.java | 10 -- .../rules/expression/FoldConstantTest.java | 92 ++++++++++++ .../string_functions/test_string_function.out | Bin 4644 -> 4890 bytes .../fold_constant_string_arithmatic.groovy | 118 ++++++++++++++- 8 files changed, 284 insertions(+), 223 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 08e5d6268b..9b2a1bfa56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -400,8 +400,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StPolygonfrom import org.apache.doris.nereids.trees.expressions.functions.scalar.StX; import org.apache.doris.nereids.trees.expressions.functions.scalar.StY; import org.apache.doris.nereids.trees.expressions.functions.scalar.StartsWith; -import org.apache.doris.nereids.trees.expressions.functions.scalar.StrLeft; -import org.apache.doris.nereids.trees.expressions.functions.scalar.StrRight; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.Strcmp; import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; @@ -738,7 +736,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(L2Distance.class, "l2_distance"), scalar(LastDay.class, "last_day"), scalar(Least.class, "least"), - scalar(Left.class, "left"), + scalar(Left.class, "left", "strleft"), scalar(Length.class, "length"), scalar(Crc32.class, "crc32"), scalar(Like.class, "like"), @@ -820,7 +818,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Replace.class, "replace"), scalar(ReplaceEmpty.class, "replace_empty"), scalar(Reverse.class, "reverse"), - scalar(Right.class, "right"), + scalar(Right.class, "right", "strright"), scalar(Round.class, "round"), scalar(RoundBankers.class, "round_bankers"), scalar(Rpad.class, "rpad"), @@ -880,8 +878,6 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(StY.class, "st_y"), scalar(StartsWith.class, "starts_with"), scalar(Strcmp.class, "strcmp"), - scalar(StrLeft.class, "strleft"), - scalar(StrRight.class, "strright"), scalar(StrToDate.class, "str_to_date"), scalar(SubBitmap.class, "sub_bitmap"), scalar(SubReplace.class, "sub_replace"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java index 532ec04d8a..ed3e3894ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; @@ -61,7 +62,7 @@ import java.util.regex.Pattern; * concat */ public class StringArithmetic { - private static Expression castStringLikeLiteral(StringLikeLiteral first, String value) { + private static Literal castStringLikeLiteral(StringLikeLiteral first, String value) { if (first instanceof StringLiteral) { return new StringLiteral(value); } else if (first instanceof VarcharLiteral) { @@ -80,7 +81,7 @@ public class StringArithmetic { } private static String substringImpl(String first, int second, int third) { - int stringLength = first.length(); + int stringLength = first.codePointCount(0, first.length()); if (stringLength == 0) { return ""; } @@ -102,8 +103,11 @@ public class StringArithmetic { } else { rightIndex = third + leftIndex; } + // at here leftIndex and rightIndex can not be exceeding boundary + int finalLeftIndex = first.codePointCount(0, (int) leftIndex); + int finalRightIndex = first.codePointCount(0, (int) rightIndex); // left index and right index are in integer range because of definition, so we can safely cast it to int - return first.substring((int) leftIndex, (int) rightIndex); + return first.substring(finalLeftIndex, finalRightIndex); } /** @@ -294,12 +298,14 @@ public class StringArithmetic { */ @ExecFunction(name = "left") public static Expression left(StringLikeLiteral first, IntegerLiteral second) { + int inputLength = first.getValue().codePointCount(0, first.getValue().length()); if (second.getValue() <= 0) { return castStringLikeLiteral(first, ""); - } else if (second.getValue() < first.getValue().length()) { - return castStringLikeLiteral(first, first.getValue().substring(0, second.getValue())); - } else { + } else if (second.getValue() >= inputLength) { return first; + } else { + int index = first.getValue().codePointCount(0, second.getValue()); + return castStringLikeLiteral(first, first.getValue().substring(0, index)); } } @@ -308,17 +314,20 @@ public class StringArithmetic { */ @ExecFunction(name = "right") public static Expression right(StringLikeLiteral first, IntegerLiteral second) { - if (second.getValue() < (- first.getValue().length()) || Math.abs(second.getValue()) == 0) { + int inputLength = first.getValue().codePointCount(0, first.getValue().length()); + if (second.getValue() < (- inputLength) || Math.abs(second.getValue()) == 0) { return castStringLikeLiteral(first, ""); - } else if (second.getValue() > first.getValue().length()) { + } else if (second.getValue() >= inputLength) { return first; } else { - if (second.getValue() > 0) { + if (second.getValue() >= 0) { + int index = first.getValue().codePointCount(0, second.getValue()); return castStringLikeLiteral(first, first.getValue().substring( - first.getValue().length() - second.getValue(), first.getValue().length())); + inputLength - index, inputLength)); } else { + int index = first.getValue().codePointCount(Math.abs(second.getValue()) - 1, first.getValue().length()); return castStringLikeLiteral(first, first.getValue().substring( - Math.abs(second.getValue()) - 1, first.getValue().length())); + Math.abs(index) - 1, inputLength)); } } } @@ -338,7 +347,7 @@ public class StringArithmetic { public static Expression locate(StringLikeLiteral first, StringLikeLiteral second, IntegerLiteral third) { int result = second.getValue().indexOf(first.getValue()) + 1; if (third.getValue() <= 0 || !substringImpl(second.getValue(), third.getValue(), - second.getValue().length()).contains(first.getValue())) { + second.getValue().codePointCount(0, second.getValue().length())).contains(first.getValue())) { result = 0; } return new IntegerLiteral(result); @@ -409,7 +418,7 @@ public class StringArithmetic { */ @ExecFunction(name = "character_length") public static Expression characterLength(StringLikeLiteral first) { - return new IntegerLiteral(first.getValue().length()); + return new IntegerLiteral(first.getValue().codePointCount(0, first.getValue().length())); } private static boolean isAlphabetic(char c) { @@ -666,6 +675,23 @@ public class StringArithmetic { return new VarcharLiteral(sb.toString()); } + /** + * split by char by empty string considering emoji + * @param str input string to be split + * @return ArrayLiteral + */ + public static List splitByGrapheme(StringLikeLiteral str) { + List result = Lists.newArrayListWithExpectedSize(str.getValue().length()); + int length = str.getValue().length(); + for (int i = 0; i < length; ) { + int codePoint = str.getValue().codePointAt(i); + int charCount = Character.charCount(codePoint); + result.add(new String(new int[]{codePoint}, 0, 1)); + i += charCount; + } + return result; + } + /** * Executable arithmetic functions split_by_string */ @@ -674,11 +700,17 @@ public class StringArithmetic { if (first.getValue().isEmpty()) { return new ArrayLiteral(ImmutableList.of(), ArrayType.of(first.getDataType())); } - int limit = second.getValue().isEmpty() ? 0 : -1; - String[] result = first.getValue().split(Pattern.quote(second.getValue()), limit); + if (second.getValue().isEmpty()) { + List result = Lists.newArrayListWithExpectedSize(first.getValue().length()); + for (String resultStr : splitByGrapheme(first)) { + result.add(castStringLikeLiteral(first, resultStr)); + } + return new ArrayLiteral(result); + } + String[] result = first.getValue().split(Pattern.quote(second.getValue()), -1); List items = new ArrayList<>(); for (String s : result) { - items.add((Literal) castStringLikeLiteral(first, s)); + items.add(castStringLikeLiteral(first, s)); } return new ArrayLiteral(items); } @@ -775,60 +807,26 @@ public class StringArithmetic { } } - /** - * Executable arithmetic functions strLeft - */ - @ExecFunction(name = "strleft") - public static Expression strLeft(StringLikeLiteral first, IntegerLiteral second) { - if (second.getValue() <= 0) { - return castStringLikeLiteral(first, ""); - } else if (second.getValue() > first.getValue().length()) { - return first; - } else { - return castStringLikeLiteral(first, first.getValue().substring(0, second.getValue())); - } - } - - /** - * Executable arithmetic functions strRight - */ - @ExecFunction(name = "strright") - public static Expression strRight(StringLikeLiteral first, IntegerLiteral second) { - if (second.getValue() < (- first.getValue().length()) || Math.abs(second.getValue()) == 0) { - return castStringLikeLiteral(first, ""); - } else if (second.getValue() > first.getValue().length()) { - return first; - } else { - if (second.getValue() > 0) { - return castStringLikeLiteral(first, first.getValue().substring( - first.getValue().length() - second.getValue(), first.getValue().length())); - } else { - return castStringLikeLiteral(first, first.getValue().substring( - Math.abs(second.getValue()) - 1, first.getValue().length())); - } - } - } - /** * Executable arithmetic functions overlay */ @ExecFunction(name = "overlay") - public static Expression overlay(StringLikeLiteral first, - IntegerLiteral second, IntegerLiteral third, StringLikeLiteral four) { + public static Expression overlay(StringLikeLiteral originStr, + IntegerLiteral pos, IntegerLiteral len, StringLikeLiteral insertStr) { StringBuilder sb = new StringBuilder(); - if (second.getValue() <= 0 || second.getValue() > first.getValue().length()) { - return first; + int totalLength = originStr.getValue().codePointCount(0, originStr.getValue().length()); + if (pos.getValue() <= 0 || pos.getValue() > totalLength) { + return originStr; } else { - if (third.getValue() < 0 || third.getValue() > (first.getValue().length() - third.getValue())) { - sb.append(first.getValue().substring(0, second.getValue() - 1)); - sb.append(four.getValue()); - return castStringLikeLiteral(first, sb.toString()); + if (len.getValue() < 0 || len.getValue() > (totalLength - pos.getValue())) { + sb.append(substringImpl(originStr.getValue(), 1, pos.getValue() - 1)); + sb.append(insertStr.getValue()); + return castStringLikeLiteral(originStr, sb.toString()); } else { - sb.append(first.getValue().substring(0, second.getValue() - 1)); - sb.append(four.getValue()); - sb.append(first.getValue().substring(second.getValue() - + third.getValue() - 1, first.getValue().length())); - return castStringLikeLiteral(first, sb.toString()); + sb.append(substringImpl(originStr.getValue(), 1, pos.getValue() - 1)); + sb.append(insertStr.getValue()); + sb.append(substringImpl(originStr.getValue(), pos.getValue() + len.getValue(), totalLength)); + return castStringLikeLiteral(originStr, sb.toString()); } } } @@ -937,7 +935,7 @@ public class StringArithmetic { */ @ExecFunction(name = "append_trailing_char_if_absent") public static Expression appendTrailingCharIfAbsent(StringLikeLiteral first, StringLikeLiteral second) { - if (second.getValue().length() != 1) { + if (second.getValue().codePointCount(0, second.getValue().length()) != 1) { return new NullLiteral(first.getDataType()); } if (first.getValue().endsWith(second.getValue())) { @@ -1001,6 +999,19 @@ public class StringArithmetic { */ @ExecFunction(name = "replace_empty") public static Expression replaceEmpty(StringLikeLiteral first, StringLikeLiteral second, StringLikeLiteral third) { + if (second.getValue().isEmpty()) { + if (first.getValue().isEmpty()) { + return castStringLikeLiteral(first, third.getValue()); + } + List inputs = splitByGrapheme(first); + StringBuilder sb = new StringBuilder(); + sb.append(third.getValue()); + for (String input : inputs) { + sb.append(input); + sb.append(third.getValue()); + } + return castStringLikeLiteral(first, sb.toString()); + } return castStringLikeLiteral(first, first.getValue().replace(second.getValue(), third.getValue())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java deleted file mode 100644 index e8188dbc0f..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.VarcharType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * ScalarFunction 'strleft'. This class is generated by GenerateFunction. - */ -public class StrLeft extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT, IntegerType.INSTANCE), - FunctionSignature.ret(StringType.INSTANCE).args(StringType.INSTANCE, IntegerType.INSTANCE)); - - /** - * constructor with 2 arguments. - */ - public StrLeft(Expression arg0, Expression arg1) { - super("strleft", arg0, arg1); - } - - /** - * withChildren. - */ - @Override - public StrLeft withChildren(List children) { - Preconditions.checkArgument(children.size() == 2); - return new StrLeft(children.get(0), children.get(1)); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitStrLeft(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java deleted file mode 100644 index 0cb563ce94..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.VarcharType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * ScalarFunction 'strright'. This class is generated by GenerateFunction. - */ -public class StrRight extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT, IntegerType.INSTANCE), - FunctionSignature.ret(StringType.INSTANCE).args(StringType.INSTANCE, IntegerType.INSTANCE)); - - /** - * constructor with 2 arguments. - */ - public StrRight(Expression arg0, Expression arg1) { - super("strright", arg0, arg1); - } - - /** - * withChildren. - */ - @Override - public StrRight withChildren(List children) { - Preconditions.checkArgument(children.size() == 2); - return new StrRight(children.get(0), children.get(1)); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitStrRight(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index a45d6ccdff..30ce433c07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -398,8 +398,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StPolygonfrom import org.apache.doris.nereids.trees.expressions.functions.scalar.StX; import org.apache.doris.nereids.trees.expressions.functions.scalar.StY; import org.apache.doris.nereids.trees.expressions.functions.scalar.StartsWith; -import org.apache.doris.nereids.trees.expressions.functions.scalar.StrLeft; -import org.apache.doris.nereids.trees.expressions.functions.scalar.StrRight; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.Strcmp; import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; @@ -1958,14 +1956,6 @@ public interface ScalarFunctionVisitor { return visitScalarFunction(startsWith, context); } - default R visitStrLeft(StrLeft strLeft, C context) { - return visitScalarFunction(strLeft, context); - } - - default R visitStrRight(StrRight strRight, C context) { - return visitScalarFunction(strRight, context); - } - default R visitStrToDate(StrToDate strToDate, C context) { return visitScalarFunction(strToDate, context); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java index 3e5654580b..3a83877849 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Asin; import org.apache.doris.nereids.trees.expressions.functions.scalar.Bin; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitCount; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ceil; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CharacterLength; import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos; @@ -47,16 +48,21 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Exp; import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor; import org.apache.doris.nereids.trees.expressions.functions.scalar.FromUnixtime; import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Left; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ln; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Locate; import org.apache.doris.nereids.trees.expressions.functions.scalar.Log; import org.apache.doris.nereids.trees.expressions.functions.scalar.MinutesAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.Power; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ReplaceEmpty; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Right; import org.apache.doris.nereids.trees.expressions.functions.scalar.Round; import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sign; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sin; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sqrt; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; import org.apache.doris.nereids.trees.expressions.functions.scalar.Tan; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDays; import org.apache.doris.nereids.trees.expressions.functions.scalar.UnixTimestamp; @@ -332,6 +338,92 @@ class FoldConstantTest extends ExpressionRewriteTestHelper { AppendTrailingCharIfAbsent a = new AppendTrailingCharIfAbsent(StringLiteral.of("1"), StringLiteral.of("3")); rewritten = executor.rewrite(a, context); Assertions.assertEquals(new StringLiteral("13"), rewritten); + + Left left = new Left(StringLiteral.of("hello world"), IntegerLiteral.of(5)); + rewritten = executor.rewrite(left, context); + Assertions.assertEquals(new StringLiteral("hello"), rewritten); + left = new Left(StringLiteral.of("test"), IntegerLiteral.of(10)); + rewritten = executor.rewrite(left, context); + Assertions.assertEquals(new StringLiteral("test"), rewritten); + left = new Left(StringLiteral.of("data"), IntegerLiteral.of(0)); + rewritten = executor.rewrite(left, context); + Assertions.assertEquals(new StringLiteral(""), rewritten); + left = new Left(StringLiteral.of("data"), IntegerLiteral.of(-3)); + rewritten = executor.rewrite(left, context); + Assertions.assertEquals(new StringLiteral(""), rewritten); + + Right right = new Right(StringLiteral.of("hello world"), IntegerLiteral.of(5)); + rewritten = executor.rewrite(right, context); + Assertions.assertEquals(new StringLiteral("world"), rewritten); + right = new Right(StringLiteral.of("test"), IntegerLiteral.of(10)); + rewritten = executor.rewrite(right, context); + Assertions.assertEquals(new StringLiteral("test"), rewritten); + right = new Right(StringLiteral.of("data"), IntegerLiteral.of(0)); + rewritten = executor.rewrite(right, context); + Assertions.assertEquals(new StringLiteral(""), rewritten); + right = new Right(StringLiteral.of("data"), IntegerLiteral.of(-3)); + rewritten = executor.rewrite(right, context); + Assertions.assertEquals(new StringLiteral("ata"), rewritten); + + Substring substr = new Substring( + StringLiteral.of("database"), + IntegerLiteral.of(1), + IntegerLiteral.of(4) + ); + rewritten = executor.rewrite(substr, context); + Assertions.assertEquals(new StringLiteral("data"), rewritten); + substr = new Substring( + StringLiteral.of("database"), + IntegerLiteral.of(-4), + IntegerLiteral.of(4) + ); + rewritten = executor.rewrite(substr, context); + Assertions.assertEquals(new StringLiteral("base"), rewritten); + substr = new Substring( + StringLiteral.of("example"), + IntegerLiteral.of(3), + IntegerLiteral.of(10) + ); + rewritten = executor.rewrite(substr, context); + Assertions.assertEquals(new StringLiteral("ample"), rewritten); + + Locate locate = new Locate( + StringLiteral.of("world"), + StringLiteral.of("hello world") + ); + rewritten = executor.rewrite(locate, context); + Assertions.assertEquals(new IntegerLiteral(7), rewritten); + locate = new Locate( + StringLiteral.of("test"), + StringLiteral.of("hello world") + ); + rewritten = executor.rewrite(locate, context); + Assertions.assertEquals(new IntegerLiteral(0), rewritten); + locate = new Locate( + StringLiteral.of("l"), + StringLiteral.of("hello world"), + IntegerLiteral.of(3) + ); + rewritten = executor.rewrite(locate, context); + Assertions.assertEquals(new IntegerLiteral(3), rewritten); + + CharacterLength len = new CharacterLength(StringLiteral.of("hello")); + rewritten = executor.rewrite(len, context); + Assertions.assertEquals(new IntegerLiteral(5), rewritten); + len = new CharacterLength(StringLiteral.of("")); + rewritten = executor.rewrite(len, context); + Assertions.assertEquals(new IntegerLiteral(0), rewritten); + len = new CharacterLength(StringLiteral.of("๐Ÿ˜Š")); + rewritten = executor.rewrite(len, context); + Assertions.assertEquals(new IntegerLiteral(1), rewritten); + + ReplaceEmpty replace = new ReplaceEmpty( + StringLiteral.of(""), + StringLiteral.of(""), + StringLiteral.of("default") + ); + rewritten = executor.rewrite(replace, context); + Assertions.assertEquals(new StringLiteral("default"), rewritten); } @Test diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out index 86953f95ab0e8944bf6f13e4337f5619001c766d..dacc36966a26418d4d24efa91f7251401541b73b 100644 GIT binary patch delta 286 zcmZ3YGD~ek631j+7NN~$9QsW4rAfslMVWc&@uekc7V*V}ISRVETwGkbx(bTLSmj=< rpNdcV*|hd&Tf6b8Krx_xz$`D!i7zfq+Ps>JgKhH#fe^;Y>jY#0WqD(D delta 29 lcmbQGwnSw^5(iVh!sa9nN2blQx%t>8GYLytest')") testFoldConst("select initcap('ๆ•ฐๅญ—123test')") testFoldConst("select initcap(' leading space')") @@ -498,7 +497,7 @@ suite("fold_constant_string_arithmatic") { testFoldConst("select null_or_empty(NULL)") testFoldConst("select null_or_empty('\b')") testFoldConst("select null_or_empty(' \b')") - + // parse_url testFoldConst("select parse_url(cast('http://www.example.com/path?query=abc' as string), cast('HOST' as string))") testFoldConst("select parse_url('http://www.example.com/path?query=abc', 'HOST')") @@ -934,6 +933,60 @@ suite("fold_constant_string_arithmatic") { testFoldConst("select strleft(' Hello World', 5)") testFoldConst("select strleft('Hello World ', 50)") testFoldConst("select strleft(NULL, 1)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘', 2)") + testFoldConst("select strleft('ฮฑฮฒฮณฮด', 3)") + testFoldConst("select strleft('ไฝ ๅฅฝไธ–็•Œ', 4)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏไธ–็•Œ', 5)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š”', 3)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚', 4)") + testFoldConst("select strleft('ืฉืœื•ื', 3)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰', 4)") + testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถ', 4)") + testFoldConst("select strleft('ไฝ ๅฅฝๅ‘€๏ผŒไธ–็•Œ', 6)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏใ€็ด ๆ™ดใ‚‰ใ—ใ„ไธ€ๆ—ฅ', 7)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š” ์—ฌ๋Ÿฌ๋ถ„', 5)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚ ะผะธั€', 6)") + testFoldConst("select strleft('ืฉืœื•ื ืขื•ืœื', 4)") + testFoldConst("select strleft(null, 2)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘', 0)") + testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถฮท', -1)") + testFoldConst("select strleft('ไฝ ๅฅฝ๏ผŒ็พŽๅฅฝ็š„ไธ€ๅคฉ', -2)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏใ€็ด ๆ™ดใ‚‰ใ—ใ„ไธ€ๆ—ฅ', -3)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š” ์—ฌ๋Ÿฌ๋ถ„ ์•ˆ๋…•ํžˆ๊ฐ€์„ธ์š”', -4)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚ ะฒัะตะผ ะดั€ัƒะทัŒัะผ', -5)") + testFoldConst("select strleft('ืฉืœื•ื ืœื›ืœ ื”ื—ื‘ืจื™ื', -3)") + testFoldConst("select strleft('', 2)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰', -1)") + testFoldConst("select strleft('ฮฑฮฒ', 0)") + testFoldConst("select strleft('ไฝ ๅฅฝ', -1)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏ', 0)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š”', -1)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚', 0)") + testFoldConst("select strleft('ืฉืœื•ื', -1)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š', 5)") + testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถฮทฮธ', 5)") + testFoldConst("select strleft('ไฝ ๅฅฝ๏ผŒไธ–็•Œ๏ผๆฌข่ฟŽ', 6)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏใ€ไธ–็•Œ๏ผใ‚ˆใ†ใ“ใ', 7)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š” ์„ธ๊ณ„!', 5)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚, ะผะธั€!', 6)") + testFoldConst("select strleft('ืฉืœื•ื ืขื•ืœื!', 4)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰', 6)") + testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถฮทฮธฮน', 6)") + testFoldConst("select strleft('ไฝ ๅฅฝๅ‘€๏ผŒ็พŽๅฅฝ็š„ไธ–็•Œ', 7)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏใ€็ด ๆ™ดใ‚‰ใ—ใ„ไธ–็•Œใ‚ˆ', 8)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š”, ์•„๋ฆ„๋‹ค์šด ์„ธ์ƒ', 7)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚, ะฟั€ะตะบั€ะฐัะฝั‹ะน ะผะธั€', 8)") + testFoldConst("select strleft('ืฉืœื•ื ืœืขื•ืœื ื™ืคื”', 5)") + testFoldConst("select strleft('', -1)") + testFoldConst("select strleft('๐Ÿ˜Š๐Ÿ˜‰', 0)") + testFoldConst("select strleft('ฮฑฮฒ', -1)") + testFoldConst("select strleft('ไฝ ๅฅฝ', 0)") + testFoldConst("select strleft('ใ“ใ‚“ใซใกใฏ', -1)") + testFoldConst("select strleft('์•ˆ๋…•ํ•˜์„ธ์š”', 0)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚', -1)") + testFoldConst("select strleft('ืฉืœื•ื', 0)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚', 2147483647)") + testFoldConst("select strleft('ะฟั€ะธะฒะตั‚', 2147483648)") // strright testFoldConst("select strright('good morning', NULL)") @@ -944,7 +997,61 @@ suite("fold_constant_string_arithmatic") { testFoldConst("select strright(' Hello World', 5)") testFoldConst("select strright('Hello World ', 5)") testFoldConst("select strright(NULL, 1)") - + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘', 2)") + testFoldConst("select strright('ฮฑฮฒฮณฮด', 3)") + testFoldConst("select strright('ไฝ ๅฅฝไธ–็•Œ', 2)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏไธ–็•Œ', 3)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š”', 3)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚', 4)") + testFoldConst("select strright('ืฉืœื•ื', 3)") + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰', 4)") + testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถ', 4)") + testFoldConst("select strright('ไฝ ๅฅฝๅ‘€๏ผŒไธ–็•Œ', 6)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏใ€็ด ๆ™ดใ‚‰ใ—ใ„ไธ€ๆ—ฅ', 7)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š” ์—ฌ๋Ÿฌ๋ถ„', 5)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚ ะผะธั€', 6)") + testFoldConst("select strright('ืฉืœื•ื ืœื›ืœ ื”ื—ื‘ืจื™ื', 10)") + testFoldConst("select strright(null, 2)") + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘', 0)") + testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถฮท', -1)") + testFoldConst("select strright('ไฝ ๅฅฝ๏ผŒ็พŽๅฅฝ็š„ไธ€ๅคฉ', -2)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏใ€็ด ๆ™ดใ‚‰ใ—ใ„ไธ€ๆ—ฅ', -3)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š” ์—ฌ๋Ÿฌ๋ถ„ ์•ˆ๋…•ํžˆ๊ฐ€์„ธ์š”', -4)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚ ะฒัะตะผ ะดั€ัƒะทัŒัะผ', -5)") + testFoldConst("select strright('ืฉืœื•ื ืขื•ืœื!', -3)") + testFoldConst("select strright('', 2)") + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰', -1)") + testFoldConst("select strright('ฮฑฮฒ', 0)") + testFoldConst("select strright('ไฝ ๅฅฝ', -1)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏ', 0)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š”', -1)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚', 0)") + testFoldConst("select strright('ืฉืœื•ื', -1)") + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š', 5)") + testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถฮทฮธ', 5)") + testFoldConst("select strright('ไฝ ๅฅฝ๏ผŒไธ–็•Œ๏ผๆฌข่ฟŽ', 6)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏใ€ไธ–็•Œ๏ผใ‚ˆใ†ใ“ใ', 7)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š” ์„ธ๊ณ„!', 5)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚, ะผะธั€!', 6)") + testFoldConst("select strright('ืฉืœื•ื ืขื•ืœื!', 4)") + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰๐Ÿ‘๐Ÿ˜Š๐Ÿ˜‰', 6)") + testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถฮทฮธฮน', 6)") + testFoldConst("select strright('ไฝ ๅฅฝๅ‘€๏ผŒ็พŽๅฅฝ็š„ไธ–็•Œ', 7)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏใ€็ด ๆ™ดใ‚‰ใ—ใ„ไธ–็•Œใ‚ˆ', 8)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š”, ์•„๋ฆ„๋‹ค์šด ์„ธ์ƒ', 7)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚, ะฟั€ะตะบั€ะฐัะฝั‹ะน ะผะธั€', 8)") + testFoldConst("select strright('ืฉืœื•ื ืœืขื•ืœื ื™ืคื”', 5)") + testFoldConst("select strright('', -1)") + testFoldConst("select strright('๐Ÿ˜Š๐Ÿ˜‰', 0)") + testFoldConst("select strright('ฮฑฮฒ', -1)") + testFoldConst("select strright('ไฝ ๅฅฝ', 0)") + testFoldConst("select strright('ใ“ใ‚“ใซใกใฏ', -1)") + testFoldConst("select strright('์•ˆ๋…•ํ•˜์„ธ์š”', 0)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚', -1)") + testFoldConst("select strright('ืฉืœื•ื', 0)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚', 2147483647)") + testFoldConst("select strright('ะฟั€ะธะฒะตั‚', 2147483648)") + // sub_replace testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS STRING), 1, 2)") testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS STRING), 1, 2)") @@ -1602,4 +1709,9 @@ suite("fold_constant_string_arithmatic") { testFoldConst("select extract_url_parameter('http://user:pwd@www.baidu.com?๐ŸŒ=b&c=d&e=f&g=h&i=j&k=l', '')") testFoldConst("select extract_url_parameter('http://user:pwd@www.baidu.com?๐ŸŒ=b&c=d&e=f&g=h&i=j&k=l', null)") + // emoji + testFoldConst("select replace_empty('๐Ÿ˜€abc', '', 'def')") + testFoldConst("select split_by_string('a๐Ÿ˜a๐Ÿ˜a', '')") + testFoldConst("select character_length('a๐Ÿ˜a๐Ÿ˜a')") + testFoldConst("select replace_empty('a๐Ÿ˜a๐Ÿ˜a', '', '2')") }