database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Planner) fix varchar does not show real length (#25171)

Browse Source 
Problem:
when we create table with datatype varchar(), we regard it to be max length by default. But when we desc, it does not show
real length but show varchar()
Reason:
when we upgrade version from 2.0.1 to 2.0.2, we support new feature of creating varchar(), and it shows the same way with
ddl schema. So user would confuse of the length of varchar
Solved:
change the showing of varchar() to varchar(65533), which in compatible with hive
This commit is contained in:
LiBinfeng
2023-11-14 10:49:21 +08:00
committed by GitHub
parent e0934166f5
commit 0a9d71ebd2
21 changed files with 43 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -367,6 +367,7 @@ import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.RelationUtil;
@ -1655,6 +1656,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
private Expression processCast(Expression cast, DataType dataType) {
if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
if (dataType.isVarcharType() && ((VarcharType) dataType).isWildcardVarchar()) {
return cast;
}
List<Expression> args = ImmutableList.of(
cast,
new TinyIntLiteral((byte) 1),

4
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/LogicalPlanTrinoBuilder.java
View File

@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.CharacterType;
import com.google.common.collect.ImmutableList;
@ -176,6 +177,9 @@ public class LogicalPlanTrinoBuilder extends io.trino.sql.tree.AstVisitor<Object
DataType dataType = mappingType(node.getType());
Expression cast = new Cast(expr, dataType);
if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
if (dataType.isVarcharType() && ((VarcharType) dataType).isWildcardVarchar()) {
return cast;
}
List<Expression> args = ImmutableList.of(
cast,
new TinyIntLiteral((byte) 1),

6
fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java
View File

@ -67,7 +67,7 @@ public class VarcharType extends CharacterType {
@Override
public String toSql() {
if (len == -1) {
return "VARCHAR(*)";
return "VARCHAR(" + MAX_VARCHAR_LENGTH + ")";
}
return "VARCHAR(" + len + ")";
}
@ -85,4 +85,8 @@ public class VarcharType extends CharacterType {
public int hashCode() {
return Objects.hash(super.hashCode(), len);
}
public boolean isWildcardVarchar() {
return len == -1 || len == MAX_VARCHAR_LENGTH;
}
}

10
fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
View File

@ -114,7 +114,7 @@ public class CreateFunctionTest {
queryStr = "select db1.id_masking(k1) from db1.tbl1";
Assert.assertTrue(
dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 4))"));
dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
// create alias function with cast
// cast any type to decimal with specific precision and scale
@ -149,7 +149,7 @@ public class CreateFunctionTest {
// cast any type to varchar with fixed length
createFuncStr = "create alias function db1.varchar(all) with parameter(text) as "
+ "cast(text as varchar(*));";
+ "cast(text as varchar(65533));";
createFunctionStmt = (CreateFunctionStmt) UtFrameUtils.parseAndAnalyzeStmt(createFuncStr, ctx);
Env.getCurrentEnv().createFunction(createFunctionStmt);
@ -172,7 +172,7 @@ public class CreateFunctionTest {
Assert.assertTrue(constExprLists.get(0).get(0) instanceof StringLiteral);
queryStr = "select db1.varchar(k1, 4) from db1.tbl1;";
Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(*))"));
Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(65533))"));
// cast any type to char with fixed length
createFuncStr = "create alias function db1.to_char(all, int) with parameter(text, length) as "
@ -235,7 +235,7 @@ public class CreateFunctionTest {
queryStr = "select id_masking(k1) from db2.tbl1";
Assert.assertTrue(
dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(*)), 4))"));
dorisAssert.query(queryStr).explainQuery().contains("concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
// 4. create alias function with cast
// cast any type to decimal with specific precision and scale
@ -270,7 +270,7 @@ public class CreateFunctionTest {
testFunctionQuery(ctx, queryStr, true);
queryStr = "select varchar(k1, 4) from db2.tbl1;";
Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(*))"));
Assert.assertTrue(dorisAssert.query(queryStr).explainQuery().contains("CAST(`k1` AS VARCHAR(65533))"));
// 6. cast any type to char with fixed length
createFuncStr = "create global alias function db2.to_char(all, int) with parameter(text, length) as "

4
fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java
View File

@ -773,7 +773,7 @@ public class MaterializedViewFunctionTest {
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
dorisAssert.withMaterializedView(createUserTagMVSql);
String query = "select ndv(tag_id) from " + USER_TAG_TABLE_NAME + ";";
String query = "select /*+ SET_VAR(enable_fallback_to_original_planner=false) */ndv(tag_id) from " + USER_TAG_TABLE_NAME + ";";
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg");
}
@ -782,7 +782,7 @@ public class MaterializedViewFunctionTest {
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
dorisAssert.withMaterializedView(createUserTagMVSql);
String query = "select approx_count_distinct(tag_id) from " + USER_TAG_TABLE_NAME + ";";
String query = "select /*+ SET_VAR(enable_fallback_to_original_planner=false) */ approx_count_distinct(tag_id) from " + USER_TAG_TABLE_NAME + ";";
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg");
}

4
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -564,11 +564,11 @@ public class QueryPlanTest extends TestWithFeService {
// disable cast hll/bitmap to string
assertSQLPlanOrErrorMsgContains(
"select cast(id2 as varchar) from test.hll_table;",
"Invalid type cast of `id2` from HLL to VARCHAR(*)"
"Invalid type cast of `id2` from HLL to VARCHAR(65533)"
);
assertSQLPlanOrErrorMsgContains(
"select cast(id2 as varchar) from test.bitmap_table;",
"Invalid type cast of `id2` from BITMAP to VARCHAR(*)"
"Invalid type cast of `id2` from BITMAP to VARCHAR(65533)"
);
// disable implicit cast hll/bitmap to string
assertSQLPlanOrErrorMsgContains(

10
fe/fe-core/src/test/java/org/apache/doris/rewrite/ExtractCommonFactorsRuleFunctionTest.java
View File

@ -273,27 +273,27 @@ public class ExtractCommonFactorsRuleFunctionTest {
// tinyint
String sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k1 like '%4%';";
LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
dorisAssert.query(sql).explainContains("CAST(`k1` AS VARCHAR(*)) LIKE '%4%'");
dorisAssert.query(sql).explainContains("CAST(`k1` AS VARCHAR(65533)) LIKE '%4%'");
// smallint
sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k2 like '%4%';";
LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
dorisAssert.query(sql).explainContains("CAST(`k2` AS VARCHAR(*)) LIKE '%4%'");
dorisAssert.query(sql).explainContains("CAST(`k2` AS VARCHAR(65533)) LIKE '%4%'");
// int
sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k3 like '%4%';";
LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
dorisAssert.query(sql).explainContains("CAST(`k3` AS VARCHAR(*)) LIKE '%4%'");
dorisAssert.query(sql).explainContains("CAST(`k3` AS VARCHAR(65533)) LIKE '%4%'");
// bigint
sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k4 like '%4%';";
LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
dorisAssert.query(sql).explainContains("CAST(`k4` AS VARCHAR(*)) LIKE '%4%'");
dorisAssert.query(sql).explainContains("CAST(`k4` AS VARCHAR(65533)) LIKE '%4%'");
// largeint
sql = "select /*+ SET_VAR(enable_nereids_planner=false) */ * from tb3 where k5 like '%4%';";
LOG.info("EXPLAIN:{}", dorisAssert.query(sql).explainQuery());
dorisAssert.query(sql).explainContains("CAST(`k5` AS VARCHAR(*)) LIKE '%4%'");
dorisAssert.query(sql).explainContains("CAST(`k5` AS VARCHAR(65533)) LIKE '%4%'");
}
@Test