database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](planner) use string for varchar in ctas if original table is not olap (#30323)

Browse Source
This commit is contained in:
morrySnow
2024-01-29 14:09:15 +08:00
committed by yiguolei
parent 930e3bb701
commit 779a9a1fbb
22 changed files with 109 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -1228,6 +1228,11 @@ public class InternalCatalog implements CatalogIf<Database> {
default:
throw new DdlException("Unsupported string type for ctas");
}
if (resultExpr.getSrcSlotRef() != null
&& resultExpr.getSrcSlotRef().getTable() != null
&& !resultExpr.getSrcSlotRef().getTable().isManagedTable()) {
typeDef = new TypeDef(ScalarType.createStringType());
}
} else if (resultType.isDecimalV2() && resultType.equals(ScalarType.DECIMALV2)) {
typeDef = new TypeDef(ScalarType.createDecimalType(27, 9));
} else if (resultType.isDecimalV3()) {

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ElementAtToSlot.java
View File

@ -72,7 +72,7 @@ public class ElementAtToSlot extends DefaultExpressionRewriter<ExpressionRewrite
}
SlotReference slotRef = new SlotReference(StatementScopeIdGenerator.newExprId(),
topColumnSlot.getName(), topColumnSlot.getDataType(),
topColumnSlot.nullable(), topColumnSlot.getQualifier(),
topColumnSlot.nullable(), topColumnSlot.getQualifier(), topColumnSlot.getTable().get(),
topColumnSlot.getColumn().get(), Optional.of(topColumnSlot.getInternalName()),
fullPaths);
ctx.addPathSlotRef(topColumnSlot, fullPaths, slotRef, elementAt);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java
View File

@ -83,7 +83,7 @@ public class DeferMaterializeTopNResult implements RewriteRuleFactory {
LogicalTopN<? extends Plan> logicalTopN, Optional<LogicalFilter<? extends Plan>> logicalFilter,
LogicalOlapScan logicalOlapScan) {
Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column");
SlotReference columnId = SlotReference.fromColumn(rowId,
SlotReference columnId = SlotReference.fromColumn(logicalOlapScan.getTable(), rowId,
logicalOlapScan.getQualifier(), logicalOlapScan);
Set<ExprId> deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet());
logicalFilter.ifPresent(filter -> filter.getConjuncts()

3
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
View File

@ -74,6 +74,9 @@ public class Alias extends NamedExpression implements UnaryExpression {
SlotReference slotReference = child() instanceof SlotReference
? (SlotReference) child() : null;
return new SlotReference(exprId, name, child().getDataType(), child().nullable(), qualifier,
slotReference != null
? ((SlotReference) child()).getTable().orElse(null)
: null,
slotReference != null
? slotReference.getColumn().orElse(null)
: null,

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
View File

@ -142,7 +142,7 @@ public class ArrayItemReference extends NamedExpression implements ExpectsInputT
* @param nullable true if nullable
*/
public ArrayItemSlot(ExprId exprId, String name, DataType dataType, boolean nullable) {
super(exprId, name, dataType, nullable, ImmutableList.of(), null, Optional.empty(), null);
super(exprId, name, dataType, nullable, ImmutableList.of(), null, null, Optional.empty(), null);
}
@Override

61
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.types.DataType;
@ -50,34 +51,36 @@ public class SlotReference extends Slot {
// TODO: remove this member variable after mv selection is refactored
protected final Optional<String> internalName;
private final TableIf table;
private final Column column;
public SlotReference(String name, DataType dataType) {
this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(),
null, Optional.empty(), null);
null, null, Optional.empty(), null);
}
public SlotReference(String name, DataType dataType, boolean nullable) {
this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, ImmutableList.of(),
null, Optional.empty(), null);
null, null, Optional.empty(), null);
}
public SlotReference(String name, DataType dataType, boolean nullable, List<String> qualifier) {
this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, qualifier, null, Optional.empty(), null);
this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable,
qualifier, null, null, Optional.empty(), null);
}
public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List<String> qualifier) {
this(exprId, name, dataType, nullable, qualifier, null, Optional.empty(), null);
this(exprId, name, dataType, nullable, qualifier, null, null, Optional.empty(), null);
}
public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable,
List<String> qualifier, @Nullable Column column) {
this(exprId, name, dataType, nullable, qualifier, column, Optional.empty(), null);
List<String> qualifier, @Nullable TableIf table, @Nullable Column column) {
this(exprId, name, dataType, nullable, qualifier, table, column, Optional.empty(), null);
}
public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable,
List<String> qualifier, @Nullable Column column, Optional<String> internalName) {
this(exprId, name, dataType, nullable, qualifier, column, internalName, null);
List<String> qualifier, @Nullable TableIf table, @Nullable Column column, Optional<String> internalName) {
this(exprId, name, dataType, nullable, qualifier, table, column, internalName, null);
}
/**
@ -93,13 +96,14 @@ public class SlotReference extends Slot {
* @param subColLabels subColumn access labels
*/
public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable,
List<String> qualifier, @Nullable Column column, Optional<String> internalName,
List<String> subColLabels) {
List<String> qualifier, @Nullable TableIf table, @Nullable Column column,
Optional<String> internalName, List<String> subColLabels) {
this.exprId = exprId;
this.name = name;
this.dataType = dataType;
this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null"));
this.nullable = nullable;
this.table = table;
this.column = column;
this.subColPath = subColLabels;
if (subColLabels != null && !this.subColPath.isEmpty()) {
@ -122,10 +126,10 @@ public class SlotReference extends Slot {
* @param qualifier the qualifier of SlotReference
* @param relation the relation which column is from
*/
public static SlotReference fromColumn(Column column, List<String> qualifier, Relation relation) {
public static SlotReference fromColumn(TableIf table, Column column, List<String> qualifier, Relation relation) {
DataType dataType = DataType.fromCatalogType(column.getType());
SlotReference slot = new SlotReference(StatementScopeIdGenerator.newExprId(), column.getName(), dataType,
column.isAllowNull(), qualifier, column, Optional.empty(), null);
column.isAllowNull(), qualifier, table, column, Optional.empty(), null);
if (relation != null && ConnectContext.get() != null
&& ConnectContext.get().getStatementContext() != null) {
ConnectContext.get().getStatementContext().addSlotToRelation(slot, relation);
@ -133,16 +137,10 @@ public class SlotReference extends Slot {
return slot;
}
public static SlotReference fromColumn(Column column, String name, List<String> qualifier) {
public static SlotReference fromColumn(TableIf table, Column column, String name, List<String> qualifier) {
DataType dataType = DataType.fromCatalogType(column.getType());
return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType,
column.isAllowNull(), qualifier, column, Optional.empty(), null);
}
public static boolean containsPathsSlotReference(Expression expression) {
return expression.collectToList(SlotReference.class::isInstance)
.stream().anyMatch(expr -> {
return ((SlotReference) expr).hasSubColPath(); });
column.isAllowNull(), qualifier, table, column, Optional.empty(), null);
}
@Override
@ -175,6 +173,14 @@ public class SlotReference extends Slot {
return internalName.get();
}
public Optional<Column> getColumn() {
return Optional.ofNullable(column);
}
public Optional<TableIf> getTable() {
return Optional.ofNullable(table);
}
@Override
public String toSql() {
return name;
@ -223,10 +229,6 @@ public class SlotReference extends Slot {
return exprId.asInt();
}
public Optional<Column> getColumn() {
return Optional.ofNullable(column);
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitSlotReference(this, context);
@ -234,7 +236,7 @@ public class SlotReference extends Slot {
@Override
public SlotReference withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 0);
Preconditions.checkArgument(children.isEmpty());
return this;
}
@ -243,22 +245,23 @@ public class SlotReference extends Slot {
if (this.nullable == newNullable) {
return this;
}
return new SlotReference(exprId, name, dataType, newNullable, qualifier, column, internalName, subColPath);
return new SlotReference(exprId, name, dataType, newNullable,
qualifier, table, column, internalName, subColPath);
}
@Override
public SlotReference withQualifier(List<String> qualifier) {
return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, subColPath);
return new SlotReference(exprId, name, dataType, nullable, qualifier, table, column, internalName, subColPath);
}
@Override
public SlotReference withName(String name) {
return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, subColPath);
return new SlotReference(exprId, name, dataType, nullable, qualifier, table, column, internalName, subColPath);
}
@Override
public SlotReference withExprId(ExprId exprId) {
return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, subColPath);
return new SlotReference(exprId, name, dataType, nullable, qualifier, table, column, internalName, subColPath);
}
public boolean isVisible() {

33
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
@ -38,11 +39,15 @@ import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.CharType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
@ -110,16 +115,24 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
for (int i = 0; i < slots.size(); i++) {
Slot s = slots.get(i);
DataType dataType = s.getDataType().conversion();
if (dataType.isNullType()) {
dataType = TinyIntType.INSTANCE;
} else if (dataType.isDecimalV2Type()) {
dataType = DecimalV2Type.SYSTEM_DEFAULT;
} else if (i == 0 && dataType.isStringType()) {
if (i == 0 && dataType.isStringType()) {
dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH);
} else if (dataType instanceof CharacterType) {
// if column is not come from table, we should set varchar length to max
if (!s.isColumnFromTable()) {
dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH);
} else {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
NullType.class, TinyIntType.INSTANCE);
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
DecimalV2Type.class, DecimalV2Type.SYSTEM_DEFAULT);
if (s.isColumnFromTable()) {
if (!((SlotReference) s).getTable().isPresent()
|| !((SlotReference) s).getTable().get().isManagedTable()) {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, StringType.INSTANCE);
}
} else {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
VarcharType.class, VarcharType.MAX_VARCHAR_TYPE);
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharType.class, VarcharType.MAX_VARCHAR_TYPE);
}
}
// if the column is an expression, we set it to nullable, otherwise according to the nullable of the slot.
@ -151,7 +164,7 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
void handleFallbackFailedCtas(ConnectContext ctx) {
try {
Env.getCurrentEnv().dropTable(new DropTableStmt(false,
new TableName(Env.getCurrentEnv().getCurrentCatalog().getName(),
new TableName(createTableInfo.getCtlName(),
createTableInfo.getDbName(), createTableInfo.getTableName()), true));
} catch (Exception e) {
// TODO: refactor it with normal error process.

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
View File

@ -198,7 +198,13 @@ public class CreateTableInfo {
return tableName;
}
/**
* full qualifier table name.
*/
public List<String> getTableNameParts() {
if (ctlName != null && dbName != null) {
return ImmutableList.of(ctlName, dbName, tableName);
}
if (dbName != null) {
return ImmutableList.of(dbName, tableName);
}
@ -860,7 +866,7 @@ public class CreateTableInfo {
}
return new CreateTableStmt(ifNotExists, isExternal,
new TableName(Env.getCurrentEnv().getCurrentCatalog().getName(), dbName, tableName),
new TableName(ctlName, dbName, tableName),
catalogColumns, catalogIndexes, engineName,
new KeysDesc(keysType, keys, clusterKeysColumnNames, clusterKeysColumnIds),
partitionDesc, distributionDesc, Maps.newHashMap(properties), extProperties,

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java
View File

@ -99,6 +99,7 @@ public class LogicalCTEConsumer extends LogicalRelation implements BlockFuncDeps
Slot consumerSlot = new SlotReference(StatementScopeIdGenerator.newExprId(),
producerOutputSlot.getName(), producerOutputSlot.getDataType(),
producerOutputSlot.nullable(), ImmutableList.of(name),
slotRef != null ? (slotRef.getTable().isPresent() ? slotRef.getTable().get() : null) : null,
slotRef != null ? (slotRef.getColumn().isPresent() ? slotRef.getColumn().get() : null) : null,
slotRef != null ? Optional.of(slotRef.getInternalName()) : Optional.empty());
producerToConsumerOutputMap.put(producerOutputSlot, consumerSlot);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
View File

@ -93,7 +93,7 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements
public List<Slot> computeOutput() {
return table.getBaseSchema()
.stream()
.map(col -> SlotReference.fromColumn(col, qualified(), this))
.map(col -> SlotReference.fromColumn(table, col, qualified(), this))
.collect(ImmutableList.toImmutableList());
}

12
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -334,7 +334,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
if (cacheSlotWithSlotName.containsKey(Pair.of(selectedIndexId, col.getName()))) {
return cacheSlotWithSlotName.get(Pair.of(selectedIndexId, col.getName()));
}
Slot slot = SlotReference.fromColumn(col, qualified(), this);
Slot slot = SlotReference.fromColumn(table, col, qualified(), this);
cacheSlotWithSlotName.put(Pair.of(selectedIndexId, col.getName()), slot);
return slot;
}).collect(ImmutableList.toImmutableList());
@ -357,22 +357,24 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
// when we have a partitioned table without any partition, visible index is empty
if (-1 == indexId || olapTable.getIndexMetaByIndexId(indexId) == null) {
return olapTable.getIndexMetaByIndexId(indexId).getSchema().stream()
.map(s -> generateUniqueSlot(s, indexId == ((OlapTable) table).getBaseIndexId(), indexId))
.map(c -> generateUniqueSlot(olapTable, c,
indexId == ((OlapTable) table).getBaseIndexId(), indexId))
.collect(Collectors.toList());
}
return olapTable.getIndexMetaByIndexId(indexId).getSchema().stream()
.map(s -> generateUniqueSlot(s, indexId == ((OlapTable) table).getBaseIndexId(), indexId))
.map(s -> generateUniqueSlot(olapTable, s,
indexId == ((OlapTable) table).getBaseIndexId(), indexId))
.collect(ImmutableList.toImmutableList());
}
private Slot generateUniqueSlot(Column column, boolean isBaseIndex, long indexId) {
private Slot generateUniqueSlot(OlapTable table, Column column, boolean isBaseIndex, long indexId) {
String name = isBaseIndex ? column.getName()
: AbstractSelectMaterializedIndexRule.parseMvColumnToMvName(column.getName(),
column.isAggregated() ? Optional.of(column.getAggregationType().toSql()) : Optional.empty());
if (cacheSlotWithSlotName.containsKey(Pair.of(indexId, name))) {
return cacheSlotWithSlotName.get(Pair.of(indexId, name));
}
Slot slot = SlotReference.fromColumn(column, name, qualified());
Slot slot = SlotReference.fromColumn(table, column, name, qualified());
cacheSlotWithSlotName.put(Pair.of(indexId, name), slot);
return slot;
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
View File

@ -99,7 +99,7 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
public List<Slot> computeOutput() {
return function.getTable().getBaseSchema()
.stream()
.map(col -> SlotReference.fromColumn(col, qualifier, this))
.map(col -> SlotReference.fromColumn(function.getTable(), col, qualifier, this))
.collect(ImmutableList.toImmutableList());
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java
View File

@ -101,7 +101,7 @@ public abstract class PhysicalCatalogRelation extends PhysicalRelation implement
public List<Slot> computeOutput() {
return table.getBaseSchema()
.stream()
.map(col -> SlotReference.fromColumn(col, qualified(), this))
.map(col -> SlotReference.fromColumn(table, col, qualified(), this))
.collect(ImmutableList.toImmutableList());
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java
View File

@ -107,7 +107,7 @@ public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation
public List<Slot> computeOutput() {
return function.getTable().getBaseSchema()
.stream()
.map(col -> SlotReference.fromColumn(col, ImmutableList.of(), this))
.map(col -> SlotReference.fromColumn(function.getTable(), col, ImmutableList.of(), this))
.collect(ImmutableList.toImmutableList());
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java
View File

@ -28,8 +28,9 @@ import java.util.Objects;
*/
public class VarcharType extends CharacterType {
public static final VarcharType SYSTEM_DEFAULT = new VarcharType(-1);
public static final int MAX_VARCHAR_LENGTH = ScalarType.MAX_VARCHAR_LENGTH;
public static final VarcharType SYSTEM_DEFAULT = new VarcharType(-1);
public static final VarcharType MAX_VARCHAR_TYPE = new VarcharType(MAX_VARCHAR_LENGTH);
public VarcharType(int len) {
super(len);
@ -40,9 +41,14 @@ public class VarcharType extends CharacterType {
return len;
}
/**
* create varchar type from length.
*/
public static VarcharType createVarcharType(int len) {
if (len == SYSTEM_DEFAULT.len) {
return SYSTEM_DEFAULT;
} else if (len == MAX_VARCHAR_LENGTH) {
return MAX_VARCHAR_TYPE;
}
return new VarcharType(len);
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
View File

@ -332,7 +332,10 @@ public class TypeCoercionUtils {
return replaceSpecifiedType(dataType, DateTimeV2Type.class, DateTimeV2Type.MAX);
}
private static DataType replaceSpecifiedType(DataType dataType,
/**
* replace specifiedType in dataType to newType.
*/
public static DataType replaceSpecifiedType(DataType dataType,
Class<? extends DataType> specifiedType, DataType newType) {
if (dataType instanceof ArrayType) {
return ArrayType.of(replaceSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType, newType));