database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1] pick some prs (#39860)

Browse Source 
## Proposed changes

Issue Number: close #xxx

https://github.com/apache/doris/pull/38385 optimize parsing datetime
https://github.com/apache/doris/pull/38978 make stream load failure
message more clear and disable some error's stacktrace by default
https://github.com/apache/doris/pull/39255 fix random function coredump
https://github.com/apache/doris/pull/39324 fix function corr
inconsistency with doc
https://github.com/apache/doris/pull/39449 check auto partitoin nullity
when creating partition
https://github.com/apache/doris/pull/39695 make
DynamicPartitionScheduler immediately know interval's change
https://github.com/apache/doris/pull/39754 Add some partition expr check
on creating table
This commit is contained in:
zclllhhjj
2024-08-24 17:26:42 +08:00
committed by GitHub
parent 564d3cd647
commit 460605ae3c
23 changed files with 536 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -291,6 +291,7 @@ import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
@ -328,6 +329,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@ -553,6 +555,10 @@ public class Env {
private final SplitSourceManager splitSourceManager;
// if a config is relative to a daemon thread. record the relation here. we will proactively change interval of it.
private final Map<String, Supplier<MasterDaemon>> configtoThreads = ImmutableMap
.of("dynamic_partition_check_interval_seconds", this::getDynamicPartitionScheduler);
public List<TFrontendInfo> getFrontendInfos() {
List<TFrontendInfo> res = new ArrayList<>();
@ -5538,13 +5544,30 @@ public class Env {
globalFunctionMgr.replayDropFunction(functionSearchDesc);
}
/**
* we can't set callback which is in fe-core to config items which are in fe-common. so wrap them here. it's not so
* good but is best for us now.
*/
public void setMutableConfigwithCallback(String key, String value) throws ConfigException {
ConfigBase.setMutableConfig(key, value);
if (configtoThreads.get(key) != null) {
try {
configtoThreads.get(key).get().setInterval(Config.getField(key).getLong(null) * 1000L);
configtoThreads.get(key).get().interrupt();
LOG.info("set config " + key + " to " + value);
} catch (IllegalAccessException e) {
LOG.warn("set config " + key + " failed: " + e.getMessage());
}
}
}
public void setConfig(AdminSetConfigStmt stmt) throws Exception {
Map<String, String> configs = stmt.getConfigs();
Preconditions.checkState(configs.size() == 1);
for (Map.Entry<String, String> entry : configs.entrySet()) {
try {
ConfigBase.setMutableConfig(entry.getKey(), entry.getValue());
setMutableConfigwithCallback(entry.getKey(), entry.getValue());
} catch (ConfigException e) {
throw new DdlException(e.getMessage());
}

138
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -44,6 +44,8 @@ import org.apache.doris.analysis.KeysDesc;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionKeyDesc.PartitionKeyValueType;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.RecoverDbStmt;
import org.apache.doris.analysis.RecoverPartitionStmt;
@ -2117,6 +2119,108 @@ public class InternalCatalog implements CatalogIf<Database> {
throw new AnalysisException("Cannot find column `" + name + "` in table's columns");
}
private boolean findAllowNullforSlotRef(List<Column> baseSchema, SlotRef slot) throws AnalysisException {
for (Column col : baseSchema) {
if (col.nameEquals(slot.getColumnName(), true)) {
return col.isAllowNull();
}
}
throw new AnalysisException("Unknown partition column name:" + slot.getColumnName());
}
private void checkNullityEqual(ArrayList<Boolean> partitionSlotNullables, List<PartitionValue> item)
throws AnalysisException {
// for MAX_VALUE or somethings
if (item == null) {
return;
}
for (int i = 0; i < item.size(); i++) {
try {
if (!partitionSlotNullables.get(i) && item.get(i).isNullPartition()) {
throw new AnalysisException("Can't have null partition is for NOT NULL partition "
+ "column in partition expr's index " + i);
}
} catch (IndexOutOfBoundsException e) {
throw new AnalysisException("partition item's size out of partition columns: " + e.getMessage());
}
}
}
private void checkPartitionNullity(List<Column> baseSchema, PartitionDesc partitionDesc,
SinglePartitionDesc partition)
throws AnalysisException {
// in creating OlapTable, expr.desc is null. so we should find the column ourself.
ArrayList<Expr> partitionExprs = partitionDesc.getPartitionExprs();
ArrayList<Boolean> partitionSlotNullables = new ArrayList<Boolean>();
for (Expr expr : partitionExprs) {
if (expr instanceof SlotRef) {
partitionSlotNullables.add(findAllowNullforSlotRef(baseSchema, (SlotRef) expr));
} else if (expr instanceof FunctionCallExpr) {
partitionSlotNullables.add(Expr.isNullable(((FunctionCallExpr) expr).getFn(), expr.getChildren()));
} else {
throw new AnalysisException("Unknown partition expr type:" + expr.getExprName());
}
}
if (partition.getPartitionKeyDesc().getPartitionType() == PartitionKeyValueType.IN) {
List<List<PartitionValue>> inValues = partition.getPartitionKeyDesc().getInValues();
for (List<PartitionValue> item : inValues) {
checkNullityEqual(partitionSlotNullables, item);
}
} else if (partition.getPartitionKeyDesc().getPartitionType() == PartitionKeyValueType.LESS_THAN) {
// only upper
List<PartitionValue> upperValues = partition.getPartitionKeyDesc().getUpperValues();
checkNullityEqual(partitionSlotNullables, upperValues);
} else {
// fixed. upper and lower
List<PartitionValue> lowerValues = partition.getPartitionKeyDesc().getLowerValues();
List<PartitionValue> upperValues = partition.getPartitionKeyDesc().getUpperValues();
checkNullityEqual(partitionSlotNullables, lowerValues);
checkNullityEqual(partitionSlotNullables, upperValues);
}
}
private void checkLegalityofPartitionExprs(CreateTableStmt stmt, PartitionDesc partitionDesc)
throws AnalysisException {
for (Expr expr : partitionDesc.getPartitionExprs()) {
if (expr instanceof FunctionCallExpr) { // test them
if (!partitionDesc.isAutoCreatePartitions() || partitionDesc.getType() != PartitionType.RANGE) {
throw new AnalysisException("only Auto Range Partition support FunctionCallExpr");
}
FunctionCallExpr func = (FunctionCallExpr) expr;
ArrayList<Expr> children = func.getChildren();
Type[] childTypes = new Type[children.size()];
for (int i = 0; i < children.size(); i++) {
if (children.get(i) instanceof LiteralExpr) {
childTypes[i] = children.get(i).getType();
} else if (children.get(i) instanceof SlotRef) {
childTypes[i] = getChildTypeByName(children.get(i).getExprName(), stmt);
} else {
throw new AnalysisException(String.format(
"partition expr %s has unrecognized parameter in slot %d", func.getExprName(), i));
}
}
Function fn = null;
try {
fn = func.getBuiltinFunction(func.getFnName().getFunction(), childTypes,
Function.CompareMode.IS_INDISTINGUISHABLE); // only for test
} catch (Exception e) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
}
if (fn == null) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
}
} else if (expr instanceof SlotRef) {
if (partitionDesc.isAutoCreatePartitions() && partitionDesc.getType() == PartitionType.RANGE) {
throw new AnalysisException("Auto Range Partition need FunctionCallExpr");
}
} else {
throw new AnalysisException("partition expr " + expr.getExprName() + " is illegal!");
}
}
}
// Create olap table and related base index synchronously.
private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserException {
String tableName = stmt.getTableName();
@ -2162,43 +2266,21 @@ public class InternalCatalog implements CatalogIf<Database> {
// create partition info
PartitionDesc partitionDesc = stmt.getPartitionDesc();
// check legality of partiton exprs
ConnectContext ctx = ConnectContext.get();
Env env = Env.getCurrentEnv();
// check legality of partiton exprs.
if (ctx != null && env != null && partitionDesc != null && partitionDesc.getPartitionExprs() != null) {
for (Expr expr : partitionDesc.getPartitionExprs()) {
if (expr != null && expr instanceof FunctionCallExpr) { // test them
FunctionCallExpr func = (FunctionCallExpr) expr;
ArrayList<Expr> children = func.getChildren();
Type[] childTypes = new Type[children.size()];
for (int i = 0; i < children.size(); i++) {
if (children.get(i) instanceof LiteralExpr) {
childTypes[i] = children.get(i).getType();
} else if (children.get(i) instanceof SlotRef) {
childTypes[i] = getChildTypeByName(children.get(i).getExprName(), stmt);
} else {
throw new AnalysisException(String.format(
"partition expr %s has unrecognized parameter in slot %d", func.getExprName(), i));
}
}
Function fn = null;
try {
fn = func.getBuiltinFunction(func.getFnName().getFunction(), childTypes,
Function.CompareMode.IS_INDISTINGUISHABLE); // only for test
} catch (Exception e) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
}
if (fn == null) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
}
}
}
checkLegalityofPartitionExprs(stmt, partitionDesc);
}
PartitionInfo partitionInfo = null;
Map<String, Long> partitionNameToId = Maps.newHashMap();
if (partitionDesc != null) {
for (SinglePartitionDesc desc : partitionDesc.getSinglePartitionDescs()) {
// check legality of nullity of partition items.
checkPartitionNullity(baseSchema, partitionDesc, desc);
long partitionId = idGeneratorBuffer.getNextId();
partitionNameToId.put(desc.getPartitionName(), partitionId);
}

3
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/SetConfigAction.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.httpv2.rest;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.ConfigException;
import org.apache.doris.common.DdlException;
@ -93,7 +94,7 @@ public class SetConfigAction extends RestBaseController {
try {
if (confValue != null && confValue.length == 1) {
try {
ConfigBase.setMutableConfig(confKey, confValue[0]);
Env.getCurrentEnv().setMutableConfigwithCallback(confKey, confValue[0]);
} catch (ConfigException e) {
throw new DdlException(e.getMessage());
}

11
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Random.java
View File

@ -65,10 +65,17 @@ public class Random extends ScalarFunction
*/
public Random(Expression lchild, Expression rchild) {
super("random", lchild, rchild);
}
@Override
public void checkLegalityBeforeTypeCoercion() {
// align with original planner behavior, refer to:
// org/apache/doris/analysis/Expr.getBuiltinFunction()
Preconditions.checkState(lchild instanceof Literal && rchild instanceof Literal,
"The param of rand function must be literal");
for (Expression child : children()) {
if (!child.isLiteral()) {
throw new AnalysisException("The param of rand function must be literal ");
}
}
}
/**

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
View File

@ -97,11 +97,8 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
LOG.debug("Nereids start to execute the create table command, query id: {}, tableName: {}",
ctx.queryId(), createTableInfo.getTableName());
}
try {
Env.getCurrentEnv().createTable(createTableStmt);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
Env.getCurrentEnv().createTable(createTableStmt);
return;
}
LogicalPlan query = ctasQuery.get();