database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](dynamic partition) fix dynamic partition thread met uncatch exception #35778 (#36166)

Browse Source 
cherry pick from #35778
This commit is contained in:
yujun
2024-06-12 22:16:51 +08:00
committed by GitHub
parent f1e83f5656
commit 6d54527395
8 changed files with 181 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
View File

@ -89,12 +89,13 @@ public class ColocateGroupSchema implements Writable {
HashDistributionInfo info = (HashDistributionInfo) distributionInfo;
// buckets num
if (info.getBucketNum() != bucketsNum) {
ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM, bucketsNum);
ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM,
info.getBucketNum(), bucketsNum);
}
// distribution col size
if (info.getDistributionColumns().size() != distributionColTypes.size()) {
ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_SIZE,
distributionColTypes.size());
info.getDistributionColumns().size(), distributionColTypes.size());
}
// distribution col type
for (int i = 0; i < distributionColTypes.size(); i++) {

91
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View File

@ -40,7 +40,6 @@ import org.apache.doris.catalog.PartitionKey;
import org.apache.doris.catalog.RangePartitionInfo;
import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
@ -102,9 +101,9 @@ public class DynamicPartitionScheduler extends MasterDaemon {
this.initialize = false;
}
public void executeDynamicPartitionFirstTime(Long dbId, Long tableId) {
public void executeDynamicPartitionFirstTime(Long dbId, Long tableId) throws DdlException {
List<Pair<Long, Long>> tempDynamicPartitionTableInfo = Lists.newArrayList(Pair.of(dbId, tableId));
executeDynamicPartition(tempDynamicPartitionTableInfo);
executeDynamicPartition(tempDynamicPartitionTableInfo, true);
}
public void registerDynamicPartitionTable(Long dbId, Long tableId) {
@ -186,8 +185,10 @@ public class DynamicPartitionScheduler extends MasterDaemon {
}
}
private static int getBucketsNum(DynamicPartitionProperty property, OlapTable table, String nowPartitionName) {
if (!table.isAutoBucket()) {
private static int getBucketsNum(DynamicPartitionProperty property, OlapTable table,
String nowPartitionName, boolean executeFirstTime) {
// if execute first time, all partitions no contain data
if (!table.isAutoBucket() || executeFirstTime) {
return property.getBuckets();
}
@ -217,7 +218,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
}
private ArrayList<AddPartitionClause> getAddPartitionClause(Database db, OlapTable olapTable,
Column partitionColumn, String partitionFormat) {
Column partitionColumn, String partitionFormat, boolean executeFirstTime) throws DdlException {
ArrayList<AddPartitionClause> addPartitionClauses = new ArrayList<>();
DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty();
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo();
@ -225,15 +226,10 @@ public class DynamicPartitionScheduler extends MasterDaemon {
boolean createHistoryPartition = dynamicPartitionProperty.isCreateHistoryPartition();
int idx;
int start = dynamicPartitionProperty.getStart();
int historyPartitionNum = dynamicPartitionProperty.getHistoryPartitionNum();
// When enable create_history_partition, will check the valid value from start and history_partition_num.
if (createHistoryPartition) {
if (historyPartitionNum == DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM) {
idx = start;
} else {
idx = Math.max(start, -historyPartitionNum);
}
idx = DynamicPartitionUtil.getRealStart(dynamicPartitionProperty.getStart(),
dynamicPartitionProperty.getHistoryPartitionNum());
} else {
idx = 0;
}
@ -262,12 +258,14 @@ public class DynamicPartitionScheduler extends MasterDaemon {
PartitionKey upperBound = PartitionKey.createPartitionKey(Collections.singletonList(upperValue),
Collections.singletonList(partitionColumn));
addPartitionKeyRange = Range.closedOpen(lowerBound, upperBound);
} catch (AnalysisException | IllegalArgumentException e) {
} catch (Exception e) {
// AnalysisException: keys.size is always equal to column.size, cannot reach this exception
// IllegalArgumentException: lb is greater than ub
LOG.warn("Error in gen addPartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(),
db.getFullName(), olapTable.getName());
continue;
LOG.warn("Error in gen addPartitionKeyRange. db: {}, table: {}, partition idx: {}",
db.getFullName(), olapTable.getName(), idx, e);
recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(),
e.getMessage(), olapTable.getId());
throw new DdlException(e.getMessage());
}
for (PartitionItem partitionItem : rangePartitionInfo.getIdToItem(false).values()) {
// only support single column partition now
@ -277,13 +275,16 @@ public class DynamicPartitionScheduler extends MasterDaemon {
isPartitionExists = true;
if (addPartitionKeyRange.equals(partitionItem.getItems())) {
if (LOG.isDebugEnabled()) {
LOG.debug("partition range {} exist in table {}, clear fail msg",
addPartitionKeyRange, olapTable.getName());
LOG.debug("partition range {} exist in db {} table {} partition idx {}, clear fail msg",
addPartitionKeyRange, db.getFullName(), olapTable.getName(), idx);
}
clearCreatePartitionFailedMsg(olapTable.getId());
} else {
LOG.warn("check partition range {} in db {} table {} partiton idx {} fail",
addPartitionKeyRange, db.getFullName(), olapTable.getName(), idx, e);
recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(),
e.getMessage(), olapTable.getId());
throw new DdlException(e.getMessage());
}
break;
}
@ -319,7 +320,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
DistributionDesc distributionDesc = null;
DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo();
int bucketsNum = getBucketsNum(dynamicPartitionProperty, olapTable, nowPartitionName);
int bucketsNum = getBucketsNum(dynamicPartitionProperty, olapTable, nowPartitionName, executeFirstTime);
if (distributionInfo.getType() == DistributionInfo.DistributionInfoType.HASH) {
HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo;
List<String> distColumnNames = new ArrayList<>();
@ -393,11 +394,11 @@ public class DynamicPartitionScheduler extends MasterDaemon {
PartitionKey upperBorderBound = PartitionKey.createPartitionKey(
Collections.singletonList(upperBorderPartitionValue), Collections.singletonList(partitionColumn));
reservedHistoryPartitionKeyRange = Range.closed(lowerBorderBound, upperBorderBound);
} catch (AnalysisException e) {
} catch (org.apache.doris.common.AnalysisException | org.apache.doris.nereids.exceptions.AnalysisException e) {
// AnalysisException: keys.size is always equal to column.size, cannot reach this exception
// IllegalArgumentException: lb is greater than ub
LOG.warn("Error in gen reservePartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(),
db.getFullName(), olapTable.getName());
LOG.warn("Error in gen reservePartitionKeyRange. {}, table: {}",
db.getFullName(), olapTable.getName(), e);
}
return reservedHistoryPartitionKeyRange;
}
@ -415,9 +416,11 @@ public class DynamicPartitionScheduler extends MasterDaemon {
return dropPartitionClauses;
}
int realStart = DynamicPartitionUtil.getRealStart(dynamicPartitionProperty.getStart(),
dynamicPartitionProperty.getHistoryPartitionNum());
ZonedDateTime now = ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
String lowerBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
now, dynamicPartitionProperty.getStart(), partitionFormat);
now, realStart, partitionFormat);
PartitionValue lowerPartitionValue = new PartitionValue(lowerBorder);
List<Range<PartitionKey>> reservedHistoryPartitionKeyRangeList = new ArrayList<Range<PartitionKey>>();
Range<PartitionKey> reservePartitionKeyRange;
@ -426,11 +429,12 @@ public class DynamicPartitionScheduler extends MasterDaemon {
Collections.singletonList(partitionColumn));
reservePartitionKeyRange = Range.atLeast(lowerBound);
reservedHistoryPartitionKeyRangeList.add(reservePartitionKeyRange);
} catch (AnalysisException | IllegalArgumentException e) {
} catch (Exception e) {
// AnalysisException: keys.size is always equal to column.size, cannot reach this exception
// IllegalArgumentException: lb is greater than ub
LOG.warn("Error in gen reservePartitionKeyRange. Error={}, db: {}, table: {}", e.getMessage(),
db.getFullName(), olapTable.getName());
LOG.warn("Error in gen reservePartitionKeyRange. db: {}, table: {}",
db.getFullName(), olapTable.getName(), e);
recordDropPartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId());
return dropPartitionClauses;
}
@ -451,7 +455,8 @@ public class DynamicPartitionScheduler extends MasterDaemon {
reservedHistoryPartitionKeyRangeList.add(reservedHistoryPartitionKeyRange);
} catch (IllegalArgumentException e) {
return dropPartitionClauses;
} catch (AnalysisException e) {
} catch (org.apache.doris.common.AnalysisException
| org.apache.doris.nereids.exceptions.AnalysisException e) {
throw new DdlException(e.getMessage());
}
}
@ -483,7 +488,8 @@ public class DynamicPartitionScheduler extends MasterDaemon {
return dropPartitionClauses;
}
private void executeDynamicPartition(Collection<Pair<Long, Long>> dynamicPartitionTableInfoCol) {
private void executeDynamicPartition(Collection<Pair<Long, Long>> dynamicPartitionTableInfoCol,
boolean executeFirstTime) throws DdlException {
Iterator<Pair<Long, Long>> iterator = dynamicPartitionTableInfoCol.iterator();
while (iterator.hasNext()) {
Pair<Long, Long> tableInfo = iterator.next();
@ -543,12 +549,16 @@ public class DynamicPartitionScheduler extends MasterDaemon {
}
if (!skipAddPartition) {
addPartitionClauses = getAddPartitionClause(db, olapTable, partitionColumn, partitionFormat);
addPartitionClauses = getAddPartitionClause(db, olapTable, partitionColumn, partitionFormat,
executeFirstTime);
}
dropPartitionClauses = getDropPartitionClause(db, olapTable, partitionColumn, partitionFormat);
tableName = olapTable.getName();
} catch (DdlException e) {
LOG.warn("should not happen", e);
} catch (Exception e) {
LOG.warn("has error", e);
if (executeFirstTime) {
throw new DdlException(e.getMessage());
}
} finally {
olapTable.readUnlock();
}
@ -562,6 +572,10 @@ public class DynamicPartitionScheduler extends MasterDaemon {
clearDropPartitionFailedMsg(olapTable.getId());
} catch (Exception e) {
recordDropPartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId());
LOG.warn("has error", e);
if (executeFirstTime) {
throw new DdlException(e.getMessage());
}
} finally {
olapTable.writeUnlock();
}
@ -574,6 +588,10 @@ public class DynamicPartitionScheduler extends MasterDaemon {
clearCreatePartitionFailedMsg(olapTable.getId());
} catch (Exception e) {
recordCreatePartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId());
LOG.warn("has error", e);
if (executeFirstTime) {
throw new DdlException(e.getMessage());
}
}
}
}
@ -631,7 +649,14 @@ public class DynamicPartitionScheduler extends MasterDaemon {
}
setInterval(Config.dynamic_partition_check_interval_seconds * 1000L);
if (Config.dynamic_partition_enable) {
executeDynamicPartition(dynamicPartitionTableInfo);
try {
executeDynamicPartition(dynamicPartitionTableInfo, false);
} catch (Exception e) {
// previous had log DdlException
if (LOG.isDebugEnabled()) {
LOG.debug("dynamic partition has error: ", e);
}
}
}
}
}

4
fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
View File

@ -1121,9 +1121,9 @@ public enum ErrorCode {
ERR_COLOCATE_TABLE_MUST_HAS_SAME_REPLICATION_ALLOCATION(5063, new byte[]{'4', '2', '0', '0', '0'},
"Colocate tables must have same replication allocation: { %s } should be { %s }"),
ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM(5063, new byte[]{'4', '2', '0', '0', '0'},
"Colocate tables must have same bucket num: %s"),
"Colocate tables must have same bucket num: %s should be %s"),
ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_SIZE(5063, new byte[]{'4', '2', '0', '0', '0'},
"Colocate tables distribution columns size must be same : %s"),
"Colocate tables distribution columns size must be same: %s should be %s"),
ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_TYPE(5063, new byte[]{'4', '2', '0', '0', '0'},
"Colocate tables distribution columns must have the same data type: %s should be %s"),
ERR_COLOCATE_NOT_COLOCATE_TABLE(5064, new byte[]{'4', '2', '0', '0', '0'},

23
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -583,21 +583,18 @@ public class DynamicPartitionUtil {
long expectCreatePartitionNum = 0;
if (!createHistoryPartition) {
start = 0;
expectCreatePartitionNum = (long) end - start;
} else {
int historyPartitionNum = Integer.parseInt(analyzedProperties.getOrDefault(
DynamicPartitionProperty.HISTORY_PARTITION_NUM,
String.valueOf(DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM)));
if (historyPartitionNum != DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM) {
expectCreatePartitionNum = (long) end - Math.max(start, -historyPartitionNum);
} else {
if (start == Integer.MIN_VALUE) {
throw new DdlException("Provide start or history_partition_num property"
+ " when create_history_partition=true. Otherwise set create_history_partition=false");
}
expectCreatePartitionNum = (long) end - start;
start = getRealStart(start, historyPartitionNum);
if (start == Integer.MIN_VALUE) {
throw new DdlException("Provide start or history_partition_num property"
+ " when create_history_partition=true. Otherwise set create_history_partition=false");
}
}
expectCreatePartitionNum = (long) end - start;
if (hasEnd && (expectCreatePartitionNum > Config.max_dynamic_partition_num)
&& Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))) {
throw new DdlException("Too many dynamic partitions: "
@ -683,6 +680,14 @@ public class DynamicPartitionUtil {
return analyzedProperties;
}
public static int getRealStart(int start, int historyPartitionNum) {
if (historyPartitionNum == DynamicPartitionProperty.NOT_SET_HISTORY_PARTITION_NUM) {
return start;
} else {
return Math.max(start, -historyPartitionNum);
}
}
public static void checkAlterAllowed(OlapTable olapTable) throws DdlException {
TableProperty tableProperty = olapTable.getTableProperty();
if (tableProperty != null && tableProperty.getDynamicPartitionProperty() != null