database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[imporvement](table property) support for alter table property disable_auto_compaction (#27961)

Browse Source 
in some case, some tablets may cause coredump or OOM when compaction, and it is necessary to manually close the compaction of a specific table by 'disable_auto_compaction' to make be service available

This commit allow modify disable_auto_compaction table property in schema change.

---------

Signed-off-by: nextdreamblue <zxw520blue1@163.com>
This commit is contained in:
xueweizhang
2023-12-07 15:08:39 +08:00
committed by GitHub
parent cb9a6f63ab
commit 8526b9ffbe
9 changed files with 61 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View File

@ -504,6 +504,8 @@ public class Alter {
.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD));
((SchemaChangeHandler) schemaChangeHandler).updateTableProperties(db, tableName, properties);

20
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -2194,6 +2194,7 @@ public class SchemaChangeHandler extends AlterHandler {
if (isInMemory < 0 && storagePolicyId < 0 && compactionPolicy == null && timeSeriesCompactionConfig.isEmpty()
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
LOG.info("Properties already up-to-date");
@ -2206,6 +2207,12 @@ public class SchemaChangeHandler extends AlterHandler {
enableSingleCompaction = Boolean.parseBoolean(singleCompaction) ? 1 : 0;
}
String disableAutoCompactionBoolean = properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION);
int disableAutoCompaction = -1; // < 0 means don't update
if (disableAutoCompactionBoolean != null) {
disableAutoCompaction = Boolean.parseBoolean(disableAutoCompactionBoolean) ? 1 : 0;
}
String skipWriteIndexOnLoad = properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD);
int skip = -1; // < 0 means don't update
if (skipWriteIndexOnLoad != null) {
@ -2214,7 +2221,8 @@ public class SchemaChangeHandler extends AlterHandler {
for (Partition partition : partitions) {
updatePartitionProperties(db, olapTable.getName(), partition.getName(), storagePolicyId, isInMemory,
null, compactionPolicy, timeSeriesCompactionConfig, enableSingleCompaction, skip);
null, compactionPolicy, timeSeriesCompactionConfig, enableSingleCompaction, skip,
disableAutoCompaction);
}
olapTable.writeLockOrDdlException();
@ -2257,7 +2265,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (String partitionName : partitionNames) {
try {
updatePartitionProperties(db, olapTable.getName(), partitionName, storagePolicyId,
isInMemory, null, null, null, -1, -1);
isInMemory, null, null, null, -1, -1, -1);
} catch (Exception e) {
String errMsg = "Failed to update partition[" + partitionName + "]'s 'in_memory' property. "
+ "The reason is [" + e.getMessage() + "]";
@ -2273,7 +2281,8 @@ public class SchemaChangeHandler extends AlterHandler {
public void updatePartitionProperties(Database db, String tableName, String partitionName, long storagePolicyId,
int isInMemory, BinlogConfig binlogConfig, String compactionPolicy,
Map<String, Long> timeSeriesCompactionConfig,
int enableSingleCompaction, int skipWriteIndexOnLoad) throws UserException {
int enableSingleCompaction, int skipWriteIndexOnLoad,
int disableAutoCompaction) throws UserException {
// be id -> <tablet id,schemaHash>
Map<Long, Set<Pair<Long, Integer>>> beIdToTabletIdWithHash = Maps.newHashMap();
OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP);
@ -2306,7 +2315,8 @@ public class SchemaChangeHandler extends AlterHandler {
countDownLatch.addMark(kv.getKey(), kv.getValue());
UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), isInMemory,
storagePolicyId, binlogConfig, countDownLatch, compactionPolicy,
timeSeriesCompactionConfig, enableSingleCompaction, skipWriteIndexOnLoad);
timeSeriesCompactionConfig, enableSingleCompaction, skipWriteIndexOnLoad,
disableAutoCompaction);
batchTask.addTask(task);
}
if (!FeConstants.runningUnitTest) {
@ -3011,7 +3021,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Partition partition : partitions) {
updatePartitionProperties(db, olapTable.getName(), partition.getName(), -1, -1,
newBinlogConfig, null, null, -1, -1);
newBinlogConfig, null, null, -1, -1, -1);
}
olapTable.writeLockOrDdlException();

10
fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
View File

@ -226,6 +226,16 @@ public class ModifyTablePropertiesClause extends AlterTableClause {
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)) {
if (!properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("true")
&& !properties.get(PropertyAnalyzer
.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("false")) {
throw new AnalysisException(
"Property "
+ PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION + " should be set to true or false");
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)) {
long groupCommitIntervalMs;
String groupCommitIntervalMsStr = properties.get(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS);

1
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -4698,6 +4698,7 @@ public class Env {
.buildTimeSeriesCompactionFileCountThreshold()
.buildTimeSeriesCompactionTimeThresholdSeconds()
.buildSkipWriteIndexOnLoad()
.buildDisableAutoCompaction()
.buildEnableSingleReplicaCompaction();
// need to update partition info meta

1
fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View File

@ -131,6 +131,7 @@ public class TableProperty implements Writable {
buildTimeSeriesCompactionTimeThresholdSeconds();
buildSkipWriteIndexOnLoad();
buildEnableSingleReplicaCompaction();
buildDisableAutoCompaction();
break;
default:
break;

8
fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
View File

@ -53,6 +53,7 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
// < 0 means not to update property, > 0 means true, == 0 means false
private int enableSingleReplicaCompaction = -1;
private int skipWriteIndexOnLoad = -1;
private int disableAutoCompaction = -1;
public UpdateTabletMetaInfoTask(long backendId, Set<Pair<Long, Integer>> tableIdWithSchemaHash) {
super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO,
@ -87,12 +88,14 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
String compactionPolicy,
Map<String, Long> timeSeriesCompactionConfig,
int enableSingleReplicaCompaction,
int skipWriteIndexOnLoad) {
int skipWriteIndexOnLoad,
int disableAutoCompaction) {
this(backendId, tableIdWithSchemaHash, inMemory, storagePolicyId, binlogConfig, latch);
this.compactionPolicy = compactionPolicy;
this.timeSeriesCompactionConfig = timeSeriesCompactionConfig;
this.enableSingleReplicaCompaction = enableSingleReplicaCompaction;
this.skipWriteIndexOnLoad = skipWriteIndexOnLoad;
this.disableAutoCompaction = disableAutoCompaction;
}
public void countDownLatch(long backendId, Set<Pair<Long, Integer>> tablets) {
@ -159,6 +162,9 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
if (skipWriteIndexOnLoad >= 0) {
metaInfo.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad > 0);
}
if (disableAutoCompaction >= 0) {
metaInfo.setDisableAutoCompaction(disableAutoCompaction > 0);
}
updateTabletMetaInfoReq.addToTabletMetaInfos(metaInfo);
}
} else {