database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[cherry-pick](branch-2.1) Pick "[Fix](delete command) Mark delete sign when do delete command in MoW table (#35917)" (#37594)

Browse Source 
Pick #35917 and #37151
This commit is contained in:
abmdocrt
2024-07-15 18:54:01 +08:00
committed by GitHub
parent 03e21dddff
commit 63c2d22513
22 changed files with 635 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View File

@ -518,6 +518,8 @@ public class Alter {
.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
|| properties
.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_LEVEL_THRESHOLD));
((SchemaChangeHandler) schemaChangeHandler).updateTableProperties(db, tableName, properties);

13
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -2263,6 +2263,7 @@ public class SchemaChangeHandler extends AlterHandler {
if (isInMemory < 0 && storagePolicyId < 0 && compactionPolicy == null && timeSeriesCompactionConfig.isEmpty()
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)
@ -2278,6 +2279,18 @@ public class SchemaChangeHandler extends AlterHandler {
enableSingleCompaction = Boolean.parseBoolean(singleCompaction) ? 1 : 0;
}
if (enableUniqueKeyMergeOnWrite && Boolean.parseBoolean(singleCompaction)) {
throw new UserException(
"enable_single_replica_compaction property is not supported for merge-on-write table");
}
String enableMowLightDelete = properties.get(
PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
if (enableMowLightDelete != null && !enableUniqueKeyMergeOnWrite) {
throw new UserException(
"enable_mow_light_delete property is only supported for unique merge-on-write table");
}
String disableAutoCompactionBoolean = properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION);
int disableAutoCompaction = -1; // < 0 means don't update
if (disableAutoCompactionBoolean != null) {

3
fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
View File

@ -123,7 +123,8 @@ public class DeleteStmt extends DdlStmt {
}
// analyze predicate
if (fromClause == null) {
if ((fromClause == null && !((OlapTable) targetTable).getEnableUniqueKeyMergeOnWrite())
|| (fromClause == null && ((OlapTable) targetTable).getEnableMowLightDelete())) {
if (wherePredicate == null) {
throw new AnalysisException("Where clause is not set");
}

30
fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.alter.AlterOpType;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableProperty;
@ -267,13 +268,36 @@ public class ModifyTablePropertiesClause extends AlterTableClause {
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)) {
if (!properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
.equalsIgnoreCase("true")
&& !properties.get(PropertyAnalyzer
.PROPERTIES_ENABLE_MOW_LIGHT_DELETE).equalsIgnoreCase("false")) {
throw new AnalysisException(
"Property "
+ PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE
+ " should be set to true or false");
}
OlapTable table = null;
if (tableName != null) {
table = (OlapTable) (Env.getCurrentInternalCatalog().getDbOrAnalysisException(tableName.getDb())
.getTableOrAnalysisException(tableName.getTbl()));
}
if (table == null || !table.getEnableUniqueKeyMergeOnWrite()) {
throw new AnalysisException(
"enable_mow_light_delete property is "
+ "only supported for unique merge-on-write table");
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)) {
if (!properties.get(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("true")
&& !properties.get(PropertyAnalyzer
.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("false")) {
.PROPERTIES_DISABLE_AUTO_COMPACTION).equalsIgnoreCase("false")) {
throw new AnalysisException(
"Property "
+ PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION + " should be set to true or false");
"Property "
+ PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION
+ " should be set to true or false");
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;

7
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -3460,6 +3460,13 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_DATA_BYTES).append("\" = \"");
sb.append(olapTable.getGroupCommitDataBytes()).append("\"");
// enable delete on delete predicate
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
.append("\" = \"");
sb.append(olapTable.getEnableMowLightDelete()).append("\"");
}
// enable duplicate without keys by default
if (olapTable.isDuplicateWithoutKey()) {
sb.append(",\n\"")

8
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -1201,6 +1201,14 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
return null;
}
public void setEnableMowLightDelete(boolean enable) {
getOrCreatTableProperty().setEnableMowLightDelete(enable);
}
public boolean getEnableMowLightDelete() {
return getOrCreatTableProperty().getEnableMowLightDelete();
}
public void setGroupCommitIntervalMs(int groupCommitInterValMs) {
getOrCreatTableProperty().setGroupCommitIntervalMs(groupCommitInterValMs);
}

10
fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View File

@ -532,6 +532,16 @@ public class TableProperty implements Writable {
PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"));
}
public void setEnableMowLightDelete(boolean enable) {
properties.put(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE, Boolean.toString(enable));
}
public boolean getEnableMowLightDelete() {
return Boolean.parseBoolean(properties.getOrDefault(
PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE,
Boolean.toString(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE_DEFAULT_VALUE)));
}
public void setSequenceMapCol(String colName) {
properties.put(PropertyAnalyzer.PROPERTIES_FUNCTION_COLUMN + "."
+ PropertyAnalyzer.PROPERTIES_SEQUENCE_COL, colName);

24
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -198,6 +198,11 @@ public class PropertyAnalyzer {
public static final int PROPERTIES_GROUP_COMMIT_DATA_BYTES_DEFAULT_VALUE
= Config.group_commit_data_bytes_default_value;
public static final String PROPERTIES_ENABLE_MOW_LIGHT_DELETE =
"enable_mow_light_delete";
public static final boolean PROPERTIES_ENABLE_MOW_LIGHT_DELETE_DEFAULT_VALUE
= Config.enable_mow_light_delete;
// compaction policy
public static final String SIZE_BASED_COMPACTION_POLICY = "size_based";
public static final String TIME_SERIES_COMPACTION_POLICY = "time_series";
@ -1277,6 +1282,25 @@ public class PropertyAnalyzer {
throw new AnalysisException(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " must be `true` or `false`");
}
public static boolean analyzeEnableDeleteOnDeletePredicate(Map<String, String> properties)
throws AnalysisException {
if (properties == null || properties.isEmpty()) {
return false;
}
String value = properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
if (value == null) {
return false;
}
properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
if (value.equals("true")) {
return true;
} else if (value.equals("false")) {
return false;
}
throw new AnalysisException(
PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE + " must be `true` or `false`");
}
/**
* Found property with "group_commit_interval_ms" prefix and return a time in ms.
* e.g.

12
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2350,6 +2350,18 @@ public class InternalCatalog implements CatalogIf<Database> {
}
olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite);
boolean enableDeleteOnDeletePredicate = false;
try {
enableDeleteOnDeletePredicate = PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) {
throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE
+ " property is only supported for unique merge-on-write table");
}
olapTable.setEnableMowLightDelete(enableDeleteOnDeletePredicate);
boolean enableSingleReplicaCompaction = false;
try {
enableSingleReplicaCompaction = PropertyAnalyzer.analyzeEnableSingleReplicaCompaction(properties);

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
View File

@ -166,6 +166,13 @@ public class DeleteFromCommand extends Command implements ForwardWithSync, Expla
}
}
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite()
&& !olapTable.getEnableMowLightDelete()) {
new DeleteFromUsingCommand(nameParts, tableAlias, isTempPart, partitions,
logicalQuery, Optional.empty()).run(ctx, executor);
return;
}
// call delete handler to process
List<Predicate> predicates = planner.getScanNodes().get(0).getConjuncts().stream()
.filter(c -> {