database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](merge-on-write) enable merge on write by default (#19017)

Browse Source
This commit is contained in:
zhannngchen
2023-05-11 11:10:48 +08:00
committed by GitHub
parent 1d421a26d9
commit 5167dc1251
27 changed files with 262 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -554,10 +554,14 @@ public class MaterializedViewHandler extends AlterHandler {
}
}
if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasDeleteSign()) {
newMVColumns.add(new Column(olapTable.getDeleteSignColumn()));
Column newColumn = new Column(olapTable.getDeleteSignColumn());
newColumn.setAggregationType(AggregateType.REPLACE, true);
newMVColumns.add(newColumn);
}
if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasSequenceCol()) {
newMVColumns.add(new Column(olapTable.getSequenceCol()));
Column newColumn = new Column(olapTable.getSequenceCol());
newColumn.setAggregationType(AggregateType.REPLACE, true);
newMVColumns.add(newColumn);
}
// if the column is complex type, we forbid to create materialized view
for (Column column : newMVColumns) {

32
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View File

@ -316,14 +316,6 @@ public class CreateTableStmt extends DdlStmt {
analyzeEngineName();
// `analyzeXXX` would modify `properties`, which will be used later,
// so we just clone a properties map here.
boolean enableUniqueKeyMergeOnWrite = false;
boolean enableStoreRowColumn = false;
if (properties != null) {
enableUniqueKeyMergeOnWrite = PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(new HashMap<>(properties));
enableStoreRowColumn = PropertyAnalyzer.analyzeStoreRowColumn(new HashMap<>(properties));
}
//pre-block creation with column type ALL
for (ColumnDef columnDef : columnDefs) {
if (Objects.equals(columnDef.getType(), Type.ALL)) {
@ -337,6 +329,9 @@ public class CreateTableStmt extends DdlStmt {
+ "please use `DECIMALV3`.");
}
}
boolean enableUniqueKeyMergeOnWrite = false;
boolean enableStoreRowColumn = false;
// analyze key desc
if (engineName.equalsIgnoreCase("olap")) {
// olap table
@ -401,6 +396,23 @@ public class CreateTableStmt extends DdlStmt {
}
}
if (properties != null && properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE)
&& keysDesc.getKeysType() != KeysType.UNIQUE_KEYS) {
throw new AnalysisException(
PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " property only support unique key table");
}
if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
enableUniqueKeyMergeOnWrite = true;
if (properties != null) {
// `analyzeXXX` would modify `properties`, which will be used later,
// so we just clone a properties map here.
enableUniqueKeyMergeOnWrite = PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(
new HashMap<>(properties));
enableStoreRowColumn = PropertyAnalyzer.analyzeStoreRowColumn(new HashMap<>(properties));
}
}
keysDesc.analyze(columnDefs);
for (int i = 0; i < keysDesc.keysColumnSize(); ++i) {
columnDefs.get(i).setIsKey(true);
@ -410,7 +422,7 @@ public class CreateTableStmt extends DdlStmt {
if (keysDesc.getKeysType() == KeysType.DUP_KEYS) {
type = AggregateType.NONE;
}
if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite) {
if (enableUniqueKeyMergeOnWrite) {
type = AggregateType.NONE;
}
for (int i = keysDesc.keysColumnSize(); i < columnDefs.size(); ++i) {
@ -508,7 +520,7 @@ public class CreateTableStmt extends DdlStmt {
if (partitionDesc != null) {
if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc
|| partitionDesc instanceof ColumnPartitionDesc) {
partitionDesc.analyze(columnDefs, properties);
partitionDesc.analyze(columnDefs, properties, keysDesc);
} else {
throw new AnalysisException("Currently only support range"
+ " and list partition with engine type olap");

20
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
View File

@ -17,14 +17,13 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
@ -76,18 +75,12 @@ public class PartitionDesc {
return partitionColNames;
}
public void analyze(List<ColumnDef> columnDefs, Map<String, String> otherProperties) throws AnalysisException {
public void analyze(List<ColumnDef> columnDefs, Map<String, String> otherProperties, KeysDesc keysDesc)
throws AnalysisException {
if (partitionColNames == null || partitionColNames.isEmpty()) {
throw new AnalysisException("No partition columns.");
}
// `analyzeUniqueKeyMergeOnWrite` would modify `properties`, which will be used later,
// so we just clone a properties map here.
boolean enableUniqueKeyMergeOnWrite = false;
if (otherProperties != null) {
enableUniqueKeyMergeOnWrite =
PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(Maps.newHashMap(otherProperties));
}
Set<String> partColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
for (String partitionCol : partitionColNames) {
if (!partColNames.add(partitionCol)) {
@ -97,9 +90,10 @@ public class PartitionDesc {
boolean found = false;
for (ColumnDef columnDef : columnDefs) {
if (columnDef.getName().equals(partitionCol)) {
if (!columnDef.isKey() && (columnDef.getAggregateType() != AggregateType.NONE
|| enableUniqueKeyMergeOnWrite)) {
throw new AnalysisException("The partition column could not be aggregated column");
if (!columnDef.isKey() && keysDesc.getKeysType() != KeysType.DUP_KEYS) {
throw new AnalysisException(
"The partition column must be key column in " + keysDesc.getKeysType().toSql()
+ " table");
}
if (columnDef.getType().isFloatingPointType()) {
throw new AnalysisException("Floating point type column can not be partition column");

2
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -2964,7 +2964,7 @@ public class Env {
}
// unique key table with merge on write
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite()) {
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && !olapTable.getEnableUniqueKeyMergeOnWrite()) {
sb.append(",\n\"").append(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE).append("\" = \"");
sb.append(olapTable.getEnableUniqueKeyMergeOnWrite()).append("\"");
}

8
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -776,12 +776,10 @@ public class PropertyAnalyzer {
public static boolean analyzeUniqueKeyMergeOnWrite(Map<String, String> properties) throws AnalysisException {
if (properties == null || properties.isEmpty()) {
return false;
}
String value = properties.get(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE);
if (value == null) {
return false;
// enable merge on write by default
return true;
}
String value = properties.getOrDefault(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "true");
properties.remove(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE);
if (value.equals("true")) {
return true;