database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhance](resource group)db support replication_allocation (#25195)

Browse Source 
- db support replication_allocation,when create table,if not set `replication_num` or `replication_allocation `,will use it in db
- fix partition property will disappear when table partition is not null
This commit is contained in:
zhangdong
2023-10-13 10:24:01 +08:00
committed by GitHub
parent 26f50f4f0f
commit 11bbeb9a21
12 changed files with 317 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
View File

@ -70,9 +70,6 @@ public class AlterDatabasePropertyStmt extends DdlStmt {
// clone properties for analyse
Map<String, String> analysisProperties = new HashMap<String, String>(properties);
PropertyAnalyzer.analyzeBinlogConfig(analysisProperties);
if (!analysisProperties.isEmpty()) {
throw new UserException("Invalid property name or value: " + analysisProperties);
}
}
@Override

49
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.analysis.IndexDef.IndexType;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
@ -40,6 +41,7 @@ import org.apache.doris.common.util.ParseUtil;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@ -49,6 +51,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -349,7 +352,7 @@ public class CreateTableStmt extends DdlStmt {
boolean enableDuplicateWithoutKeysByDefault = false;
if (properties != null) {
enableDuplicateWithoutKeysByDefault =
PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
}
//pre-block creation with column type ALL
for (ColumnDef columnDef : columnDefs) {
@ -647,9 +650,10 @@ public class CreateTableStmt extends DdlStmt {
}
}
private Map<String, String> rewriteReplicaAllocationProperties(Map<String, String> properties) {
private Map<String, String> rewriteReplicaAllocationProperties(Map<String, String> properties)
throws AnalysisException {
if (Config.force_olap_table_replication_num <= 0) {
return properties;
return rewriteReplicaAllocationPropertiesByDatabase(properties);
}
// if force_olap_table_replication_num is set, use this value to rewrite the replication_num or
// replication_allocation properties
@ -675,6 +679,45 @@ public class CreateTableStmt extends DdlStmt {
return newProperties;
}
private Map<String, String> rewriteReplicaAllocationPropertiesByDatabase(Map<String, String> properties)
throws AnalysisException {
// if table contain `replication_allocation` or `replication_allocation`,not need rewrite by db
if (properties != null && (properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)
|| properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM))) {
return properties;
}
CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogNullable(tableName.getCtl());
if (catalog == null) {
return properties;
}
DatabaseIf db = catalog.getDbNullable(tableName.getDb());
if (db == null) {
return properties;
}
// if db not have properties,not need rewrite
if (db.getDbProperties() == null) {
return properties;
}
Map<String, String> dbProperties = db.getDbProperties().getProperties();
if (dbProperties == null) {
return properties;
}
if (properties == null) {
properties = Maps.newHashMap();
}
if (dbProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION) && StringUtils
.isNotEmpty(dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION))) {
properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION));
}
if (dbProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM) && StringUtils
.isNotEmpty(dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM))) {
properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM));
}
return properties;
}
private void analyzeEngineName() throws AnalysisException {
if (Strings.isNullOrEmpty(engineName)) {
engineName = "olap";

10
fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Joiner;
import com.google.common.base.Joiner.MapJoiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import java.util.Map;
@ -148,9 +149,16 @@ public class SinglePartitionDesc implements AllPartitionDesc {
partitionKeyDesc.analyze(partColNum);
Map<String, String> mergedMap = Maps.newHashMap();
// Should putAll `otherProperties` before `this.properties`,
// because the priority of partition is higher than table
if (otherProperties != null) {
this.properties = otherProperties;
mergedMap.putAll(otherProperties);
}
if (this.properties != null) {
mergedMap.putAll(this.properties);
}
this.properties = mergedMap;
// analyze data property
partitionDataProperty = PropertyAnalyzer.analyzeDataProperty(properties,

47
fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.persist.CreateTableInfo;
import org.apache.doris.persist.gson.GsonUtils;
@ -858,33 +859,35 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
public void replayUpdateDbProperties(Map<String, String> properties) {
dbProperties.updateProperties(properties);
binlogConfig = dbProperties.getBinlogConfig();
if (PropertyAnalyzer.hasBinlogConfig(properties)) {
binlogConfig = dbProperties.getBinlogConfig();
}
}
public boolean updateDbProperties(Map<String, String> properties) throws DdlException {
BinlogConfig oldBinlogConfig = getBinlogConfig();
BinlogConfig newBinlogConfig = BinlogConfig.fromProperties(properties);
if (oldBinlogConfig.equals(newBinlogConfig)) {
return false;
}
if (PropertyAnalyzer.hasBinlogConfig(properties)) {
BinlogConfig oldBinlogConfig = getBinlogConfig();
BinlogConfig newBinlogConfig = BinlogConfig.fromProperties(properties);
if (newBinlogConfig.isEnable() && !oldBinlogConfig.isEnable()) {
// check all tables binlog enable is true
for (Table table : idToTable.values()) {
if (table.getType() != TableType.OLAP) {
continue;
}
OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
if (!olapTable.getBinlogConfig().isEnable()) {
String errMsg = String.format("binlog is not enable in table[%s] in db [%s]", table.getName(),
getFullName());
throw new DdlException(errMsg);
if (newBinlogConfig.isEnable() && !oldBinlogConfig.isEnable()) {
// check all tables binlog enable is true
for (Table table : idToTable.values()) {
if (table.getType() != TableType.OLAP) {
continue;
}
OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
if (!olapTable.getBinlogConfig().isEnable()) {
String errMsg = String
.format("binlog is not enable in table[%s] in db [%s]", table.getName(),
getFullName());
throw new DdlException(errMsg);
}
} finally {
olapTable.readUnlock();
}
} finally {
olapTable.readUnlock();
}
}
}

13
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -943,6 +943,19 @@ public class PropertyAnalyzer {
return tagMap;
}
public static boolean hasBinlogConfig(Map<String, String> properties) {
if (properties == null || properties.isEmpty()) {
return false;
}
for (String key : properties.keySet()) {
if (key.startsWith(PROPERTIES_BINLOG_PREFIX)) {
return true;
}
}
return false;
}
public static Map<String, String> analyzeBinlogConfig(Map<String, String> properties) throws AnalysisException {
if (properties == null || properties.isEmpty()) {
return null;