database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](alter backends)backend modify & drop & decommission by ids (#25444)

Browse Source
This commit is contained in:
Guangdong Liu
2023-10-25 14:32:30 +08:00
committed by GitHub
parent 49b73483fd
commit 4f17c2a8b1
17 changed files with 269 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View File

@ -86,7 +86,7 @@ import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@ -739,7 +739,7 @@ public class Alter {
}
public void processAlterCluster(AlterSystemStmt stmt) throws UserException {
clusterHandler.process(Arrays.asList(stmt.getAlterClause()), stmt.getClusterName(), null, null);
clusterHandler.process(Collections.singletonList(stmt.getAlterClause()), stmt.getClusterName(), null, null);
}
private void processRename(Database db, OlapTable table, List<AlterClause> alterClauses) throws DdlException {

34
fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java
View File

@ -125,7 +125,13 @@ public class SystemHandler extends AlterHandler {
+ "All data on this backend will be discarded permanently. "
+ "If you insist, use DROPP instead of DROP");
}
Env.getCurrentSystemInfo().dropBackends(dropBackendClause.getHostInfos());
if (dropBackendClause.getHostInfos().isEmpty()) {
// drop by id
Env.getCurrentSystemInfo().dropBackendsByIds(dropBackendClause.getIds());
} else {
// drop by host
Env.getCurrentSystemInfo().dropBackends(dropBackendClause.getHostInfos());
}
} else if (alterClause instanceof DecommissionBackendClause) {
// decommission
DecommissionBackendClause decommissionBackendClause = (DecommissionBackendClause) alterClause;
@ -192,6 +198,9 @@ public class SystemHandler extends AlterHandler {
private List<Backend> checkDecommission(DecommissionBackendClause decommissionBackendClause)
throws DdlException {
if (decommissionBackendClause.getHostInfos().isEmpty()) {
return checkDecommissionByIds(decommissionBackendClause.getIds());
}
return checkDecommission(decommissionBackendClause.getHostInfos());
}
@ -226,6 +235,29 @@ public class SystemHandler extends AlterHandler {
return decommissionBackends;
}
public static List<Backend> checkDecommissionByIds(List<String> ids)
throws DdlException {
SystemInfoService infoService = Env.getCurrentSystemInfo();
List<Backend> decommissionBackends = Lists.newArrayList();
// check if exist
for (String id : ids) {
Backend backend = infoService.getBackend(Long.parseLong(id));
if (backend == null) {
throw new DdlException("Backend does not exist, backend id is " + id);
}
if (backend.isDecommissioned()) {
// already under decommission, ignore it
continue;
}
decommissionBackends.add(backend);
}
// TODO(cmy): check if replication num can be met
// TODO(cmy): check remaining space
return decommissionBackends;
}
@Override
public synchronized void cancel(CancelStmt stmt) throws DdlException {
CancelAlterSystemStmt cancelAlterSystemStmt = (CancelAlterSystemStmt) stmt;

12
fe/fe-core/src/main/java/org/apache/doris/analysis/AddBackendClause.java
View File

@ -23,12 +23,14 @@ import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.resource.Tag;
import com.google.common.collect.Maps;
import lombok.Getter;
import java.util.List;
import java.util.Map;
public class AddBackendClause extends BackendClause {
protected Map<String, String> properties = Maps.newHashMap();
@Getter
private Map<String, String> tagMap;
public AddBackendClause(List<String> hostPorts) {
@ -43,10 +45,6 @@ public class AddBackendClause extends BackendClause {
}
}
public Map<String, String> getTagMap() {
return tagMap;
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
@ -69,9 +67,9 @@ public class AddBackendClause extends BackendClause {
StringBuilder sb = new StringBuilder();
sb.append("ADD ");
sb.append("BACKEND ");
for (int i = 0; i < hostPorts.size(); i++) {
sb.append("\"").append(hostPorts.get(i)).append("\"");
if (i != hostPorts.size() - 1) {
for (int i = 0; i < params.size(); i++) {
sb.append("\"").append(params.get(i)).append("\"");
if (i != params.size() - 1) {
sb.append(", ");
}
}

12
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSystemStmt.java
View File

@ -25,19 +25,17 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import lombok.Getter;
@Getter
public class AlterSystemStmt extends DdlStmt {
private AlterClause alterClause;
private final AlterClause alterClause;
public AlterSystemStmt(AlterClause alterClause) {
this.alterClause = alterClause;
}
public AlterClause getAlterClause() {
return alterClause;
}
@Override
public void analyze(Analyzer analyzer) throws UserException {
@ -65,9 +63,7 @@ public class AlterSystemStmt extends DdlStmt {
@Override
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("ALTER SYSTEM ").append(alterClause.toSql());
return sb.toString();
return "ALTER SYSTEM " + alterClause.toSql();
}
@Override

31
fe/fe-core/src/main/java/org/apache/doris/analysis/BackendClause.java
View File

@ -24,37 +24,46 @@ import org.apache.doris.system.SystemInfoService.HostInfo;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import lombok.Getter;
import org.apache.commons.lang3.NotImplementedException;
import java.util.List;
import java.util.Map;
public class BackendClause extends AlterClause {
protected List<String> hostPorts;
protected List<String> params;
@Getter
protected List<HostInfo> hostInfos;
@Getter
protected List<String> ids;
public static final String MUTLI_TAG_DISABLED_MSG = "Not support multi tags for Backend now. "
+ "You can set 'enable_multi_tags=true' in fe.conf to enable this feature.";
public static final String NEED_LOCATION_TAG_MSG
= "Backend must have location type tag. Eg: 'tag.location' = 'xxx'.";
protected BackendClause(List<String> hostPorts) {
protected BackendClause(List<String> params) {
super(AlterOpType.ALTER_OTHER);
this.hostPorts = hostPorts;
this.params = params;
this.ids = Lists.newArrayList();
this.hostInfos = Lists.newArrayList();
}
public List<HostInfo> getHostInfos() {
return hostInfos;
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
for (String hostPort : hostPorts) {
HostInfo hostInfo = SystemInfoService.getHostAndPort(hostPort);
hostInfos.add(hostInfo);
for (String param : params) {
if (!param.contains(":")) {
ids.add(param);
} else {
HostInfo hostInfo = SystemInfoService.getHostAndPort(param);
this.hostInfos.add(hostInfo);
}
}
Preconditions.checkState(!hostInfos.isEmpty());
Preconditions.checkState(!this.hostInfos.isEmpty() || !this.ids.isEmpty(),
"hostInfos or ids can not be empty");
}
@Override

6
fe/fe-core/src/main/java/org/apache/doris/analysis/DecommissionBackendClause.java
View File

@ -29,9 +29,9 @@ public class DecommissionBackendClause extends BackendClause {
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("DECOMMISSION BACKEND ");
for (int i = 0; i < hostPorts.size(); i++) {
sb.append("\"").append(hostPorts.get(i)).append("\"");
if (i != hostPorts.size() - 1) {
for (int i = 0; i < params.size(); i++) {
sb.append("\"").append(params.get(i)).append("\"");
if (i != params.size() - 1) {
sb.append(", ");
}
}

41
fe/fe-core/src/main/java/org/apache/doris/analysis/DropBackendClause.java
View File

@ -17,52 +17,31 @@
package org.apache.doris.analysis;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.system.SystemInfoService.HostInfo;
import com.google.common.base.Preconditions;
import lombok.Getter;
import java.util.List;
@Getter
public class DropBackendClause extends BackendClause {
private boolean force;
private final boolean force;
public DropBackendClause(List<String> hostPorts) {
super(hostPorts);
public DropBackendClause(List<String> params) {
super(params);
this.force = true;
}
public DropBackendClause(List<String> hostPorts, boolean force) {
super(hostPorts);
public DropBackendClause(List<String> params, boolean force) {
super(params);
this.force = force;
}
public boolean isForce() {
return force;
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
if (Config.enable_fqdn_mode) {
for (String hostPort : hostPorts) {
HostInfo hostInfo = SystemInfoService.getHostAndPort(hostPort);
hostInfos.add(hostInfo);
}
Preconditions.checkState(!hostInfos.isEmpty());
} else {
super.analyze(analyzer);
}
}
@Override
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("DROP BACKEND ");
for (int i = 0; i < hostPorts.size(); i++) {
sb.append("\"").append(hostPorts.get(i)).append("\"");
if (i != hostPorts.size() - 1) {
for (int i = 0; i < params.size(); i++) {
sb.append("\"").append(params.get(i)).append("\"");
if (i != params.size() - 1) {
sb.append(", ");
}
}

14
fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyBackendClause.java
View File

@ -23,14 +23,16 @@ import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.resource.Tag;
import com.google.common.collect.Maps;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Map;
public class ModifyBackendClause extends BackendClause {
protected Map<String, String> properties = Maps.newHashMap();
protected Map<String, String> properties;
protected Map<String, String> analyzedProperties = Maps.newHashMap();
@Getter
private Map<String, String> tagMap = null;
private Boolean isQueryDisabled = null;
private Boolean isLoadDisabled = null;
@ -74,10 +76,6 @@ public class ModifyBackendClause extends BackendClause {
}
}
public Map<String, String> getTagMap() {
return tagMap;
}
public Boolean isQueryDisabled() {
return isQueryDisabled;
}
@ -90,9 +88,9 @@ public class ModifyBackendClause extends BackendClause {
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("MODIFY BACKEND ");
for (int i = 0; i < hostPorts.size(); i++) {
sb.append("\"").append(hostPorts.get(i)).append("\"");
if (i != hostPorts.size() - 1) {
for (int i = 0; i < params.size(); i++) {
sb.append("\"").append(params.get(i)).append("\"");
if (i != params.size() - 1) {
sb.append(", ");
}
}

39
fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
View File

@ -235,12 +235,21 @@ public class SystemInfoService {
.getHostPortInAccessibleFormat(hostInfo.getHost(), hostInfo.getPort());
throw new DdlException("backend does not exists[" + backendIdentifier + "]");
}
}
for (HostInfo hostInfo : hostInfos) {
dropBackend(hostInfo.getHost(), hostInfo.getPort());
}
}
public void dropBackendsByIds(List<String> ids) throws DdlException {
for (String id : ids) {
if (getBackend(Long.parseLong(id)) == null) {
throw new DdlException("backend does not exists[" + id + "]");
}
dropBackend(Long.parseLong(id));
}
}
// for decommission
public void dropBackend(long backendId) throws DdlException {
Backend backend = getBackend(backendId);
@ -896,14 +905,26 @@ public class SystemInfoService {
public void modifyBackends(ModifyBackendClause alterClause) throws UserException {
List<HostInfo> hostInfos = alterClause.getHostInfos();
List<Backend> backends = Lists.newArrayList();
for (HostInfo hostInfo : hostInfos) {
Backend be = getBackendWithHeartbeatPort(hostInfo.getHost(), hostInfo.getPort());
if (be == null) {
throw new DdlException(
"backend does not exists[" + NetUtils
.getHostPortInAccessibleFormat(hostInfo.getHost(), hostInfo.getPort()) + "]");
if (hostInfos.isEmpty()) {
List<String> ids = alterClause.getIds();
for (String id : ids) {
long backendId = Long.parseLong(id);
Backend be = getBackend(backendId);
if (be == null) {
throw new DdlException("backend does not exists[" + backendId + "]");
}
backends.add(be);
}
} else {
for (HostInfo hostInfo : hostInfos) {
Backend be = getBackendWithHeartbeatPort(hostInfo.getHost(), hostInfo.getPort());
if (be == null) {
throw new DdlException(
"backend does not exists[" + NetUtils
.getHostPortInAccessibleFormat(hostInfo.getHost(), hostInfo.getPort()) + "]");
}
backends.add(be);
}
backends.add(be);
}
for (Backend be : backends) {