database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Revert "[Enhancement](show-backends-disks) Add show backends disks (#24229)" (#25389)

Browse Source 
This reverts commit 21223e65c59c23cfcb9e8ab610ea321168bcb75a.
This commit is contained in:
yiguolei
2023-10-13 14:08:45 +08:00
committed by GitHub
parent 6f9a084d99
commit 6757d2f361
27 changed files with 20 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
@ -29,22 +28,11 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.collect.ImmutableList;
public class ShowBackendsStmt extends ShowStmt {
private String type;
public ShowBackendsStmt() {
}
public ShowBackendsStmt(String type) {
this.type = type;
}
public String getType() {
return type;
}
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
@ -54,22 +42,12 @@ public class ShowBackendsStmt extends ShowStmt {
PrivPredicate.OPERATOR)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR");
}
if (type != null && !type.equals("disks")) {
throw new AnalysisException("Show backends with extra info only support show backends disks");
}
}
@Override
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
ImmutableList<String> titles = BackendsProcDir.TITLE_NAMES;
if (type != null && type.equals("disks")) {
titles = BackendsProcDir.DISK_TITLE_NAMES;
}
for (String title : titles) {
for (String title : BackendsProcDir.TITLE_NAMES) {
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
}
return builder.build();

13
fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
View File

@ -55,8 +55,6 @@ public class DiskInfo implements Writable {
private long diskAvailableCapacityB;
@SerializedName("state")
private DiskState state;
@SerializedName("dirType")
private String dirType;
// path hash and storage medium are reported from Backend and no need to persist
private long pathHash = 0;
private TStorageMedium storageMedium;
@ -72,7 +70,6 @@ public class DiskInfo implements Writable {
this.trashUsedCapacityB = 0;
this.diskAvailableCapacityB = DEFAULT_CAPACITY_B;
this.state = DiskState.ONLINE;
this.dirType = "STORAGE";
this.pathHash = 0;
this.storageMedium = TStorageMedium.HDD;
}
@ -133,10 +130,6 @@ public class DiskInfo implements Writable {
return state;
}
public String getDirType() {
return dirType;
}
// return true if changed
public boolean setState(DiskState state) {
if (this.state != state) {
@ -146,10 +139,6 @@ public class DiskInfo implements Writable {
return false;
}
public void setDirType(String dirType) {
this.dirType = dirType;
}
public long getPathHash() {
return pathHash;
}
@ -196,7 +185,7 @@ public class DiskInfo implements Writable {
return "DiskInfo [rootPath=" + rootPath + "(" + pathHash + "), totalCapacityB=" + totalCapacityB
+ ", dataUsedCapacityB=" + dataUsedCapacityB + ", trashUsedCapacityB=" + trashUsedCapacityB
+ ", diskAvailableCapacityB=" + diskAvailableCapacityB + ", state=" + state
+ ", dirType=" + dirType + ", medium: " + storageMedium + "]";
+ ", medium: " + storageMedium + "]";
}
@Override

74
fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.common.proc;
import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
@ -31,7 +30,6 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import org.apache.logging.log4j.LogManager;
@ -80,17 +78,6 @@ public class BackendsProcDir implements ProcDirInterface {
return result;
}
public static List<List<String>> getBackendInfos(String type) {
List<List<String>> backendInfos = new LinkedList<>();
if (type == null) {
backendInfos = getBackendInfos();
} else if (type.equals("disks")) {
backendInfos = getBackendsDiskInfos();
}
return backendInfos;
}
/**
* get backends info
*
@ -201,67 +188,6 @@ public class BackendsProcDir implements ProcDirInterface {
return backendInfos;
}
/**
* get backends disk info
*
* @return
*/
public static List<List<String>> getBackendsDiskInfos() {
final SystemInfoService systemInfoService = Env.getCurrentSystemInfo();
List<List<String>> backendsDiskInfos = new LinkedList<>();
List<Long> backendIds = systemInfoService.getAllBackendIds(false);
if (backendIds == null) {
return backendsDiskInfos;
}
List<List<Comparable>> comparableBackendsDiskInfos = new LinkedList<>();
for (long backendId : backendIds) {
Backend backend = systemInfoService.getBackend(backendId);
if (backend == null) {
continue;
}
ImmutableMap<String, DiskInfo> disksRef = backend.getAllDisks();
for (DiskInfo disk : disksRef.values()) {
List<Comparable> backendsDiskInfo = Lists.newArrayList();
backendsDiskInfo.add(String.valueOf(backendId));
backendsDiskInfo.add(backend.getHost());
// add disk info to backendsDiskInfo
backendsDiskInfo.add(disk.getRootPath());
backendsDiskInfo.add(disk.getDirType());
backendsDiskInfo.add(disk.getState());
long totalCapacityB = disk.getTotalCapacityB();
Pair<Double, String> totalCapacity = DebugUtil.getByteUint(totalCapacityB);
backendsDiskInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(
totalCapacity.first) + " " + totalCapacity.second);
long diskUsedCapacityB = disk.getDiskUsedCapacityB();
Pair<Double, String> diskUsedCapacity = DebugUtil.getByteUint(diskUsedCapacityB);
backendsDiskInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(
diskUsedCapacity.first) + " " + diskUsedCapacity.second);
long availableCapacityB = disk.getAvailableCapacityB();
Pair<Double, String> availableCapacity = DebugUtil.getByteUint(availableCapacityB);
backendsDiskInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(
availableCapacity.first) + " " + availableCapacity.second);
backendsDiskInfo.add(String.format("%.2f", disk.getUsedPct() * 100) + " %");
comparableBackendsDiskInfos.add(backendsDiskInfo);
}
}
// sort by host name
ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(1);
comparableBackendsDiskInfos.sort(comparator);
for (List<Comparable> backendsDiskInfo : comparableBackendsDiskInfos) {
List<String> oneInfo = new ArrayList<String>(backendsDiskInfo.size());
for (Comparable element : backendsDiskInfo) {
oneInfo.add(element.toString());
}
backendsDiskInfos.add(oneInfo);
}
return backendsDiskInfos;
}
@Override
public boolean register(String name, ProcNodeInterface node) {
return false;

2
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -1964,7 +1964,7 @@ public class ShowExecutor {
private void handleShowBackends() {
final ShowBackendsStmt showStmt = (ShowBackendsStmt) stmt;
List<List<String>> backendInfos = BackendsProcDir.getBackendInfos(showStmt.getType());
List<List<String>> backendInfos = BackendsProcDir.getBackendInfos();
backendInfos.sort(new Comparator<List<String>>() {
@Override

50
fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
View File

@ -59,7 +59,6 @@ public class Backend implements Writable {
// Represent a meaningless IP
public static final String DUMMY_IP = "0.0.0.0";
public static final String DATA_DIR_TYPE = "STORAGE";
@SerializedName("id")
private long id;
@ -375,44 +374,22 @@ public class Backend implements Writable {
}
public ImmutableMap<String, DiskInfo> getDisks() {
Map<String, DiskInfo> disks = Maps.newHashMap();
for (Map.Entry<String, DiskInfo> entry : disksRef.entrySet()) {
if (entry.getValue().getDirType().equals(DATA_DIR_TYPE)) {
disks.put(entry.getKey(), entry.getValue());
}
}
return ImmutableMap.copyOf(disks);
}
public ImmutableMap<String, DiskInfo> getAllDisks() {
return this.disksRef;
}
public boolean hasPathHash() {
Map<String, DiskInfo> disks = Maps.newHashMap();
for (Map.Entry<String, DiskInfo> entry : disksRef.entrySet()) {
if (entry.getValue().getDirType().equals(DATA_DIR_TYPE)) {
disks.put(entry.getKey(), entry.getValue());
}
}
return disks.values().stream().allMatch(DiskInfo::hasPathHash);
return disksRef.values().stream().allMatch(DiskInfo::hasPathHash);
}
public boolean hasSpecifiedStorageMedium(TStorageMedium storageMedium) {
Map<String, DiskInfo> disks = Maps.newHashMap();
for (Map.Entry<String, DiskInfo> entry : disksRef.entrySet()) {
if (entry.getValue().getDirType().equals(DATA_DIR_TYPE)) {
disks.put(entry.getKey(), entry.getValue());
}
}
return disks.values().stream().anyMatch(d -> d.isStorageMediumMatch(storageMedium));
return disksRef.values().stream().anyMatch(d -> d.isStorageMediumMatch(storageMedium));
}
public long getTotalCapacityB() {
ImmutableMap<String, DiskInfo> disks = disksRef;
long totalCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
if (diskInfo.getState() == DiskState.ONLINE) {
totalCapacityB += diskInfo.getTotalCapacityB();
}
}
@ -424,7 +401,7 @@ public class Backend implements Writable {
ImmutableMap<String, DiskInfo> disks = disksRef;
long availableCapacityB = 1L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
if (diskInfo.getState() == DiskState.ONLINE) {
availableCapacityB += diskInfo.getAvailableCapacityB();
}
}
@ -435,7 +412,7 @@ public class Backend implements Writable {
ImmutableMap<String, DiskInfo> disks = disksRef;
long dataUsedCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
if (diskInfo.getState() == DiskState.ONLINE) {
dataUsedCapacityB += diskInfo.getDataUsedCapacityB();
}
}
@ -446,7 +423,7 @@ public class Backend implements Writable {
ImmutableMap<String, DiskInfo> disks = disksRef;
long trashUsedCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
if (diskInfo.getState() == DiskState.ONLINE) {
trashUsedCapacityB += diskInfo.getTrashUsedCapacityB();
}
}
@ -457,7 +434,7 @@ public class Backend implements Writable {
ImmutableMap<String, DiskInfo> disks = disksRef;
long totalRemoteUsedCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
if (diskInfo.getState() == DiskState.ONLINE) {
totalRemoteUsedCapacityB += diskInfo.getRemoteUsedCapacity();
}
}
@ -468,7 +445,7 @@ public class Backend implements Writable {
ImmutableMap<String, DiskInfo> disks = disksRef;
double maxPct = 0.0;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
if (diskInfo.getState() == DiskState.ONLINE) {
double percent = diskInfo.getUsedPct();
if (percent > maxPct) {
maxPct = percent;
@ -486,7 +463,7 @@ public class Backend implements Writable {
boolean exceedLimit = true;
for (DiskInfo diskInfo : diskInfos.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getStorageMedium()
== storageMedium && !diskInfo.exceedLimit(true) && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
== storageMedium && !diskInfo.exceedLimit(true)) {
exceedLimit = false;
break;
}
@ -501,8 +478,7 @@ public class Backend implements Writable {
ImmutableMap<String, DiskInfo> diskInfos = disksRef;
boolean exceedLimit = true;
for (DiskInfo diskInfo : diskInfos.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)
&& !diskInfo.exceedLimit(true)) {
if (diskInfo.getState() == DiskState.ONLINE && !diskInfo.exceedLimit(true)) {
exceedLimit = false;
break;
}
@ -516,7 +492,7 @@ public class Backend implements Writable {
if (!initPathInfo) {
boolean allPathHashUpdated = true;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getDirType().equals(DATA_DIR_TYPE) && diskInfo.getPathHash() == 0) {
if (diskInfo.getPathHash() == 0) {
allPathHashUpdated = false;
break;
}
@ -543,7 +519,6 @@ public class Backend implements Writable {
long trashUsedCapacityB = tDisk.getTrashUsedCapacity();
long diskAvailableCapacityB = tDisk.getDiskAvailableCapacity();
boolean isUsed = tDisk.isUsed();
String dirType = tDisk.getDirType().toString();
DiskInfo diskInfo = disks.get(rootPath);
if (diskInfo == null) {
diskInfo = new DiskInfo(rootPath);
@ -557,8 +532,6 @@ public class Backend implements Writable {
diskInfo.setDataUsedCapacityB(dataUsedCapacityB);
diskInfo.setTrashUsedCapacityB(trashUsedCapacityB);
diskInfo.setAvailableCapacityB(diskAvailableCapacityB);
diskInfo.setDirType(dirType);
if (tDisk.isSetRemoteUsedCapacity()) {
diskInfo.setRemoteUsedCapacity(tDisk.getRemoteUsedCapacity());
}
@ -580,7 +553,6 @@ public class Backend implements Writable {
isChanged = true;
}
}
LOG.debug("update disk info. backendId: {}, diskInfo: {}", id, diskInfo.toString());
}

8
fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
View File

@ -868,14 +868,10 @@ public class SystemInfoService {
public void updatePathInfo(List<DiskInfo> addedDisks, List<DiskInfo> removedDisks) {
Map<Long, DiskInfo> copiedPathInfos = Maps.newHashMap(pathHashToDiskInfoRef);
for (DiskInfo diskInfo : addedDisks) {
if (diskInfo.getDirType().equals("STORAGE")) {
copiedPathInfos.put(diskInfo.getPathHash(), diskInfo);
}
copiedPathInfos.put(diskInfo.getPathHash(), diskInfo);
}
for (DiskInfo diskInfo : removedDisks) {
if (diskInfo.getDirType().equals("STORAGE")) {
copiedPathInfos.remove(diskInfo.getPathHash());
}
copiedPathInfos.remove(diskInfo.getPathHash());
}
ImmutableMap<Long, DiskInfo> newPathInfos = ImmutableMap.copyOf(copiedPathInfos);
pathHashToDiskInfoRef = newPathInfos;