From 079141e14a9e5f1feff598cd878b503ac6194cce Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 23 Jan 2019 14:08:33 +0800 Subject: [PATCH] Add disk usage percent in SHOW BACKEND stmt (#571) --- .../Contents/Administration/admin_stmt.md | 50 +++++++++++- .../org/apache/doris/catalog/BrokerMgr.java | 7 +- .../org/apache/doris/catalog/FsBroker.java | 4 +- .../doris/common/proc/BackendProcNode.java | 77 ++++++++++--------- .../doris/common/proc/BackendsProcDir.java | 22 ++++-- .../doris/common/proc/FrontendsProcNode.java | 9 ++- .../java/org/apache/doris/system/Backend.java | 13 ---- .../org/apache/doris/system/Frontend.java | 2 +- .../common/proc/BackendProcNodeTest.java | 14 +++- 9 files changed, 125 insertions(+), 73 deletions(-) diff --git a/docs/help/Contents/Administration/admin_stmt.md b/docs/help/Contents/Administration/admin_stmt.md index e16359b4b1..c0e9c0fe3e 100644 --- a/docs/help/Contents/Administration/admin_stmt.md +++ b/docs/help/Contents/Administration/admin_stmt.md @@ -265,13 +265,59 @@ # SHOW BACKENDS ## description - 该语句用于查看cluster内的节点 + 该语句用于查看 cluster 内的 BE 节点 语法： - SHOW BACKENDS + SHOW BACKENDS; + + 说明： + 1. LastStartTime 表示最近一次 BE 启动时间。 + 2. LastHeartbeat 表示最近一次心跳。 + 3. Alive 表示节点是否存活。 + 4. SystemDecommissioned 为 true 表示节点正在安全下线中。 + 5. ClusterDecommissioned 为 true 表示节点正在冲当前cluster中下线。 + 6. TabletNum 表示该节点上分片数量。 + 7. DataUsedCapacity 表示实际用户数据所占用的空间。 + 8. AvailCapacity 表示磁盘的可使用空间。 + 9. TotalCapacity 表示总磁盘空间。TotalCapacity = AvailCapacity + DataUsedCapacity + 其他非用户数据文件占用空间。 + 10. UsedPct 表示磁盘已使用量百分比。 + 11. ErrMsg 用于显示心跳失败时的错误信息。 ## keyword SHOW, BACKENDS +# SHOW FRONTENDS +## description + 该语句用于查看 FE 节点 + 语法： + SHOW FRONTENDS; + + 说明： + 1. name 表示该 FE 节点在 bdbje 中的名称。 + 2. Join 为 true 表示该节点曾经加入过集群。但不代表当前还在集群内（可能已失联） + 3. Alive 表示节点是否存活。 + 4. ReplayedJournalId 表示该节点当前已经回放的最大元数据日志id。 + 5. LastHeartbeat 是最近一次心跳。 + 6. IsHelper 表示该节点是否是 bdbje 中的 helper 节点。 + 7. ErrMsg 用于显示心跳失败时的错误信息。 + +## keyword + SHOW, FRONTENDS + +# SHOW BROKER +## description + 该语句用于查看当前存在的 broker + 语法： + SHOW BROKER; + + 说明： + 1. LastStartTime 表示最近一次 BE 启动时间。 + 2. LastHeartbeat 表示最近一次心跳。 + 3. Alive 表示节点是否存活。 + 4. ErrMsg 用于显示心跳失败时的错误信息。 + +## keyword + SHOW, BROKER + # ADMIN SET CONFIG ## description diff --git a/fe/src/main/java/org/apache/doris/catalog/BrokerMgr.java b/fe/src/main/java/org/apache/doris/catalog/BrokerMgr.java index 7cb57ec404..9f68c26d3e 100644 --- a/fe/src/main/java/org/apache/doris/catalog/BrokerMgr.java +++ b/fe/src/main/java/org/apache/doris/catalog/BrokerMgr.java @@ -40,7 +40,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.concurrent.locks.ReentrantLock; /** @@ -48,12 +47,10 @@ import java.util.concurrent.locks.ReentrantLock; */ public class BrokerMgr { public static final ImmutableList BROKER_PROC_NODE_TITLE_NAMES = new ImmutableList.Builder() - .add("Name").add("IP").add("Port").add("IsAlive") - .add("LstStartTime").add("LstUpdateTime").add("ErrMsg") + .add("Name").add("IP").add("Port").add("Alive") + .add("LastStartTime").add("LastUpdateTime").add("ErrMsg") .build(); - private final Random random = new Random(System.currentTimeMillis()); - // we need IP to find the co-location broker. // { BrokerName -> { IP -> [FsBroker] } } private final Map> brokersMap = Maps.newHashMap(); diff --git a/fe/src/main/java/org/apache/doris/catalog/FsBroker.java b/fe/src/main/java/org/apache/doris/catalog/FsBroker.java index 967cabcf01..a8b00436ec 100644 --- a/fe/src/main/java/org/apache/doris/catalog/FsBroker.java +++ b/fe/src/main/java/org/apache/doris/catalog/FsBroker.java @@ -32,7 +32,7 @@ public class FsBroker implements Writable, Comparable { // msg for ping result public String msg; public long lastUpdateTime; - public long lastStartTime; + public long lastStartTime = -1; public boolean isAlive; @@ -55,6 +55,8 @@ public class FsBroker implements Writable, Comparable { isAlive = true; isChanged = true; lastStartTime = hbResponse.getHbTime(); + } else if (lastStartTime == -1) { + lastStartTime = hbResponse.getHbTime(); } lastUpdateTime = hbResponse.getHbTime(); msg = ""; diff --git a/fe/src/main/java/org/apache/doris/common/proc/BackendProcNode.java b/fe/src/main/java/org/apache/doris/common/proc/BackendProcNode.java index 0daf9cf0e5..9de723e40f 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/BackendProcNode.java +++ b/fe/src/main/java/org/apache/doris/common/proc/BackendProcNode.java @@ -17,19 +17,23 @@ package org.apache.doris.common.proc; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - +import org.apache.doris.catalog.DiskInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.system.Backend; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.Map; + public class BackendProcNode implements ProcNodeInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("RootPath").add("TotalCapacity").add("DataUsedCapacity").add("DiskAvailableCapacity") - .add("State").add("PathHash") + .add("RootPath").add("DataUsedCapacity").add("AvailCapacity").add("TotalCapacity") + .add("UsedPct").add("State").add("PathHash") .build(); private Backend backend; @@ -46,41 +50,38 @@ public class BackendProcNode implements ProcNodeInterface { result.setNames(TITLE_NAMES); - for (String infoString : backend.getDiskInfosAsString()) { - String[] infos = infoString.split("\\|"); - Preconditions.checkState(infos.length == 6); + for (Map.Entry entry : backend.getDisks().entrySet()) { + List info = Lists.newArrayList(); + info.add(entry.getKey()); + + // data used + long dataUsedB = entry.getValue().getDataUsedCapacityB(); + Pair dataUsedUnitPair = DebugUtil.getByteUint(dataUsedB); + info.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(dataUsedUnitPair.first) + " " + + dataUsedUnitPair.second); + + // avail + long availB = entry.getValue().getAvailableCapacityB(); + Pair availUnitPair = DebugUtil.getByteUint(availB); + info.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(availUnitPair.first) + " " + availUnitPair.second); + + // total + long totalB = entry.getValue().getTotalCapacityB(); + Pair totalUnitPair = DebugUtil.getByteUint(totalB); + info.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalUnitPair.first) + " " + totalUnitPair.second); + + // used percent + double used = 0.0; + if (totalB <= 0) { + used = 0.0; + } else { + used = (double) (totalB - availB) * 100 / totalB; + } + info.add(String.format("%.2f", used) + " %"); - Pair totalUnitPair = DebugUtil.getByteUint(Long.valueOf(infos[1])); - Pair dataUsedUnitPair = DebugUtil.getByteUint(Long.valueOf(infos[2])); - Pair diskAvailableUnitPair = DebugUtil.getByteUint(Long.valueOf(infos[3])); - - String readableTotalCapacity = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalUnitPair.first) + " " - + totalUnitPair.second; - String readableDataUsedCapacity = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(dataUsedUnitPair.first) + " " - + dataUsedUnitPair.second; - String readableDiskAvailableCapacity = DebugUtil.DECIMAL_FORMAT_SCALE_3.format( - diskAvailableUnitPair.first) + " " + diskAvailableUnitPair.second; - - result.addRow(Lists.newArrayList(infos[0], readableTotalCapacity, readableDataUsedCapacity, - readableDiskAvailableCapacity, infos[4], infos[5])); + result.addRow(info); } - long totalCapacityB = backend.getTotalCapacityB(); - Pair unitPair = DebugUtil.getByteUint(totalCapacityB); - String readableTotalCapacity = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(unitPair.first) + " " + unitPair.second; - - long dataUsedCapacityB = backend.getDataUsedCapacityB(); - unitPair = DebugUtil.getByteUint(dataUsedCapacityB); - String readableDataUsedCapacity = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(unitPair.first) + " " - + unitPair.second; - - long diskAvailableCapacityB = backend.getAvailableCapacityB(); - unitPair = DebugUtil.getByteUint(diskAvailableCapacityB); - String readableDiskAvailableCapacity = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(unitPair.first) + " " - + unitPair.second; - result.addRow(Lists.newArrayList("Total", readableTotalCapacity, readableDataUsedCapacity, - readableDiskAvailableCapacity, "", "")); - return result; } diff --git a/fe/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java index c534c9bbe0..feeb912d6f 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java @@ -50,9 +50,9 @@ public class BackendsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("BackendId").add("Cluster").add("IP").add("HostName").add("HeartbeatPort") - .add("BePort").add("HttpPort").add("brpcPort").add("LastStartTime").add("LastHeartbeat").add("Alive") + .add("BePort").add("HttpPort").add("BrpcPort").add("LastStartTime").add("LastHeartbeat").add("Alive") .add("SystemDecommissioned").add("ClusterDecommissioned").add("TabletNum") - .add("DataUsedCapacity").add("TotalCapacity").add("UsedSpace").add("ErrMsg") + .add("DataUsedCapacity").add("AvailCapacity").add("TotalCapacity").add("UsedPct").add("ErrMsg") .build(); public static final int IP_INDEX = 2; @@ -153,17 +153,25 @@ public class BackendsProcDir implements ProcDirInterface { backendInfo.add(tabletNum.toString()); // capacity - Pair usedCapacity = DebugUtil.getByteUint(backend.getDataUsedCapacityB()); + // data used + long dataUsedB = backend.getDataUsedCapacityB(); + Pair usedCapacity = DebugUtil.getByteUint(dataUsedB); backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(usedCapacity.first) + " " + usedCapacity.second); - Pair totalCapacity = DebugUtil.getByteUint(backend.getTotalCapacityB()); + // available + long availB = backend.getAvailableCapacityB(); + Pair availCapacity = DebugUtil.getByteUint(availB); + backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(availCapacity.first) + " " + availCapacity.second); + // total + long totalB = backend.getTotalCapacityB(); + Pair totalCapacity = DebugUtil.getByteUint(totalB); backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalCapacity.first) + " " + totalCapacity.second); - // used space + // used percent double used = 0.0; - if (backend.getTotalCapacityB() <= 0) { + if (totalB <= 0) { used = 0.0; } else { - used = (double) backend.getDataUsedCapacityB() * 100 / backend.getTotalCapacityB(); + used = (double) (totalB - availB) * 100 / totalB; } backendInfo.add(String.format("%.2f", used) + " %"); diff --git a/fe/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java b/fe/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java index e6be11fb5a..45bf349083 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java +++ b/fe/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java @@ -1,3 +1,4 @@ + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -36,9 +37,9 @@ import java.util.List; */ public class FrontendsProcNode implements ProcNodeInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("name").add("Host").add("EditLogPort").add("HttpPort").add("QueryPort").add("RpcPort") - .add("Role").add("IsMaster").add("ClusterId").add("Join").add("IsAlive") - .add("ReplayedJournalId").add("LstUpdateTime").add("IsHelper") + .add("Name").add("Host").add("EditLogPort").add("HttpPort").add("QueryPort").add("RpcPort") + .add("Role").add("IsMaster").add("ClusterId").add("Join").add("Alive") + .add("ReplayedJournalId").add("LastHeartbeat").add("IsHelper").add("ErrMsg") .build(); private Catalog catalog; @@ -107,6 +108,8 @@ public class FrontendsProcNode implements ProcNodeInterface { info.add(String.valueOf(isHelperNode(helperNodes, fe))); + info.add(fe.getMsg()); + infos.add(info); } } diff --git a/fe/src/main/java/org/apache/doris/system/Backend.java b/fe/src/main/java/org/apache/doris/system/Backend.java index cc532ad94b..c214183b38 100644 --- a/fe/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/src/main/java/org/apache/doris/system/Backend.java @@ -37,8 +37,6 @@ import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -283,17 +281,6 @@ public class Backend implements Writable { return disksRef.get().values().stream().allMatch(v -> v.hasPathHash()); } - public List getDiskInfosAsString() { - ImmutableMap disks = disksRef.get(); - List diskInfoStrings = new LinkedList(); - for (DiskInfo diskInfo : disks.values()) { - diskInfoStrings.add(diskInfo.getRootPath() + "|" + diskInfo.getTotalCapacityB() + "|" - + diskInfo.getDataUsedCapacityB() + "|" + diskInfo.getAvailableCapacityB() + "|" - + diskInfo.getState().name() + "|" + diskInfo.getPathHash()); - } - return diskInfoStrings; - } - public long getTotalCapacityB() { ImmutableMap disks = disksRef.get(); long totalCapacityB = 0L; diff --git a/fe/src/main/java/org/apache/doris/system/Frontend.java b/fe/src/main/java/org/apache/doris/system/Frontend.java index 32f7a8f2f8..d6ae74da8a 100644 --- a/fe/src/main/java/org/apache/doris/system/Frontend.java +++ b/fe/src/main/java/org/apache/doris/system/Frontend.java @@ -40,7 +40,7 @@ public class Frontend implements Writable { private long replayedJournalId; private long lastUpdateTime; - private String msg; + private String msg = ""; private boolean isAlive = false; diff --git a/fe/src/test/java/org/apache/doris/common/proc/BackendProcNodeTest.java b/fe/src/test/java/org/apache/doris/common/proc/BackendProcNodeTest.java index a3d32d1a86..53c5ce240d 100644 --- a/fe/src/test/java/org/apache/doris/common/proc/BackendProcNodeTest.java +++ b/fe/src/test/java/org/apache/doris/common/proc/BackendProcNodeTest.java @@ -18,11 +18,14 @@ package org.apache.doris.common.proc; import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.DiskInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.persist.EditLog; import org.apache.doris.system.Backend; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.easymock.EasyMock; import org.junit.After; @@ -35,6 +38,8 @@ import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.util.Map; + @RunWith(PowerMockRunner.class) @PowerMockIgnore({ "org.apache.log4j.*", "javax.management.*" }) @PrepareForTest(Catalog.class) @@ -67,6 +72,10 @@ public class BackendProcNodeTest { b1 = new Backend(1000, "host1", 10000); b1.updateOnce(10001, 10003, 10005); + Map disks = Maps.newHashMap(); + disks.put("/home/disk1", new DiskInfo("/home/disk1")); + ImmutableMap immutableMap = ImmutableMap.copyOf(disks); + b1.setDisks(immutableMap); } @After @@ -84,9 +93,8 @@ public class BackendProcNodeTest { Assert.assertTrue(result instanceof BaseProcResult); Assert.assertTrue(result.getRows().size() >= 1); - Assert.assertEquals(Lists.newArrayList("RootPath", "TotalCapacity", "DataUsedCapacity", - "DiskAvailableCapacity", "State", "PathHash"), - result.getColumnNames()); + Assert.assertEquals(Lists.newArrayList("RootPath", "DataUsedCapacity", "AvailCapacity", + "TotalCapacity", "UsedPct", "State", "PathHash"), result.getColumnNames()); } }