From 3975360a7cc01571be72a41fceb304a64929ff5f Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 3 Jan 2018 16:13:41 +0800 Subject: [PATCH] add jvm monitor (#170) SHOW PROC '/monitor/jvm'; to get info --- fe/src/com/baidu/palo/common/Config.java | 2 +- .../baidu/palo/common/proc/JvmProcDir.java | 90 ++++ .../palo/common/proc/MonitorProcDir.java | 51 ++ .../baidu/palo/common/proc/ProcService.java | 1 + .../com/baidu/palo/monitor/jvm/GcNames.java | 59 +++ .../com/baidu/palo/monitor/jvm/JvmInfo.java | 466 +++++++++++++++++ .../baidu/palo/monitor/jvm/JvmService.java | 117 +++++ .../com/baidu/palo/monitor/jvm/JvmStats.java | 478 ++++++++++++++++++ .../baidu/palo/monitor/unit/ByteSizeUnit.java | 252 +++++++++ .../palo/monitor/unit/ByteSizeValue.java | 251 +++++++++ .../baidu/palo/monitor/unit/TimeValue.java | 363 +++++++++++++ .../com/baidu/palo/monitor/utils/Strings.java | 59 +++ 12 files changed, 2188 insertions(+), 1 deletion(-) create mode 100644 fe/src/com/baidu/palo/common/proc/JvmProcDir.java create mode 100644 fe/src/com/baidu/palo/common/proc/MonitorProcDir.java create mode 100644 fe/src/com/baidu/palo/monitor/jvm/GcNames.java create mode 100644 fe/src/com/baidu/palo/monitor/jvm/JvmInfo.java create mode 100644 fe/src/com/baidu/palo/monitor/jvm/JvmService.java create mode 100644 fe/src/com/baidu/palo/monitor/jvm/JvmStats.java create mode 100644 fe/src/com/baidu/palo/monitor/unit/ByteSizeUnit.java create mode 100644 fe/src/com/baidu/palo/monitor/unit/ByteSizeValue.java create mode 100644 fe/src/com/baidu/palo/monitor/unit/TimeValue.java create mode 100644 fe/src/com/baidu/palo/monitor/utils/Strings.java diff --git a/fe/src/com/baidu/palo/common/Config.java b/fe/src/com/baidu/palo/common/Config.java index 732fd3e563..2b26667b22 100644 --- a/fe/src/com/baidu/palo/common/Config.java +++ b/fe/src/com/baidu/palo/common/Config.java @@ -344,7 +344,7 @@ public class Config extends ConfigBase { * When create a table(or partition), you can specify its storage media(HDD or SSD). * If set to SSD, this specifies the default duration that tablets will stay on SSD. * After that, tablets will be moved to HDD automatically. - * You can set storage cooldown time in LOAD stmt. + * You can set storage cooldown time in CREATE TABLE stmt. */ @ConfField public static long storage_cooldown_second = 30 * 24 * 3600L; // 30 days /* diff --git a/fe/src/com/baidu/palo/common/proc/JvmProcDir.java b/fe/src/com/baidu/palo/common/proc/JvmProcDir.java new file mode 100644 index 0000000000..e32fd30ca3 --- /dev/null +++ b/fe/src/com/baidu/palo/common/proc/JvmProcDir.java @@ -0,0 +1,90 @@ +package com.baidu.palo.common.proc; + +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.monitor.jvm.JvmInfo; +import com.baidu.palo.monitor.jvm.JvmService; +import com.baidu.palo.monitor.jvm.JvmStats; +import com.baidu.palo.monitor.jvm.JvmStats.BufferPool; +import com.baidu.palo.monitor.jvm.JvmStats.GarbageCollector; +import com.baidu.palo.monitor.jvm.JvmStats.MemoryPool; +import com.baidu.palo.monitor.jvm.JvmStats.Threads; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.Iterator; +import java.util.List; + +public class JvmProcDir implements ProcNodeInterface { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("Name").add("Value") + .build(); + + @Override + public ProcResult fetchResult() throws AnalysisException { + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + JvmService jvmService = new JvmService(); + + // 1. jvm info + JvmInfo jvmInfo = jvmService.info(); + result.addRow(genRow("jvm start time", TimeUtils.longToTimeString(jvmInfo.getStartTime()))); + result.addRow(genRow("jvm version info", Joiner.on(" ").join(jvmInfo.getVersion(), + jvmInfo.getVmName(), + jvmInfo.getVmVendor(), + jvmInfo.getVmVersion()))); + + result.addRow(genRow("configured init heap size", jvmInfo.getConfiguredInitialHeapSize())); + result.addRow(genRow("configured max heap size", jvmInfo.getConfiguredMaxHeapSize())); + result.addRow(genRow("frontend pid", jvmInfo.getPid())); + + // 2. jvm stats + JvmStats jvmStats = jvmService.stats(); + result.addRow(genRow("classes loaded", jvmStats.getClasses().getLoadedClassCount())); + result.addRow(genRow("classes total loaded", jvmStats.getClasses().getTotalLoadedClassCount())); + result.addRow(genRow("classes unloaded", jvmStats.getClasses().getUnloadedClassCount())); + + result.addRow(genRow("mem heap committed", jvmStats.getMem().getHeapCommitted().getBytes())); + result.addRow(genRow("mem heap used", jvmStats.getMem().getHeapUsed().getBytes())); + result.addRow(genRow("mem non heap committed", jvmStats.getMem().getNonHeapCommitted().getBytes())); + result.addRow(genRow("mem non heap used", jvmStats.getMem().getNonHeapUsed().getBytes())); + + Iterator memIter = jvmStats.getMem().iterator(); + while (memIter.hasNext()) { + MemoryPool memPool = memIter.next(); + result.addRow(genRow("mem pool " + memPool.getName() + " used", memPool.getUsed().getBytes())); + result.addRow(genRow("mem pool " + memPool.getName() + " max", memPool.getMax().getBytes())); + result.addRow(genRow("mem pool " + memPool.getName() + " peak used", memPool.getPeakUsed().getBytes())); + result.addRow(genRow("mem pool " + memPool.getName() + " peak max", memPool.getPeakMax().getBytes())); + } + + for (BufferPool bp : jvmStats.getBufferPools()) { + result.addRow(genRow("buffer pool " + bp.getName() + " count", bp.getCount())); + result.addRow(genRow("buffer pool " + bp.getName() + " used", bp.getUsed().getBytes())); + result.addRow(genRow("buffer pool " + bp.getName() + " capacity", bp.getTotalCapacity().getBytes())); + } + + Iterator gcIter = jvmStats.getGc().iterator(); + while (gcIter.hasNext()) { + GarbageCollector gc = gcIter.next(); + result.addRow(genRow("gc " + gc.getName() + " collection count", gc.getCollectionCount())); + result.addRow(genRow("gc " + gc.getName() + " collection time", gc.getCollectionTime().getMillis())); + } + + Threads threads = jvmStats.getThreads(); + result.addRow(genRow("threads count", threads.getCount())); + result.addRow(genRow("threads peak count", threads.getPeakCount())); + + return result; + } + + private List genRow(String key, Object value) { + List row = Lists.newArrayList(); + row.add(key); + row.add(String.valueOf(value)); + return row; + } +} diff --git a/fe/src/com/baidu/palo/common/proc/MonitorProcDir.java b/fe/src/com/baidu/palo/common/proc/MonitorProcDir.java new file mode 100644 index 0000000000..0b172f664a --- /dev/null +++ b/fe/src/com/baidu/palo/common/proc/MonitorProcDir.java @@ -0,0 +1,51 @@ +package com.baidu.palo.common.proc; + +import com.baidu.palo.common.AnalysisException; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +public class MonitorProcDir implements ProcDirInterface { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("Name").add("Info") + .build(); + + public MonitorProcDir() { + + } + + @Override + public ProcResult fetchResult() throws AnalysisException { + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + List jvmRow = Lists.newArrayList(); + jvmRow.add("jvm"); + jvmRow.add(" "); + result.addRow(jvmRow); + + return result; + } + + @Override + public boolean register(String name, ProcNodeInterface node) { + return false; + } + + @Override + public ProcNodeInterface lookup(String name) throws AnalysisException { + if (Strings.isNullOrEmpty(name)) { + throw new AnalysisException("name is null"); + } + + if (name.equalsIgnoreCase("jvm")) { + return new JvmProcDir(); + } else { + throw new AnalysisException("unknown name: " + name); + } + } + +} diff --git a/fe/src/com/baidu/palo/common/proc/ProcService.java b/fe/src/com/baidu/palo/common/proc/ProcService.java index 6b2f70675a..3666545458 100644 --- a/fe/src/com/baidu/palo/common/proc/ProcService.java +++ b/fe/src/com/baidu/palo/common/proc/ProcService.java @@ -46,6 +46,7 @@ public final class ProcService { root.register("frontends", new FrontendsProcNode(Catalog.getInstance())); root.register("brokers", Catalog.getInstance().getBrokerMgr().getProcNode()); root.register("load_error_hub_url", new LoadErrorProcNode(Catalog.getInstance())); + root.register("monitor", new MonitorProcDir()); } // 通过指定的路径获得对应的PROC Node diff --git a/fe/src/com/baidu/palo/monitor/jvm/GcNames.java b/fe/src/com/baidu/palo/monitor/jvm/GcNames.java new file mode 100644 index 0000000000..9c6f938efb --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/jvm/GcNames.java @@ -0,0 +1,59 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.jvm; + +/** + * + * JVM heap generation name + * + */ +public class GcNames { + + public static final String YOUNG = "young"; + public static final String OLD = "old"; + public static final String SURVIVOR = "survivor"; + + /** + * Resolves the GC type by its memory pool name ({@link java.lang.management.MemoryPoolMXBean#getName()}. + */ + public static String getByMemoryPoolName(String poolName, String defaultName) { + if ("Eden Space".equals(poolName) || "PS Eden Space".equals(poolName) || "Par Eden Space".equals(poolName) + || "G1 Eden Space".equals(poolName)) { + return YOUNG; + } + if ("Survivor Space".equals(poolName) || "PS Survivor Space".equals(poolName) + || "Par Survivor Space".equals(poolName) || "G1 Survivor Space".equals(poolName)) { + return SURVIVOR; + } + if ("Tenured Gen".equals(poolName) || "PS Old Gen".equals(poolName) + || "CMS Old Gen".equals(poolName) || "G1 Old Gen".equals(poolName)) { + return OLD; + } + return defaultName; + } + + public static String getByGcName(String gcName, String defaultName) { + if ("Copy".equals(gcName) || "PS Scavenge".equals(gcName) || "ParNew".equals(gcName) + || "G1 Young Generation".equals(gcName)) { + return YOUNG; + } + if ("MarkSweepCompact".equals(gcName) || "PS MarkSweep".equals(gcName) + || "ConcurrentMarkSweep".equals(gcName) || "G1 Old Generation".equals(gcName)) { + return OLD; + } + return defaultName; + } +} diff --git a/fe/src/com/baidu/palo/monitor/jvm/JvmInfo.java b/fe/src/com/baidu/palo/monitor/jvm/JvmInfo.java new file mode 100644 index 0000000000..37cae88a57 --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/jvm/JvmInfo.java @@ -0,0 +1,466 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.jvm; + +import com.baidu.palo.monitor.unit.ByteSizeValue; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.ManagementPermission; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.PlatformManagedObject; +import java.lang.management.RuntimeMXBean; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class JvmInfo { + + private static JvmInfo INSTANCE; + + static { + RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + + // returns the @ + long pid; + String xPid = runtimeMXBean.getName(); + try { + xPid = xPid.split("@")[0]; + pid = Long.parseLong(xPid); + } catch (Exception e) { + pid = -1; + } + + long heapInit = memoryMXBean.getHeapMemoryUsage().getInit() < 0 ? + 0 : memoryMXBean.getHeapMemoryUsage().getInit(); + long heapMax = memoryMXBean.getHeapMemoryUsage().getMax() < 0 ? + 0 : memoryMXBean.getHeapMemoryUsage().getMax(); + long nonHeapInit = memoryMXBean.getNonHeapMemoryUsage().getInit() < 0 ? + 0 : memoryMXBean.getNonHeapMemoryUsage().getInit(); + long nonHeapMax = memoryMXBean.getNonHeapMemoryUsage().getMax() < 0 ? + 0 : memoryMXBean.getNonHeapMemoryUsage().getMax(); + long directMemoryMax = 0; + try { + Class vmClass = Class.forName("sun.misc.VM"); + directMemoryMax = (Long) vmClass.getMethod("maxDirectMemory").invoke(null); + } catch (Exception t) { + // ignore + } + String[] inputArguments = runtimeMXBean.getInputArguments() + .toArray(new String[runtimeMXBean.getInputArguments().size()]); + Mem mem = new Mem(heapInit, heapMax, nonHeapInit, nonHeapMax, directMemoryMax); + + String bootClassPath; + try { + bootClassPath = runtimeMXBean.getBootClassPath(); + } catch (UnsupportedOperationException e) { + // oracle java 9 + bootClassPath = System.getProperty("sun.boot.class.path"); + if (bootClassPath == null) { + // something else + bootClassPath = ""; + } + } + String classPath = runtimeMXBean.getClassPath(); + Map systemProperties = Collections.unmodifiableMap(runtimeMXBean.getSystemProperties()); + + List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); + String[] gcCollectors = new String[gcMxBeans.size()]; + for (int i = 0; i < gcMxBeans.size(); i++) { + GarbageCollectorMXBean gcMxBean = gcMxBeans.get(i); + gcCollectors[i] = gcMxBean.getName(); + } + + List memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans(); + String[] memoryPools = new String[memoryPoolMXBeans.size()]; + for (int i = 0; i < memoryPoolMXBeans.size(); i++) { + MemoryPoolMXBean memoryPoolMXBean = memoryPoolMXBeans.get(i); + memoryPools[i] = memoryPoolMXBean.getName(); + } + + String onError = null; + String onOutOfMemoryError = null; + String useCompressedOops = "unknown"; + String useG1GC = "unknown"; + String useSerialGC = "unknown"; + long configuredInitialHeapSize = -1; + long configuredMaxHeapSize = -1; + try { + @SuppressWarnings("unchecked") Class clazz = + (Class) + Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); + Class vmOptionClazz = Class.forName("com.sun.management.VMOption"); + PlatformManagedObject hotSpotDiagnosticMXBean = ManagementFactory.getPlatformMXBean(clazz); + Method vmOptionMethod = clazz.getMethod("getVMOption", String.class); + Method valueMethod = vmOptionClazz.getMethod("getValue"); + + try { + Object onErrorObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError"); + onError = (String) valueMethod.invoke(onErrorObject); + } catch (Exception ignored) { + } + + try { + Object onOutOfMemoryErrorObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError"); + onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryErrorObject); + } catch (Exception ignored) { + } + + try { + Object useCompressedOopsVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, + "UseCompressedOops"); + useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOptionObject); + } catch (Exception ignored) { + } + + try { + Object useG1GCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC"); + useG1GC = (String) valueMethod.invoke(useG1GCVmOptionObject); + } catch (Exception ignored) { + } + + try { + Object initialHeapSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, + "InitialHeapSize"); + configuredInitialHeapSize = Long.parseLong((String) valueMethod.invoke(initialHeapSizeVmOptionObject)); + } catch (Exception ignored) { + } + + try { + Object maxHeapSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxHeapSize"); + configuredMaxHeapSize = Long.parseLong((String) valueMethod.invoke(maxHeapSizeVmOptionObject)); + } catch (Exception ignored) { + } + + try { + Object useSerialGCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseSerialGC"); + useSerialGC = (String) valueMethod.invoke(useSerialGCVmOptionObject); + } catch (Exception ignored) { + } + + } catch (Exception ignored) { + + } + + INSTANCE = new JvmInfo(pid, System.getProperty("java.version"), runtimeMXBean.getVmName(), + runtimeMXBean.getVmVersion(), + runtimeMXBean.getVmVendor(), runtimeMXBean.getStartTime(), configuredInitialHeapSize, + configuredMaxHeapSize, + mem, inputArguments, bootClassPath, classPath, systemProperties, + gcCollectors, memoryPools, onError, onOutOfMemoryError, + useCompressedOops, useG1GC, useSerialGC); + } + + public static JvmInfo jvmInfo() { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new ManagementPermission("monitor")); + sm.checkPropertyAccess("*"); + } + return INSTANCE; + } + + private final long pid; + private final String version; + private final String vmName; + private final String vmVersion; + private final String vmVendor; + private final long startTime; + private final long configuredInitialHeapSize; + private final long configuredMaxHeapSize; + private final Mem mem; + private final String[] inputArguments; + private final String bootClassPath; + private final String classPath; + private final Map systemProperties; + private final String[] gcCollectors; + private final String[] memoryPools; + private final String onError; + private final String onOutOfMemoryError; + private final String useCompressedOops; + private final String useG1GC; + private final String useSerialGC; + + private JvmInfo(long pid, String version, String vmName, String vmVersion, String vmVendor, long startTime, + long configuredInitialHeapSize, long configuredMaxHeapSize, Mem mem, + String[] inputArguments, String bootClassPath, + String classPath, Map systemProperties, String[] gcCollectors, + String[] memoryPools, String onError, + String onOutOfMemoryError, String useCompressedOops, String useG1GC, String useSerialGC) { + this.pid = pid; + this.version = version; + this.vmName = vmName; + this.vmVersion = vmVersion; + this.vmVendor = vmVendor; + this.startTime = startTime; + this.configuredInitialHeapSize = configuredInitialHeapSize; + this.configuredMaxHeapSize = configuredMaxHeapSize; + this.mem = mem; + this.inputArguments = inputArguments; + this.bootClassPath = bootClassPath; + this.classPath = classPath; + this.systemProperties = systemProperties; + this.gcCollectors = gcCollectors; + this.memoryPools = memoryPools; + this.onError = onError; + this.onOutOfMemoryError = onOutOfMemoryError; + this.useCompressedOops = useCompressedOops; + this.useG1GC = useG1GC; + this.useSerialGC = useSerialGC; + } + + /** + * The process id. + */ + public long pid() { + return this.pid; + } + + /** + * The process id. + */ + public long getPid() { + return pid; + } + + public String version() { + return this.version; + } + + public String getVersion() { + return this.version; + } + + public int versionAsInteger() { + try { + int i = 0; + String sVersion = ""; + for (; i < version.length(); i++) { + if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { + break; + } + if (version.charAt(i) != '.') { + sVersion += version.charAt(i); + } + } + if (i == 0) { + return -1; + } + return Integer.parseInt(sVersion); + } catch (Exception e) { + return -1; + } + } + + public int versionUpdatePack() { + try { + int i = 0; + String sVersion = ""; + for (; i < version.length(); i++) { + if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { + break; + } + if (version.charAt(i) != '.') { + sVersion += version.charAt(i); + } + } + if (i == 0) { + return -1; + } + Integer.parseInt(sVersion); + int from; + if (version.charAt(i) == '_') { + // 1.7.0_4 + from = ++i; + } else if (version.charAt(i) == '-' && version.charAt(i + 1) == 'u') { + // 1.7.0-u2-b21 + i = i + 2; + from = i; + } else { + return -1; + } + for (; i < version.length(); i++) { + if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') { + break; + } + } + if (from == i) { + return -1; + } + return Integer.parseInt(version.substring(from, i)); + } catch (Exception e) { + return -1; + } + } + + public String getVmName() { + return this.vmName; + } + + public String getVmVersion() { + return this.vmVersion; + } + + public String getVmVendor() { + return this.vmVendor; + } + + public long getStartTime() { + return this.startTime; + } + + public Mem getMem() { + return this.mem; + } + + public String[] getInputArguments() { + return this.inputArguments; + } + + public String getBootClassPath() { + return this.bootClassPath; + } + + public String getClassPath() { + return this.classPath; + } + + public Map getSystemProperties() { + return this.systemProperties; + } + + public long getConfiguredInitialHeapSize() { + return configuredInitialHeapSize; + } + + public long getConfiguredMaxHeapSize() { + return configuredMaxHeapSize; + } + + public String onError() { + return onError; + } + + public String onOutOfMemoryError() { + return onOutOfMemoryError; + } + + /** + * The value of the JVM flag UseCompressedOops, if available otherwise + * "unknown". The value "unknown" indicates that an attempt was + * made to obtain the value of the flag on this JVM and the attempt + * failed. + * + * @return the value of the JVM flag UseCompressedOops or "unknown" + */ + public String useCompressedOops() { + return this.useCompressedOops; + } + + public String useG1GC() { + return this.useG1GC; + } + + public String useSerialGC() { + return this.useSerialGC; + } + + public String[] getGcCollectors() { + return gcCollectors; + } + + public String[] getMemoryPools() { + return memoryPools; + } + + static final class Fields { + static final String JVM = "jvm"; + static final String PID = "pid"; + static final String VERSION = "version"; + static final String VM_NAME = "vm_name"; + static final String VM_VERSION = "vm_version"; + static final String VM_VENDOR = "vm_vendor"; + static final String START_TIME = "start_time"; + static final String START_TIME_IN_MILLIS = "start_time_in_millis"; + + static final String MEM = "mem"; + static final String HEAP_INIT = "heap_init"; + static final String HEAP_INIT_IN_BYTES = "heap_init_in_bytes"; + static final String HEAP_MAX = "heap_max"; + static final String HEAP_MAX_IN_BYTES = "heap_max_in_bytes"; + static final String NON_HEAP_INIT = "non_heap_init"; + static final String NON_HEAP_INIT_IN_BYTES = "non_heap_init_in_bytes"; + static final String NON_HEAP_MAX = "non_heap_max"; + static final String NON_HEAP_MAX_IN_BYTES = "non_heap_max_in_bytes"; + static final String DIRECT_MAX = "direct_max"; + static final String DIRECT_MAX_IN_BYTES = "direct_max_in_bytes"; + static final String GC_COLLECTORS = "gc_collectors"; + static final String MEMORY_POOLS = "memory_pools"; + static final String USING_COMPRESSED_OOPS = "using_compressed_ordinary_object_pointers"; + static final String INPUT_ARGUMENTS = "input_arguments"; + } + + public static class Mem { + + private final long heapInit; + private final long heapMax; + private final long nonHeapInit; + private final long nonHeapMax; + private final long directMemoryMax; + + public Mem(long heapInit, long heapMax, long nonHeapInit, long nonHeapMax, long directMemoryMax) { + this.heapInit = heapInit; + this.heapMax = heapMax; + this.nonHeapInit = nonHeapInit; + this.nonHeapMax = nonHeapMax; + this.directMemoryMax = directMemoryMax; + } + + public ByteSizeValue getHeapInit() { + return new ByteSizeValue(heapInit); + } + + public ByteSizeValue getHeapMax() { + return new ByteSizeValue(heapMax); + } + + public ByteSizeValue getNonHeapInit() { + return new ByteSizeValue(nonHeapInit); + } + + public ByteSizeValue getNonHeapMax() { + return new ByteSizeValue(nonHeapMax); + } + + public ByteSizeValue getDirectMemoryMax() { + return new ByteSizeValue(directMemoryMax); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("heap init: ").append(getHeapInit().toString()); + sb.append(", heap max: ").append(getHeapMax().toString()); + sb.append(", non heap init: ").append(getNonHeapInit().toString()); + sb.append(", non heap max: ").append(getNonHeapMax().toString()); + sb.append(" direct mem max: ").append(getDirectMemoryMax().toString()); + return sb.toString(); + } + } +} diff --git a/fe/src/com/baidu/palo/monitor/jvm/JvmService.java b/fe/src/com/baidu/palo/monitor/jvm/JvmService.java new file mode 100644 index 0000000000..15ed457c26 --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/jvm/JvmService.java @@ -0,0 +1,117 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.jvm; + +import com.baidu.palo.monitor.jvm.JvmStats.BufferPool; +import com.baidu.palo.monitor.jvm.JvmStats.Classes; +import com.baidu.palo.monitor.jvm.JvmStats.GarbageCollector; +import com.baidu.palo.monitor.jvm.JvmStats.GarbageCollectors; +import com.baidu.palo.monitor.jvm.JvmStats.Mem; +import com.baidu.palo.monitor.jvm.JvmStats.Threads; + +import com.google.common.base.Joiner; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; + +/** + * Obtain JVMInfo and JvmStats + */ +public class JvmService { + private static final Logger LOG = LogManager.getLogger(JvmService.class); + + private final JvmInfo jvmInfo; + + private JvmStats jvmStats; + + public JvmService() { + this.jvmInfo = JvmInfo.jvmInfo(); + this.jvmStats = JvmStats.jvmStats(); + } + + public JvmInfo info() { + return this.jvmInfo; + } + + public synchronized JvmStats stats() { + jvmStats = JvmStats.jvmStats(); + return jvmStats; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + // 1. jvm stats + JvmStats jvmStats = stats(); + + // buffer pool + List bufferPools = jvmStats.getBufferPools(); + sb.append("JVM Stats: \nBuffer pools:\n"); + for (BufferPool bufferPool : bufferPools) { + sb.append("\t").append(bufferPool.toString()).append("\n"); + } + + // classes + Classes classes = jvmStats.getClasses(); + sb.append(classes.toString()).append("\n"); + + // gc + GarbageCollectors gc = jvmStats.getGc(); + GarbageCollector[] gcs = gc.getCollectors(); + sb.append("Garbage Collectors: \n"); + for (GarbageCollector garbageCollector : gcs) { + sb.append("\t").append(garbageCollector.toString()); + } + + // mem + Mem mem = jvmStats.getMem(); + sb.append("\nMem: ").append(mem.toString()); + + // threads + Threads threads = jvmStats.getThreads(); + sb.append("\nThreads: ").append(threads.toString()); + + sb.append("\nUpTime: ").append(jvmStats.getUptime().toString()); + sb.append("\nTimestamp: ").append(jvmStats.getTimestamp()); + + LOG.info(sb.toString()); + + // 2. jvm info + JvmInfo jvmInfo = info(); + + sb.append("\nJVM Info: \nboot class path: ").append(jvmInfo.getBootClassPath()); + sb.append("\nclass path: ").append(jvmInfo.getClassPath()); + sb.append("\nconfigured init heap size: ").append(jvmInfo.getConfiguredInitialHeapSize()); + sb.append("\nconfigured max heap size: ").append(jvmInfo.getConfiguredMaxHeapSize()); + sb.append("\npid: ").append(jvmInfo.getPid()); + sb.append("\nstart time: ").append(jvmInfo.getStartTime()); + sb.append("\nversion: ").append(jvmInfo.getVersion()); + sb.append("\nvm name").append(jvmInfo.getVmName()); + sb.append("\nvm vendor: ").append(jvmInfo.getVmVendor()); + sb.append("\nvm version").append(jvmInfo.getVmVersion()); + + sb.append("\ngcs: ").append(Joiner.on(", ").join(jvmInfo.getGcCollectors())); + sb.append("\ninput arguments: ").append(Joiner.on(", ").join(jvmInfo.getInputArguments())); + sb.append("\nmem: ").append(jvmInfo.getMem().toString()); + sb.append("\nmem pools: ").append(Joiner.on(", ").join(jvmInfo.getMemoryPools())); + sb.append("\nsystem props: ").append(jvmInfo.getSystemProperties()); + + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/monitor/jvm/JvmStats.java b/fe/src/com/baidu/palo/monitor/jvm/JvmStats.java new file mode 100644 index 0000000000..e22e0ff003 --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/jvm/JvmStats.java @@ -0,0 +1,478 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.jvm; + +import com.baidu.palo.monitor.unit.ByteSizeValue; +import com.baidu.palo.monitor.unit.TimeValue; + +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.RuntimeMXBean; +import java.lang.management.ThreadMXBean; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class JvmStats { + + private static final RuntimeMXBean runtimeMXBean; + private static final MemoryMXBean memoryMXBean; + private static final ThreadMXBean threadMXBean; + private static final ClassLoadingMXBean classLoadingMXBean; + + static { + runtimeMXBean = ManagementFactory.getRuntimeMXBean(); + memoryMXBean = ManagementFactory.getMemoryMXBean(); + threadMXBean = ManagementFactory.getThreadMXBean(); + classLoadingMXBean = ManagementFactory.getClassLoadingMXBean(); + } + + public static JvmStats jvmStats() { + MemoryUsage memUsage = memoryMXBean.getHeapMemoryUsage(); + long heapUsed = memUsage.getUsed() < 0 ? 0 : memUsage.getUsed(); + long heapCommitted = memUsage.getCommitted() < 0 ? 0 : memUsage.getCommitted(); + long heapMax = memUsage.getMax() < 0 ? 0 : memUsage.getMax(); + memUsage = memoryMXBean.getNonHeapMemoryUsage(); + long nonHeapUsed = memUsage.getUsed() < 0 ? 0 : memUsage.getUsed(); + long nonHeapCommitted = memUsage.getCommitted() < 0 ? 0 : memUsage.getCommitted(); + List memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans(); + List pools = new ArrayList<>(); + for (MemoryPoolMXBean memoryPoolMXBean : memoryPoolMXBeans) { + try { + MemoryUsage usage = memoryPoolMXBean.getUsage(); + MemoryUsage peakUsage = memoryPoolMXBean.getPeakUsage(); + String name = GcNames.getByMemoryPoolName(memoryPoolMXBean.getName(), null); + if (name == null) { // if we can't resolve it, its not interesting.... (Per Gen, Code Cache) + continue; + } + pools.add(new MemoryPool(name, + usage.getUsed() < 0 ? 0 : usage.getUsed(), + usage.getMax() < 0 ? 0 : usage.getMax(), + peakUsage.getUsed() < 0 ? 0 : peakUsage.getUsed(), + peakUsage.getMax() < 0 ? 0 : peakUsage.getMax() + )); + } catch (Exception ex) { + /* ignore some JVMs might barf here with: + * java.lang.InternalError: Memory Pool not found + * we just omit the pool in that case!*/ + } + } + Mem mem = new Mem(heapCommitted, heapUsed, heapMax, nonHeapCommitted, nonHeapUsed, + Collections.unmodifiableList(pools)); + Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount()); + + List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); + GarbageCollector[] collectors = new GarbageCollector[gcMxBeans.size()]; + for (int i = 0; i < collectors.length; i++) { + GarbageCollectorMXBean gcMxBean = gcMxBeans.get(i); + collectors[i] = new GarbageCollector(GcNames.getByGcName(gcMxBean.getName(), gcMxBean.getName()), + gcMxBean.getCollectionCount(), gcMxBean.getCollectionTime()); + } + GarbageCollectors garbageCollectors = new GarbageCollectors(collectors); + List bufferPoolsList = Collections.emptyList(); + try { + List bufferPools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + bufferPoolsList = new ArrayList<>(bufferPools.size()); + for (BufferPoolMXBean bufferPool : bufferPools) { + bufferPoolsList.add(new BufferPool(bufferPool.getName(), bufferPool.getCount(), + bufferPool.getTotalCapacity(), bufferPool.getMemoryUsed())); + } + } catch (Exception e) { + // buffer pools are not available + } + + Classes classes = new Classes(classLoadingMXBean.getLoadedClassCount(), + classLoadingMXBean.getTotalLoadedClassCount(), + classLoadingMXBean.getUnloadedClassCount()); + + return new JvmStats(System.currentTimeMillis(), runtimeMXBean.getUptime(), mem, threads, + garbageCollectors, bufferPoolsList, classes); + } + + private final long timestamp; + private final long uptime; + private final Mem mem; + private final Threads threads; + private final GarbageCollectors gc; + private final List bufferPools; + private final Classes classes; + + public JvmStats(long timestamp, long uptime, Mem mem, Threads threads, GarbageCollectors gc, + List bufferPools, Classes classes) { + this.timestamp = timestamp; + this.uptime = uptime; + this.mem = mem; + this.threads = threads; + this.gc = gc; + this.bufferPools = bufferPools; + this.classes = classes; + } + + public long getTimestamp() { + return timestamp; + } + + public TimeValue getUptime() { + return new TimeValue(uptime); + } + + public Mem getMem() { + return this.mem; + } + + public Threads getThreads() { + return threads; + } + + public GarbageCollectors getGc() { + return gc; + } + + public List getBufferPools() { + return bufferPools; + } + + public Classes getClasses() { + return classes; + } + + static final class Fields { + static final String JVM = "jvm"; + static final String TIMESTAMP = "timestamp"; + static final String UPTIME = "uptime"; + static final String UPTIME_IN_MILLIS = "uptime_in_millis"; + + static final String MEM = "mem"; + static final String HEAP_USED = "heap_used"; + static final String HEAP_USED_IN_BYTES = "heap_used_in_bytes"; + static final String HEAP_USED_PERCENT = "heap_used_percent"; + static final String HEAP_MAX = "heap_max"; + static final String HEAP_MAX_IN_BYTES = "heap_max_in_bytes"; + static final String HEAP_COMMITTED = "heap_committed"; + static final String HEAP_COMMITTED_IN_BYTES = "heap_committed_in_bytes"; + + static final String NON_HEAP_USED = "non_heap_used"; + static final String NON_HEAP_USED_IN_BYTES = "non_heap_used_in_bytes"; + static final String NON_HEAP_COMMITTED = "non_heap_committed"; + static final String NON_HEAP_COMMITTED_IN_BYTES = "non_heap_committed_in_bytes"; + + static final String POOLS = "pools"; + static final String USED = "used"; + static final String USED_IN_BYTES = "used_in_bytes"; + static final String MAX = "max"; + static final String MAX_IN_BYTES = "max_in_bytes"; + static final String PEAK_USED = "peak_used"; + static final String PEAK_USED_IN_BYTES = "peak_used_in_bytes"; + static final String PEAK_MAX = "peak_max"; + static final String PEAK_MAX_IN_BYTES = "peak_max_in_bytes"; + + static final String THREADS = "threads"; + static final String COUNT = "count"; + static final String PEAK_COUNT = "peak_count"; + + static final String GC = "gc"; + static final String COLLECTORS = "collectors"; + static final String COLLECTION_COUNT = "collection_count"; + static final String COLLECTION_TIME = "collection_time"; + static final String COLLECTION_TIME_IN_MILLIS = "collection_time_in_millis"; + + static final String BUFFER_POOLS = "buffer_pools"; + static final String TOTAL_CAPACITY = "total_capacity"; + static final String TOTAL_CAPACITY_IN_BYTES = "total_capacity_in_bytes"; + + static final String CLASSES = "classes"; + static final String CURRENT_LOADED_COUNT = "current_loaded_count"; + static final String TOTAL_LOADED_COUNT = "total_loaded_count"; + static final String TOTAL_UNLOADED_COUNT = "total_unloaded_count"; + } + + public static class GarbageCollectors implements Iterable { + + private final GarbageCollector[] collectors; + + public GarbageCollectors(GarbageCollector[] collectors) { + this.collectors = collectors; + } + + public GarbageCollector[] getCollectors() { + return this.collectors; + } + + @Override + public Iterator iterator() { + return Arrays.stream(collectors).iterator(); + } + } + + public static class GarbageCollector { + + private final String name; + private final long collectionCount; + private final long collectionTime; + + public GarbageCollector(String name, long collectionCount, long collectionTime) { + this.name = name; + this.collectionCount = collectionCount; + this.collectionTime = collectionTime; + } + + public String getName() { + return this.name; + } + + public long getCollectionCount() { + return this.collectionCount; + } + + public TimeValue getCollectionTime() { + return new TimeValue(collectionTime, TimeUnit.MILLISECONDS); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(name); + sb.append(": collection count: ").append(collectionCount); + sb.append(", collection time: ").append(getCollectionTime().getStringRep()); + return sb.toString(); + } + } + + public static class Threads { + + private final int count; + private final int peakCount; + + public Threads(int count, int peakCount) { + this.count = count; + this.peakCount = peakCount; + } + + public int getCount() { + return count; + } + + public int getPeakCount() { + return peakCount; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("count: ").append(count).append(", peak count: ").append(peakCount); + return sb.toString(); + } + } + + public static class MemoryPool { + + private final String name; + private final long used; + private final long max; + private final long peakUsed; + private final long peakMax; + + public MemoryPool(String name, long used, long max, long peakUsed, long peakMax) { + this.name = name; + this.used = used; + this.max = max; + this.peakUsed = peakUsed; + this.peakMax = peakMax; + } + + public String getName() { + return this.name; + } + + public ByteSizeValue getUsed() { + return new ByteSizeValue(used); + } + + public ByteSizeValue getMax() { + return new ByteSizeValue(max); + } + + public ByteSizeValue getPeakUsed() { + return new ByteSizeValue(peakUsed); + } + + public ByteSizeValue getPeakMax() { + return new ByteSizeValue(peakMax); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("name: ").append(name).append(", used: ").append(getUsed().toString()); + sb.append(", max: ").append(getMax().toString()).append(", peak used: ").append(getPeakUsed().toString()); + sb.append(", peak max: ").append(getPeakMax().toString()); + return sb.toString(); + } + } + + public static class Mem implements Iterable { + + private final long heapCommitted; + private final long heapUsed; + private final long heapMax; + private final long nonHeapCommitted; + private final long nonHeapUsed; + private final List pools; + + public Mem(long heapCommitted, long heapUsed, long heapMax, + long nonHeapCommitted, long nonHeapUsed, List pools) { + this.heapCommitted = heapCommitted; + this.heapUsed = heapUsed; + this.heapMax = heapMax; + this.nonHeapCommitted = nonHeapCommitted; + this.nonHeapUsed = nonHeapUsed; + this.pools = pools; + } + + @Override + public Iterator iterator() { + return pools.iterator(); + } + + public ByteSizeValue getHeapCommitted() { + return new ByteSizeValue(heapCommitted); + } + + public ByteSizeValue getHeapUsed() { + return new ByteSizeValue(heapUsed); + } + + /** + * returns the maximum heap size. 0 bytes signals unknown. + */ + public ByteSizeValue getHeapMax() { + return new ByteSizeValue(heapMax); + } + + /** + * returns the heap usage in percent. -1 signals unknown. + */ + public short getHeapUsedPercent() { + if (heapMax == 0) { + return -1; + } + return (short) (heapUsed * 100 / heapMax); + } + + public ByteSizeValue getNonHeapCommitted() { + return new ByteSizeValue(nonHeapCommitted); + } + + public ByteSizeValue getNonHeapUsed() { + return new ByteSizeValue(nonHeapUsed); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("heap committed: ").append(getHeapCommitted().toString()); + sb.append(", heap used: ").append(getHeapUsed().toString()); + sb.append(", heap max: ").append(getHeapMax().toString()); + sb.append(", non heap committed: ").append(getNonHeapCommitted().toString()); + sb.append(", non heap used: ").append(getNonHeapUsed().toString()); + sb.append("\nMem pools: "); + for (MemoryPool memoryPool : pools) { + sb.append(memoryPool.toString()).append("\n"); + } + return sb.toString(); + } + } + + public static class BufferPool { + + private final String name; + private final long count; + private final long totalCapacity; + private final long used; + + public BufferPool(String name, long count, long totalCapacity, long used) { + this.name = name; + this.count = count; + this.totalCapacity = totalCapacity; + this.used = used; + } + + public String getName() { + return this.name; + } + + public long getCount() { + return this.count; + } + + public ByteSizeValue getTotalCapacity() { + return new ByteSizeValue(totalCapacity); + } + + public ByteSizeValue getUsed() { + return new ByteSizeValue(used); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("name: ").append(name).append(", count: ").append(count); + sb.append("total capacity: ").append(totalCapacity).append(", used: ").append(used); + return sb.toString(); + } + } + + public static class Classes { + + private final long loadedClassCount; + private final long totalLoadedClassCount; + private final long unloadedClassCount; + + public Classes(long loadedClassCount, long totalLoadedClassCount, long unloadedClassCount) { + this.loadedClassCount = loadedClassCount; + this.totalLoadedClassCount = totalLoadedClassCount; + this.unloadedClassCount = unloadedClassCount; + } + + public long getLoadedClassCount() { + return loadedClassCount; + } + + public long getTotalLoadedClassCount() { + return totalLoadedClassCount; + } + + public long getUnloadedClassCount() { + return unloadedClassCount; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Classes: ").append("loaded: ").append(loadedClassCount); + sb.append(", total loaded: ").append(totalLoadedClassCount); + sb.append(", unloaded: ").append(unloadedClassCount); + return sb.toString(); + } + } +} diff --git a/fe/src/com/baidu/palo/monitor/unit/ByteSizeUnit.java b/fe/src/com/baidu/palo/monitor/unit/ByteSizeUnit.java new file mode 100644 index 0000000000..32a503925d --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/unit/ByteSizeUnit.java @@ -0,0 +1,252 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.unit; + + +/** + * A SizeUnit represents size at a given unit of + * granularity and provides utility methods to convert across units. + * A SizeUnit does not maintain size information, but only + * helps organize and use size representations that may be maintained + * separately across various contexts. + */ +public enum ByteSizeUnit { + BYTES { + @Override + public long toBytes(long size) { + return size; + } + + @Override + public long toKB(long size) { + return size / (C1 / C0); + } + + @Override + public long toMB(long size) { + return size / (C2 / C0); + } + + @Override + public long toGB(long size) { + return size / (C3 / C0); + } + + @Override + public long toTB(long size) { + return size / (C4 / C0); + } + + @Override + public long toPB(long size) { + return size / (C5 / C0); + } + }, + KB { + @Override + public long toBytes(long size) { + return x(size, C1 / C0, MAX / (C1 / C0)); + } + + @Override + public long toKB(long size) { + return size; + } + + @Override + public long toMB(long size) { + return size / (C2 / C1); + } + + @Override + public long toGB(long size) { + return size / (C3 / C1); + } + + @Override + public long toTB(long size) { + return size / (C4 / C1); + } + + @Override + public long toPB(long size) { + return size / (C5 / C1); + } + }, + MB { + @Override + public long toBytes(long size) { + return x(size, C2 / C0, MAX / (C2 / C0)); + } + + @Override + public long toKB(long size) { + return x(size, C2 / C1, MAX / (C2 / C1)); + } + + @Override + public long toMB(long size) { + return size; + } + + @Override + public long toGB(long size) { + return size / (C3 / C2); + } + + @Override + public long toTB(long size) { + return size / (C4 / C2); + } + + @Override + public long toPB(long size) { + return size / (C5 / C2); + } + }, + GB { + @Override + public long toBytes(long size) { + return x(size, C3 / C0, MAX / (C3 / C0)); + } + + @Override + public long toKB(long size) { + return x(size, C3 / C1, MAX / (C3 / C1)); + } + + @Override + public long toMB(long size) { + return x(size, C3 / C2, MAX / (C3 / C2)); + } + + @Override + public long toGB(long size) { + return size; + } + + @Override + public long toTB(long size) { + return size / (C4 / C3); + } + + @Override + public long toPB(long size) { + return size / (C5 / C3); + } + }, + TB { + @Override + public long toBytes(long size) { + return x(size, C4 / C0, MAX / (C4 / C0)); + } + + @Override + public long toKB(long size) { + return x(size, C4 / C1, MAX / (C4 / C1)); + } + + @Override + public long toMB(long size) { + return x(size, C4 / C2, MAX / (C4 / C2)); + } + + @Override + public long toGB(long size) { + return x(size, C4 / C3, MAX / (C4 / C3)); + } + + @Override + public long toTB(long size) { + return size; + } + + @Override + public long toPB(long size) { + return size / (C5 / C4); + } + }, + PB { + @Override + public long toBytes(long size) { + return x(size, C5 / C0, MAX / (C5 / C0)); + } + + @Override + public long toKB(long size) { + return x(size, C5 / C1, MAX / (C5 / C1)); + } + + @Override + public long toMB(long size) { + return x(size, C5 / C2, MAX / (C5 / C2)); + } + + @Override + public long toGB(long size) { + return x(size, C5 / C3, MAX / (C5 / C3)); + } + + @Override + public long toTB(long size) { + return x(size, C5 / C4, MAX / (C5 / C4)); + } + + @Override + public long toPB(long size) { + return size; + } + }; + + static final long C0 = 1L; + static final long C1 = C0 * 1024L; + static final long C2 = C1 * 1024L; + static final long C3 = C2 * 1024L; + static final long C4 = C3 * 1024L; + static final long C5 = C4 * 1024L; + + static final long MAX = Long.MAX_VALUE; + + public static ByteSizeUnit fromId(int id) { + if (id < 0 || id >= values().length) { + throw new IllegalArgumentException("No byte size unit found for id [" + id + "]"); + } + return values()[id]; + } + + /** + * Scale d by m, checking for overflow. + * This has a short name to make above code more readable. + */ + static long x(long d, long m, long over) { + if (d > over) return Long.MAX_VALUE; + if (d < -over) return Long.MIN_VALUE; + return d * m; + } + + public abstract long toBytes(long size); + + public abstract long toKB(long size); + + public abstract long toMB(long size); + + public abstract long toGB(long size); + + public abstract long toTB(long size); + + public abstract long toPB(long size); + +} diff --git a/fe/src/com/baidu/palo/monitor/unit/ByteSizeValue.java b/fe/src/com/baidu/palo/monitor/unit/ByteSizeValue.java new file mode 100644 index 0000000000..f9993dbf6e --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/unit/ByteSizeValue.java @@ -0,0 +1,251 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.unit; + +import com.baidu.palo.monitor.utils.Strings; + +import java.util.Locale; +import java.util.Objects; + +public class ByteSizeValue implements Comparable { + + private final long size; + private final ByteSizeUnit unit; + + public ByteSizeValue(long bytes) { + this(bytes, ByteSizeUnit.BYTES); + } + + public ByteSizeValue(long size, ByteSizeUnit unit) { + this.size = size; + this.unit = unit; + } + + public int bytesAsInt() { + long bytes = getBytes(); + if (bytes > Integer.MAX_VALUE) { + throw new IllegalArgumentException("size [" + toString() + "] is bigger than max int"); + } + return (int) bytes; + } + + public long getBytes() { + return unit.toBytes(size); + } + + public long getKb() { + return unit.toKB(size); + } + + public long getMb() { + return unit.toMB(size); + } + + public long getGb() { + return unit.toGB(size); + } + + public long getTb() { + return unit.toTB(size); + } + + public long getPb() { + return unit.toPB(size); + } + + public double getKbFrac() { + return ((double) getBytes()) / ByteSizeUnit.C1; + } + + public double getMbFrac() { + return ((double) getBytes()) / ByteSizeUnit.C2; + } + + public double getGbFrac() { + return ((double) getBytes()) / ByteSizeUnit.C3; + } + + public double getTbFrac() { + return ((double) getBytes()) / ByteSizeUnit.C4; + } + + public double getPbFrac() { + return ((double) getBytes()) / ByteSizeUnit.C5; + } + + @Override + public String toString() { + long bytes = getBytes(); + double value = bytes; + String suffix = "b"; + if (bytes >= ByteSizeUnit.C5) { + value = getPbFrac(); + suffix = "pb"; + } else if (bytes >= ByteSizeUnit.C4) { + value = getTbFrac(); + suffix = "tb"; + } else if (bytes >= ByteSizeUnit.C3) { + value = getGbFrac(); + suffix = "gb"; + } else if (bytes >= ByteSizeUnit.C2) { + value = getMbFrac(); + suffix = "mb"; + } else if (bytes >= ByteSizeUnit.C1) { + value = getKbFrac(); + suffix = "kb"; + } + return Strings.format1Decimals(value, suffix); + } + + public static ByteSizeValue parseBytesSizeValue(String sValue, String settingName) throws Exception { + return parseBytesSizeValue(sValue, null, settingName); + } + + public static ByteSizeValue parseBytesSizeValue(String sValue, ByteSizeValue defaultValue, String settingName) + throws Exception { + settingName = Objects.requireNonNull(settingName); + if (sValue == null) { + return defaultValue; + } + long bytes; + try { + String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim(); + if (lowerSValue.endsWith("k")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 1)) * ByteSizeUnit.C1); + } else if (lowerSValue.endsWith("kb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 2)) * ByteSizeUnit.C1); + } else if (lowerSValue.endsWith("m")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 1)) * ByteSizeUnit.C2); + } else if (lowerSValue.endsWith("mb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 2)) * ByteSizeUnit.C2); + } else if (lowerSValue.endsWith("g")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 1)) * ByteSizeUnit.C3); + } else if (lowerSValue.endsWith("gb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 2)) * ByteSizeUnit.C3); + } else if (lowerSValue.endsWith("t")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 1)) * ByteSizeUnit.C4); + } else if (lowerSValue.endsWith("tb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 2)) * ByteSizeUnit.C4); + } else if (lowerSValue.endsWith("p")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 1)) * ByteSizeUnit.C5); + } else if (lowerSValue.endsWith("pb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, + lowerSValue.length() - 2)) * ByteSizeUnit.C5); + } else if (lowerSValue.endsWith("b")) { + bytes = Long.parseLong(lowerSValue.substring(0, lowerSValue.length() - 1).trim()); + } else if (lowerSValue.equals("-1")) { + // Allow this special value to be unit-less: + bytes = -1; + } else if (lowerSValue.equals("0")) { + // Allow this special value to be unit-less: + bytes = 0; + } else { + // Missing units: + throw new NumberFormatException("failed to parse string"); + } + } catch (NumberFormatException e) { + throw new NumberFormatException("failed to parse string"); + } + return new ByteSizeValue(bytes, ByteSizeUnit.BYTES); + } + + public static long simpleParseBytesSizeValue(String sValue, String settingName) throws Exception { + long bytes; + try { + String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim(); + if (lowerSValue.endsWith("k")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) + * ByteSizeUnit.C1); + } else if (lowerSValue.endsWith("kb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) + * ByteSizeUnit.C1); + } else if (lowerSValue.endsWith("m")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) + * ByteSizeUnit.C2); + } else if (lowerSValue.endsWith("mb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) + * ByteSizeUnit.C2); + } else if (lowerSValue.endsWith("g")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) + * ByteSizeUnit.C3); + } else if (lowerSValue.endsWith("gb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) + * ByteSizeUnit.C3); + } else if (lowerSValue.endsWith("t")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) + * ByteSizeUnit.C4); + } else if (lowerSValue.endsWith("tb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) + * ByteSizeUnit.C4); + } else if (lowerSValue.endsWith("p")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) + * ByteSizeUnit.C5); + } else if (lowerSValue.endsWith("pb")) { + bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) + * ByteSizeUnit.C5); + } else if (lowerSValue.endsWith("b")) { + bytes = Long.parseLong(lowerSValue.substring(0, lowerSValue.length() - 1).trim()); + } else if (lowerSValue.equals("-1")) { + // Allow this special value to be unit-less: + bytes = -1; + } else if (lowerSValue.equals("0")) { + // Allow this special value to be unit-less: + bytes = 0; + } else { + // Missing units: + throw new Exception("failed to parse setting [{}] with value" + + " as a size in bytes: unit is missing or unrecognized"); + + } + } catch (NumberFormatException e) { + throw new Exception("failed to parse"); + } + return bytes; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + return compareTo((ByteSizeValue) o) == 0; + } + + @Override + public int hashCode() { + return Double.hashCode(((double) size) * unit.toBytes(1)); + } + + @Override + public int compareTo(ByteSizeValue other) { + double thisValue = ((double) size) * unit.toBytes(1); + double otherValue = ((double) other.size) * other.unit.toBytes(1); + return Double.compare(thisValue, otherValue); + } +} diff --git a/fe/src/com/baidu/palo/monitor/unit/TimeValue.java b/fe/src/com/baidu/palo/monitor/unit/TimeValue.java new file mode 100644 index 0000000000..cc3e2272ba --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/unit/TimeValue.java @@ -0,0 +1,363 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.unit; + +import com.baidu.palo.monitor.utils.Strings; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class TimeValue implements Comparable { + + /** + * How many nano-seconds in one milli-second + */ + public static final long NSEC_PER_MSEC = TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS); + + private static Map TIME_UNIT_BYTE_MAP; + private static Map BYTE_TIME_UNIT_MAP; + + static { + final Map timeUnitByteMap = new EnumMap<>(TimeUnit.class); + timeUnitByteMap.put(TimeUnit.NANOSECONDS, (byte) 0); + timeUnitByteMap.put(TimeUnit.MICROSECONDS, (byte) 1); + timeUnitByteMap.put(TimeUnit.MILLISECONDS, (byte) 2); + timeUnitByteMap.put(TimeUnit.SECONDS, (byte) 3); + timeUnitByteMap.put(TimeUnit.MINUTES, (byte) 4); + timeUnitByteMap.put(TimeUnit.HOURS, (byte) 5); + timeUnitByteMap.put(TimeUnit.DAYS, (byte) 6); + + final Set bytes = new HashSet<>(); + for (TimeUnit value : TimeUnit.values()) { + assert timeUnitByteMap.containsKey(value) : value; + assert bytes.add(timeUnitByteMap.get(value)); + } + + final Map byteTimeUnitMap = new HashMap<>(); + for (Map.Entry entry : timeUnitByteMap.entrySet()) { + byteTimeUnitMap.put(entry.getValue(), entry.getKey()); + } + + TIME_UNIT_BYTE_MAP = Collections.unmodifiableMap(timeUnitByteMap); + BYTE_TIME_UNIT_MAP = Collections.unmodifiableMap(byteTimeUnitMap); + } + + public static final TimeValue MINUS_ONE = timeValueMillis(-1); + public static final TimeValue ZERO = timeValueMillis(0); + + public static TimeValue timeValueNanos(long nanos) { + return new TimeValue(nanos, TimeUnit.NANOSECONDS); + } + + public static TimeValue timeValueMillis(long millis) { + return new TimeValue(millis, TimeUnit.MILLISECONDS); + } + + public static TimeValue timeValueSeconds(long seconds) { + return new TimeValue(seconds, TimeUnit.SECONDS); + } + + public static TimeValue timeValueMinutes(long minutes) { + return new TimeValue(minutes, TimeUnit.MINUTES); + } + + public static TimeValue timeValueHours(long hours) { + return new TimeValue(hours, TimeUnit.HOURS); + } + + private final long duration; + + // visible for testing + long duration() { + return duration; + } + + private final TimeUnit timeUnit; + + // visible for testing + TimeUnit timeUnit() { + return timeUnit; + } + + public TimeValue(long millis) { + this(millis, TimeUnit.MILLISECONDS); + } + + public TimeValue(long duration, TimeUnit timeUnit) { + this.duration = duration; + this.timeUnit = timeUnit; + } + + + public long nanos() { + return timeUnit.toNanos(duration); + } + + public long getNanos() { + return nanos(); + } + + public long micros() { + return timeUnit.toMicros(duration); + } + + public long getMicros() { + return micros(); + } + + public long millis() { + return timeUnit.toMillis(duration); + } + + public long getMillis() { + return millis(); + } + + public long seconds() { + return timeUnit.toSeconds(duration); + } + + public long getSeconds() { + return seconds(); + } + + public long minutes() { + return timeUnit.toMinutes(duration); + } + + public long getMinutes() { + return minutes(); + } + + public long hours() { + return timeUnit.toHours(duration); + } + + public long getHours() { + return hours(); + } + + public long days() { + return timeUnit.toDays(duration); + } + + public long getDays() { + return days(); + } + + public double microsFrac() { + return ((double) nanos()) / C1; + } + + public double getMicrosFrac() { + return microsFrac(); + } + + public double millisFrac() { + return ((double) nanos()) / C2; + } + + public double getMillisFrac() { + return millisFrac(); + } + + public double secondsFrac() { + return ((double) nanos()) / C3; + } + + public double getSecondsFrac() { + return secondsFrac(); + } + + public double minutesFrac() { + return ((double) nanos()) / C4; + } + + public double getMinutesFrac() { + return minutesFrac(); + } + + public double hoursFrac() { + return ((double) nanos()) / C5; + } + + public double getHoursFrac() { + return hoursFrac(); + } + + public double daysFrac() { + return ((double) nanos()) / C6; + } + + public double getDaysFrac() { + return daysFrac(); + } + + /** + * Returns a {@link String} representation of the current {@link TimeValue}. + *

+ * Note that this method might produce fractional time values (ex 1.6m) which cannot be + * parsed by method like {@link TimeValue#parse(String, String, String)}. + */ + @Override + public String toString() { + if (duration < 0) { + return Long.toString(duration); + } + long nanos = nanos(); + if (nanos == 0) { + return "0s"; + } + double value = nanos; + String suffix = "nanos"; + if (nanos >= C6) { + value = daysFrac(); + suffix = "d"; + } else if (nanos >= C5) { + value = hoursFrac(); + suffix = "h"; + } else if (nanos >= C4) { + value = minutesFrac(); + suffix = "m"; + } else if (nanos >= C3) { + value = secondsFrac(); + suffix = "s"; + } else if (nanos >= C2) { + value = millisFrac(); + suffix = "ms"; + } else if (nanos >= C1) { + value = microsFrac(); + suffix = "micros"; + } + return Strings.format1Decimals(value, suffix); + } + + public String getStringRep() { + if (duration < 0) { + return Long.toString(duration); + } + switch (timeUnit) { + case NANOSECONDS: + return duration + "nanos"; + case MICROSECONDS: + return duration + "micros"; + case MILLISECONDS: + return duration + "ms"; + case SECONDS: + return duration + "s"; + case MINUTES: + return duration + "m"; + case HOURS: + return duration + "h"; + case DAYS: + return duration + "d"; + default: + throw new IllegalArgumentException("unknown time unit: " + timeUnit.name()); + } + } + + public static TimeValue parseTimeValue(String sValue, String settingName) throws Exception { + Objects.requireNonNull(settingName); + Objects.requireNonNull(sValue); + return parseTimeValue(sValue, null, settingName); + } + + public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, String settingName) + throws Exception { + settingName = Objects.requireNonNull(settingName); + if (sValue == null) { + return defaultValue; + } + final String normalized = sValue.toLowerCase(Locale.ROOT).trim(); + if (normalized.endsWith("nanos")) { + return new TimeValue(parse(sValue, normalized, "nanos"), TimeUnit.NANOSECONDS); + } else if (normalized.endsWith("micros")) { + return new TimeValue(parse(sValue, normalized, "micros"), TimeUnit.MICROSECONDS); + } else if (normalized.endsWith("ms")) { + return new TimeValue(parse(sValue, normalized, "ms"), TimeUnit.MILLISECONDS); + } else if (normalized.endsWith("s")) { + return new TimeValue(parse(sValue, normalized, "s"), TimeUnit.SECONDS); + } else if (sValue.endsWith("m")) { + // parsing minutes should be case-sensitive as 'M' means "months", not "minutes"; this is the only special case. + return new TimeValue(parse(sValue, normalized, "m"), TimeUnit.MINUTES); + } else if (normalized.endsWith("h")) { + return new TimeValue(parse(sValue, normalized, "h"), TimeUnit.HOURS); + } else if (normalized.endsWith("d")) { + return new TimeValue(parse(sValue, normalized, "d"), TimeUnit.DAYS); + } else if (normalized.matches("-0*1")) { + return TimeValue.MINUS_ONE; + } else if (normalized.matches("0+")) { + return TimeValue.ZERO; + } else { + // Missing units: + throw new Exception( + "failed to parse setting [{}] with value [{}] as a time value: unit is missing or unrecognized"); + } + } + + private static long parse(final String initialInput, final String normalized, final String suffix) { + final String s = normalized.substring(0, normalized.length() - suffix.length()).trim(); + try { + return Long.parseLong(s); + } catch (final NumberFormatException e) { + try { + @SuppressWarnings("unused") final double ignored = Double.parseDouble(s); + throw new NumberFormatException("failed to parse, fractional time values are not supported"); + } catch (final NumberFormatException ignored) { + throw new NumberFormatException("failed to parse"); + } + } + } + + private static final long C0 = 1L; + private static final long C1 = C0 * 1000L; + private static final long C2 = C1 * 1000L; + private static final long C3 = C2 * 1000L; + private static final long C4 = C3 * 60L; + private static final long C5 = C4 * 60L; + private static final long C6 = C5 * 24L; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + return this.compareTo(((TimeValue) o)) == 0; + } + + @Override + public int hashCode() { + return Double.hashCode(((double) duration) * timeUnit.toNanos(1)); + } + + public static long nsecToMSec(long ns) { + return ns / NSEC_PER_MSEC; + } + + @Override + public int compareTo(TimeValue timeValue) { + double thisValue = ((double) duration) * timeUnit.toNanos(1); + double otherValue = ((double) timeValue.duration) * timeValue.timeUnit.toNanos(1); + return Double.compare(thisValue, otherValue); + } +} diff --git a/fe/src/com/baidu/palo/monitor/utils/Strings.java b/fe/src/com/baidu/palo/monitor/utils/Strings.java new file mode 100644 index 0000000000..3455d98713 --- /dev/null +++ b/fe/src/com/baidu/palo/monitor/utils/Strings.java @@ -0,0 +1,59 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.monitor.utils; + +/** + * String utility method for jvm data + */ +public class Strings { + + /** + * Format the double value with a single decimal points, trimming trailing '.0'. + */ + public static String format1Decimals(double value, String suffix) { + String p = String.valueOf(value); + int ix = p.indexOf('.') + 1; + int ex = p.indexOf('E'); + char fraction = p.charAt(ix); + if (fraction == '0') { + if (ex != -1) { + return p.substring(0, ix - 1) + p.substring(ex) + suffix; + } else { + return p.substring(0, ix - 1) + suffix; + } + } else { + if (ex != -1) { + return p.substring(0, ix) + fraction + p.substring(ex) + suffix; + } else { + return p.substring(0, ix) + fraction + suffix; + } + } + } + + /** + * Determine whether the given array is empty: + * i.e. null or of zero length. + * + * @param array the array to check + */ + private static boolean isEmpty(Object[] array) { + return (array == null || array.length == 0); + } + + private Strings() { + } + +}