From 14620a67667dfb7fa3b603522242146e3b580a52 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Fri, 19 May 2023 08:53:57 +0800 Subject: [PATCH] [minor](log) add details for unqueryable replicas (#19792) Add a new FE config: show_details_for_unaccessible_tablet. Default is false, when set to true, if a query is unable to select a healthy replica, the detailed information of all the replicas of the tablet including the specific reason why they are unqueryable, will be printed out. --- .../java/org/apache/doris/common/Config.java | 9 ++++++++- .../java/org/apache/doris/catalog/Tablet.java | 19 ++++++++++++++++++ .../apache/doris/planner/OlapScanNode.java | 16 ++++++++------- .../java/org/apache/doris/system/Backend.java | 20 +++++++++++++++---- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 2fd6b7dfff..9922034adc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1968,6 +1968,13 @@ public class Config extends ConfigBase { * optimization of table structures * */ - @ConfField(mutable = true, masterOnly = false) + @ConfField(mutable = true) public static boolean enable_query_hit_stats = false; + + @ConfField(mutable = true, description = { + "设置为 true，如果查询无法选择到健康副本时，会打印出该tablet所有副本的详细信息，" + "以及不可查询的具体原因。", + "When set to true, if a query is unable to select a healthy replica, " + + "the detailed information of all the replicas of the tablet," + + " including the specific reason why they are unqueryable, will be printed out."}) + public static boolean show_details_for_unaccessible_tablet = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index a78d12d65e..b8f93a9091 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -277,6 +277,25 @@ public class Tablet extends MetaObject implements Writable { return allQueryableReplica; } + public String getDetailsStatusForQuery(long visibleVersion) { + StringBuilder sb = new StringBuilder("Visible Replicas:"); + sb.append("Visible version: ").append(visibleVersion); + sb.append(", Replicas: "); + for (Replica replica : replicas) { + sb.append(replica.toString()); + } + sb.append(", Backends: "); + for (Replica replica : replicas) { + Backend be = Env.getCurrentSystemInfo().getBackend(replica.getBackendId()); + if (be == null) { + sb.append("Backend [id=" + id + ", not exists]"); + } else { + sb.append(be.getHealthyStatus()); + } + } + return sb.toString(); + } + public Replica getReplicaById(long replicaId) { for (Replica replica : replicas) { if (replica.getId() == replicaId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index ea4be00ba0..cc24c48bd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -737,14 +737,16 @@ public class OlapScanNode extends ScanNode { // random shuffle List && only collect one copy List replicas = tablet.getQueryableReplicas(visibleVersion); if (replicas.isEmpty()) { - LOG.error("no queryable replica found in tablet {}. visible version {}", - tabletId, visibleVersion); - if (LOG.isDebugEnabled()) { - for (Replica replica : tablet.getReplicas()) { - LOG.debug("tablet {}, replica: {}", tabletId, replica.toString()); - } + LOG.warn("no queryable replica found in tablet {}. visible version {}", tabletId, visibleVersion); + StringBuilder sb = new StringBuilder( + "Failed to get scan range, no queryable replica found in tablet: " + tabletId); + if (Config.show_details_for_unaccessible_tablet) { + sb.append(". Reason: ").append(tablet.getDetailsStatusForQuery(visibleVersion)); } - throw new UserException("Failed to get scan range, no queryable replica found in tablet: " + tabletId); + if (LOG.isDebugEnabled()) { + LOG.debug(sb.toString()); + } + throw new UserException(sb.toString()); } int useFixReplica = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java index cc4b1376ae..77bdb1d66b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java @@ -601,15 +601,20 @@ public class Backend implements Writable { Backend backend = (Backend) obj; - return (id == backend.id) && (host.equals(backend.host)) && (heartbeatPort == backend.heartbeatPort) - && (bePort == backend.bePort) && (isAlive.get() == backend.isAlive.get()); + return (id == backend.id) && (host.equals(backend.host)) && (heartbeatPort == backend.heartbeatPort) && (bePort + == backend.bePort) && (isAlive.get() == backend.isAlive.get()); } @Override public String toString() { return "Backend [id=" + id + ", host=" + host + ", heartbeatPort=" + heartbeatPort + ", alive=" + isAlive.get() - + ", lastStartTime=" + TimeUtils.longToTimeString(lastStartTime) - + ", tags: " + tagMap + "]"; + + ", lastStartTime=" + TimeUtils.longToTimeString(lastStartTime) + ", tags: " + tagMap + "]"; + } + + public String getHealthyStatus() { + return "Backend [id=" + id + ", isDecommission: " + isDecommissioned + ", backendState: " + backendState + + ", backendStatus: " + backendStatus + ", isAlive: " + isAlive.get() + ", lastUpdateTime: " + + TimeUtils.longToTimeString(lastUpdateMs); } public String getOwnerClusterName() { @@ -746,6 +751,13 @@ public class Backend implements Writable { public volatile boolean isQueryDisabled = false; @SerializedName("isLoadDisabled") public volatile boolean isLoadDisabled = false; + + @Override + public String toString() { + return "[" + "lastSuccessReportTabletsTime='" + lastSuccessReportTabletsTime + '\'' + + ", lastStreamLoadTime=" + lastStreamLoadTime + ", isQueryDisabled=" + isQueryDisabled + + ", isLoadDisabled=" + isLoadDisabled + "]"; + } } public Tag getLocationTag() {