From 5c7c4d90b41f50d5d39a78ba1683c00ae2c8dd77 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Thu, 20 Jul 2023 10:42:29 +0800 Subject: [PATCH] [improvement](catalog) return the root cause of error when forwarding init request to master FE (#22001) --- .../org/apache/doris/common/util/Util.java | 18 +++++++++++++ .../doris/qe/MasterCatalogExecutor.java | 6 +++++ .../doris/service/FrontendServiceImpl.java | 25 ++++++++++++++----- .../doris/common/util/DebugUtilTest.java | 16 ++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index f83be84e29..8d9c2c6b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -41,6 +41,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -616,4 +618,20 @@ public class Util { } return rootCause; } + + // Return the stack of the root cause + public static String getRootCauseStack(Throwable t) { + String rootStack = "unknown"; + if (t == null) { + return rootStack; + } + Throwable p = t; + while (p.getCause() != null) { + p = p.getCause(); + } + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + p.printStackTrace(pw); + return sw.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java index fe21f04066..2bd72642fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterCatalogExecutor.java @@ -19,6 +19,7 @@ package org.apache.doris.qe; import org.apache.doris.catalog.Env; import org.apache.doris.common.ClientPool; +import org.apache.doris.common.UserException; import org.apache.doris.thrift.FrontendService; import org.apache.doris.thrift.TInitExternalCtlMetaRequest; import org.apache.doris.thrift.TInitExternalCtlMetaResult; @@ -35,6 +36,8 @@ import org.apache.logging.log4j.Logger; public class MasterCatalogExecutor { private static final Logger LOG = LogManager.getLogger(MasterCatalogExecutor.class); + public static final String STATUS_OK = "OK"; + private int waitTimeoutMs; public MasterCatalogExecutor(int waitTimeoutMs) { @@ -64,6 +67,9 @@ public class MasterCatalogExecutor { try { TInitExternalCtlMetaResult result = client.initExternalCtlMeta(request); ConnectContext.get().getEnv().getJournalObservable().waitOn(result.maxJournalId, waitTimeoutMs); + if (!result.getStatus().equalsIgnoreCase(STATUS_OK)) { + throw new UserException(result.getStatus()); + } isReturnToPool = true; } catch (Exception e) { LOG.warn("Failed to finish forward init operation, please try again. ", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index bb6fbe38b3..f792c561cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -57,6 +57,7 @@ import org.apache.doris.common.ThriftServerEventProcessor; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.annotation.LogException; +import org.apache.doris.common.util.Util; import org.apache.doris.cooldown.CooldownDelete; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -69,6 +70,7 @@ import org.apache.doris.planner.StreamLoadPlanner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectProcessor; import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.MasterCatalogExecutor; import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.VariableMgr; @@ -1987,10 +1989,15 @@ public class FrontendServiceImpl implements FrontendService.Iface { if (!(catalog instanceof ExternalCatalog)) { throw new TException("Only support forward ExternalCatalog init operation."); } - ((ExternalCatalog) catalog).makeSureInitialized(); TInitExternalCtlMetaResult result = new TInitExternalCtlMetaResult(); - result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); - result.setStatus("OK"); + try { + ((ExternalCatalog) catalog).makeSureInitialized(); + result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); + result.setStatus(MasterCatalogExecutor.STATUS_OK); + } catch (Throwable t) { + LOG.warn("init catalog failed. catalog: {}", catalog.getName(), t); + result.setStatus(Util.getRootCauseStack(t)); + } return result; } @@ -2006,10 +2013,16 @@ public class FrontendServiceImpl implements FrontendService.Iface { if (!(db instanceof ExternalDatabase)) { throw new TException("Only support forward ExternalDatabase init operation."); } - ((ExternalDatabase) db).makeSureInitialized(); + TInitExternalCtlMetaResult result = new TInitExternalCtlMetaResult(); - result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); - result.setStatus("OK"); + try { + ((ExternalDatabase) db).makeSureInitialized(); + result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); + result.setStatus(MasterCatalogExecutor.STATUS_OK); + } catch (Throwable t) { + LOG.warn("init database failed. catalog.database: {}", catalog.getName(), db.getFullName(), t); + result.setStatus(Util.getRootCauseStack(t)); + } return result; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java index 0dec8873f1..bebe65cd2e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/DebugUtilTest.java @@ -17,6 +17,7 @@ package org.apache.doris.common.util; +import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.junit.Assert; @@ -81,4 +82,19 @@ public class DebugUtilTest { Assert.assertEquals(result.first, Double.valueOf(1.1497809458523989)); Assert.assertEquals(result.second, "GB"); } + + @Test + public void testUtilGetStackTrace() { + Exception e1 = new Exception("exception1"); + DdlException e2 = new DdlException("exception2", e1); + e2.printStackTrace(); + System.out.println(Util.getRootCauseStack(e2)); + Assert.assertTrue(Util.getRootCauseStack(e2).contains("java.lang.Exception: exception1")); + + DdlException e3 = new DdlException("only one exception"); + System.out.println(Util.getRootCauseStack(e3)); + Assert.assertTrue(Util.getRootCauseStack(e3) + .contains("org.apache.doris.common.DdlException: errCode = 2, detailMessage = only one exception")); + Assert.assertEquals("unknown", Util.getRootCauseStack(null)); + } }