From 7e77b5ed7f29bf26e66509ba16342a6fa2d98341 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Thu, 15 Jul 2021 11:13:51 +0800 Subject: [PATCH] [Optimize] Using custom conf dir to save log config of Spring (#6205) The log4j-config.xml will be generated at startup of FE and also when modifying FE config. But in some deploy environment such as k8s, the conf dir is not writable. So change the dir of log4j-config.xml to Config.custom_conf_dir. Also fix some small bugs: 1. Typo "less then" -> "less than" 2. Duplicated `exec_mem_limit` showed in SHOW ROUTINE LOAD 3. Allow MAXVALUE in single partition column table. 4. Add IP info for "intolerate index channel failure" msg. Change-Id: Ib4e1182084219c41eae44d3a28110c0315fdbd7d Co-authored-by: chenmingyu --- be/src/exec/tablet_sink.cpp | 3 +++ .../java/org/apache/doris/alter/RollupJobV2.java | 4 ++-- .../org/apache/doris/alter/SchemaChangeJobV2.java | 4 ++-- .../doris/analysis/CreateMaterializedViewStmt.java | 2 +- .../java/org/apache/doris/analysis/InsertStmt.java | 10 +++++++++- .../apache/doris/analysis/PartitionKeyDesc.java | 14 ++++++++------ .../java/org/apache/doris/catalog/Database.java | 2 +- .../java/org/apache/doris/catalog/OlapTable.java | 2 +- .../java/org/apache/doris/common/Log4jConfig.java | 5 ++++- .../java/org/apache/doris/httpv2/HttpServer.java | 3 ++- .../doris/httpv2/config/SpringLog4j2Config.java | 2 +- .../apache/doris/load/loadv2/LoadLoadingTask.java | 2 +- .../doris/load/routineload/RoutineLoadJob.java | 7 +------ .../load/routineload/RoutineLoadScheduler.java | 2 +- .../apache/doris/planner/AssertNumRowsNode.java | 2 +- .../java/org/apache/doris/qe/ConnectProcessor.java | 11 +++++++++-- .../transaction/TabletQuorumFailedException.java | 2 +- .../org/apache/doris/utframe/UtFrameUtils.java | 5 +++++ 18 files changed, 53 insertions(+), 29 deletions(-) diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index d7d24d1d96..e978f996c0 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -26,6 +26,7 @@ #include "runtime/row_batch.h" #include "runtime/runtime_state.h" #include "runtime/tuple_row.h" +#include "service/backend_options.h" #include "service/brpc.h" #include "util/brpc_stub_cache.h" #include "util/debug/sanitizer_scopes.h" @@ -473,6 +474,8 @@ Status IndexChannel::add_row(Tuple* tuple, int64_t tablet_id) { } if (has_intolerable_failure()) { + std::stringstream ss; + ss << "index channel has intolerable failure. " << BackendOptions::get_localhost(); return Status::InternalError(ss.str()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 470d4c5269..dbd020476f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -751,8 +751,8 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable { } /** - * This method is only used to deserialize the text mate which version is less then 86. - * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather then here. + * This method is only used to deserialize the text mate which version is less than 86. + * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather than here. */ public static RollupJobV2 read(DataInput in) throws IOException { Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_86); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index bd22ba2a20..3292d86ff2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -979,8 +979,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 { } /** - * This method is only used to deserialize the text mate which version is less then 86. - * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather then here. + * This method is only used to deserialize the text mate which version is less than 86. + * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather than here. */ public static SchemaChangeJobV2 read(DataInput in) throws IOException { Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_86); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index c5607bd399..f29daba87d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -245,7 +245,7 @@ public class CreateMaterializedViewStmt extends DdlStmt { List orderByElements = selectStmt.getOrderByElements(); if (orderByElements.size() > mvColumnItemList.size()) { - throw new AnalysisException("The number of columns in order clause must be less then " + "the number of " + throw new AnalysisException("The number of columns in order clause must be less than " + "the number of " + "columns in select clause"); } if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java index bd6f0acfca..8ded93dc07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -129,6 +129,12 @@ public class InsertStmt extends DdlStmt { */ private boolean isTransactionBegin = false; + private boolean isValuesOrConstantSelect = false; + + public boolean isValuesOrConstantSelect() { + return isValuesOrConstantSelect; + } + public InsertStmt(InsertTarget target, String label, List cols, InsertSource source, List hints) { this.tblName = target.getTblName(); this.targetPartitionNames = target.getPartitionNames(); @@ -140,6 +146,8 @@ public class InsertStmt extends DdlStmt { if (!Strings.isNullOrEmpty(label)) { isUserSpecifiedLabel = true; } + + this.isValuesOrConstantSelect = (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()); } // Ctor for CreateTableAsSelectStmt @@ -488,7 +496,7 @@ public class InsertStmt extends DdlStmt { checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ; // handle VALUES() or SELECT constant list - if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) { + if (isValuesOrConstantSelect) { SelectStmt selectStmt = (SelectStmt) queryStmt; if (selectStmt.getValueList() != null) { // INSERT INTO VALUES(...) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java index 3237ee25b1..da019d716b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java @@ -106,20 +106,22 @@ public class PartitionKeyDesc { } } - // currently, we do not support MAXVALUE in partition range values. eg: ("100", "200", MAXVALUE); - // maybe support later. - if (lowerValues != null) { + // Currently, we do not support MAXVALUE in multi partition range values. eg: ("100", "200", MAXVALUE); + // Because we still don’t support expressing such values on the BE side. + // Maybe support later. + // But we can support MAXVALUE in single partition values. + if (lowerValues != null && lowerValues.size() > 1) { for (PartitionValue lowerVal : lowerValues) { if (lowerVal.isMax()) { - throw new AnalysisException("Not support MAXVALUE in partition range values."); + throw new AnalysisException("Not support MAXVALUE in multi partition range values."); } } } - if (upperValues != null) { + if (upperValues != null && upperValues.size() > 1) { for (PartitionValue upperVal : upperValues) { if (upperVal.isMax()) { - throw new AnalysisException("Not support MAXVALUE in partition range values."); + throw new AnalysisException("Not support MAXVALUE in multi partition range values."); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index f1b7e82972..2226cd1745 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -74,7 +74,7 @@ public class Database extends MetaObject implements Writable { private static final Logger LOG = LogManager.getLogger(Database.class); // empirical value. - // assume that the time a lock is held by thread is less then 100ms + // assume that the time a lock is held by thread is less than 100ms public static final long TRY_LOCK_TIMEOUT_MS = 100L; private long id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 478e076225..a2376272f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -281,7 +281,7 @@ public class OlapTable extends Table { indexName = getIndexNameById(indexId); Preconditions.checkState(indexName != null); } - // Nullable when meta is less then VERSION_74 + // Nullable when meta is less than VERSION_74 if (keysType == null) { keysType = this.keysType; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 98944fd2b4..71b22f11a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -108,6 +108,8 @@ public class Log4jConfig extends XmlConfiguration { private static String logXmlConfTemplate; // dir of fe.conf public static String confDir; + // custom conf dir + public static String customConfDir; private static void reconfig() throws IOException { String newXmlConfTemplate = xmlConfTemplate; @@ -180,7 +182,7 @@ public class Log4jConfig extends XmlConfiguration { System.out.println(newXmlConfTemplate); System.out.println("====="); logXmlConfTemplate = newXmlConfTemplate; - SpringLog4j2Config.writeSpringLogConf(confDir); + SpringLog4j2Config.writeSpringLogConf(customConfDir); // new SimpleLog4jConfiguration with xmlConfTemplate ByteArrayInputStream bis = new ByteArrayInputStream(newXmlConfTemplate.getBytes("UTF-8")); @@ -221,6 +223,7 @@ public class Log4jConfig extends XmlConfiguration { verboseModules = Config.sys_log_verbose_modules; auditModules = Config.audit_log_modules; confDir = dorisConfDir; + customConfDir = Config.custom_config_dir; reconfig(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java index 62af813d54..0a1b6543aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java @@ -18,6 +18,7 @@ package org.apache.doris.httpv2; import org.apache.doris.PaloFe; +import org.apache.doris.common.Config; import org.apache.doris.httpv2.config.SpringLog4j2Config; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -85,7 +86,7 @@ public class HttpServer extends SpringBootServletInitializer { // To avoid some unexpected behavior. System.setProperty("spring.devtools.restart.enabled", "false"); System.setProperty("spring.http.multipart.location", PaloFe.DORIS_HOME_DIR); - properties.put("logging.config", dorisHome + "/conf/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); + properties.put("logging.config", Config.custom_config_dir + "/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); new SpringApplicationBuilder() .sources(HttpServer.class) .properties(properties) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java index d18c2b50bc..21842c53eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java @@ -35,7 +35,7 @@ public class SpringLog4j2Config { Writer writer = null; try { // log4j2-spring.xml file path - File file = new File(confDir + SPRING_LOG_XML_FILE); + File file = new File(confDir + "/" + SPRING_LOG_XML_FILE); if (!file.exists()) { file.createNewFile(); //write file diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index f9ad624eff..2e0c7bb60e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -140,7 +140,7 @@ public class LoadLoadingTask extends LoadTask { private void actualExecute(Coordinator curCoordinator) throws Exception { int waitSecond = (int) (getLeftTimeMs() / 1000); if (waitSecond <= 0) { - throw new LoadException("failed to execute plan when the left time is less then 0"); + throw new LoadException("failed to execute plan when the left time is less than 0"); } if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index f55e904e18..8213514dcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -302,7 +302,6 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl } jobProperties.put(LoadStmt.TIMEZONE, stmt.getTimezone()); jobProperties.put(LoadStmt.STRICT_MODE, String.valueOf(stmt.isStrictMode())); - jobProperties.put(LoadStmt.EXEC_MEM_LIMIT, String.valueOf(stmt.getExecMemLimit())); if (Strings.isNullOrEmpty(stmt.getFormat()) || stmt.getFormat().equals("csv")) { jobProperties.put(PROPS_FORMAT, "csv"); jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false"); @@ -522,11 +521,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl @Override public long getMemLimit() { - String value = jobProperties.get(LoadStmt.EXEC_MEM_LIMIT); - if (value == null) { - return DEFAULT_EXEC_MEM_LIMIT; - } - return Long.valueOf(value); + return execMemLimit; } public String getTimezone() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java index 5454481ea7..df15bdc5d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java @@ -82,7 +82,7 @@ public class RoutineLoadScheduler extends MasterDaemon { if (desiredConcurrentTaskNum <= 0) { // the job will be rescheduled later. LOG.info(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, routineLoadJob.getId()) - .add("msg", "the current concurrent num is less then or equal to zero, " + .add("msg", "the current concurrent num is less than or equal to zero, " + "job will be rescheduled later") .build()); continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java index f8196e5df6..388f6fa096 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java @@ -29,7 +29,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; /** - * Assert num rows node is used to determine whether the number of rows is less then desired num of rows. + * Assert num rows node is used to determine whether the number of rows is less than desired num of rows. * The rows are the result of subqueryString. * If the number of rows is more than the desired num of rows, the query will be cancelled. * The cancelled reason will be reported by Backend and displayed back to the user. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 727559ae11..6aae692c5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -17,7 +17,7 @@ package org.apache.doris.qe; -import avro.shaded.com.google.common.collect.Lists; +import org.apache.doris.analysis.InsertStmt; import org.apache.doris.analysis.KillStmt; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; @@ -50,6 +50,7 @@ import org.apache.doris.thrift.TMasterOpRequest; import org.apache.doris.thrift.TMasterOpResult; import org.apache.doris.thrift.TUniqueId; +import com.google.common.collect.Lists; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -147,7 +148,13 @@ public class ConnectProcessor { if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql()); } else { - ctx.getAuditEventBuilder().setStmt(origStmt); + if (parsedStmt instanceof InsertStmt && ((InsertStmt)parsedStmt).isValuesOrConstantSelect()) { + // INSERT INTO VALUES may be very long, so we only log at most 1K bytes. + int length = Math.min(1024, origStmt.length()); + ctx.getAuditEventBuilder().setStmt(origStmt.substring(0, length)); + } else { + ctx.getAuditEventBuilder().setStmt(origStmt); + } } Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java index cf49602948..2aa3a3b9a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java @@ -25,7 +25,7 @@ import java.util.Set; public class TabletQuorumFailedException extends TransactionException { private static final String TABLET_QUORUM_FAILED_MSG = "Failed to commit txn %s. " - + "Tablet [%s] success replica num %s is less then quorum " + + "Tablet [%s] success replica num %s is less than quorum " + "replica num %s while error backends %s"; private long tabletId; diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java index 8b416dc963..7686c0fd29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java @@ -130,6 +130,11 @@ public class UtFrameUtils { dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString(); } Config.plugin_dir = dorisHome + "/plugins"; + Config.custom_config_dir = dorisHome + "/conf"; + File file = new File(Config.custom_config_dir); + if (!file.exists()) { + file.mkdir(); + } int fe_http_port = findValidPort(); int fe_rpc_port = findValidPort();