diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index d7d24d1d96..e978f996c0 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -26,6 +26,7 @@ #include "runtime/row_batch.h" #include "runtime/runtime_state.h" #include "runtime/tuple_row.h" +#include "service/backend_options.h" #include "service/brpc.h" #include "util/brpc_stub_cache.h" #include "util/debug/sanitizer_scopes.h" @@ -473,6 +474,8 @@ Status IndexChannel::add_row(Tuple* tuple, int64_t tablet_id) { } if (has_intolerable_failure()) { + std::stringstream ss; + ss << "index channel has intolerable failure. " << BackendOptions::get_localhost(); return Status::InternalError(ss.str()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 470d4c5269..dbd020476f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -751,8 +751,8 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable { } /** - * This method is only used to deserialize the text mate which version is less then 86. - * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather then here. + * This method is only used to deserialize the text mate which version is less than 86. + * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather than here. */ public static RollupJobV2 read(DataInput in) throws IOException { Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_86); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index bd22ba2a20..3292d86ff2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -979,8 +979,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 { } /** - * This method is only used to deserialize the text mate which version is less then 86. - * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather then here. + * This method is only used to deserialize the text mate which version is less than 86. + * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather than here. */ public static SchemaChangeJobV2 read(DataInput in) throws IOException { Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_86); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index c5607bd399..f29daba87d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -245,7 +245,7 @@ public class CreateMaterializedViewStmt extends DdlStmt { List orderByElements = selectStmt.getOrderByElements(); if (orderByElements.size() > mvColumnItemList.size()) { - throw new AnalysisException("The number of columns in order clause must be less then " + "the number of " + throw new AnalysisException("The number of columns in order clause must be less than " + "the number of " + "columns in select clause"); } if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java index bd6f0acfca..8ded93dc07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -129,6 +129,12 @@ public class InsertStmt extends DdlStmt { */ private boolean isTransactionBegin = false; + private boolean isValuesOrConstantSelect = false; + + public boolean isValuesOrConstantSelect() { + return isValuesOrConstantSelect; + } + public InsertStmt(InsertTarget target, String label, List cols, InsertSource source, List hints) { this.tblName = target.getTblName(); this.targetPartitionNames = target.getPartitionNames(); @@ -140,6 +146,8 @@ public class InsertStmt extends DdlStmt { if (!Strings.isNullOrEmpty(label)) { isUserSpecifiedLabel = true; } + + this.isValuesOrConstantSelect = (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()); } // Ctor for CreateTableAsSelectStmt @@ -488,7 +496,7 @@ public class InsertStmt extends DdlStmt { checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ; // handle VALUES() or SELECT constant list - if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) { + if (isValuesOrConstantSelect) { SelectStmt selectStmt = (SelectStmt) queryStmt; if (selectStmt.getValueList() != null) { // INSERT INTO VALUES(...) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java index 3237ee25b1..da019d716b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java @@ -106,20 +106,22 @@ public class PartitionKeyDesc { } } - // currently, we do not support MAXVALUE in partition range values. eg: ("100", "200", MAXVALUE); - // maybe support later. - if (lowerValues != null) { + // Currently, we do not support MAXVALUE in multi partition range values. eg: ("100", "200", MAXVALUE); + // Because we still don’t support expressing such values on the BE side. + // Maybe support later. + // But we can support MAXVALUE in single partition values. + if (lowerValues != null && lowerValues.size() > 1) { for (PartitionValue lowerVal : lowerValues) { if (lowerVal.isMax()) { - throw new AnalysisException("Not support MAXVALUE in partition range values."); + throw new AnalysisException("Not support MAXVALUE in multi partition range values."); } } } - if (upperValues != null) { + if (upperValues != null && upperValues.size() > 1) { for (PartitionValue upperVal : upperValues) { if (upperVal.isMax()) { - throw new AnalysisException("Not support MAXVALUE in partition range values."); + throw new AnalysisException("Not support MAXVALUE in multi partition range values."); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index f1b7e82972..2226cd1745 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -74,7 +74,7 @@ public class Database extends MetaObject implements Writable { private static final Logger LOG = LogManager.getLogger(Database.class); // empirical value. - // assume that the time a lock is held by thread is less then 100ms + // assume that the time a lock is held by thread is less than 100ms public static final long TRY_LOCK_TIMEOUT_MS = 100L; private long id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 478e076225..a2376272f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -281,7 +281,7 @@ public class OlapTable extends Table { indexName = getIndexNameById(indexId); Preconditions.checkState(indexName != null); } - // Nullable when meta is less then VERSION_74 + // Nullable when meta is less than VERSION_74 if (keysType == null) { keysType = this.keysType; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 98944fd2b4..71b22f11a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -108,6 +108,8 @@ public class Log4jConfig extends XmlConfiguration { private static String logXmlConfTemplate; // dir of fe.conf public static String confDir; + // custom conf dir + public static String customConfDir; private static void reconfig() throws IOException { String newXmlConfTemplate = xmlConfTemplate; @@ -180,7 +182,7 @@ public class Log4jConfig extends XmlConfiguration { System.out.println(newXmlConfTemplate); System.out.println("====="); logXmlConfTemplate = newXmlConfTemplate; - SpringLog4j2Config.writeSpringLogConf(confDir); + SpringLog4j2Config.writeSpringLogConf(customConfDir); // new SimpleLog4jConfiguration with xmlConfTemplate ByteArrayInputStream bis = new ByteArrayInputStream(newXmlConfTemplate.getBytes("UTF-8")); @@ -221,6 +223,7 @@ public class Log4jConfig extends XmlConfiguration { verboseModules = Config.sys_log_verbose_modules; auditModules = Config.audit_log_modules; confDir = dorisConfDir; + customConfDir = Config.custom_config_dir; reconfig(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java index 62af813d54..0a1b6543aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java @@ -18,6 +18,7 @@ package org.apache.doris.httpv2; import org.apache.doris.PaloFe; +import org.apache.doris.common.Config; import org.apache.doris.httpv2.config.SpringLog4j2Config; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -85,7 +86,7 @@ public class HttpServer extends SpringBootServletInitializer { // To avoid some unexpected behavior. System.setProperty("spring.devtools.restart.enabled", "false"); System.setProperty("spring.http.multipart.location", PaloFe.DORIS_HOME_DIR); - properties.put("logging.config", dorisHome + "/conf/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); + properties.put("logging.config", Config.custom_config_dir + "/" + SpringLog4j2Config.SPRING_LOG_XML_FILE); new SpringApplicationBuilder() .sources(HttpServer.class) .properties(properties) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java index d18c2b50bc..21842c53eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java @@ -35,7 +35,7 @@ public class SpringLog4j2Config { Writer writer = null; try { // log4j2-spring.xml file path - File file = new File(confDir + SPRING_LOG_XML_FILE); + File file = new File(confDir + "/" + SPRING_LOG_XML_FILE); if (!file.exists()) { file.createNewFile(); //write file diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index f9ad624eff..2e0c7bb60e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -140,7 +140,7 @@ public class LoadLoadingTask extends LoadTask { private void actualExecute(Coordinator curCoordinator) throws Exception { int waitSecond = (int) (getLeftTimeMs() / 1000); if (waitSecond <= 0) { - throw new LoadException("failed to execute plan when the left time is less then 0"); + throw new LoadException("failed to execute plan when the left time is less than 0"); } if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index f55e904e18..8213514dcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -302,7 +302,6 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl } jobProperties.put(LoadStmt.TIMEZONE, stmt.getTimezone()); jobProperties.put(LoadStmt.STRICT_MODE, String.valueOf(stmt.isStrictMode())); - jobProperties.put(LoadStmt.EXEC_MEM_LIMIT, String.valueOf(stmt.getExecMemLimit())); if (Strings.isNullOrEmpty(stmt.getFormat()) || stmt.getFormat().equals("csv")) { jobProperties.put(PROPS_FORMAT, "csv"); jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false"); @@ -522,11 +521,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl @Override public long getMemLimit() { - String value = jobProperties.get(LoadStmt.EXEC_MEM_LIMIT); - if (value == null) { - return DEFAULT_EXEC_MEM_LIMIT; - } - return Long.valueOf(value); + return execMemLimit; } public String getTimezone() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java index 5454481ea7..df15bdc5d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java @@ -82,7 +82,7 @@ public class RoutineLoadScheduler extends MasterDaemon { if (desiredConcurrentTaskNum <= 0) { // the job will be rescheduled later. LOG.info(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, routineLoadJob.getId()) - .add("msg", "the current concurrent num is less then or equal to zero, " + .add("msg", "the current concurrent num is less than or equal to zero, " + "job will be rescheduled later") .build()); continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java index f8196e5df6..388f6fa096 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java @@ -29,7 +29,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; /** - * Assert num rows node is used to determine whether the number of rows is less then desired num of rows. + * Assert num rows node is used to determine whether the number of rows is less than desired num of rows. * The rows are the result of subqueryString. * If the number of rows is more than the desired num of rows, the query will be cancelled. * The cancelled reason will be reported by Backend and displayed back to the user. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 727559ae11..6aae692c5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -17,7 +17,7 @@ package org.apache.doris.qe; -import avro.shaded.com.google.common.collect.Lists; +import org.apache.doris.analysis.InsertStmt; import org.apache.doris.analysis.KillStmt; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; @@ -50,6 +50,7 @@ import org.apache.doris.thrift.TMasterOpRequest; import org.apache.doris.thrift.TMasterOpResult; import org.apache.doris.thrift.TUniqueId; +import com.google.common.collect.Lists; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -147,7 +148,13 @@ public class ConnectProcessor { if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql()); } else { - ctx.getAuditEventBuilder().setStmt(origStmt); + if (parsedStmt instanceof InsertStmt && ((InsertStmt)parsedStmt).isValuesOrConstantSelect()) { + // INSERT INTO VALUES may be very long, so we only log at most 1K bytes. + int length = Math.min(1024, origStmt.length()); + ctx.getAuditEventBuilder().setStmt(origStmt.substring(0, length)); + } else { + ctx.getAuditEventBuilder().setStmt(origStmt); + } } Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java index cf49602948..2aa3a3b9a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java @@ -25,7 +25,7 @@ import java.util.Set; public class TabletQuorumFailedException extends TransactionException { private static final String TABLET_QUORUM_FAILED_MSG = "Failed to commit txn %s. " - + "Tablet [%s] success replica num %s is less then quorum " + + "Tablet [%s] success replica num %s is less than quorum " + "replica num %s while error backends %s"; private long tabletId; diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java index 8b416dc963..7686c0fd29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java @@ -130,6 +130,11 @@ public class UtFrameUtils { dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString(); } Config.plugin_dir = dorisHome + "/plugins"; + Config.custom_config_dir = dorisHome + "/conf"; + File file = new File(Config.custom_config_dir); + if (!file.exists()) { + file.mkdir(); + } int fe_http_port = findValidPort(); int fe_rpc_port = findValidPort();