[Optimize] Using custom conf dir to save log config of Spring (#6205)

The log4j-config.xml will be generated at startup of FE and also when modifying FE config.
But in some deploy environment such as k8s, the conf dir is not writable.

So change the dir of log4j-config.xml to Config.custom_conf_dir.

Also fix some small bugs:

1. Typo "less then" -> "less than"
2. Duplicated `exec_mem_limit` showed in SHOW ROUTINE LOAD
3. Allow MAXVALUE in single partition column table.
4. Add IP info for "intolerate index channel failure" msg.

Change-Id: Ib4e1182084219c41eae44d3a28110c0315fdbd7d

Co-authored-by: chenmingyu <chenmingyu@baidu.com>

fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java

@@ -751,8 +751,8 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
     }
 
     /**
-     * This method is only used to deserialize the text mate which version is less then 86.
-     * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather then here.
+     * This method is only used to deserialize the text mate which version is less than 86.
+     * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather than here.
      */
     public static RollupJobV2 read(DataInput in) throws IOException {
         Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_86);

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java

@@ -979,8 +979,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
     }
 
     /**
-     * This method is only used to deserialize the text mate which version is less then 86.
-     * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather then here.
+     * This method is only used to deserialize the text mate which version is less than 86.
+     * If the meta version >=86, it will be deserialized by the `read` of AlterJobV2 rather than here.
      */
     public static SchemaChangeJobV2 read(DataInput in) throws IOException {
         Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_86);

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java

@@ -245,7 +245,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
 
         List<OrderByElement> orderByElements = selectStmt.getOrderByElements();
         if (orderByElements.size() > mvColumnItemList.size()) {
-            throw new AnalysisException("The number of columns in order clause must be less then " + "the number of "
+            throw new AnalysisException("The number of columns in order clause must be less than " + "the number of "
                                                 + "columns in select clause");
         }
         if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) {

fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java

@@ -129,6 +129,12 @@ public class InsertStmt extends DdlStmt {
      */
     private boolean isTransactionBegin = false;
 
+    private boolean isValuesOrConstantSelect = false;
+
+    public boolean isValuesOrConstantSelect() {
+        return isValuesOrConstantSelect;
+    }
+
     public InsertStmt(InsertTarget target, String label, List<String> cols, InsertSource source, List<String> hints) {
         this.tblName = target.getTblName();
         this.targetPartitionNames = target.getPartitionNames();
@@ -140,6 +146,8 @@ public class InsertStmt extends DdlStmt {
         if (!Strings.isNullOrEmpty(label)) {
             isUserSpecifiedLabel = true;
         }
+
+        this.isValuesOrConstantSelect = (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty());
     }
 
     // Ctor for CreateTableAsSelectStmt
@@ -488,7 +496,7 @@ public class InsertStmt extends DdlStmt {
         checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ;
         
         // handle VALUES() or SELECT constant list
-        if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) {
+        if (isValuesOrConstantSelect) {
             SelectStmt selectStmt = (SelectStmt) queryStmt;
             if (selectStmt.getValueList() != null) {
                 // INSERT INTO VALUES(...)

fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java

@@ -106,20 +106,22 @@ public class PartitionKeyDesc {
             }
         }
 
-        // currently, we do not support MAXVALUE in partition range values. eg: ("100", "200", MAXVALUE);
-        // maybe support later.
-        if (lowerValues != null) {
+        // Currently, we do not support MAXVALUE in multi partition range values. eg: ("100", "200", MAXVALUE);
+        // Because we still don’t support expressing such values on the BE side.
+        // Maybe support later.
+        // But we can support MAXVALUE in single partition values.
+        if (lowerValues != null && lowerValues.size() > 1) {
             for (PartitionValue lowerVal : lowerValues) {
                 if (lowerVal.isMax()) {
-                    throw new AnalysisException("Not support MAXVALUE in partition range values.");
+                    throw new AnalysisException("Not support MAXVALUE in multi partition range values.");
                 }
             }
         }
 
-        if (upperValues != null) {
+        if (upperValues != null && upperValues.size() > 1) {
             for (PartitionValue upperVal : upperValues) {
                 if (upperVal.isMax()) {
-                    throw new AnalysisException("Not support MAXVALUE in partition range values.");
+                    throw new AnalysisException("Not support MAXVALUE in multi partition range values.");
                 }
             }
         }

fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java

@@ -74,7 +74,7 @@ public class Database extends MetaObject implements Writable {
     private static final Logger LOG = LogManager.getLogger(Database.class);
 
     // empirical value.
-    // assume that the time a lock is held by thread is less then 100ms
+    // assume that the time a lock is held by thread is less than 100ms
     public static final long TRY_LOCK_TIMEOUT_MS = 100L;
 
     private long id;

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -281,7 +281,7 @@ public class OlapTable extends Table {
             indexName = getIndexNameById(indexId);
             Preconditions.checkState(indexName != null);
         }
-        // Nullable when meta is less then VERSION_74
+        // Nullable when meta is less than VERSION_74
         if (keysType == null) {
             keysType = this.keysType;
         }

fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java

@@ -108,6 +108,8 @@ public class Log4jConfig extends XmlConfiguration {
     private static String logXmlConfTemplate;
     // dir of fe.conf
     public static String confDir;
+    // custom conf dir
+    public static String customConfDir;
 
     private static void reconfig() throws IOException {
         String newXmlConfTemplate = xmlConfTemplate;
@@ -180,7 +182,7 @@ public class Log4jConfig extends XmlConfiguration {
         System.out.println(newXmlConfTemplate);
         System.out.println("=====");
         logXmlConfTemplate = newXmlConfTemplate;
-        SpringLog4j2Config.writeSpringLogConf(confDir);
+        SpringLog4j2Config.writeSpringLogConf(customConfDir);
 
         // new SimpleLog4jConfiguration with xmlConfTemplate
         ByteArrayInputStream bis = new ByteArrayInputStream(newXmlConfTemplate.getBytes("UTF-8"));
@@ -221,6 +223,7 @@ public class Log4jConfig extends XmlConfiguration {
         verboseModules = Config.sys_log_verbose_modules;
         auditModules = Config.audit_log_modules;
         confDir = dorisConfDir;
+        customConfDir = Config.custom_config_dir;
         reconfig();
     }
 

fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java

@@ -18,6 +18,7 @@
 package org.apache.doris.httpv2;
 
 import org.apache.doris.PaloFe;
+import org.apache.doris.common.Config;
 import org.apache.doris.httpv2.config.SpringLog4j2Config;
 
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -85,7 +86,7 @@ public class HttpServer extends SpringBootServletInitializer {
         // To avoid some unexpected behavior.
         System.setProperty("spring.devtools.restart.enabled", "false");
         System.setProperty("spring.http.multipart.location", PaloFe.DORIS_HOME_DIR);
-        properties.put("logging.config", dorisHome + "/conf/" + SpringLog4j2Config.SPRING_LOG_XML_FILE);
+        properties.put("logging.config", Config.custom_config_dir + "/" + SpringLog4j2Config.SPRING_LOG_XML_FILE);
         new SpringApplicationBuilder()
                 .sources(HttpServer.class)
                 .properties(properties)

fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java

@@ -35,7 +35,7 @@ public class SpringLog4j2Config {
         Writer writer = null;
         try {
             // log4j2-spring.xml file path
-            File file = new File(confDir + SPRING_LOG_XML_FILE);
+            File file = new File(confDir + "/" + SPRING_LOG_XML_FILE);
             if (!file.exists()) {
                 file.createNewFile();
                 //write file

fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java

@@ -140,7 +140,7 @@ public class LoadLoadingTask extends LoadTask {
     private void actualExecute(Coordinator curCoordinator) throws Exception {
         int waitSecond = (int) (getLeftTimeMs() / 1000);
         if (waitSecond <= 0) {
-            throw new LoadException("failed to execute plan when the left time is less then 0");
+            throw new LoadException("failed to execute plan when the left time is less than 0");
         }
 
         if (LOG.isDebugEnabled()) {

fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java

@@ -302,7 +302,6 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
         }
         jobProperties.put(LoadStmt.TIMEZONE, stmt.getTimezone());
         jobProperties.put(LoadStmt.STRICT_MODE, String.valueOf(stmt.isStrictMode()));
-        jobProperties.put(LoadStmt.EXEC_MEM_LIMIT, String.valueOf(stmt.getExecMemLimit()));
         if (Strings.isNullOrEmpty(stmt.getFormat()) || stmt.getFormat().equals("csv")) {
             jobProperties.put(PROPS_FORMAT, "csv");
             jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
@@ -522,11 +521,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
 
     @Override
     public long getMemLimit() {
-        String value = jobProperties.get(LoadStmt.EXEC_MEM_LIMIT);
-        if (value == null) {
-            return DEFAULT_EXEC_MEM_LIMIT;
-        }
-        return Long.valueOf(value);
+        return execMemLimit;
     }
 
     public String getTimezone() {

fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java

@@ -82,7 +82,7 @@ public class RoutineLoadScheduler extends MasterDaemon {
                 if (desiredConcurrentTaskNum <= 0) {
                     // the job will be rescheduled later.
                     LOG.info(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, routineLoadJob.getId())
-                                     .add("msg", "the current concurrent num is less then or equal to zero, "
+                                     .add("msg", "the current concurrent num is less than or equal to zero, "
                                              + "job will be rescheduled later")
                                      .build());
                     continue;

fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java

@@ -29,7 +29,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 /**
- * Assert num rows node is used to determine whether the number of rows is less then desired num of rows.
+ * Assert num rows node is used to determine whether the number of rows is less than desired num of rows.
  * The rows are the result of subqueryString.
  * If the number of rows is more than the desired num of rows, the query will be cancelled.
  * The cancelled reason will be reported by Backend and displayed back to the user.

fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java

@@ -17,7 +17,7 @@
 
 package org.apache.doris.qe;
 
-import avro.shaded.com.google.common.collect.Lists;
+import org.apache.doris.analysis.InsertStmt;
 import org.apache.doris.analysis.KillStmt;
 import org.apache.doris.analysis.SqlParser;
 import org.apache.doris.analysis.SqlScanner;
@@ -50,6 +50,7 @@ import org.apache.doris.thrift.TMasterOpRequest;
 import org.apache.doris.thrift.TMasterOpResult;
 import org.apache.doris.thrift.TUniqueId;
 
+import com.google.common.collect.Lists;
 import com.google.common.base.Strings;
 
 import org.apache.logging.log4j.LogManager;
@@ -147,7 +148,13 @@ public class ConnectProcessor {
         if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) {
             ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql());
         } else {
-            ctx.getAuditEventBuilder().setStmt(origStmt);
+            if (parsedStmt instanceof InsertStmt && ((InsertStmt)parsedStmt).isValuesOrConstantSelect()) {
+                // INSERT INTO VALUES may be very long, so we only log at most 1K bytes.
+                int length = Math.min(1024, origStmt.length());
+                ctx.getAuditEventBuilder().setStmt(origStmt.substring(0, length));
+            } else {
+                ctx.getAuditEventBuilder().setStmt(origStmt);
+            }
         }
         
         Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build());

fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java

@@ -25,7 +25,7 @@ import java.util.Set;
 public class TabletQuorumFailedException extends TransactionException {
 
     private static final String TABLET_QUORUM_FAILED_MSG = "Failed to commit txn %s. "
-            + "Tablet [%s] success replica num %s is less then quorum "
+            + "Tablet [%s] success replica num %s is less than quorum "
             + "replica num %s while error backends %s";
 
     private long tabletId;

fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java

@@ -130,6 +130,11 @@ public class UtFrameUtils {
             dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString();
         }
         Config.plugin_dir = dorisHome + "/plugins";
+        Config.custom_config_dir = dorisHome + "/conf";
+        File file = new File(Config.custom_config_dir);
+        if (!file.exists()) {
+            file.mkdir();
+        }
 
         int fe_http_port = findValidPort();
         int fe_rpc_port = findValidPort();