database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](fe) Fx SimpleDateFormatter thread unsafe issue by replacing to DateTimeFormatter. (#19265)

Browse Source 
DateTimeFormatter replace SimpleDateFormat in fe module because SimpleDateFormat is not thread-safe.
This commit is contained in:
Chuang Li
2023-05-11 22:50:24 +08:00
committed by GitHub
parent d58498841a
commit a041f8eabe
28 changed files with 247 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/check/checkstyle/import-control.xml
View File

@ -33,6 +33,7 @@ under the License.
<disallow pkg="com.alibaba.fastjson2" />
<disallow pkg="org.apache.log4j.LogManager" />
<disallow pkg="org.apache.log4j.Logger" />
<disallow pkg="java.text.SimpleDateFormat"/>
<subpackage name="nereids">
<allow pkg="org.junit.jupiter"/>
<disallow pkg="org.junit"/>

20
fe/fe-common/src/main/java/org/apache/doris/common/property/PropertySchema.java
View File

@ -25,8 +25,10 @@ import com.google.common.collect.ImmutableMap;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Arrays;
import java.util.Date;
import java.util.Optional;
@ -224,14 +226,14 @@ public abstract class PropertySchema<T> {
}
public static final class DateProperty extends PropertySchema<Date> {
SimpleDateFormat dateFormat;
DateTimeFormatter dateFormat;
public DateProperty(String name, SimpleDateFormat dateFormat) {
public DateProperty(String name, DateTimeFormatter dateFormat) {
super(name);
this.dateFormat = dateFormat;
}
DateProperty(String name, SimpleDateFormat dateFormat, boolean isRequired) {
DateProperty(String name, DateTimeFormatter dateFormat, boolean isRequired) {
super(name, isRequired);
this.dateFormat = dateFormat;
}
@ -266,15 +268,15 @@ public abstract class PropertySchema<T> {
public Date readTimeFormat(String timeStr) throws IllegalArgumentException {
try {
return this.dateFormat.parse(timeStr);
} catch (ParseException e) {
return Date.from(LocalDateTime.parse(timeStr, dateFormat).atZone(ZoneId.systemDefault()).toInstant());
} catch (DateTimeParseException e) {
throw new IllegalArgumentException("Invalid time format, time param need "
+ "to be " + this.dateFormat.toPattern());
+ "to be " + this.dateFormat.toString());
}
}
public String writeTimeFormat(Date timeDate) throws IllegalArgumentException {
return this.dateFormat.format(timeDate.getTime());
return LocalDateTime.ofInstant(timeDate.toInstant(), ZoneId.systemDefault()).format(this.dateFormat);
}
}

23
fe/fe-common/src/test/java/org/apache/doris/common/property/PropertySchemaTest.java
View File

@ -31,7 +31,9 @@ import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Date;
public class PropertySchemaTest {
@ -220,13 +222,16 @@ public class PropertySchemaTest {
@Test
public void testDatePropNormal() throws Exception {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault());
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.DateProperty prop =
new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
prop.write(dateFormat.parse("2021-06-30 20:34:51"), output);
new PropertySchema.DateProperty("key",
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault()));
prop.write(Date.from(
LocalDateTime.parse("2021-06-30 20:34:51", dateFormat).atZone(ZoneId.systemDefault()).toInstant()),
output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
@ -244,8 +249,9 @@ public class PropertySchemaTest {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid time format"));
PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd "
+ "HH:mm:ss"));
PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key",
DateTimeFormatter.ofPattern("yyyy-MM-dd "
+ "HH:mm:ss").withZone(ZoneId.systemDefault()));
prop.read("2021-06-30");
}
@ -254,8 +260,9 @@ public class PropertySchemaTest {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid time format"));
PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd "
+ "HH:mm:ss"));
PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key",
DateTimeFormatter.ofPattern("yyyy-MM-dd "
+ "HH:mm:ss").withZone(ZoneId.systemDefault()));
prop.read((String) null);
}

6
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -86,7 +86,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Calendar;
@ -635,10 +634,11 @@ public class Analyzer {
* Create query global parameters to be set in each TPlanExecRequest.
*/
public static TQueryGlobals createQueryGlobals() {
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSSSSS");
TQueryGlobals queryGlobals = new TQueryGlobals();
Calendar currentDate = Calendar.getInstance();
String nowStr = formatter.format(currentDate.getTime());
LocalDateTime localDateTime = LocalDateTime.ofInstant(currentDate.toInstant(),
currentDate.getTimeZone().toZoneId());
String nowStr = localDateTime.format(TimeUtils.DATETIME_NS_FORMAT);
queryGlobals.setNowString(nowStr);
queryGlobals.setNanoSeconds(LocalDateTime.now().getNano());
return queryGlobals;

4
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
View File

@ -67,7 +67,6 @@ import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@ -647,8 +646,7 @@ public class BackupJob extends AbstractJob {
}
private void saveMetaInfo() {
String createTimeStr = TimeUtils.longToTimeString(createTime, new SimpleDateFormat(
"yyyy-MM-dd-HH-mm-ss"));
String createTimeStr = TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
// local job dir: backup/label__createtime/
localJobDirPath = Paths.get(BackupHandler.BACKUP_ROOT_DIR.toString(),
label + "__" + createTimeStr).normalize();

3
fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
View File

@ -57,7 +57,6 @@ import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.List;
/*
@ -136,7 +135,7 @@ public class Repository implements Writable {
return PREFIX_JOB_INFO;
} else {
return PREFIX_JOB_INFO
+ TimeUtils.longToTimeString(createTime, new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"));
+ TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
}
}

14
fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
View File

@ -75,8 +75,10 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import shade.doris.hive.org.apache.thrift.TException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Deque;
@ -545,7 +547,8 @@ public class HiveMetaStoreClientHelper {
return boolLiteral.getValue();
} else if (expr instanceof DateLiteral) {
DateLiteral dateLiteral = (DateLiteral) expr;
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss")
.withZone(ZoneId.systemDefault());
StringBuilder sb = new StringBuilder();
sb.append(dateLiteral.getYear())
.append(dateLiteral.getMonth())
@ -555,8 +558,9 @@ public class HiveMetaStoreClientHelper {
.append(dateLiteral.getSecond());
Date date;
try {
date = formatter.parse(sb.toString());
} catch (ParseException e) {
date = Date.from(
LocalDateTime.parse(sb.toString(), formatter).atZone(ZoneId.systemDefault()).toInstant());
} catch (DateTimeParseException e) {
return null;
}
return date.getTime();

38
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -49,13 +49,14 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Month;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Comparator;
@ -320,7 +321,7 @@ public class DynamicPartitionUtil {
List<Range> reservedHistoryPeriodsToRangeList = convertStringToPeriodsList(reservedHistoryPeriods, timeUnit);
Integer sizeOfPeriods = reservedHistoryPeriods.split("],\\[").length;
SimpleDateFormat sdf = getSimpleDateFormat(timeUnit);
DateTimeFormatter sdf = getDateTimeFormatter(timeUnit);
if (reservedHistoryPeriodsToRangeList.size() != sizeOfPeriods) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_RESERVED_HISTORY_PERIODS_INVALID,
@ -338,7 +339,7 @@ public class DynamicPartitionUtil {
+ " \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.");
}
}
} catch (ParseException e) {
} catch (DateTimeParseException e) {
throw new DdlException("Invalid " + DynamicPartitionProperty.RESERVED_HISTORY_PERIODS
+ " value. It must be like \"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\""
+ " while time_unit is DAY/WEEK/MONTH "
@ -377,11 +378,11 @@ public class DynamicPartitionUtil {
}
}
private static SimpleDateFormat getSimpleDateFormat(String timeUnit) {
private static DateTimeFormatter getDateTimeFormatter(String timeUnit) {
if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return new SimpleDateFormat(DATETIME_FORMAT);
return TimeUtils.DATETIME_FORMAT;
} else {
return new SimpleDateFormat(DATE_FORMAT);
return TimeUtils.DATE_FORMAT;
}
}
@ -721,8 +722,10 @@ public class DynamicPartitionUtil {
formattedDateStr = formattedDateStr.substring(0, 8);
Calendar calendar = Calendar.getInstance(tz);
try {
calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
calendar.setTime(Date.from(
LocalDate.parse(formattedDateStr, formatter).atStartOfDay().atZone(tz.toZoneId()).toInstant()));
} catch (DateTimeParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
return formattedDateStr;
}
@ -756,22 +759,29 @@ public class DynamicPartitionUtil {
public static String getHistoryPartitionRangeString(DynamicPartitionProperty dynamicPartitionProperty,
String time, String format) throws AnalysisException {
ZoneId zoneId = dynamicPartitionProperty.getTimeZone().toZoneId();
Date date = null;
LocalDateTime dateTime = null;
Timestamp timestamp = null;
String timeUnit = dynamicPartitionProperty.getTimeUnit();
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s").withZone(zoneId);
SimpleDateFormat simpleDateFormat = getSimpleDateFormat(timeUnit);
try {
date = simpleDateFormat.parse(time);
} catch (ParseException e) {
dateTime = getDateTimeByTimeUnit(time, timeUnit);
} catch (DateTimeParseException e) {
LOG.warn("Parse dynamic partition periods error. Error={}", e.getMessage());
throw new AnalysisException("Parse dynamic partition periods error. Error=" + e.getMessage());
}
timestamp = new Timestamp(date.getTime());
timestamp = Timestamp.valueOf(dateTime);
return getFormattedTimeWithoutMinuteSecond(
ZonedDateTime.parse(timestamp.toString(), dateTimeFormatter), format);
}
private static LocalDateTime getDateTimeByTimeUnit(String time, String timeUnit) {
if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return LocalDateTime.parse(time, TimeUtils.DATETIME_FORMAT);
} else {
return LocalDate.from(TimeUtils.DATE_FORMAT.parse(time)).atStartOfDay();
}
}
/**
* return formatted string of partition range in HOUR granularity.
* offset: The offset from the current hour. 0 means current hour, 1 means next hour, -1 last hour.

137
fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
View File

@ -32,42 +32,35 @@ import com.google.common.collect.ImmutableMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.time.DateTimeException;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.Date;
import java.util.SimpleTimeZone;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
// TODO(dhc) add nanosecond timer for coordinator's root profile
public class TimeUtils {
private static final Logger LOG = LogManager.getLogger(TimeUtils.class);
public static final String UTC_TIME_ZONE = "Europe/London"; // This is just a Country to represent UTC offset +00:00
public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
private static final TimeZone TIME_ZONE;
public static final ZoneId TIME_ZONE;
// set CST to +08:00 instead of America/Chicago
public static final ImmutableMap<String, String> timeZoneAliasMap = ImmutableMap.of(
"CST", DEFAULT_TIME_ZONE, "PRC", DEFAULT_TIME_ZONE, "UTC", UTC_TIME_ZONE);
// NOTICE: Date formats are not synchronized.
// it must be used as synchronized externally.
private static final SimpleDateFormat DATE_FORMAT;
private static final SimpleDateFormat DATETIME_FORMAT;
private static final SimpleDateFormat TIME_FORMAT;
public static final DateTimeFormatter DATE_FORMAT;
public static final DateTimeFormatter DATETIME_FORMAT;
public static final DateTimeFormatter TIME_FORMAT;
public static final Pattern DATETIME_FORMAT_REG =
Pattern.compile("^((\\d{2}(([02468][048])|([13579][26]))[\\-\\/\\s]?((((0?[13578])|(1[02]))[\\-\\/\\s]?"
+ "((0?[1-9])|([1-2][0-9])|(3[01])))|(((0?[469])|(11))[\\-\\/\\s]?"
@ -76,41 +69,46 @@ public class TimeUtils {
+ "[\\-\\/\\s]?((0?[1-9])|([1-2][0-9])|(3[01])))|(((0?[469])|(11))[\\-\\/\\s]?"
+ "((0?[1-9])|([1-2][0-9])|(30)))|(0?2[\\-\\/\\s]?((0?[1-9])|(1[0-9])|(2[0-8]))))))"
+ "(\\s(((0?[0-9])|([1][0-9])|([2][0-3]))\\:([0-5]?[0-9])((\\s)|(\\:([0-5]?[0-9])))))?$");
public static final DateTimeFormatter DATETIME_MS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
.withZone(ZoneId.systemDefault());
public static final DateTimeFormatter DATETIME_NS_FORMAT = DateTimeFormatter.ofPattern(
"yyyy-MM-dd HH:mm:ss.SSSSSSSSS")
.withZone(ZoneId.systemDefault());
public static final DateTimeFormatter DATETIME_FORMAT_WITH_HYPHEN = DateTimeFormatter.ofPattern(
"yyyy-MM-dd-HH-mm-ss")
.withZone(ZoneId.systemDefault());
private static final Logger LOG = LogManager.getLogger(TimeUtils.class);
private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]?\\d{1,2}:\\d{2}$");
public static Date MIN_DATE = null;
public static Date MAX_DATE = null;
public static Date MIN_DATETIME = null;
public static Date MAX_DATETIME = null;
private static ThreadLocal<SimpleDateFormat> datetimeFormatThreadLocal =
ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
private static ThreadLocal<SimpleDateFormat> datetimeMSFormatThreadLocal =
ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"));
static {
TIME_ZONE = new SimpleTimeZone(8 * 3600 * 1000, "");
TIME_ZONE = ZoneId.of("UTC+8");
DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
DATE_FORMAT.setTimeZone(TIME_ZONE);
DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
DATE_FORMAT.withZone(TIME_ZONE);
DATETIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
DATETIME_FORMAT.setTimeZone(TIME_ZONE);
DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
DATETIME_FORMAT.withZone(TIME_ZONE);
TIME_FORMAT = new SimpleDateFormat("HH");
TIME_FORMAT.setTimeZone(TIME_ZONE);
TIME_FORMAT = DateTimeFormatter.ofPattern("HH");
TIME_FORMAT.withZone(TIME_ZONE);
try {
MIN_DATE = DATE_FORMAT.parse("0000-01-01");
MAX_DATE = DATE_FORMAT.parse("9999-12-31");
MIN_DATETIME = DATETIME_FORMAT.parse("0000-01-01 00:00:00");
MAX_DATETIME = DATETIME_FORMAT.parse("9999-12-31 23:59:59");
MIN_DATE = Date.from(
LocalDate.parse("0001-01-01", DATE_FORMAT).atStartOfDay().atZone(TIME_ZONE).toInstant());
MAX_DATE = Date.from(
LocalDate.parse("9999-12-31", DATE_FORMAT).atStartOfDay().atZone(TIME_ZONE).toInstant());
} catch (ParseException e) {
MIN_DATETIME = Date.from(
LocalDateTime.parse("0001-01-01 00:00:00", DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
MAX_DATETIME = Date.from(
LocalDateTime.parse("9999-12-31 23:59:59", DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
} catch (DateTimeParseException e) {
LOG.error("invalid date format", e);
System.exit(-1);
}
@ -124,8 +122,8 @@ public class TimeUtils {
return System.currentTimeMillis() - startTime;
}
public static synchronized String getCurrentFormatTime() {
return DATETIME_FORMAT.format(new Date());
public static String getCurrentFormatTime() {
return LocalDateTime.now().format(DATETIME_FORMAT);
}
public static TimeZone getTimeZone() {
@ -151,55 +149,55 @@ public class TimeUtils {
return TimeZone.getTimeZone(ZoneId.of(timeZone, timeZoneAliasMap));
}
public static String longToTimeString(long timeStamp, SimpleDateFormat dateFormat) {
public static String longToTimeString(long timeStamp, DateTimeFormatter dateFormat) {
if (timeStamp <= 0L) {
return FeConstants.null_string;
}
return dateFormat.format(new Date(timeStamp));
return dateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), ZoneId.systemDefault()));
}
public static String longToTimeStringWithFormat(long timeStamp, SimpleDateFormat datetimeFormatTimeZone) {
public static String longToTimeStringWithFormat(long timeStamp, DateTimeFormatter datetimeFormatTimeZone) {
TimeZone timeZone = getTimeZone();
datetimeFormatTimeZone.setTimeZone(timeZone);
datetimeFormatTimeZone.withZone(timeZone.toZoneId());
return longToTimeString(timeStamp, datetimeFormatTimeZone);
}
public static String longToTimeString(long timeStamp) {
SimpleDateFormat datetimeFormatTimeZone = datetimeFormatThreadLocal.get();
return longToTimeStringWithFormat(timeStamp, datetimeFormatTimeZone);
return longToTimeStringWithFormat(timeStamp, DATETIME_FORMAT);
}
public static String longToTimeStringWithms(long timeStamp) {
SimpleDateFormat datatimeFormatTimeZone = datetimeMSFormatThreadLocal.get();
return longToTimeStringWithFormat(timeStamp, datatimeFormatTimeZone);
return longToTimeStringWithFormat(timeStamp, DATETIME_MS_FORMAT);
}
public static synchronized Date getTimeAsDate(String timeString) {
public static Date getTimeAsDate(String timeString) {
try {
Date date = TIME_FORMAT.parse(timeString);
return date;
} catch (ParseException e) {
return Date.from(
LocalTime.parse(timeString, TIME_FORMAT).atDate(LocalDate.now()).atZone(TIME_ZONE).toInstant());
} catch (DateTimeParseException e) {
LOG.warn("invalid time format: {}", timeString);
return null;
}
}
public static synchronized Date parseDate(String dateStr, PrimitiveType type) throws AnalysisException {
public static Date parseDate(String dateStr, PrimitiveType type) throws AnalysisException {
Date date = null;
Matcher matcher = DATETIME_FORMAT_REG.matcher(dateStr);
if (!matcher.matches()) {
throw new AnalysisException("Invalid date string: " + dateStr);
}
dateStr = formatDateStr(dateStr);
if (type == PrimitiveType.DATE) {
ParsePosition pos = new ParsePosition(0);
date = DATE_FORMAT.parse(dateStr, pos);
date = Date.from(
LocalDate.from(DATE_FORMAT.parse(dateStr, pos)).atStartOfDay().atZone(TIME_ZONE).toInstant());
if (pos.getIndex() != dateStr.length() || date == null) {
throw new AnalysisException("Invalid date string: " + dateStr);
}
} else if (type == PrimitiveType.DATETIME) {
try {
date = DATETIME_FORMAT.parse(dateStr);
} catch (ParseException e) {
date = Date.from(LocalDateTime.parse(dateStr, DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
} catch (DateTimeParseException e) {
throw new AnalysisException("Invalid date string: " + dateStr);
}
} else {
@ -209,44 +207,44 @@ public class TimeUtils {
return date;
}
public static synchronized Date parseDate(String dateStr, Type type) throws AnalysisException {
public static Date parseDate(String dateStr, Type type) throws AnalysisException {
return parseDate(dateStr, type.getPrimitiveType());
}
public static synchronized String format(Date date, PrimitiveType type) {
public static String format(Date date, PrimitiveType type) {
if (type == PrimitiveType.DATE) {
return DATE_FORMAT.format(date);
return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).format(DATE_FORMAT);
} else if (type == PrimitiveType.DATETIME) {
return DATETIME_FORMAT.format(date);
return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).format(DATETIME_FORMAT);
} else {
return "INVALID";
}
}
public static synchronized String format(Date date, Type type) {
public static String format(Date date, Type type) {
return format(date, type.getPrimitiveType());
}
public static synchronized long timeStringToLong(String timeStr) {
public static long timeStringToLong(String timeStr) {
Date d;
try {
d = DATETIME_FORMAT.parse(timeStr);
} catch (ParseException e) {
d = Date.from(LocalDateTime.parse(timeStr, DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
} catch (DateTimeParseException e) {
return -1;
}
return d.getTime();
}
public static long timeStringToLong(String timeStr, TimeZone timeZone) {
SimpleDateFormat dateFormatTimeZone = datetimeFormatThreadLocal.get();
dateFormatTimeZone.setTimeZone(timeZone);
Date d;
DateTimeFormatter dateFormatTimeZone = DATETIME_FORMAT;
dateFormatTimeZone.withZone(timeZone.toZoneId());
LocalDateTime d;
try {
d = dateFormatTimeZone.parse(timeStr);
} catch (ParseException e) {
d = LocalDateTime.parse(timeStr, dateFormatTimeZone);
} catch (DateTimeParseException e) {
return -1;
}
return d.getTime();
return d.atZone(timeZone.toZoneId()).toInstant().toEpochMilli();
}
// Check if the time zone_value is valid
@ -305,4 +303,11 @@ public class TimeUtils {
LocalTime.of(hour, minute, second, milliSecond * 1000000));
}
private static String formatDateStr(String dateStr) {
String[] parts = dateStr.trim().split("[ :-]+");
return String.format("%s-%02d-%02d%s", parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2]),
parts.length > 3 ? String.format(" %02d:%02d:%02d", Integer.parseInt(parts[3]),
Integer.parseInt(parts[4]), Integer.parseInt(parts[5])) : "");
}
}

4
fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java vendored
View File

@ -28,13 +28,13 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.SystemIdGenerator;
import org.apache.doris.common.property.PropertySchema;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.common.util.TimeUtils;
import com.google.common.collect.Maps;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
@ -132,7 +132,7 @@ public class IcebergTableCreationRecordMgr extends MasterDaemon {
@Override
protected void runAfterCatalogReady() {
PropertySchema.DateProperty prop =
new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
new PropertySchema.DateProperty("key", TimeUtils.DATETIME_FORMAT);
// list iceberg tables in dbs
// When listing table is done, remove database from icebergDbs.
for (Iterator<Map.Entry<Long, Database>> it = icebergDbs.entrySet().iterator(); it.hasNext(); it.remove()) {

14
fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java vendored
View File

@ -59,8 +59,10 @@ import org.apache.iceberg.types.Types;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -370,7 +372,8 @@ public class IcebergUtils {
return boolLiteral.getValue();
} else if (expr instanceof DateLiteral) {
DateLiteral dateLiteral = (DateLiteral) expr;
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss")
.withZone(ZoneId.systemDefault());
StringBuilder sb = new StringBuilder();
sb.append(dateLiteral.getYear())
.append(dateLiteral.getMonth())
@ -380,8 +383,9 @@ public class IcebergUtils {
.append(dateLiteral.getSecond());
Date date;
try {
date = formatter.parse(sb.toString());
} catch (ParseException e) {
date = Date.from(
LocalDateTime.parse(sb.toString(), formatter).atZone(ZoneId.systemDefault()).toInstant());
} catch (DateTimeParseException e) {
return null;
}
return date.getTime();

5
fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java
View File

@ -50,7 +50,6 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
@ -248,9 +247,9 @@ public class StreamLoadRecordMgr extends MasterDaemon {
for (Map.Entry<String, TStreamLoadRecord> entry : streamLoadRecordBatch.entrySet()) {
TStreamLoadRecord streamLoadItem = entry.getValue();
String startTime = TimeUtils.longToTimeString(streamLoadItem.getStartTime(),
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"));
TimeUtils.DATETIME_MS_FORMAT);
String finishTime = TimeUtils.longToTimeString(streamLoadItem.getFinishTime(),
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"));
TimeUtils.DATETIME_MS_FORMAT);
if (LOG.isDebugEnabled()) {
LOG.debug("receive stream load record info from backend: {}."
+ " label: {}, db: {}, tbl: {}, user: {}, user_ip: {},"

11
fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java
View File

@ -41,11 +41,13 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -440,10 +442,11 @@ public class CanalSyncDataConsumer extends SyncDataConsumer {
long ackTime = positionMeta.getAckTime();
StringBuilder sb = new StringBuilder();
if (ackPosition != null) {
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
DateTimeFormatter format = DateTimeFormatter.ofPattern(DATE_FORMAT).withZone(ZoneId.systemDefault());
long executeTime = ackPosition.getExecuteTime();
long delayTime = ackTime - executeTime;
Date date = new Date(executeTime);
LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()),
ZoneId.systemDefault());
sb.append("position:").append(ackPosition)
.append(", executeTime:[").append(format.format(date)).append("], ")
.append("delay:").append(delayTime).append("ms");

31
fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.load.sync.canal;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.load.sync.model.Events;
import org.apache.doris.load.sync.position.EntryPosition;
@ -31,16 +32,15 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
public class CanalUtils {
private static Logger logger = LogManager.getLogger(CanalUtils.class);
private static final String SEP = SystemUtils.LINE_SEPARATOR;
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(DATE_FORMAT);
private static String context_format = null;
private static String row_format = null;
@ -68,9 +68,8 @@ public class CanalUtils {
}
String startPosition = buildPositionForDump(entries.get(0));
String endPosition = buildPositionForDump(entries.get(entries.size() - 1));
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
logger.info(context_format, dataEvents.getId(), entries.size(), dataEvents.getMemSize(),
format.format(new Date()), startPosition, endPosition);
TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()), startPosition, endPosition);
}
public static void printSummary(Message message, int size, long memsize) {
@ -80,16 +79,14 @@ public class CanalUtils {
}
String startPosition = buildPositionForDump(message.getEntries().get(0));
String endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
logger.info(context_format, message.getId(), size, memsize,
format.format(new Date()), startPosition, endPosition);
TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()), startPosition, endPosition);
}
public static String buildPositionForDump(CanalEntry.Entry entry) {
CanalEntry.Header header = entry.getHeader();
long time = entry.getHeader().getExecuteTime();
Date date = new Date(time);
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneId.systemDefault());
StringBuilder sb = new StringBuilder();
sb.append(header.getLogfileName())
.append(":")
@ -97,7 +94,7 @@ public class CanalUtils {
.append(":")
.append(header.getExecuteTime())
.append("(")
.append(format.format(date))
.append(TimeUtils.DATETIME_FORMAT.format(date))
.append(")");
if (StringUtils.isNotEmpty(entry.getHeader().getGtid())) {
sb.append(" gtid(").append(entry.getHeader().getGtid())
@ -118,12 +115,12 @@ public class CanalUtils {
public static void printRow(CanalEntry.RowChange rowChange, CanalEntry.Header header) {
long executeTime = header.getExecuteTime();
long delayTime = System.currentTimeMillis() - executeTime;
Date date = new Date(executeTime);
LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(executeTime), ZoneId.systemDefault());
CanalEntry.EventType eventType = rowChange.getEventType();
logger.info(row_format, header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), header.getSchemaName(),
header.getTableName(), eventType,
String.valueOf(header.getExecuteTime()), simpleDateFormat.format(date),
String.valueOf(header.getExecuteTime()), TimeUtils.DATETIME_FORMAT.format(date),
header.getGtid(), String.valueOf(delayTime));
if (eventType == CanalEntry.EventType.QUERY || rowChange.getIsDdl()) {
logger.info(" sql ----> " + rowChange.getSql() + SEP);
@ -190,7 +187,7 @@ public class CanalUtils {
public static void transactionBegin(CanalEntry.Entry entry) {
long executeTime = entry.getHeader().getExecuteTime();
long delayTime = System.currentTimeMillis() - executeTime;
Date date = new Date(executeTime);
LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(executeTime), ZoneId.systemDefault());
CanalEntry.TransactionBegin begin = null;
try {
begin = CanalEntry.TransactionBegin.parseFrom(entry.getStoreValue());
@ -200,7 +197,7 @@ public class CanalUtils {
// print transaction begin info, thread ID, time consumption
logger.info(transaction_format, entry.getHeader().getLogfileName(),
String.valueOf(entry.getHeader().getLogfileOffset()),
String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date),
String.valueOf(entry.getHeader().getExecuteTime()), TimeUtils.DATETIME_FORMAT.format(date),
entry.getHeader().getGtid(), String.valueOf(delayTime));
logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
printXAInfo(begin.getPropsList());
@ -209,7 +206,7 @@ public class CanalUtils {
public static void transactionEnd(CanalEntry.Entry entry) {
long executeTime = entry.getHeader().getExecuteTime();
long delayTime = System.currentTimeMillis() - executeTime;
Date date = new Date(executeTime);
LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(executeTime), ZoneId.systemDefault());
CanalEntry.TransactionEnd end = null;
try {
end = CanalEntry.TransactionEnd.parseFrom(entry.getStoreValue());
@ -222,7 +219,7 @@ public class CanalUtils {
printXAInfo(end.getPropsList());
logger.info(transaction_format, entry.getHeader().getLogfileName(),
String.valueOf(entry.getHeader().getLogfileOffset()),
String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date),
String.valueOf(entry.getHeader().getExecuteTime()), TimeUtils.DATETIME_FORMAT.format(date),
entry.getHeader().getGtid(), String.valueOf(delayTime));
}

11
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.analysis.MVRefreshIntervalTriggerInfo;
import org.apache.doris.analysis.MVRefreshTriggerInfo;
import org.apache.doris.catalog.MaterializedView;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.mtmv.MTMVUtils.TriggerMode;
import org.apache.doris.mtmv.metadata.MTMVJob;
import org.apache.doris.mtmv.metadata.MTMVJob.JobSchedule;
@ -31,8 +32,8 @@ import org.apache.doris.mtmv.metadata.MTMVJob.JobSchedule;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@ -97,11 +98,11 @@ public class MTMVJobFactory {
private static JobSchedule genJobSchedule(MaterializedView materializedView) {
MVRefreshIntervalTriggerInfo info = materializedView.getRefreshInfo().getTriggerInfo().getIntervalTrigger();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long startTime;
try {
startTime = format.parse(info.getStartTime()).getTime() / 1000;
} catch (ParseException e) {
LocalDateTime dateTime = LocalDateTime.parse(info.getStartTime(), TimeUtils.DATETIME_FORMAT);
startTime = dateTime.toEpochSecond(TimeUtils.TIME_ZONE.getRules().getOffset(dateTime));
} catch (DateTimeParseException e) {
throw new RuntimeException(e);
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
View File

@ -53,7 +53,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -69,7 +68,6 @@ import java.util.concurrent.TimeUnit;
*/
public class FoldConstantRuleOnBE extends AbstractExpressionRewriteRule {
private static final Logger LOG = LogManager.getLogger(FoldConstantRuleOnBE.class);
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private final IdGenerator<ExprId> idGenerator = ExprId.createGenerator();
@Override
@ -159,7 +157,7 @@ public class FoldConstantRuleOnBE extends AbstractExpressionRewriteRule {
TNetworkAddress brpcAddress = new TNetworkAddress(be.getHost(), be.getBrpcPort());
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(DATE_FORMAT.format(LocalDateTime.now()));
queryGlobals.setNowString(TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()));
queryGlobals.setTimestampMs(System.currentTimeMillis());
queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
if (context.getSessionVariable().getTimeZone().equals("CST")) {

7
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
View File

@ -43,6 +43,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.load.BrokerFileGroup;
import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.load.routineload.RoutineLoadJob;
@ -66,10 +67,7 @@ import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -79,7 +77,6 @@ import java.util.Map;
// TODO(zc): support other type table
public class StreamLoadPlanner {
private static final Logger LOG = LogManager.getLogger(StreamLoadPlanner.class);
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// destination Db and table get from request
// Data will load to this table
@ -301,7 +298,7 @@ public class StreamLoadPlanner {
params.setQueryOptions(queryOptions);
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
queryGlobals.setNowString(TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()));
queryGlobals.setTimestampMs(System.currentTimeMillis());
queryGlobals.setTimeZone(taskInfo.getTimezone());
queryGlobals.setLoadZeroTolerance(taskInfo.getMaxFilterRatio() <= 0.0);

18
fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java
View File

@ -37,8 +37,8 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -161,12 +161,12 @@ public class StoragePolicy extends Policy {
throw new AnalysisException(COOLDOWN_DATETIME + " or " + COOLDOWN_TTL + " must be set");
}
if (hasCooldownDatetime) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
this.cooldownTimestampMs = df.parse(props.get(COOLDOWN_DATETIME)).getTime();
} catch (ParseException e) {
this.cooldownTimestampMs = LocalDateTime.parse(props.get(COOLDOWN_DATETIME), TimeUtils.DATETIME_FORMAT)
.atZone(TimeUtils.TIME_ZONE).toInstant().toEpochMilli();
} catch (DateTimeParseException e) {
throw new AnalysisException(String.format("cooldown_datetime format error: %s",
props.get(COOLDOWN_DATETIME)), e);
props.get(COOLDOWN_DATETIME)), e);
}
// ttl would be set as -1 when using datetime
this.cooldownTtl = -1;
@ -325,10 +325,10 @@ public class StoragePolicy extends Policy {
if (properties.get(COOLDOWN_DATETIME).isEmpty()) {
cooldownTimestampMs = -1;
} else {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
cooldownTimestampMs = df.parse(properties.get(COOLDOWN_DATETIME)).getTime();
} catch (ParseException e) {
cooldownTimestampMs = LocalDateTime.parse(properties.get(COOLDOWN_DATETIME),
TimeUtils.DATETIME_FORMAT).atZone(TimeUtils.TIME_ZONE).toInstant().toEpochMilli();
} catch (DateTimeParseException e) {
throw new RuntimeException(e);
}
}

9
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -130,13 +130,10 @@ import org.apache.thrift.TException;
import org.jetbrains.annotations.NotNull;
import java.nio.charset.StandardCharsets;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -156,8 +153,6 @@ import java.util.stream.Collectors;
public class Coordinator {
private static final Logger LOG = LogManager.getLogger(Coordinator.class);
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static final String localIP = FrontendOptions.getLocalHostAddress();
// Random is used to shuffle instances of partitioned
@ -336,7 +331,7 @@ public class Coordinator {
setFromUserProperty(context);
this.queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
this.queryGlobals.setNowString(TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()));
this.queryGlobals.setTimestampMs(System.currentTimeMillis());
this.queryGlobals.setNanoSeconds(LocalDateTime.now().getNano());
this.queryGlobals.setLoadZeroTolerance(false);
@ -365,7 +360,7 @@ public class Coordinator {
this.fragments = fragments;
this.scanNodes = scanNodes;
this.queryOptions = new TQueryOptions();
this.queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
this.queryGlobals.setNowString(TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()));
this.queryGlobals.setTimestampMs(System.currentTimeMillis());
this.queryGlobals.setTimeZone(timezone);
this.queryGlobals.setLoadZeroTolerance(loadZeroTolerance);

19
fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java vendored
View File

@ -41,7 +41,10 @@ import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -139,8 +142,8 @@ public class PartitionRange {
}
public static class PartitionKeyType {
private SimpleDateFormat df8 = new SimpleDateFormat("yyyyMMdd");
private SimpleDateFormat df10 = new SimpleDateFormat("yyyy-MM-dd");
private DateTimeFormatter df8 = DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneId.systemDefault());
private DateTimeFormatter df10 = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault());
public KeyType keyType = KeyType.DEFAULT;
public long value;
@ -150,7 +153,8 @@ public class PartitionRange {
switch (type.getPrimitiveType()) {
case DATE:
try {
date = df10.parse(str);
date = Date.from(
LocalDate.parse(str, df10).atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
} catch (Exception e) {
LOG.warn("parse error str{}.", str);
return false;
@ -226,7 +230,7 @@ public class PartitionRange {
if (keyType == KeyType.DEFAULT) {
return "";
} else if (keyType == KeyType.DATE) {
return df10.format(date);
return df10.format(LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()));
} else {
return String.valueOf(value);
}
@ -234,7 +238,7 @@ public class PartitionRange {
public long realValue() {
if (keyType == KeyType.DATE) {
return Long.parseLong(df8.format(date));
return Long.parseLong(df8.format(LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault())));
} else {
return value;
}
@ -244,7 +248,8 @@ public class PartitionRange {
value = expr.getLongValue() / 1000000;
Date dt = null;
try {
dt = df8.parse(String.valueOf(value));
dt = Date.from(LocalDate.parse(String.valueOf(value), df8).atStartOfDay().atZone(ZoneId.systemDefault())
.toInstant());
} catch (Exception e) {
// CHECKSTYLE IGNORE THIS LINE
}

6
fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
View File

@ -57,11 +57,8 @@ import com.google.common.base.Predicates;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -85,7 +82,6 @@ import java.util.concurrent.TimeUnit;
*/
public class FoldConstantsRule implements ExprRewriteRule {
private static final Logger LOG = LogManager.getLogger(FoldConstantsRule.class);
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
public static ExprRewriteRule INSTANCE = new FoldConstantsRule();
@ -357,7 +353,7 @@ public class FoldConstantsRule implements ExprRewriteRule {
brpcAddress = new TNetworkAddress(be.getHost(), be.getBrpcPort());
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
queryGlobals.setNowString(TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()));
queryGlobals.setTimestampMs(System.currentTimeMillis());
queryGlobals.setNanoSeconds(LocalDateTime.now().getNano());
queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);

11
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@ -54,11 +55,12 @@ import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -542,15 +544,14 @@ public class AnalysisManager {
ImmutableList<String> titleNames = stmt.getTitleNames();
List<ResultRow> resultRows = StatisticsUtil.execStatisticQuery(executeSql);
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
for (ResultRow resultRow : resultRows) {
List<Comparable> result = Lists.newArrayList();
for (String column : titleNames) {
String value = resultRow.getColumnValue(column);
if (LAST_EXEC_TIME_IN_MS.equals(column)) {
long timeMillis = Long.parseLong(value);
value = dateFormat.format(new Date(timeMillis));
value = TimeUtils.DATETIME_FORMAT.format(
LocalDateTime.ofInstant(Instant.ofEpochMilli(timeMillis), ZoneId.systemDefault()));
}
result.add(value);
}

11
fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.StmtExecutor;
@ -40,10 +41,11 @@ import org.apache.logging.log4j.Logger;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -274,8 +276,9 @@ public class HiveAnalysisTask extends HMSAnalysisTask {
timestamp = parameters.get(TIMESTAMP);
}
params.put("numRows", numRows);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
params.put("update_time", sdf.format(new Date(Long.parseLong(timestamp) * 1000)));
params.put("update_time", TimeUtils.DATETIME_FORMAT.format(
LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(timestamp) * 1000),
ZoneId.systemDefault())));
}
private String genColumnStatId(long tableId, long indexId, String columnName, String partitionName) {

7
fe/fe-core/src/main/java/org/apache/doris/statistics/IcebergAnalysisTask.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.StmtExecutor;
@ -34,8 +35,7 @@ import org.apache.iceberg.TableScan;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.types.Types;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
@ -107,8 +107,7 @@ public class IcebergAnalysisTask extends HMSAnalysisTask {
params.put("numRows", String.valueOf(numRows));
params.put("nulls", String.valueOf(numNulls));
params.put("dataSize", String.valueOf(dataSize));
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
params.put("update_time", sdf.format(new Date()));
params.put("update_time", TimeUtils.DATETIME_FORMAT.format(LocalDateTime.now()));
// Update table level stats info of this column.
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);

16
fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
View File

@ -39,7 +39,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
@ -506,9 +508,10 @@ public class DynamicPartitionTableTest {
Collections.sort(partNames);
int partitionCount = 0;
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
DateTimeFormatter sdf = DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneId.systemDefault());
for (String partName : partNames) {
Date partitionDate = sdf.parse(partName);
Date partitionDate = Date.from(
LocalDate.parse(partName, sdf).atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
Date date = new Date();
Calendar calendar = new GregorianCalendar();
calendar.setTime(date);
@ -562,9 +565,10 @@ public class DynamicPartitionTableTest {
Collections.sort(partNames);
int partitionCount = -3;
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
DateTimeFormatter sdf = DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneId.systemDefault());
for (String partName : partNames) {
Date partitionDate = sdf.parse(partName);
Date partitionDate = Date.from(
LocalDate.parse(partName, sdf).atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
Date date = new Date();
Calendar calendar = new GregorianCalendar();
calendar.setTime(date);
@ -1405,7 +1409,7 @@ public class DynamicPartitionTableTest {
+ ");";
ExceptionChecker.expectThrowsWithMsg(DdlException.class,
"errCode = 2, detailMessage = Invalid dynamic_partition.reserved_history_periods value. "
+ "It must be correct DATE value "
+ "It must be like "
+ "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH or "
+ "\"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.",
() -> createTable(createOlapTblStmt2));

7
fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
View File

@ -39,7 +39,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -142,7 +144,8 @@ public class PropertyAnalyzerTest {
@Test
public void testStorageMedium() throws AnalysisException {
long tomorrowTs = System.currentTimeMillis() / 1000 + 86400;
String tomorrowTimeStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000);
String tomorrowTimeStr = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault())
.format(Instant.ofEpochMilli(tomorrowTs * 1000));
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD");

8
fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
View File

@ -58,9 +58,9 @@ public class TimeUtilsTest {
Assert.assertNotNull(TimeUtils.getStartTimeMs());
Assert.assertTrue(TimeUtils.getElapsedTimeMs(0L) > 0);
Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATE.getTime());
Assert.assertEquals(-62135625600000L, TimeUtils.MIN_DATE.getTime());
Assert.assertEquals(253402185600000L, TimeUtils.MAX_DATE.getTime());
Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATETIME.getTime());
Assert.assertEquals(-62135625600000L, TimeUtils.MIN_DATETIME.getTime());
Assert.assertEquals(253402271999000L, TimeUtils.MAX_DATETIME.getTime());
}
@ -75,7 +75,7 @@ public class TimeUtilsTest {
validDateList.add("9999-12-31");
validDateList.add("1900-01-01");
validDateList.add("2013-2-28");
validDateList.add("0000-01-01");
validDateList.add("0001-01-01");
for (String validDate : validDateList) {
try {
TimeUtils.parseDate(validDate, PrimitiveType.DATE);
@ -114,7 +114,7 @@ public class TimeUtilsTest {
validDateTimeList.add("2013-2-28 23:59:59");
validDateTimeList.add("2013-2-28 2:3:4");
validDateTimeList.add("2014-05-07 19:8:50");
validDateTimeList.add("0000-01-01 00:00:00");
validDateTimeList.add("0001-01-01 00:00:00");
for (String validDateTime : validDateTimeList) {
try {
TimeUtils.parseDate(validDateTime, PrimitiveType.DATETIME);

16
fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
View File

@ -17,15 +17,15 @@
package org.apache.doris.plugin.audit;
import com.google.common.collect.Queues;
import org.apache.doris.common.Config;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditPlugin;
import org.apache.doris.plugin.Plugin;
import org.apache.doris.plugin.PluginContext;
import org.apache.doris.plugin.PluginException;
import org.apache.doris.plugin.PluginInfo;
import org.apache.doris.common.Config;
import com.google.common.collect.Queues;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -39,8 +39,10 @@ import java.nio.charset.CodingErrorAction;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
@ -54,8 +56,8 @@ import java.util.stream.Collectors;
public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
private final static Logger LOG = LogManager.getLogger(AuditLoaderPlugin.class);
private static final ThreadLocal<SimpleDateFormat> dateFormatContainer = ThreadLocal.withInitial(
() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.systemDefault());
private StringBuilder auditLogBuffer = new StringBuilder();
private StringBuilder slowLogBuffer = new StringBuilder();
@ -339,6 +341,6 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
if (timeStamp <= 0L) {
return "1900-01-01 00:00:00";
}
return dateFormatContainer.get().format(new Date(timeStamp));
return DATETIME_FORMAT.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), ZoneId.systemDefault()));
}
}