From 11b78008cdee3582de30edb4e925e251878b3f02 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Fri, 20 Dec 2019 07:45:29 +0800 Subject: [PATCH] Timezone variable support one digital time (#2513) Support time zone variable like "-8:00","+8:00","8:00" Time zone variable like "-8:00" is illegal in time-zone ID ,so we mush transfer it to standard format --- .../doris/analysis/CreateRoutineLoadStmt.java | 3 +-- .../org/apache/doris/analysis/LoadStmt.java | 3 ++- .../apache/doris/common/util/TimeUtils.java | 14 +++++++++++-- .../java/org/apache/doris/qe/VariableMgr.java | 5 ++++- .../org/apache/doris/task/StreamLoadTask.java | 3 +-- .../analysis/CreateRoutineLoadStmtTest.java | 3 +++ .../doris/common/util/TimeUtilsTest.java | 20 +++++++++++++++++++ .../org/apache/doris/qe/VariableMgrTest.java | 12 +++++++++++ 8 files changed, 55 insertions(+), 8 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java index 06a51ae9db..9e83d41cdf 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java @@ -328,8 +328,7 @@ public class CreateRoutineLoadStmt extends DdlStmt { if (ConnectContext.get() != null) { timezone = ConnectContext.get().getSessionVariable().getTimeZone(); } - timezone = jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone); - TimeUtils.checkTimeZoneValid(timezone); + timezone = TimeUtils.checkTimeZoneValidAndStandardize(jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone)); } private void checkDataSourceProperties() throws AnalysisException { diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java index 957e0dfff8..a423d9ca72 100644 --- a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -201,7 +201,8 @@ public class LoadStmt extends DdlStmt { // time zone final String timezone = properties.get(TIMEZONE); if (timezone != null) { - TimeUtils.checkTimeZoneValid(timezone); + properties.put(TIMEZONE, TimeUtils.checkTimeZoneValidAndStandardize( + properties.getOrDefault(LoadStmt.TIMEZONE, TimeUtils.DEFAULT_TIME_ZONE))); } } diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java index 571ca2b7a5..37e1c4fa73 100644 --- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -70,7 +70,7 @@ public class TimeUtils { + "((0?[1-9])|([1-2][0-9])|(30)))|(0?2[\\-\\/\\s]?((0?[1-9])|(1[0-9])|(2[0-8]))))))" + "(\\s(((0?[0-9])|([1][0-9])|([2][0-3]))\\:([0-5]?[0-9])((\\s)|(\\:([0-5]?[0-9])))))?$"); - private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]{1}\\d{2}\\:\\d{2}$"); + private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]{0,1}\\d{1,2}\\:\\d{2}$"); public static Date MIN_DATE = null; public static Date MAX_DATE = null; @@ -235,8 +235,11 @@ public class TimeUtils { } // Check if the time zone_value is valid - public static void checkTimeZoneValid(String value) throws DdlException { + public static String checkTimeZoneValidAndStandardize(String value) throws DdlException { try { + if (value == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, "null"); + } // match offset type, such as +08:00, -07:00 Matcher matcher = TIMEZONE_OFFSET_FORMAT_REG.matcher(value); // it supports offset and region timezone type, "CST" use here is compatibility purposes. @@ -245,6 +248,11 @@ public class TimeUtils { ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, value); } if (match) { + boolean postive = value.charAt(0) != '-'; + value = (postive ? "+" : "-") + String.format("%02d:%02d", + Integer.parseInt(value.replaceAll("[+-]", "").split(":")[0]), + Integer.parseInt(value.replaceAll("[+-]", "").split(":")[1])); + // timezone offsets around the world extended from -12:00 to +14:00 int tz = Integer.parseInt(value.substring(1, 3)) * 100 + Integer.parseInt(value.substring(4, 6)); if (value.charAt(0) == '-' && tz > 1200) { @@ -254,8 +262,10 @@ public class TimeUtils { } } ZoneId.of(value, timeZoneAliasMap); + return value; } catch (DateTimeException ex) { ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TIME_ZONE, value); } + throw new DdlException("Parse time zone " + value + " error"); } } diff --git a/fe/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/src/main/java/org/apache/doris/qe/VariableMgr.java index a9fdd2a360..4bc821a03b 100644 --- a/fe/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -19,6 +19,7 @@ package org.apache.doris.qe; import org.apache.doris.analysis.SetType; import org.apache.doris.analysis.SetVar; +import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.SysVariableDesc; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Type; @@ -215,7 +216,9 @@ public class VariableMgr { checkUpdate(setVar, ctx.getFlag()); // Check variable time_zone value is valid if (setVar.getVariable().toLowerCase().equals("time_zone")) { - TimeUtils.checkTimeZoneValid(setVar.getValue().getStringValue()); + setVar = new SetVar( + setVar.getType(), setVar.getVariable(), + new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(setVar.getValue().getStringValue()))); } // To modify to default value. diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index 02b0fd9ae9..3afa2b37bb 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -160,8 +160,7 @@ public class StreamLoadTask { strictMode = request.isStrictMode(); } if (request.isSetTimezone()) { - timezone = request.getTimezone(); - TimeUtils.checkTimeZoneValid(timezone); + timezone = TimeUtils.checkTimeZoneValidAndStandardize(request.getTimezone()); } if (request.isSetExecMemLimit()) { execMemLimit = request.getExecMemLimit(); diff --git a/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java index 574f515122..33360798f4 100644 --- a/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java @@ -96,6 +96,7 @@ public class CreateRoutineLoadStmtTest { String topicName = "topic1"; String serverAddress = "127.0.0.1:8080"; String kafkaPartitionString = "1,2,3"; + String timeZone = "8:00"; List partitionNameString = Lists.newArrayList(); partitionNameString.add("p1"); PartitionNames partitionNames = new PartitionNames(partitionNameString); @@ -108,6 +109,7 @@ public class CreateRoutineLoadStmtTest { loadPropertyList.add(partitionNames); Map properties = Maps.newHashMap(); properties.put(CreateRoutineLoadStmt.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); + properties.put(LoadStmt.TIMEZONE, timeZone); String typeName = LoadDataSourceType.KAFKA.name(); Map customProperties = Maps.newHashMap(); @@ -134,6 +136,7 @@ public class CreateRoutineLoadStmtTest { Assert.assertEquals(0, createRoutineLoadStmt.getMaxErrorNum()); Assert.assertEquals(serverAddress, createRoutineLoadStmt.getKafkaBrokerList()); Assert.assertEquals(topicName, createRoutineLoadStmt.getKafkaTopic()); + Assert.assertEquals("+08:00", createRoutineLoadStmt.getTimezone()); } } diff --git a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java index 1cbabecb8d..d3d849f46b 100644 --- a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java +++ b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.junit.Assert; import org.junit.Test; @@ -134,4 +135,23 @@ public class TimeUtilsTest { Assert.assertEquals(20150301120000L, datetime.getRealValue()); } + @Test + public void testTimezone() throws AnalysisException { + try { + Assert.assertEquals("CST", TimeUtils.checkTimeZoneValidAndStandardize("CST")); + Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+08:00")); + Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+8:00")); + Assert.assertEquals("-08:00", TimeUtils.checkTimeZoneValidAndStandardize("-8:00")); + Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("8:00")); + } catch (DdlException ex) { + Assert.fail(); + } + try { + TimeUtils.checkTimeZoneValidAndStandardize("FOO"); + Assert.fail(); + } catch (DdlException ex) { + Assert.assertTrue(ex.getMessage().contains("Unknown or incorrect time zone: 'FOO'")); + } + } + } diff --git a/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java index 2ee5ee865b..b8d56fc1fd 100644 --- a/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java +++ b/fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java @@ -124,6 +124,18 @@ public class VariableMgrTest { VariableMgr.setVar(var, setVar4); Assert.assertEquals(2L, var.getSqlMode()); + // Test checkTimeZoneValidAndStandardize + SetVar setVar5 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("+8:00")); + VariableMgr.setVar(var, setVar5); + Assert.assertEquals("+08:00", VariableMgr.newSessionVariable().getTimeZone()); + + SetVar setVar6 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("8:00")); + VariableMgr.setVar(var, setVar6); + Assert.assertEquals("+08:00", VariableMgr.newSessionVariable().getTimeZone()); + + SetVar setVar7 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("-8:00")); + VariableMgr.setVar(var, setVar7); + Assert.assertEquals("-08:00", VariableMgr.newSessionVariable().getTimeZone()); } @Test(expected = DdlException.class)