From 40bc3fed536f4a0a7e863fa14486c3accceaf13d Mon Sep 17 00:00:00 2001 From: Stephen-Robin <77377842+Stephen-Robin@users.noreply.github.com> Date: Thu, 8 Jul 2021 09:55:07 +0800 Subject: [PATCH] [Code] basic property related classes supports create, query, read, write, etc. (#6153) Provides basic property related classes supports create, query, read, write, etc. Currently, Doris FE mostly uses `if` statement to check properties in SQL. There is a lot of redundancy in the code. The `PropertySet` class can be used in the analysis phase of `Statement`. The validation and correctness of the input properties are automatic verified. It can simplify the code and improve the readability of the code. Usage: 1. Create a custom class that implements `SchemaGroup` interface. 2. Define the properties to be used. If it's a required parameter, there is no need to set the default value. 3. According the the requirements, in the logic called `readFromStrMap` and other functions to check and obtain parameters. Demo: Class definition ``` public class FileFormat implements PropertySchema.SchemaGroup { public static final PropertySchema FILE_FORMAT_TYPE = new PropertySchema.EnumProperty<>("type", FileFormat.Type.class).setDefauleValue(FileFormat.Type.CSV); public static final PropertySchema RECORD_DELIMITER = new PropertySchema.StringProperty("record_delimiter").setDefauleValue("\n"); public static final PropertySchema FIELD_DELIMITER = new PropertySchema.StringProperty("field_delimiter").setDefauleValue("|"); public static final PropertySchema SKIP_HEADER = new PropertySchema.IntProperty("skip_header", true).setMin(0).setDefauleValue(0); private static final FileFormat INSTANCE = new FileFormat(); private ImmutableMap schemas = PropertySchema.createSchemas( FILE_FORMAT_TYPE, RECORD_DELIMITER, FIELD_DELIMITER, SKIP_HEADER); public ImmutableMap getSchemas() { return schemas; } public static FileFormat get() { return INSTANCE; } } ``` Usage ``` public class CreateXXXStmt extends DdlStmt { private PropertiesSet analyzedFileFormat = PropertiesSet.empty(FileFormat.get()); private final Map fileFormatOptions; ... public void analyze(Analyzer analyzer) throws UserException { ... if (fileFormatOptions != null) { try { analyzedFileFormat = PropertiesSet.readFromStrMap(FileFormat.get(), fileFormatOptions); } catch (IllegalArgumentException e) { ... } } // 1. Get property value String recordDelimiter = analyzedFileFormat.get(FileFormat.RECORD_DELIMITER) // 2. Check the validity of parameters PropertiesSet.verifyKey(FileFormat.get(), fileFormatOptions); ... } } ``` --- .../doris/common/property/PropertiesSet.java | 179 ++++++++ .../doris/common/property/PropertySchema.java | 399 ++++++++++++++++++ .../common/property/PropertiesSetTest.java | 163 +++++++ .../common/property/PropertySchemaTest.java | 297 +++++++++++++ gensrc/thrift/FrontendService.thrift | 7 + 5 files changed, 1045 insertions(+) create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/property/PropertiesSet.java create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/property/PropertySchema.java create mode 100644 fe/fe-common/src/test/java/org/apache/doris/common/property/PropertiesSetTest.java create mode 100644 fe/fe-common/src/test/java/org/apache/doris/common/property/PropertySchemaTest.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/property/PropertiesSet.java b/fe/fe-common/src/main/java/org/apache/doris/common/property/PropertiesSet.java new file mode 100644 index 0000000000..0eb32c6ad7 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/property/PropertiesSet.java @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.property; + +import org.apache.doris.common.io.Text; +import org.apache.doris.thrift.TPropertyVal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +public class PropertiesSet { + private static final Map emptyInstances = new HashMap<>(); + + private final T schemaGroup; + private final Map properties; + private List modifiedSchemas; + + private PropertiesSet(T schemaGroup, Map properties) { + this.schemaGroup = schemaGroup; + this.properties = properties; + } + + @SuppressWarnings("unchecked") + public U get(PropertySchema prop) throws NoSuchElementException { + return properties.containsKey(prop.getName()) ? + (U) properties.get(prop.getName()) : prop.getDefaultValue().get(); + } + + public List getModifiedSchemas() { + if (modifiedSchemas == null) { + synchronized (this) { + modifiedSchemas = properties.keySet().stream() + .map(key -> schemaGroup.getSchemas().get(key)) + .collect(Collectors.toList()); + } + } + return modifiedSchemas; + } + + private static void checkRequiredKey( + TSchemaGroup schemaGroup, Map rawProperties) throws NoSuchElementException { + List requiredKey = schemaGroup.getSchemas().values().stream() + .filter(propertySchema -> !propertySchema.getDefaultValue().isPresent()) + .map(PropertySchema::getName) + .collect(Collectors.toList()); + + List missingKeys = requiredKey.stream() + .filter(key -> !rawProperties.containsKey(key)) + .collect(Collectors.toList()); + + if (!missingKeys.isEmpty()) { + throw new NoSuchElementException("Missing " + missingKeys); + } + } + + public static void verifyKey( + TSchemaGroup schemaGroup, List rawProperties) + throws IllegalArgumentException { + rawProperties.forEach(entry -> { + if (!schemaGroup.getSchemas().containsKey(entry.toLowerCase())) { + throw new IllegalArgumentException("Invalid property " + entry); + } + }); + } + + @SuppressWarnings("unchecked") + public Map writeToThrift() { + Map ret = new HashMap<>(properties.size()); + properties.forEach((key, value) -> { + TPropertyVal out = new TPropertyVal(); + schemaGroup.getSchemas().get(key).write(value, out); + ret.put(key, out); + }); + return ret; + } + + @SuppressWarnings("unchecked") + public void writeToData(DataOutput out) throws IOException { + out.writeInt(properties.size()); + for (Map.Entry entry : properties.entrySet()) { + Text.writeString(out, entry.getKey()); + schemaGroup.getSchemas().get(entry.getKey()).write(entry.getValue(), out); + } + } + + private interface ReadLambda { + TParsed accept(PropertySchema schema, TRaw raw); + } + + @SuppressWarnings("unchecked") + private static PropertiesSet read( + TSchemaGroup schemaGroup, Map rawProperties, ReadLambda reader) + throws IllegalArgumentException, NoSuchElementException { + checkRequiredKey(schemaGroup, rawProperties); + Map properties = new HashMap<>(rawProperties.size()); + + rawProperties.forEach((key, value) -> { + String entryKey = key.toLowerCase(); + if (!schemaGroup.getSchemas().containsKey(entryKey)) { + throw new IllegalArgumentException("Invalid property " + key); + } + PropertySchema schema = schemaGroup.getSchemas().get(entryKey); + properties.put(entryKey, reader.accept(schema, value)); + }); + + return new PropertiesSet(schemaGroup, properties); + } + + @SuppressWarnings("unchecked") + public static PropertiesSet empty( + TSchemaGroup schemaGroup) { + if (!emptyInstances.containsKey(schemaGroup)) { + synchronized (PropertiesSet.class) { + if (!emptyInstances.containsKey(schemaGroup)) { + emptyInstances.put(schemaGroup, new PropertiesSet(schemaGroup, Collections.emptyMap())); + } + } + } + + return emptyInstances.get(schemaGroup); + } + + public static PropertiesSet readFromStrMap( + TSchemaGroup schemaGroup, Map rawProperties) + throws IllegalArgumentException { + + return read(schemaGroup, rawProperties, PropertySchema::read); + } + + public static PropertiesSet readFromThrift( + TSchemaGroup schemaGroup, Map rawProperties) + throws IllegalArgumentException { + + return read(schemaGroup, rawProperties, PropertySchema::read); + } + + @SuppressWarnings("unchecked") + public static PropertiesSet readFromData( + TSchemaGroup schemaGroup, DataInput input) + throws IllegalArgumentException, IOException { + int size = input.readInt(); + Map properties = new HashMap<>(size); + + for (int i = 0; i < size; i++) { + String key = Text.readString(input).toLowerCase(); + Object val = schemaGroup.getSchemas().get(key).read(input); + properties.put(key, val); + } + + return new PropertiesSet(schemaGroup, properties); + } + + @Override + public String toString() { + return properties.toString(); + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/property/PropertySchema.java b/fe/fe-common/src/main/java/org/apache/doris/common/property/PropertySchema.java new file mode 100644 index 0000000000..ca9063ef66 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/property/PropertySchema.java @@ -0,0 +1,399 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.property; + +import com.google.common.collect.ImmutableMap; +import org.apache.doris.common.io.Text; +import org.apache.doris.thrift.TPropertyVal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.Optional; + +@SuppressWarnings({"unchecked","rawtypes"}) +public abstract class PropertySchema { + private final String name; + private final boolean required; + private Optional defaultValue = Optional.empty(); + private Optional maxValue = Optional.empty(); + private Optional minValue = Optional.empty(); + + protected PropertySchema(String name) { + this(name, false); + } + + public PropertySchema(String name, boolean required) { + this.name = name; + this.required = required; + } + + public static ImmutableMap createSchemas(PropertySchema ... schemas) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + Arrays.stream(schemas).forEach(s -> builder.put(s.getName(), s)); + return builder.build(); + } + + public interface SchemaGroup { + ImmutableMap getSchemas(); + } + + public static final class StringProperty extends ComparableProperty { + StringProperty(String name) { + super(name); + } + + StringProperty(String name, boolean isRequired) { + super(name, isRequired); + } + + @Override + public String read(String rawVal) throws IllegalArgumentException { + verifyRange(rawVal); + return rawVal; + } + + @Override + public String read(TPropertyVal tVal) throws IllegalArgumentException { + verifyRange(tVal.getStrVal()); + return tVal.getStrVal(); + } + + @Override + public String read(DataInput input) throws IOException { + return Text.readString(input); + } + + @Override + public void write(String val, TPropertyVal out) { + out.setStrVal(val); + } + + @Override + public void write(String val, DataOutput out) throws IOException { + Text.writeString(out, val); + } + } + + public static final class IntProperty extends ComparableProperty { + IntProperty(String name) { + super(name); + } + + IntProperty(String name, boolean isRequired) { + super(name, isRequired); + } + + @Override + public Integer read(String rawVal) { + try { + Integer val = Integer.parseInt(rawVal); + verifyRange(val); + return val; + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format("Invalid integer %s: %s", rawVal, e.getMessage())); + } + } + + @Override + public Integer read(TPropertyVal tVal) throws IllegalArgumentException { + verifyRange(tVal.getIntVal()); + return tVal.getIntVal(); + } + + @Override + public Integer read(DataInput input) throws IOException { + return input.readInt(); + } + + @Override + public void write(Integer val, TPropertyVal out) { + out.setIntVal(val); + } + + @Override + public void write(Integer val, DataOutput out) throws IOException { + out.writeInt(val); + } + } + + public static final class LongProperty extends ComparableProperty { + LongProperty(String name) { + super(name); + } + + LongProperty(String name, boolean isRequired) { + super(name, isRequired); + } + + @Override + public Long read(String rawVal) { + try { + Long val = Long.parseLong(rawVal); + verifyRange(val); + return val; + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format("Invalid long %s: %s", rawVal, e.getMessage())); + } + } + + @Override + public Long read(TPropertyVal tVal) throws IllegalArgumentException { + verifyRange(tVal.getLongVal()); + return tVal.getLongVal(); + } + + @Override + public Long read(DataInput input) throws IOException { + return input.readLong(); + } + + @Override + public void write(Long val, TPropertyVal out) { + out.setLongVal(val); + } + + @Override + public void write(Long val, DataOutput out) throws IOException { + out.writeLong(val); + } + } + + public static final class BooleanProperty extends ComparableProperty { + BooleanProperty(String name) { + super(name); + } + + BooleanProperty(String name, boolean isRequired) { + super(name, isRequired); + } + + @Override + public Boolean read(String rawVal) { + if (rawVal == null || + (!rawVal.equalsIgnoreCase("true") && !rawVal.equalsIgnoreCase("false"))) { + throw new IllegalArgumentException(String.format("Invalid boolean : %s, use true or false", rawVal)); + } + + try { + return Boolean.parseBoolean(rawVal); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format("Invalid boolean %s: %s", rawVal, e.getMessage())); + } + } + + @Override + public Boolean read(TPropertyVal tVal) throws IllegalArgumentException { + return tVal.isBoolVal(); + } + + @Override + public Boolean read(DataInput input) throws IOException { + return input.readBoolean(); + } + + @Override + public void write(Boolean val, TPropertyVal out) { + out.setBoolVal(val); + } + + @Override + public void write(Boolean val, DataOutput out) throws IOException { + out.writeBoolean(val); + } + } + + public static final class DateProperty extends PropertySchema { + SimpleDateFormat dateFormat; + + DateProperty(String name, SimpleDateFormat dateFormat) { + super(name); + this.dateFormat = dateFormat; + } + + DateProperty(String name, SimpleDateFormat dateFormat, boolean isRequired) { + super(name, isRequired); + this.dateFormat = dateFormat; + } + + @Override + public Date read(String rawVal) throws IllegalArgumentException { + if (rawVal == null) { + throw new IllegalArgumentException("Invalid time format, time param can not is null"); + } + return readTimeFormat(rawVal); + } + + @Override + public Date read(TPropertyVal tVal) throws IllegalArgumentException { + return readTimeFormat(tVal.getStrVal()); + } + + @Override + public Date read(DataInput input) throws IOException { + return readTimeFormat(Text.readString(input)); + } + + @Override + public void write(Date val, TPropertyVal out) { + out.setStrVal(writeTimeFormat(val)); + } + + @Override + public void write(Date val, DataOutput out) throws IOException { + Text.writeString(out, writeTimeFormat(val)); + } + + public Date readTimeFormat(String timeStr) throws IllegalArgumentException { + try { + return this.dateFormat.parse(timeStr); + } catch (ParseException e) { + throw new IllegalArgumentException("Invalid time format, time param need " + + "to be " + this.dateFormat.toPattern()); + } + } + + public String writeTimeFormat(Date timeDate) throws IllegalArgumentException { + return this.dateFormat.format(timeDate.getTime()); + } + } + + public static final class EnumProperty> extends PropertySchema { + private final Class enumClass; + + EnumProperty(String name, Class enumClass) { + super(name); + this.enumClass = enumClass; + } + + EnumProperty(String name, Class enumClass, boolean isRequired) { + super(name, isRequired); + this.enumClass = enumClass; + } + + @Override + public T read(String rawVal) { + if (rawVal == null || rawVal.length() == 0) { + throw new IllegalArgumentException(formatError(rawVal)); + } + try { + return T.valueOf(enumClass, rawVal.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(formatError(rawVal)); + } + } + + @Override + public T read(TPropertyVal tVal) throws IllegalArgumentException { + return T.valueOf(enumClass, tVal.getStrVal()); + } + + @Override + public T read(DataInput input) throws IOException { + return T.valueOf(enumClass, Text.readString(input)); + } + + @Override + public void write(T val, TPropertyVal out) { + out.setStrVal(val.name()); + } + + @Override + public void write(T val, DataOutput out) throws IOException { + Text.writeString(out, val.name()); + } + + private String formatError(String rawVal) { + String enumsStr = Arrays.stream(enumClass.getEnumConstants()) + .map(Enum::toString) + .reduce((sa, sb) -> sa + ", " + sb) + .orElse(""); + return String.format("Expected values are [%s], while [%s] provided", enumsStr, rawVal); + } + } + + private static abstract class ComparableProperty extends PropertySchema { + protected ComparableProperty(String name) { + super(name); + } + + protected ComparableProperty(String name, boolean isRequired) { + super(name, isRequired); + } + + protected void verifyRange(T val) throws IllegalArgumentException { + if (getMinValue().isPresent() && (val == null || getMinValue().get().compareTo(val) > 0)) { + throw new IllegalArgumentException(val + " should not be less than " + getMinValue().get()); + } + + if (getMaxValue().isPresent() && (val == null || getMaxValue().get().compareTo(val) < 0)) { + throw new IllegalArgumentException(val + " should not be greater than " + getMaxValue().get()); + } + } + } + + PropertySchema setDefauleValue(T val) { + this.defaultValue = Optional.of(val); + return this; + } + + PropertySchema setMin(T min) { + this.minValue = Optional.of(min); + return this; + } + + PropertySchema setMax(T max) { + this.maxValue = Optional.of(max); + return this; + } + + public String getName() { + return name; + } + + public boolean isRequired() { + return required; + } + + public Optional getDefaultValue() { + return defaultValue; + } + + public Optional getMinValue() { + return minValue; + } + + public Optional getMaxValue() { + return maxValue; + } + + public abstract T read(String rawVal) throws IllegalArgumentException; + + public abstract T read(TPropertyVal tVal) throws IllegalArgumentException; + + public abstract T read(DataInput input) throws IOException; + + public abstract void write(T val, TPropertyVal out); + + public abstract void write(T val, DataOutput out) throws IOException; +} + diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/property/PropertiesSetTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/property/PropertiesSetTest.java new file mode 100644 index 0000000000..a9a9623041 --- /dev/null +++ b/fe/fe-common/src/test/java/org/apache/doris/common/property/PropertiesSetTest.java @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.property; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.apache.doris.thrift.TPropertyVal; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; + +import static org.junit.Assert.fail; + +public class PropertiesSetTest { + @Test + public void testReadFromStr() { + Map raw = new HashMap<>(); + raw.put("skip_header", "0"); + PropertiesSet properties = PropertiesSet.readFromStrMap(FileFormat.get(), raw); + Assert.assertEquals(FileFormat.Type.CSV, properties.get(FileFormat.FILE_FORMAT_TYPE)); + Assert.assertEquals("\n", properties.get(FileFormat.RECORD_DELIMITER)); + Assert.assertEquals(1, properties.writeToThrift().size()); + + properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps()); + verifyVariableProps(properties); + Assert.assertEquals(3, properties.writeToThrift().size()); + } + + @Test + public void testThriftSerde() { + PropertiesSet properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps()); + Map thriftMap = properties.writeToThrift(); + Assert.assertEquals("JSON", thriftMap.get(FileFormat.FILE_FORMAT_TYPE.getName()).strVal); + Assert.assertEquals("\r", thriftMap.get(FileFormat.RECORD_DELIMITER.getName()).strVal); + Assert.assertEquals(3, thriftMap.get(FileFormat.SKIP_HEADER.getName()).intVal); + + properties = PropertiesSet.readFromThrift(FileFormat.get(), thriftMap); + verifyVariableProps(properties); + } + + @Test + public void testDataOutputSerde() throws Exception { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertiesSet properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps()); + properties.writeToData(output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + + properties = PropertiesSet.readFromData(FileFormat.get(), input); + verifyVariableProps(properties); + } + + @Test + public void testEmpty() { + PropertiesSet p1 = PropertiesSet.empty(FileFormat.get()); + PropertiesSet p2 = PropertiesSet.empty(FileFormat.get()); + + Assert.assertEquals(p1, p2); + } + + @Test + public void testModifiedSchemas() { + PropertiesSet properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps()); + Assert.assertEquals(3, properties.getModifiedSchemas().size()); + } + + @Test + public void testToString() { + PropertiesSet properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps()); + String str = properties.toString(); + + Assert.assertTrue(str.contains(FileFormat.FILE_FORMAT_TYPE.getName())); + Assert.assertTrue(str.contains("JSON")); + Assert.assertTrue(str.contains(FileFormat.RECORD_DELIMITER.getName())); + Assert.assertTrue(str.contains("\r")); + Assert.assertTrue(str.contains(FileFormat.SKIP_HEADER.getName())); + Assert.assertTrue(str.contains("3")); + } + + private Map rawVariableProps() { + Map raw = new HashMap<>(); + raw.put(FileFormat.FILE_FORMAT_TYPE.getName(), "Json"); + raw.put(FileFormat.RECORD_DELIMITER.getName(), "\r"); + raw.put(FileFormat.SKIP_HEADER.getName(), "3"); + return raw; + } + + @Test + public void testCheckRequiredOpts() { + try { + PropertiesSet.readFromStrMap(FileFormat.get(), Maps.newHashMap()); + fail("Expected an NoSuchElementException to be thrown"); + } catch (NoSuchElementException e) { + Assert.assertTrue(e.getMessage().contains("Missing")); + } + } + + @SuppressWarnings("unchecked") + private void verifyVariableProps(PropertiesSet properties) { + Assert.assertEquals(FileFormat.Type.JSON, properties.get(FileFormat.FILE_FORMAT_TYPE)); + Assert.assertEquals("\r", properties.get(FileFormat.RECORD_DELIMITER)); + Assert.assertEquals(3, properties.get(FileFormat.SKIP_HEADER)); + } + + private static class FileFormat implements PropertySchema.SchemaGroup { + public static PropertySchema FILE_FORMAT_TYPE = + new PropertySchema.EnumProperty<>("type", Type.class) + .setDefauleValue(Type.CSV); + public static PropertySchema RECORD_DELIMITER = + new PropertySchema.StringProperty("record_delimiter").setDefauleValue("\n"); + public static PropertySchema FIELD_DELIMITER = + new PropertySchema.StringProperty("field_delimiter").setDefauleValue("|"); + public static PropertySchema SKIP_HEADER = + new PropertySchema.IntProperty("skip_header", true).setMin(0); + + private static final FileFormat INSTANCE = new FileFormat(); + + private final ImmutableMap schemas = PropertySchema.createSchemas( + FILE_FORMAT_TYPE, + RECORD_DELIMITER, + FIELD_DELIMITER, + SKIP_HEADER); + + public ImmutableMap getSchemas() { + return schemas; + } + + public static FileFormat get() { + return INSTANCE; + } + + public enum Type { + CSV, JSON, ORC, PARQUET + } + } +} diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/property/PropertySchemaTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/property/PropertySchemaTest.java new file mode 100644 index 0000000000..e9ff8e70a0 --- /dev/null +++ b/fe/fe-common/src/test/java/org/apache/doris/common/property/PropertySchemaTest.java @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.property; + +import org.apache.doris.thrift.TPropertyVal; +import org.hamcrest.Matchers; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class PropertySchemaTest { + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void testStringPropNormal() throws Exception { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key"); + prop.write("val", output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + Assert.assertEquals("val", prop.read(input)); + + TPropertyVal tProp = new TPropertyVal(); + prop.write("val", tProp); + Assert.assertEquals("val", prop.read(tProp)); + + prop.setMin("b"); + Assert.assertEquals("c", prop.read("c")); + Assert.assertEquals("b", prop.read("b")); + + prop.setMax("x"); + Assert.assertEquals("w", prop.read("w")); + Assert.assertEquals("x", prop.read("x")); + } + + @Test + public void testStringPropMinExceeded() { + PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key"); + prop.setMin("b"); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("should not be less than")); + prop.read("a"); + } + + @Test + public void testStringPropMinNull() { + PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key"); + prop.setMin("b"); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("should not be less than")); + prop.read((String) null); + } + + @Test + public void testStringPropMaxExceeded() { + PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key"); + prop.setMax("b"); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("should not be greater than")); + prop.read("c"); + } + + @Test + public void testStringPropMaxNull() { + PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key"); + prop.setMax("b"); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("should not be greater than")); + prop.read((String) null); + } + + @Test + public void testIntPropNormal() throws Exception { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertySchema.IntProperty prop = new PropertySchema.IntProperty("key"); + prop.write(5, output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + Assert.assertEquals(Integer.valueOf(5), prop.read(input)); + + TPropertyVal tProp = new TPropertyVal(); + prop.write(6, tProp); + Assert.assertEquals(Integer.valueOf(6), prop.read(tProp)); + + Assert.assertEquals(Integer.valueOf(7), prop.read("7")); + } + + @Test + public void testIntPropInvalidString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid integer")); + + PropertySchema.IntProperty prop = new PropertySchema.IntProperty("key"); + prop.read("23j"); + } + + @Test + public void testIntPropNullString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid integer")); + + PropertySchema.IntProperty prop = new PropertySchema.IntProperty("key"); + prop.read((String) null); + } + + @Test + public void testEnumPropNormal() throws Exception { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertySchema.EnumProperty prop = new PropertySchema.EnumProperty<>("key", Color.class); + prop.write(Color.RED, output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + Assert.assertEquals(Color.RED, prop.read(input)); + + TPropertyVal tProp = new TPropertyVal(); + prop.write(Color.GREEN, tProp); + Assert.assertEquals(Color.GREEN, prop.read(tProp)); + + Assert.assertEquals(Color.BLUE, prop.read("BLUE")); + Assert.assertEquals(Color.BLUE, prop.read("blue")); + Assert.assertEquals(Color.BLUE, prop.read("Blue")); + } + + @Test + public void testEnumPropInvalidString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString( + "Expected values are [RED, GREEN, BLUE], while [invalid] provided")); + + PropertySchema.EnumProperty prop = new PropertySchema.EnumProperty<>("key", Color.class); + prop.read("invalid"); + } + + @Test + public void testEnumPropNullString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString( + "Expected values are [RED, GREEN, BLUE], while [null] provided")); + + PropertySchema.EnumProperty prop = new PropertySchema.EnumProperty<>("key", Color.class); + prop.read((String) null); + } + + private enum Color { + RED, GREEN, BLUE + } + + @Test + public void testLongPropNormal() throws Exception { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertySchema.LongProperty prop = new PropertySchema.LongProperty("key"); + prop.write(5L, output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + Assert.assertEquals(Long.valueOf(5), prop.read(input)); + + TPropertyVal tProp = new TPropertyVal(); + prop.write(6L, tProp); + Assert.assertEquals(Long.valueOf(6), prop.read(tProp)); + + Assert.assertEquals(Long.valueOf(7), prop.read("7")); + } + + @Test + public void testLongPropInvalidString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid long")); + + PropertySchema.LongProperty prop = new PropertySchema.LongProperty("key"); + prop.read("23j"); + } + + @Test + public void testLongPropNullString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid long")); + + PropertySchema.LongProperty prop = new PropertySchema.LongProperty("key"); + prop.read((String) null); + } + + @Test + public void testDatePropNormal() throws Exception { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertySchema.DateProperty prop = + new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + prop.write(dateFormat.parse("2021-06-30 20:34:51"), output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + Assert.assertEquals(1625056491000L, prop.read(input).getTime()); + + TPropertyVal tProp = new TPropertyVal(); + prop.write(new Date(1625056491000L), tProp); + Assert.assertEquals(1625056491000L, prop.read(tProp).getTime()); + + Assert.assertEquals(1625056491000L, prop.read("2021-06-30 20:34:51").getTime()); + } + + @Test + public void testDatePropInvalidString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid time format")); + + PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd " + + "HH:mm:ss")); + prop.read("2021-06-30"); + } + + @Test + public void testDatePropNullString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid time format")); + + PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd " + + "HH:mm:ss")); + prop.read((String) null); + } + + @Test + public void testBooleanPropNormal() throws Exception { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(outStream); + + PropertySchema.BooleanProperty prop = new PropertySchema.BooleanProperty("key"); + prop.write(true, output); + + ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + DataInput input = new DataInputStream(inStream); + Assert.assertEquals(true, prop.read(input)); + + TPropertyVal tProp = new TPropertyVal(); + prop.write(true, tProp); + Assert.assertEquals(true, prop.read(tProp)); + + Assert.assertEquals(true, prop.read("true")); + } + + @Test + public void testBooleanPropInvalidString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid boolean")); + + PropertySchema.BooleanProperty prop = new PropertySchema.BooleanProperty("key"); + prop.read("233"); + } + + @Test + public void testBooleanPropNullString() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(Matchers.containsString("Invalid boolean")); + + PropertySchema.BooleanProperty prop = new PropertySchema.BooleanProperty("key"); + prop.read((String) null); + } +} diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 9246392274..df87a5e17c 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -687,6 +687,13 @@ struct TFrontendPingFrontendResult { 6: required string version } +struct TPropertyVal { + 1: optional string strVal + 2: optional i32 intVal + 3: optional i64 longVal + 4: optional bool boolVal +} + service FrontendService { TGetDbsResult getDbNames(1:TGetDbsParams params) TGetTablesResult getTableNames(1:TGetTablesParams params)