database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Code] basic property related classes supports create, query, read, write, etc. (#6153)

Browse Source 
Provides basic property related classes supports create, query, read, write, etc.
Currently, Doris FE mostly uses `if` statement to check properties in SQL. There is a lot of redundancy in the code.
The `PropertySet` class can be used in the analysis phase of `Statement`. The validation and correctness of the input properties are automatic verified. It can simplify the code and improve the readability of the code.

Usage:
1. Create a custom class that implements `SchemaGroup` interface.
2. Define the properties to be used. If it's a required parameter, there is no need to set the default value.
3. According the the requirements, in the logic called `readFromStrMap` and other functions to check and obtain parameters.

Demo:

Class definition

```
public class FileFormat implements PropertySchema.SchemaGroup {
    public static final PropertySchema<FileFormat.Type> FILE_FORMAT_TYPE =
            new PropertySchema.EnumProperty<>("type", FileFormat.Type.class).setDefauleValue(FileFormat.Type.CSV);
    public static final PropertySchema<String> RECORD_DELIMITER =
            new PropertySchema.StringProperty("record_delimiter").setDefauleValue("\n");
    public static final PropertySchema<String> FIELD_DELIMITER =
            new PropertySchema.StringProperty("field_delimiter").setDefauleValue("|");
    public static final PropertySchema<Integer> SKIP_HEADER =
            new PropertySchema.IntProperty("skip_header", true).setMin(0).setDefauleValue(0);

    private static final FileFormat INSTANCE = new FileFormat();

    private ImmutableMap<String, PropertySchema> schemas = PropertySchema.createSchemas(
            FILE_FORMAT_TYPE,
            RECORD_DELIMITER,
            FIELD_DELIMITER,
            SKIP_HEADER);

    public ImmutableMap<String, PropertySchema> getSchemas() {
        return schemas;
    }

    public static FileFormat get() {
        return INSTANCE;
    }
}

```

Usage
```
public class CreateXXXStmt extends DdlStmt {
    private PropertiesSet<FileFormat> analyzedFileFormat = PropertiesSet.empty(FileFormat.get());
    private final Map<String, String> fileFormatOptions;
    ...

    public void analyze(Analyzer analyzer) throws UserException {
        ...
        if (fileFormatOptions != null) {
            try {
                analyzedFileFormat = PropertiesSet.readFromStrMap(FileFormat.get(), fileFormatOptions);
            } catch (IllegalArgumentException e) {
                ...
            }
        }

        // 1. Get property value
        String recordDelimiter = analyzedFileFormat.get(FileFormat.RECORD_DELIMITER)
        // 2. Check the validity of parameters
        PropertiesSet.verifyKey(FileFormat.get(), fileFormatOptions);
        ...
    }

}
```
This commit is contained in:
Stephen-Robin
2021-07-08 09:55:07 +08:00
committed by GitHub
parent fb123b2b4b
commit 40bc3fed53
5 changed files with 1045 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
fe/fe-common/src/main/java/org/apache/doris/common/property/PropertiesSet.java Normal file
View File

@ -0,0 +1,179 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.property;
import org.apache.doris.common.io.Text;
import org.apache.doris.thrift.TPropertyVal;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
public class PropertiesSet<T extends PropertySchema.SchemaGroup> {
private static final Map<PropertySchema.SchemaGroup, PropertiesSet> emptyInstances = new HashMap<>();
private final T schemaGroup;
private final Map<String, Object> properties;
private List<PropertySchema> modifiedSchemas;
private PropertiesSet(T schemaGroup, Map<String, Object> properties) {
this.schemaGroup = schemaGroup;
this.properties = properties;
}
@SuppressWarnings("unchecked")
public <U> U get(PropertySchema<U> prop) throws NoSuchElementException {
return properties.containsKey(prop.getName()) ?
(U) properties.get(prop.getName()) : prop.getDefaultValue().get();
}
public List<PropertySchema> getModifiedSchemas() {
if (modifiedSchemas == null) {
synchronized (this) {
modifiedSchemas = properties.keySet().stream()
.map(key -> schemaGroup.getSchemas().get(key))
.collect(Collectors.toList());
}
}
return modifiedSchemas;
}
private static <TSchemaGroup extends PropertySchema.SchemaGroup, TRaw> void checkRequiredKey(
TSchemaGroup schemaGroup, Map<String, TRaw> rawProperties) throws NoSuchElementException {
List<String> requiredKey = schemaGroup.getSchemas().values().stream()
.filter(propertySchema -> !propertySchema.getDefaultValue().isPresent())
.map(PropertySchema::getName)
.collect(Collectors.toList());
List<String> missingKeys = requiredKey.stream()
.filter(key -> !rawProperties.containsKey(key))
.collect(Collectors.toList());
if (!missingKeys.isEmpty()) {
throw new NoSuchElementException("Missing " + missingKeys);
}
}
public static <TSchemaGroup extends PropertySchema.SchemaGroup> void verifyKey(
TSchemaGroup schemaGroup, List<String> rawProperties)
throws IllegalArgumentException {
rawProperties.forEach(entry -> {
if (!schemaGroup.getSchemas().containsKey(entry.toLowerCase())) {
throw new IllegalArgumentException("Invalid property " + entry);
}
});
}
@SuppressWarnings("unchecked")
public Map<String, TPropertyVal> writeToThrift() {
Map<String, TPropertyVal> ret = new HashMap<>(properties.size());
properties.forEach((key, value) -> {
TPropertyVal out = new TPropertyVal();
schemaGroup.getSchemas().get(key).write(value, out);
ret.put(key, out);
});
return ret;
}
@SuppressWarnings("unchecked")
public void writeToData(DataOutput out) throws IOException {
out.writeInt(properties.size());
for (Map.Entry<String, Object> entry : properties.entrySet()) {
Text.writeString(out, entry.getKey());
schemaGroup.getSchemas().get(entry.getKey()).write(entry.getValue(), out);
}
}
private interface ReadLambda<TParsed, TRaw> {
TParsed accept(PropertySchema schema, TRaw raw);
}
@SuppressWarnings("unchecked")
private static <TSchemaGroup extends PropertySchema.SchemaGroup, TParsed, TRaw> PropertiesSet<TSchemaGroup> read(
TSchemaGroup schemaGroup, Map<String, TRaw> rawProperties, ReadLambda<TParsed, TRaw> reader)
throws IllegalArgumentException, NoSuchElementException {
checkRequiredKey(schemaGroup, rawProperties);
Map<String, Object> properties = new HashMap<>(rawProperties.size());
rawProperties.forEach((key, value) -> {
String entryKey = key.toLowerCase();
if (!schemaGroup.getSchemas().containsKey(entryKey)) {
throw new IllegalArgumentException("Invalid property " + key);
}
PropertySchema schema = schemaGroup.getSchemas().get(entryKey);
properties.put(entryKey, reader.accept(schema, value));
});
return new PropertiesSet(schemaGroup, properties);
}
@SuppressWarnings("unchecked")
public static <TSchemaGroup extends PropertySchema.SchemaGroup> PropertiesSet<TSchemaGroup> empty(
TSchemaGroup schemaGroup) {
if (!emptyInstances.containsKey(schemaGroup)) {
synchronized (PropertiesSet.class) {
if (!emptyInstances.containsKey(schemaGroup)) {
emptyInstances.put(schemaGroup, new PropertiesSet(schemaGroup, Collections.emptyMap()));
}
}
}
return emptyInstances.get(schemaGroup);
}
public static <TSchemaGroup extends PropertySchema.SchemaGroup> PropertiesSet<TSchemaGroup> readFromStrMap(
TSchemaGroup schemaGroup, Map<String, String> rawProperties)
throws IllegalArgumentException {
return read(schemaGroup, rawProperties, PropertySchema::read);
}
public static <TSchemaGroup extends PropertySchema.SchemaGroup> PropertiesSet<TSchemaGroup> readFromThrift(
TSchemaGroup schemaGroup, Map<String, TPropertyVal> rawProperties)
throws IllegalArgumentException {
return read(schemaGroup, rawProperties, PropertySchema::read);
}
@SuppressWarnings("unchecked")
public static <TSchemaGroup extends PropertySchema.SchemaGroup> PropertiesSet<TSchemaGroup> readFromData(
TSchemaGroup schemaGroup, DataInput input)
throws IllegalArgumentException, IOException {
int size = input.readInt();
Map<String, Object> properties = new HashMap<>(size);
for (int i = 0; i < size; i++) {
String key = Text.readString(input).toLowerCase();
Object val = schemaGroup.getSchemas().get(key).read(input);
properties.put(key, val);
}
return new PropertiesSet(schemaGroup, properties);
}
@Override
public String toString() {
return properties.toString();
}
}

399
fe/fe-common/src/main/java/org/apache/doris/common/property/PropertySchema.java Normal file
View File

@ -0,0 +1,399 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.property;
import com.google.common.collect.ImmutableMap;
import org.apache.doris.common.io.Text;
import org.apache.doris.thrift.TPropertyVal;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.Optional;
@SuppressWarnings({"unchecked","rawtypes"})
public abstract class PropertySchema<T> {
private final String name;
private final boolean required;
private Optional<T> defaultValue = Optional.empty();
private Optional<T> maxValue = Optional.empty();
private Optional<T> minValue = Optional.empty();
protected PropertySchema(String name) {
this(name, false);
}
public PropertySchema(String name, boolean required) {
this.name = name;
this.required = required;
}
public static ImmutableMap<String, PropertySchema> createSchemas(PropertySchema ... schemas) {
ImmutableMap.Builder builder = ImmutableMap.builder();
Arrays.stream(schemas).forEach(s -> builder.put(s.getName(), s));
return builder.build();
}
public interface SchemaGroup {
ImmutableMap<String, PropertySchema> getSchemas();
}
public static final class StringProperty extends ComparableProperty<String> {
StringProperty(String name) {
super(name);
}
StringProperty(String name, boolean isRequired) {
super(name, isRequired);
}
@Override
public String read(String rawVal) throws IllegalArgumentException {
verifyRange(rawVal);
return rawVal;
}
@Override
public String read(TPropertyVal tVal) throws IllegalArgumentException {
verifyRange(tVal.getStrVal());
return tVal.getStrVal();
}
@Override
public String read(DataInput input) throws IOException {
return Text.readString(input);
}
@Override
public void write(String val, TPropertyVal out) {
out.setStrVal(val);
}
@Override
public void write(String val, DataOutput out) throws IOException {
Text.writeString(out, val);
}
}
public static final class IntProperty extends ComparableProperty<Integer> {
IntProperty(String name) {
super(name);
}
IntProperty(String name, boolean isRequired) {
super(name, isRequired);
}
@Override
public Integer read(String rawVal) {
try {
Integer val = Integer.parseInt(rawVal);
verifyRange(val);
return val;
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format("Invalid integer %s: %s", rawVal, e.getMessage()));
}
}
@Override
public Integer read(TPropertyVal tVal) throws IllegalArgumentException {
verifyRange(tVal.getIntVal());
return tVal.getIntVal();
}
@Override
public Integer read(DataInput input) throws IOException {
return input.readInt();
}
@Override
public void write(Integer val, TPropertyVal out) {
out.setIntVal(val);
}
@Override
public void write(Integer val, DataOutput out) throws IOException {
out.writeInt(val);
}
}
public static final class LongProperty extends ComparableProperty<Long> {
LongProperty(String name) {
super(name);
}
LongProperty(String name, boolean isRequired) {
super(name, isRequired);
}
@Override
public Long read(String rawVal) {
try {
Long val = Long.parseLong(rawVal);
verifyRange(val);
return val;
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format("Invalid long %s: %s", rawVal, e.getMessage()));
}
}
@Override
public Long read(TPropertyVal tVal) throws IllegalArgumentException {
verifyRange(tVal.getLongVal());
return tVal.getLongVal();
}
@Override
public Long read(DataInput input) throws IOException {
return input.readLong();
}
@Override
public void write(Long val, TPropertyVal out) {
out.setLongVal(val);
}
@Override
public void write(Long val, DataOutput out) throws IOException {
out.writeLong(val);
}
}
public static final class BooleanProperty extends ComparableProperty<Boolean> {
BooleanProperty(String name) {
super(name);
}
BooleanProperty(String name, boolean isRequired) {
super(name, isRequired);
}
@Override
public Boolean read(String rawVal) {
if (rawVal == null ||
(!rawVal.equalsIgnoreCase("true") && !rawVal.equalsIgnoreCase("false"))) {
throw new IllegalArgumentException(String.format("Invalid boolean : %s, use true or false", rawVal));
}
try {
return Boolean.parseBoolean(rawVal);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format("Invalid boolean %s: %s", rawVal, e.getMessage()));
}
}
@Override
public Boolean read(TPropertyVal tVal) throws IllegalArgumentException {
return tVal.isBoolVal();
}
@Override
public Boolean read(DataInput input) throws IOException {
return input.readBoolean();
}
@Override
public void write(Boolean val, TPropertyVal out) {
out.setBoolVal(val);
}
@Override
public void write(Boolean val, DataOutput out) throws IOException {
out.writeBoolean(val);
}
}
public static final class DateProperty extends PropertySchema<Date> {
SimpleDateFormat dateFormat;
DateProperty(String name, SimpleDateFormat dateFormat) {
super(name);
this.dateFormat = dateFormat;
}
DateProperty(String name, SimpleDateFormat dateFormat, boolean isRequired) {
super(name, isRequired);
this.dateFormat = dateFormat;
}
@Override
public Date read(String rawVal) throws IllegalArgumentException {
if (rawVal == null) {
throw new IllegalArgumentException("Invalid time format, time param can not is null");
}
return readTimeFormat(rawVal);
}
@Override
public Date read(TPropertyVal tVal) throws IllegalArgumentException {
return readTimeFormat(tVal.getStrVal());
}
@Override
public Date read(DataInput input) throws IOException {
return readTimeFormat(Text.readString(input));
}
@Override
public void write(Date val, TPropertyVal out) {
out.setStrVal(writeTimeFormat(val));
}
@Override
public void write(Date val, DataOutput out) throws IOException {
Text.writeString(out, writeTimeFormat(val));
}
public Date readTimeFormat(String timeStr) throws IllegalArgumentException {
try {
return this.dateFormat.parse(timeStr);
} catch (ParseException e) {
throw new IllegalArgumentException("Invalid time format, time param need "
+ "to be " + this.dateFormat.toPattern());
}
}
public String writeTimeFormat(Date timeDate) throws IllegalArgumentException {
return this.dateFormat.format(timeDate.getTime());
}
}
public static final class EnumProperty<T extends Enum<T>> extends PropertySchema<T> {
private final Class<T> enumClass;
EnumProperty(String name, Class<T> enumClass) {
super(name);
this.enumClass = enumClass;
}
EnumProperty(String name, Class<T> enumClass, boolean isRequired) {
super(name, isRequired);
this.enumClass = enumClass;
}
@Override
public T read(String rawVal) {
if (rawVal == null || rawVal.length() == 0) {
throw new IllegalArgumentException(formatError(rawVal));
}
try {
return T.valueOf(enumClass, rawVal.toUpperCase());
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(formatError(rawVal));
}
}
@Override
public T read(TPropertyVal tVal) throws IllegalArgumentException {
return T.valueOf(enumClass, tVal.getStrVal());
}
@Override
public T read(DataInput input) throws IOException {
return T.valueOf(enumClass, Text.readString(input));
}
@Override
public void write(T val, TPropertyVal out) {
out.setStrVal(val.name());
}
@Override
public void write(T val, DataOutput out) throws IOException {
Text.writeString(out, val.name());
}
private String formatError(String rawVal) {
String enumsStr = Arrays.stream(enumClass.getEnumConstants())
.map(Enum::toString)
.reduce((sa, sb) -> sa + ", " + sb)
.orElse("");
return String.format("Expected values are [%s], while [%s] provided", enumsStr, rawVal);
}
}
private static abstract class ComparableProperty<T extends Comparable> extends PropertySchema<T> {
protected ComparableProperty(String name) {
super(name);
}
protected ComparableProperty(String name, boolean isRequired) {
super(name, isRequired);
}
protected void verifyRange(T val) throws IllegalArgumentException {
if (getMinValue().isPresent() && (val == null || getMinValue().get().compareTo(val) > 0)) {
throw new IllegalArgumentException(val + " should not be less than " + getMinValue().get());
}
if (getMaxValue().isPresent() && (val == null || getMaxValue().get().compareTo(val) < 0)) {
throw new IllegalArgumentException(val + " should not be greater than " + getMaxValue().get());
}
}
}
PropertySchema<T> setDefauleValue(T val) {
this.defaultValue = Optional.of(val);
return this;
}
PropertySchema<T> setMin(T min) {
this.minValue = Optional.of(min);
return this;
}
PropertySchema<T> setMax(T max) {
this.maxValue = Optional.of(max);
return this;
}
public String getName() {
return name;
}
public boolean isRequired() {
return required;
}
public Optional<T> getDefaultValue() {
return defaultValue;
}
public Optional<T> getMinValue() {
return minValue;
}
public Optional<T> getMaxValue() {
return maxValue;
}
public abstract T read(String rawVal) throws IllegalArgumentException;
public abstract T read(TPropertyVal tVal) throws IllegalArgumentException;
public abstract T read(DataInput input) throws IOException;
public abstract void write(T val, TPropertyVal out);
public abstract void write(T val, DataOutput out) throws IOException;
}

163
fe/fe-common/src/test/java/org/apache/doris/common/property/PropertiesSetTest.java Normal file
View File

@ -0,0 +1,163 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.property;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.doris.thrift.TPropertyVal;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import static org.junit.Assert.fail;
public class PropertiesSetTest {
@Test
public void testReadFromStr() {
Map<String, String> raw = new HashMap<>();
raw.put("skip_header", "0");
PropertiesSet<FileFormat> properties = PropertiesSet.readFromStrMap(FileFormat.get(), raw);
Assert.assertEquals(FileFormat.Type.CSV, properties.get(FileFormat.FILE_FORMAT_TYPE));
Assert.assertEquals("\n", properties.get(FileFormat.RECORD_DELIMITER));
Assert.assertEquals(1, properties.writeToThrift().size());
properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps());
verifyVariableProps(properties);
Assert.assertEquals(3, properties.writeToThrift().size());
}
@Test
public void testThriftSerde() {
PropertiesSet<FileFormat> properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps());
Map<String, TPropertyVal> thriftMap = properties.writeToThrift();
Assert.assertEquals("JSON", thriftMap.get(FileFormat.FILE_FORMAT_TYPE.getName()).strVal);
Assert.assertEquals("\r", thriftMap.get(FileFormat.RECORD_DELIMITER.getName()).strVal);
Assert.assertEquals(3, thriftMap.get(FileFormat.SKIP_HEADER.getName()).intVal);
properties = PropertiesSet.readFromThrift(FileFormat.get(), thriftMap);
verifyVariableProps(properties);
}
@Test
public void testDataOutputSerde() throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertiesSet<FileFormat> properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps());
properties.writeToData(output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
properties = PropertiesSet.readFromData(FileFormat.get(), input);
verifyVariableProps(properties);
}
@Test
public void testEmpty() {
PropertiesSet<FileFormat> p1 = PropertiesSet.empty(FileFormat.get());
PropertiesSet<FileFormat> p2 = PropertiesSet.empty(FileFormat.get());
Assert.assertEquals(p1, p2);
}
@Test
public void testModifiedSchemas() {
PropertiesSet<FileFormat> properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps());
Assert.assertEquals(3, properties.getModifiedSchemas().size());
}
@Test
public void testToString() {
PropertiesSet<FileFormat> properties = PropertiesSet.readFromStrMap(FileFormat.get(), rawVariableProps());
String str = properties.toString();
Assert.assertTrue(str.contains(FileFormat.FILE_FORMAT_TYPE.getName()));
Assert.assertTrue(str.contains("JSON"));
Assert.assertTrue(str.contains(FileFormat.RECORD_DELIMITER.getName()));
Assert.assertTrue(str.contains("\r"));
Assert.assertTrue(str.contains(FileFormat.SKIP_HEADER.getName()));
Assert.assertTrue(str.contains("3"));
}
private Map<String, String> rawVariableProps() {
Map<String, String> raw = new HashMap<>();
raw.put(FileFormat.FILE_FORMAT_TYPE.getName(), "Json");
raw.put(FileFormat.RECORD_DELIMITER.getName(), "\r");
raw.put(FileFormat.SKIP_HEADER.getName(), "3");
return raw;
}
@Test
public void testCheckRequiredOpts() {
try {
PropertiesSet.readFromStrMap(FileFormat.get(), Maps.newHashMap());
fail("Expected an NoSuchElementException to be thrown");
} catch (NoSuchElementException e) {
Assert.assertTrue(e.getMessage().contains("Missing"));
}
}
@SuppressWarnings("unchecked")
private void verifyVariableProps(PropertiesSet properties) {
Assert.assertEquals(FileFormat.Type.JSON, properties.get(FileFormat.FILE_FORMAT_TYPE));
Assert.assertEquals("\r", properties.get(FileFormat.RECORD_DELIMITER));
Assert.assertEquals(3, properties.get(FileFormat.SKIP_HEADER));
}
private static class FileFormat implements PropertySchema.SchemaGroup {
public static PropertySchema<Type> FILE_FORMAT_TYPE =
new PropertySchema.EnumProperty<>("type", Type.class)
.setDefauleValue(Type.CSV);
public static PropertySchema<String> RECORD_DELIMITER =
new PropertySchema.StringProperty("record_delimiter").setDefauleValue("\n");
public static PropertySchema<String> FIELD_DELIMITER =
new PropertySchema.StringProperty("field_delimiter").setDefauleValue("|");
public static PropertySchema<Integer> SKIP_HEADER =
new PropertySchema.IntProperty("skip_header", true).setMin(0);
private static final FileFormat INSTANCE = new FileFormat();
private final ImmutableMap<String, PropertySchema> schemas = PropertySchema.createSchemas(
FILE_FORMAT_TYPE,
RECORD_DELIMITER,
FIELD_DELIMITER,
SKIP_HEADER);
public ImmutableMap<String, PropertySchema> getSchemas() {
return schemas;
}
public static FileFormat get() {
return INSTANCE;
}
public enum Type {
CSV, JSON, ORC, PARQUET
}
}
}

297
fe/fe-common/src/test/java/org/apache/doris/common/property/PropertySchemaTest.java Normal file
View File

@ -0,0 +1,297 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.property;
import org.apache.doris.thrift.TPropertyVal;
import org.hamcrest.Matchers;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
public class PropertySchemaTest {
@Rule
public ExpectedException exceptionRule = ExpectedException.none();
@Test
public void testStringPropNormal() throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key");
prop.write("val", output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
Assert.assertEquals("val", prop.read(input));
TPropertyVal tProp = new TPropertyVal();
prop.write("val", tProp);
Assert.assertEquals("val", prop.read(tProp));
prop.setMin("b");
Assert.assertEquals("c", prop.read("c"));
Assert.assertEquals("b", prop.read("b"));
prop.setMax("x");
Assert.assertEquals("w", prop.read("w"));
Assert.assertEquals("x", prop.read("x"));
}
@Test
public void testStringPropMinExceeded() {
PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key");
prop.setMin("b");
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("should not be less than"));
prop.read("a");
}
@Test
public void testStringPropMinNull() {
PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key");
prop.setMin("b");
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("should not be less than"));
prop.read((String) null);
}
@Test
public void testStringPropMaxExceeded() {
PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key");
prop.setMax("b");
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("should not be greater than"));
prop.read("c");
}
@Test
public void testStringPropMaxNull() {
PropertySchema.StringProperty prop = new PropertySchema.StringProperty("key");
prop.setMax("b");
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("should not be greater than"));
prop.read((String) null);
}
@Test
public void testIntPropNormal() throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.IntProperty prop = new PropertySchema.IntProperty("key");
prop.write(5, output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
Assert.assertEquals(Integer.valueOf(5), prop.read(input));
TPropertyVal tProp = new TPropertyVal();
prop.write(6, tProp);
Assert.assertEquals(Integer.valueOf(6), prop.read(tProp));
Assert.assertEquals(Integer.valueOf(7), prop.read("7"));
}
@Test
public void testIntPropInvalidString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid integer"));
PropertySchema.IntProperty prop = new PropertySchema.IntProperty("key");
prop.read("23j");
}
@Test
public void testIntPropNullString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid integer"));
PropertySchema.IntProperty prop = new PropertySchema.IntProperty("key");
prop.read((String) null);
}
@Test
public void testEnumPropNormal() throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.EnumProperty<Color> prop = new PropertySchema.EnumProperty<>("key", Color.class);
prop.write(Color.RED, output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
Assert.assertEquals(Color.RED, prop.read(input));
TPropertyVal tProp = new TPropertyVal();
prop.write(Color.GREEN, tProp);
Assert.assertEquals(Color.GREEN, prop.read(tProp));
Assert.assertEquals(Color.BLUE, prop.read("BLUE"));
Assert.assertEquals(Color.BLUE, prop.read("blue"));
Assert.assertEquals(Color.BLUE, prop.read("Blue"));
}
@Test
public void testEnumPropInvalidString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString(
"Expected values are [RED, GREEN, BLUE], while [invalid] provided"));
PropertySchema.EnumProperty<Color> prop = new PropertySchema.EnumProperty<>("key", Color.class);
prop.read("invalid");
}
@Test
public void testEnumPropNullString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString(
"Expected values are [RED, GREEN, BLUE], while [null] provided"));
PropertySchema.EnumProperty<Color> prop = new PropertySchema.EnumProperty<>("key", Color.class);
prop.read((String) null);
}
private enum Color {
RED, GREEN, BLUE
}
@Test
public void testLongPropNormal() throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.LongProperty prop = new PropertySchema.LongProperty("key");
prop.write(5L, output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
Assert.assertEquals(Long.valueOf(5), prop.read(input));
TPropertyVal tProp = new TPropertyVal();
prop.write(6L, tProp);
Assert.assertEquals(Long.valueOf(6), prop.read(tProp));
Assert.assertEquals(Long.valueOf(7), prop.read("7"));
}
@Test
public void testLongPropInvalidString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid long"));
PropertySchema.LongProperty prop = new PropertySchema.LongProperty("key");
prop.read("23j");
}
@Test
public void testLongPropNullString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid long"));
PropertySchema.LongProperty prop = new PropertySchema.LongProperty("key");
prop.read((String) null);
}
@Test
public void testDatePropNormal() throws Exception {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.DateProperty prop =
new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
prop.write(dateFormat.parse("2021-06-30 20:34:51"), output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
Assert.assertEquals(1625056491000L, prop.read(input).getTime());
TPropertyVal tProp = new TPropertyVal();
prop.write(new Date(1625056491000L), tProp);
Assert.assertEquals(1625056491000L, prop.read(tProp).getTime());
Assert.assertEquals(1625056491000L, prop.read("2021-06-30 20:34:51").getTime());
}
@Test
public void testDatePropInvalidString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid time format"));
PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd "
+ "HH:mm:ss"));
prop.read("2021-06-30");
}
@Test
public void testDatePropNullString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid time format"));
PropertySchema.DateProperty prop = new PropertySchema.DateProperty("key", new SimpleDateFormat("yyyy-MM-dd "
+ "HH:mm:ss"));
prop.read((String) null);
}
@Test
public void testBooleanPropNormal() throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outStream);
PropertySchema.BooleanProperty prop = new PropertySchema.BooleanProperty("key");
prop.write(true, output);
ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
DataInput input = new DataInputStream(inStream);
Assert.assertEquals(true, prop.read(input));
TPropertyVal tProp = new TPropertyVal();
prop.write(true, tProp);
Assert.assertEquals(true, prop.read(tProp));
Assert.assertEquals(true, prop.read("true"));
}
@Test
public void testBooleanPropInvalidString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid boolean"));
PropertySchema.BooleanProperty prop = new PropertySchema.BooleanProperty("key");
prop.read("233");
}
@Test
public void testBooleanPropNullString() {
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(Matchers.containsString("Invalid boolean"));
PropertySchema.BooleanProperty prop = new PropertySchema.BooleanProperty("key");
prop.read((String) null);
}
}

7
gensrc/thrift/FrontendService.thrift
View File

@ -687,6 +687,13 @@ struct TFrontendPingFrontendResult {
6: required string version
}
struct TPropertyVal {
1: optional string strVal
2: optional i32 intVal
3: optional i64 longVal
4: optional bool boolVal
}
service FrontendService {
TGetDbsResult getDbNames(1:TGetDbsParams params)
TGetTablesResult getTableNames(1:TGetTablesParams params)