database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Meta Serialization] Support GSON serialization for class "Type" (#2709)

Browse Source 
"Type" is a abstract class, it has 4 sub classes:
1. ScalarType
2. ArrayType
3. MapType
4. StructType

This CL only support ScalarType. Other types can be added later.
This commit is contained in:
Mingyu Chen
2020-01-08 19:56:56 +08:00
committed by ZHAO Chun
parent e90170a5d0
commit d4a3b34319
10 changed files with 203 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/src/main/java/org/apache/doris/catalog/ArrayType.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;

11
fe/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -40,22 +41,30 @@ import java.io.IOException;
*/
public class Column implements Writable {
private static final Logger LOG = LogManager.getLogger(Column.class);
@SerializedName(value = "name")
private String name;
@SerializedName(value = "type")
private Type type;
// column is key: aggregate type is null
// column is not key and has no aggregate type: aggregate type is none
// column is not key and has aggregate type: aggregate type is name of aggregate function.
@SerializedName(value = "aggregationType")
private AggregateType aggregationType;
// if isAggregationTypeImplicit is true, the actual aggregation type will not be shown in show create table
// the key type of table is duplicate or unique: the isAggregationTypeImplicit of value columns are true
// other cases: the isAggregationTypeImplicit is false
@SerializedName(value = "isAggregationTypeImplicit")
private boolean isAggregationTypeImplicit;
@SerializedName(value = "isKey")
private boolean isKey;
@SerializedName(value = "isAllowNull")
private boolean isAllowNull;
@SerializedName(value = "defaultValue")
private String defaultValue;
@SerializedName(value = "comment")
private String comment;
@SerializedName(value = "stats")
private ColumnStats stats; // cardinality and selectivity etc.
public Column() {

8
fe/src/main/java/org/apache/doris/catalog/ColumnStats.java
View File

@ -20,13 +20,13 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.common.io.Writable;
import org.apache.doris.catalog.PrimitiveType;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
@ -38,9 +38,13 @@ import java.io.IOException;
public class ColumnStats implements Writable {
private final static Logger LOG = LogManager.getLogger(ColumnStats.class);
@SerializedName(value = "avgSerializedSize")
private float avgSerializedSize; // in bytes; includes serialization overhead
@SerializedName(value = "maxSize")
private long maxSize; // in bytes
@SerializedName(value = "numDistinctValues")
private long numDistinctValues;
@SerializedName(value = "numNulls")
private long numNulls;
/**

1
fe/src/main/java/org/apache/doris/catalog/MapType.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;

5
fe/src/main/java/org/apache/doris/catalog/ScalarType.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -62,9 +63,11 @@ public class ScalarType extends Type {
public static final int MAX_PRECISION = 38;
public static final int MAX_SCALE = MAX_PRECISION;
@SerializedName(value = "type")
private final PrimitiveType type;
// Only used for type CHAR.
@SerializedName(value = "len")
private int len = -1;
private boolean isAssignedStrLenInColDefinition = false;
@ -73,7 +76,9 @@ public class ScalarType extends Type {
// It is invalid to have one by -1 and not the other.
// TODO: we could use that to store DECIMAL(8,*), indicating a decimal
// with 8 digits of precision and any valid ([0-8]) scale.
@SerializedName(value = "precision")
private int precision;
@SerializedName(value = "scale")
private int scale;
protected ScalarType(PrimitiveType type) {

7
fe/src/main/java/org/apache/doris/catalog/StructType.java
View File

@ -17,19 +17,20 @@
package org.apache.doris.catalog;
import java.util.ArrayList;
import java.util.HashMap;
import org.apache.doris.thrift.TStructField;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.HashMap;
/**
* Describes a STRUCT type. STRUCT types have a list of named struct fields.
*/

11
fe/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
View File

@ -17,6 +17,8 @@
package org.apache.doris.persist.gson;
import org.apache.doris.catalog.ScalarType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashBasedTable;
@ -64,13 +66,20 @@ import java.util.Map;
*/
public class GsonUtils {
// runtime adapter for class "Type"
private static RuntimeTypeAdapterFactory<org.apache.doris.catalog.Type> columnTypeAdapterFactory = RuntimeTypeAdapterFactory
.of(org.apache.doris.catalog.Type.class, "clazz")
// TODO: register other sub type after Doris support more types.
.registerSubtype(ScalarType.class, ScalarType.class.getSimpleName());
// the builder of GSON instance.
// Add any other adapters if necessary.
private static final GsonBuilder GSON_BUILDER = new GsonBuilder()
.addSerializationExclusionStrategy(new HiddenAnnotationExclusionStrategy())
.enableComplexMapKeySerialization()
.registerTypeHierarchyAdapter(Table.class, new GuavaTableAdapter())
.registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter());
.registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter())
.registerTypeAdapterFactory(columnTypeAdapterFactory);
// this instance is thread-safe.
public static final Gson GSON = GSON_BUILDER.create();

2
fe/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java
View File

@ -252,7 +252,7 @@ public final class RuntimeTypeAdapterFactory<T> implements TypeAdapterFactory {
}
public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
if (type.getRawType() != baseType) {
if (type.getRawType() != baseType && !subtypeToLabel.containsKey(type.getRawType())) {
return null;
}

123
fe/src/test/java/org/apache/doris/catalog/ColumnGsonSerializationTest.java Normal file
View File

@ -0,0 +1,123 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
public class ColumnGsonSerializationTest {
private static String fileName = "./ColumnGsonSerializationTest";
@After
public void tearDown() {
File file = new File(fileName);
file.delete();
}
public static class ColumnList implements Writable {
@SerializedName(value = "columns")
public List<Column> columns = Lists.newArrayList();
@Override
public void write(DataOutput out) throws IOException {
String json = GsonUtils.GSON.toJson(this);
Text.writeString(out, json);
}
public static ColumnList read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, ColumnList.class);
}
}
@Test
public void testSerializeColumn() throws IOException, AnalysisException {
// 1. Write objects to file
File file = new File(fileName);
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
Column c1 = new Column("c1", Type.fromPrimitiveType(PrimitiveType.BIGINT), true, null, true, "1", "abc");
String c1Json = GsonUtils.GSON.toJson(c1);
Text.writeString(out, c1Json);
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(new FileInputStream(file));
String readJson = Text.readString(in);
Column readC1 = GsonUtils.GSON.fromJson(readJson, Column.class);
Assert.assertEquals(c1, readC1);
}
@Test
public void testSerializeColumnList() throws IOException, AnalysisException {
// 1. Write objects to file
File file = new File(fileName);
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
Column c1 = new Column("c1", Type.fromPrimitiveType(PrimitiveType.BIGINT), true, null, true, "1", "abc");
Column c2 = new Column("c2", ScalarType.createType(PrimitiveType.VARCHAR, 32, -1, -1), true, null, true, "cmy", "");
Column c3 = new Column("c3", ScalarType.createDecimalV2Type(27, 9), false, AggregateType.SUM, false, "1.1", "decimalv2");
ColumnList columnList = new ColumnList();
columnList.columns.add(c1);
columnList.columns.add(c2);
columnList.columns.add(c3);
columnList.write(out);
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(new FileInputStream(file));
ColumnList readList = ColumnList.read(in);
List<Column> columns = readList.columns;
Assert.assertEquals(3, columns.size());
Assert.assertEquals(c1, columns.get(0));
Assert.assertEquals(c2, columns.get(1));
Assert.assertEquals(c3, columns.get(2));
}
}

51
fe/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java
View File

@ -34,10 +34,6 @@ import java.util.Map;
* register 2 derived classes to the factory. And then register the factory
* to the GsonBuilder to create GSON instance.
*
* Notice that there is a special field "clazz" in ParentClass. This field is used
* to help the RuntimeTypeAdapterFactory to distinguish the kind of derived class.
* This field's name should be specified when creating the RuntimeTypeAdapterFactory.
*
*/
public class GsonDerivedClassSerializationTest {
private static String fileName = "./GsonDerivedClassSerializationTest";
@ -51,12 +47,9 @@ public class GsonDerivedClassSerializationTest {
public static class ParentClass implements Writable {
@SerializedName(value = "flag")
public int flag = 0;
@SerializedName(value = "clazz")
public String clazz; // a specified field to save the type of derived classed
public ParentClass(int flag, String clazz) {
this.flag = flag;
this.clazz = clazz;
}
@Override
@ -96,10 +89,32 @@ public class GsonDerivedClassSerializationTest {
}
}
public static class WrapperClass implements Writable {
@SerializedName(value = "tag")
public ParentClass clz;
public WrapperClass() {
clz = new ChildClassA(1, "child1");
}
@Override
public void write(DataOutput out) throws IOException {
String json = TEST_GSON.toJson(this);
System.out.println("write: " + json);
Text.writeString(out, json);
}
public static WrapperClass read(DataInput in) throws IOException {
String json = Text.readString(in);
System.out.println("read: " + json);
return TEST_GSON.fromJson(json, WrapperClass.class);
}
}
private static RuntimeTypeAdapterFactory<ParentClass> runtimeTypeAdapterFactory = RuntimeTypeAdapterFactory
// the "clazz" here is the name of "clazz" field in ParentClass.
// the "clazz" is a custom defined name
.of(ParentClass.class, "clazz")
// register 2 derived classes, the second parameter must be same to the value of field "clazz"
// register 2 derived classes, the second parameter will be the value of "clazz"
.registerSubtype(ChildClassA.class, ChildClassA.class.getSimpleName())
.registerSubtype(ChildClassB.class, ChildClassB.class.getSimpleName());
@ -152,4 +167,22 @@ public class GsonDerivedClassSerializationTest {
Assert.assertEquals("B2", ((ChildClassB) parentClass).mapB.get(2L));
}
@Test
public void testWrapperClass() throws IOException {
// 1. Write objects to file
File file = new File(fileName);
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
WrapperClass wrapperClass = new WrapperClass();
wrapperClass.write(out);
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(new FileInputStream(file));
WrapperClass readWrapperClass = WrapperClass.read(in);
Assert.assertEquals(1, ((ChildClassA) readWrapperClass.clz).flag);
}
}