diff --git a/fe/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/src/main/java/org/apache/doris/catalog/ArrayType.java index f62fa66d2d..e7b4a82856 100644 --- a/fe/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -20,6 +20,7 @@ package org.apache.doris.catalog; import org.apache.doris.thrift.TTypeDesc; import org.apache.doris.thrift.TTypeNode; import org.apache.doris.thrift.TTypeNodeType; + import com.google.common.base.Preconditions; import com.google.common.base.Strings; diff --git a/fe/src/main/java/org/apache/doris/catalog/Column.java b/fe/src/main/java/org/apache/doris/catalog/Column.java index b52576bf55..6b71611f23 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/src/main/java/org/apache/doris/catalog/Column.java @@ -27,6 +27,7 @@ import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TColumnType; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,22 +41,30 @@ import java.io.IOException; */ public class Column implements Writable { private static final Logger LOG = LogManager.getLogger(Column.class); + @SerializedName(value = "name") private String name; + @SerializedName(value = "type") private Type type; // column is key: aggregate type is null // column is not key and has no aggregate type: aggregate type is none // column is not key and has aggregate type: aggregate type is name of aggregate function. + @SerializedName(value = "aggregationType") private AggregateType aggregationType; // if isAggregationTypeImplicit is true, the actual aggregation type will not be shown in show create table // the key type of table is duplicate or unique: the isAggregationTypeImplicit of value columns are true // other cases: the isAggregationTypeImplicit is false + @SerializedName(value = "isAggregationTypeImplicit") private boolean isAggregationTypeImplicit; + @SerializedName(value = "isKey") private boolean isKey; + @SerializedName(value = "isAllowNull") private boolean isAllowNull; + @SerializedName(value = "defaultValue") private String defaultValue; + @SerializedName(value = "comment") private String comment; - + @SerializedName(value = "stats") private ColumnStats stats; // cardinality and selectivity etc. public Column() { diff --git a/fe/src/main/java/org/apache/doris/catalog/ColumnStats.java b/fe/src/main/java/org/apache/doris/catalog/ColumnStats.java index 9191649329..80b1a04e4b 100644 --- a/fe/src/main/java/org/apache/doris/catalog/ColumnStats.java +++ b/fe/src/main/java/org/apache/doris/catalog/ColumnStats.java @@ -20,13 +20,13 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.io.Writable; -import org.apache.doris.catalog.PrimitiveType; import com.google.common.base.Objects; import com.google.common.base.Preconditions; +import com.google.gson.annotations.SerializedName; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; @@ -38,9 +38,13 @@ import java.io.IOException; public class ColumnStats implements Writable { private final static Logger LOG = LogManager.getLogger(ColumnStats.class); + @SerializedName(value = "avgSerializedSize") private float avgSerializedSize; // in bytes; includes serialization overhead + @SerializedName(value = "maxSize") private long maxSize; // in bytes + @SerializedName(value = "numDistinctValues") private long numDistinctValues; + @SerializedName(value = "numNulls") private long numNulls; /** diff --git a/fe/src/main/java/org/apache/doris/catalog/MapType.java b/fe/src/main/java/org/apache/doris/catalog/MapType.java index 62865d6abe..71f2c4b679 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/src/main/java/org/apache/doris/catalog/MapType.java @@ -20,6 +20,7 @@ package org.apache.doris.catalog; import org.apache.doris.thrift.TTypeDesc; import org.apache.doris.thrift.TTypeNode; import org.apache.doris.thrift.TTypeNodeType; + import com.google.common.base.Preconditions; import com.google.common.base.Strings; diff --git a/fe/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/src/main/java/org/apache/doris/catalog/ScalarType.java index 83fcdf5fea..d43ca03f2b 100644 --- a/fe/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -25,6 +25,7 @@ import org.apache.doris.thrift.TTypeNodeType; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,9 +63,11 @@ public class ScalarType extends Type { public static final int MAX_PRECISION = 38; public static final int MAX_SCALE = MAX_PRECISION; + @SerializedName(value = "type") private final PrimitiveType type; // Only used for type CHAR. + @SerializedName(value = "len") private int len = -1; private boolean isAssignedStrLenInColDefinition = false; @@ -73,7 +76,9 @@ public class ScalarType extends Type { // It is invalid to have one by -1 and not the other. // TODO: we could use that to store DECIMAL(8,*), indicating a decimal // with 8 digits of precision and any valid ([0-8]) scale. + @SerializedName(value = "precision") private int precision; + @SerializedName(value = "scale") private int scale; protected ScalarType(PrimitiveType type) { diff --git a/fe/src/main/java/org/apache/doris/catalog/StructType.java b/fe/src/main/java/org/apache/doris/catalog/StructType.java index 9f9948d433..4e04678a10 100644 --- a/fe/src/main/java/org/apache/doris/catalog/StructType.java +++ b/fe/src/main/java/org/apache/doris/catalog/StructType.java @@ -17,19 +17,20 @@ package org.apache.doris.catalog; -import java.util.ArrayList; -import java.util.HashMap; - import org.apache.doris.thrift.TStructField; import org.apache.doris.thrift.TTypeDesc; import org.apache.doris.thrift.TTypeNode; import org.apache.doris.thrift.TTypeNodeType; + import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.HashMap; + /** * Describes a STRUCT type. STRUCT types have a list of named struct fields. */ diff --git a/fe/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index f997f91269..b1e0228d05 100644 --- a/fe/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -17,6 +17,8 @@ package org.apache.doris.persist.gson; +import org.apache.doris.catalog.ScalarType; + import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashBasedTable; @@ -64,13 +66,20 @@ import java.util.Map; */ public class GsonUtils { + // runtime adapter for class "Type" + private static RuntimeTypeAdapterFactory columnTypeAdapterFactory = RuntimeTypeAdapterFactory + .of(org.apache.doris.catalog.Type.class, "clazz") + // TODO: register other sub type after Doris support more types. + .registerSubtype(ScalarType.class, ScalarType.class.getSimpleName()); + // the builder of GSON instance. // Add any other adapters if necessary. private static final GsonBuilder GSON_BUILDER = new GsonBuilder() .addSerializationExclusionStrategy(new HiddenAnnotationExclusionStrategy()) .enableComplexMapKeySerialization() .registerTypeHierarchyAdapter(Table.class, new GuavaTableAdapter()) - .registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter()); + .registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter()) + .registerTypeAdapterFactory(columnTypeAdapterFactory); // this instance is thread-safe. public static final Gson GSON = GSON_BUILDER.create(); diff --git a/fe/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java b/fe/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java index 924dc9de4b..2839e40833 100644 --- a/fe/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java +++ b/fe/src/main/java/org/apache/doris/persist/gson/RuntimeTypeAdapterFactory.java @@ -252,7 +252,7 @@ public final class RuntimeTypeAdapterFactory implements TypeAdapterFactory { } public TypeAdapter create(Gson gson, TypeToken type) { - if (type.getRawType() != baseType) { + if (type.getRawType() != baseType && !subtypeToLabel.containsKey(type.getRawType())) { return null; } diff --git a/fe/src/test/java/org/apache/doris/catalog/ColumnGsonSerializationTest.java b/fe/src/test/java/org/apache/doris/catalog/ColumnGsonSerializationTest.java new file mode 100644 index 0000000000..c017475ab3 --- /dev/null +++ b/fe/src/test/java/org/apache/doris/catalog/ColumnGsonSerializationTest.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + +public class ColumnGsonSerializationTest { + + private static String fileName = "./ColumnGsonSerializationTest"; + + @After + public void tearDown() { + File file = new File(fileName); + file.delete(); + } + + public static class ColumnList implements Writable { + @SerializedName(value = "columns") + public List columns = Lists.newArrayList(); + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } + + public static ColumnList read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, ColumnList.class); + } + } + + @Test + public void testSerializeColumn() throws IOException, AnalysisException { + // 1. Write objects to file + File file = new File(fileName); + file.createNewFile(); + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + + Column c1 = new Column("c1", Type.fromPrimitiveType(PrimitiveType.BIGINT), true, null, true, "1", "abc"); + + String c1Json = GsonUtils.GSON.toJson(c1); + Text.writeString(out, c1Json); + out.flush(); + out.close(); + + // 2. Read objects from file + DataInputStream in = new DataInputStream(new FileInputStream(file)); + + String readJson = Text.readString(in); + Column readC1 = GsonUtils.GSON.fromJson(readJson, Column.class); + + Assert.assertEquals(c1, readC1); + } + + @Test + public void testSerializeColumnList() throws IOException, AnalysisException { + // 1. Write objects to file + File file = new File(fileName); + file.createNewFile(); + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + + Column c1 = new Column("c1", Type.fromPrimitiveType(PrimitiveType.BIGINT), true, null, true, "1", "abc"); + Column c2 = new Column("c2", ScalarType.createType(PrimitiveType.VARCHAR, 32, -1, -1), true, null, true, "cmy", ""); + Column c3 = new Column("c3", ScalarType.createDecimalV2Type(27, 9), false, AggregateType.SUM, false, "1.1", "decimalv2"); + + ColumnList columnList = new ColumnList(); + columnList.columns.add(c1); + columnList.columns.add(c2); + columnList.columns.add(c3); + + columnList.write(out); + out.flush(); + out.close(); + + // 2. Read objects from file + DataInputStream in = new DataInputStream(new FileInputStream(file)); + + ColumnList readList = ColumnList.read(in); + List columns = readList.columns; + + Assert.assertEquals(3, columns.size()); + Assert.assertEquals(c1, columns.get(0)); + Assert.assertEquals(c2, columns.get(1)); + Assert.assertEquals(c3, columns.get(2)); + } + +} diff --git a/fe/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java b/fe/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java index 825da16745..35f75d82b2 100644 --- a/fe/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java +++ b/fe/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java @@ -34,10 +34,6 @@ import java.util.Map; * register 2 derived classes to the factory. And then register the factory * to the GsonBuilder to create GSON instance. * - * Notice that there is a special field "clazz" in ParentClass. This field is used - * to help the RuntimeTypeAdapterFactory to distinguish the kind of derived class. - * This field's name should be specified when creating the RuntimeTypeAdapterFactory. - * */ public class GsonDerivedClassSerializationTest { private static String fileName = "./GsonDerivedClassSerializationTest"; @@ -51,12 +47,9 @@ public class GsonDerivedClassSerializationTest { public static class ParentClass implements Writable { @SerializedName(value = "flag") public int flag = 0; - @SerializedName(value = "clazz") - public String clazz; // a specified field to save the type of derived classed public ParentClass(int flag, String clazz) { this.flag = flag; - this.clazz = clazz; } @Override @@ -96,10 +89,32 @@ public class GsonDerivedClassSerializationTest { } } + public static class WrapperClass implements Writable { + @SerializedName(value = "tag") + public ParentClass clz; + + public WrapperClass() { + clz = new ChildClassA(1, "child1"); + } + + @Override + public void write(DataOutput out) throws IOException { + String json = TEST_GSON.toJson(this); + System.out.println("write: " + json); + Text.writeString(out, json); + } + + public static WrapperClass read(DataInput in) throws IOException { + String json = Text.readString(in); + System.out.println("read: " + json); + return TEST_GSON.fromJson(json, WrapperClass.class); + } + } + private static RuntimeTypeAdapterFactory runtimeTypeAdapterFactory = RuntimeTypeAdapterFactory - // the "clazz" here is the name of "clazz" field in ParentClass. + // the "clazz" is a custom defined name .of(ParentClass.class, "clazz") - // register 2 derived classes, the second parameter must be same to the value of field "clazz" + // register 2 derived classes, the second parameter will be the value of "clazz" .registerSubtype(ChildClassA.class, ChildClassA.class.getSimpleName()) .registerSubtype(ChildClassB.class, ChildClassB.class.getSimpleName()); @@ -152,4 +167,22 @@ public class GsonDerivedClassSerializationTest { Assert.assertEquals("B2", ((ChildClassB) parentClass).mapB.get(2L)); } + @Test + public void testWrapperClass() throws IOException { + // 1. Write objects to file + File file = new File(fileName); + file.createNewFile(); + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + + WrapperClass wrapperClass = new WrapperClass(); + wrapperClass.write(out); + out.flush(); + out.close(); + + // 2. Read objects from file + DataInputStream in = new DataInputStream(new FileInputStream(file)); + WrapperClass readWrapperClass = WrapperClass.read(in); + Assert.assertEquals(1, ((ChildClassA) readWrapperClass.clz).flag); + } + }