database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Meta Serialization] Support GSON serialization for class "Type" (#2709)

Browse Source 
"Type" is a abstract class, it has 4 sub classes:
1. ScalarType
2. ArrayType
3. MapType
4. StructType

This CL only support ScalarType. Other types can be added later.
This commit is contained in:
Mingyu Chen
2020-01-08 19:56:56 +08:00
committed by ZHAO Chun
parent e90170a5d0
commit d4a3b34319
10 changed files with 203 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
fe/src/test/java/org/apache/doris/catalog/ColumnGsonSerializationTest.java Normal file
View File

@ -0,0 +1,123 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
public class ColumnGsonSerializationTest {
private static String fileName = "./ColumnGsonSerializationTest";
@After
public void tearDown() {
File file = new File(fileName);
file.delete();
}
public static class ColumnList implements Writable {
@SerializedName(value = "columns")
public List<Column> columns = Lists.newArrayList();
@Override
public void write(DataOutput out) throws IOException {
String json = GsonUtils.GSON.toJson(this);
Text.writeString(out, json);
}
public static ColumnList read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, ColumnList.class);
}
}
@Test
public void testSerializeColumn() throws IOException, AnalysisException {
// 1. Write objects to file
File file = new File(fileName);
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
Column c1 = new Column("c1", Type.fromPrimitiveType(PrimitiveType.BIGINT), true, null, true, "1", "abc");
String c1Json = GsonUtils.GSON.toJson(c1);
Text.writeString(out, c1Json);
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(new FileInputStream(file));
String readJson = Text.readString(in);
Column readC1 = GsonUtils.GSON.fromJson(readJson, Column.class);
Assert.assertEquals(c1, readC1);
}
@Test
public void testSerializeColumnList() throws IOException, AnalysisException {
// 1. Write objects to file
File file = new File(fileName);
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
Column c1 = new Column("c1", Type.fromPrimitiveType(PrimitiveType.BIGINT), true, null, true, "1", "abc");
Column c2 = new Column("c2", ScalarType.createType(PrimitiveType.VARCHAR, 32, -1, -1), true, null, true, "cmy", "");
Column c3 = new Column("c3", ScalarType.createDecimalV2Type(27, 9), false, AggregateType.SUM, false, "1.1", "decimalv2");
ColumnList columnList = new ColumnList();
columnList.columns.add(c1);
columnList.columns.add(c2);
columnList.columns.add(c3);
columnList.write(out);
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(new FileInputStream(file));
ColumnList readList = ColumnList.read(in);
List<Column> columns = readList.columns;
Assert.assertEquals(3, columns.size());
Assert.assertEquals(c1, columns.get(0));
Assert.assertEquals(c2, columns.get(1));
Assert.assertEquals(c3, columns.get(2));
}
}

51
fe/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java
View File

@ -34,10 +34,6 @@ import java.util.Map;
* register 2 derived classes to the factory. And then register the factory
* to the GsonBuilder to create GSON instance.
*
* Notice that there is a special field "clazz" in ParentClass. This field is used
* to help the RuntimeTypeAdapterFactory to distinguish the kind of derived class.
* This field's name should be specified when creating the RuntimeTypeAdapterFactory.
*
*/
public class GsonDerivedClassSerializationTest {
private static String fileName = "./GsonDerivedClassSerializationTest";
@ -51,12 +47,9 @@ public class GsonDerivedClassSerializationTest {
public static class ParentClass implements Writable {
@SerializedName(value = "flag")
public int flag = 0;
@SerializedName(value = "clazz")
public String clazz; // a specified field to save the type of derived classed
public ParentClass(int flag, String clazz) {
this.flag = flag;
this.clazz = clazz;
}
@Override
@ -96,10 +89,32 @@ public class GsonDerivedClassSerializationTest {
}
}
public static class WrapperClass implements Writable {
@SerializedName(value = "tag")
public ParentClass clz;
public WrapperClass() {
clz = new ChildClassA(1, "child1");
}
@Override
public void write(DataOutput out) throws IOException {
String json = TEST_GSON.toJson(this);
System.out.println("write: " + json);
Text.writeString(out, json);
}
public static WrapperClass read(DataInput in) throws IOException {
String json = Text.readString(in);
System.out.println("read: " + json);
return TEST_GSON.fromJson(json, WrapperClass.class);
}
}
private static RuntimeTypeAdapterFactory<ParentClass> runtimeTypeAdapterFactory = RuntimeTypeAdapterFactory
// the "clazz" here is the name of "clazz" field in ParentClass.
// the "clazz" is a custom defined name
.of(ParentClass.class, "clazz")
// register 2 derived classes, the second parameter must be same to the value of field "clazz"
// register 2 derived classes, the second parameter will be the value of "clazz"
.registerSubtype(ChildClassA.class, ChildClassA.class.getSimpleName())
.registerSubtype(ChildClassB.class, ChildClassB.class.getSimpleName());
@ -152,4 +167,22 @@ public class GsonDerivedClassSerializationTest {
Assert.assertEquals("B2", ((ChildClassB) parentClass).mapB.get(2L));
}
@Test
public void testWrapperClass() throws IOException {
// 1. Write objects to file
File file = new File(fileName);
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
WrapperClass wrapperClass = new WrapperClass();
wrapperClass.write(out);
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(new FileInputStream(file));
WrapperClass readWrapperClass = WrapperClass.read(in);
Assert.assertEquals(1, ((ChildClassA) readWrapperClass.clz).flag);
}
}