database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](parquet-reader) reset value idx in bool rle decoder and support iceberg datetime(3) (#18245)

Browse Source 
1. Fix value  idx in bool rle decoder 
2. Iceberg table support datetimev2(3).  In the previous version, we converted hive timestamp to datetimev2(0) default.
This commit is contained in:
slothever
2023-04-01 21:00:01 +08:00
committed by GitHub
parent 9e087622ab
commit 97aab138aa
6 changed files with 50 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
View File

@ -698,6 +698,13 @@ public class HiveMetaStoreClientHelper {
* Convert hive type to doris type.
*/
public static Type hiveTypeToDorisType(String hiveType) {
return hiveTypeToDorisType(hiveType, 0);
}
/**
* Convert hive type to doris type with timescale.
*/
public static Type hiveTypeToDorisType(String hiveType, int timeScale) {
String lowerCaseType = hiveType.toLowerCase();
switch (lowerCaseType) {
case "boolean":
@ -713,7 +720,7 @@ public class HiveMetaStoreClientHelper {
case "date":
return ScalarType.createDateV2Type();
case "timestamp":
return ScalarType.createDatetimeV2Type(0);
return ScalarType.createDatetimeV2Type(timeScale);
case "float":
return Type.FLOAT;
case "double":

3
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java vendored
View File

@ -318,7 +318,8 @@ public class HMSExternalTable extends ExternalTable {
List<Column> tmpSchema = Lists.newArrayListWithCapacity(hmsSchema.size());
for (FieldSchema field : hmsSchema) {
tmpSchema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null,
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType(),
IcebergExternalTable.ICEBERG_DATETIME_SCALE_MS), true, null,
true, null, field.getComment(), true, null,
schema.caseInsensitiveFindField(field.getName()).fieldId(), null));
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java vendored
View File

@ -36,6 +36,8 @@ import java.util.List;
public class IcebergExternalTable extends ExternalTable {
public static final int ICEBERG_DATETIME_SCALE_MS = 3;
public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
}
@ -88,7 +90,7 @@ public class IcebergExternalTable extends ExternalTable {
case DATE:
return ScalarType.createDateV2Type();
case TIMESTAMP:
return ScalarType.createDatetimeV2Type(0);
return ScalarType.createDatetimeV2Type(ICEBERG_DATETIME_SCALE_MS);
case TIME:
return Type.UNSUPPORTED;
default: