database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](iceberg) fix meta persist bug of iceberg catalog (#16344)

Browse Source 
This PR #16082 forgot to update the GsonUtil for Iceberg Catalog/Database/Table
This commit is contained in:
Mingyu Chen
2023-02-02 09:30:25 +08:00
committed by GitHub
parent 1c5279d26e
commit 06db0c6a91
3 changed files with 18 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java vendored
View File

@ -29,15 +29,12 @@ import java.util.List;
public class IcebergExternalTable extends ExternalTable {
IcebergExternalCatalog icebergCatalog;
public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
icebergCatalog = catalog;
}
public String getIcebergCatalogType() {
return icebergCatalog.getIcebergCatalogType();
return ((IcebergExternalCatalog) catalog).getIcebergCatalogType();
}
protected synchronized void makeSureInitialized() {
@ -49,7 +46,7 @@ public class IcebergExternalTable extends ExternalTable {
@Override
public TTableDescriptor toThrift() {
List<Column> schema = getFullSchema();
if (icebergCatalog.getIcebergCatalogType().equals("hms")) {
if (getIcebergCatalogType().equals("hms")) {
THiveTable tHiveTable = new THiveTable(dbName, name, new HashMap<>());
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.HIVE_TABLE, schema.size(), 0,
getName(), dbName);

14
fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
View File

@ -43,6 +43,8 @@ import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalDatabase;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.catalog.external.IcebergExternalDatabase;
import org.apache.doris.catalog.external.IcebergExternalTable;
import org.apache.doris.catalog.external.JdbcExternalDatabase;
import org.apache.doris.catalog.external.JdbcExternalTable;
import org.apache.doris.datasource.CatalogIf;
@ -50,6 +52,8 @@ import org.apache.doris.datasource.EsExternalCatalog;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.JdbcExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergHMSExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergRestExternalCatalog;
import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo;
import org.apache.doris.load.loadv2.SparkLoadJob.SparkLoadJobStateUpdateInfo;
import org.apache.doris.load.sync.SyncJob;
@ -167,21 +171,25 @@ public class GsonUtils {
.registerSubtype(InternalCatalog.class, InternalCatalog.class.getSimpleName())
.registerSubtype(HMSExternalCatalog.class, HMSExternalCatalog.class.getSimpleName())
.registerSubtype(EsExternalCatalog.class, EsExternalCatalog.class.getSimpleName())
.registerSubtype(JdbcExternalCatalog.class, JdbcExternalCatalog.class.getSimpleName());
.registerSubtype(JdbcExternalCatalog.class, JdbcExternalCatalog.class.getSimpleName())
.registerSubtype(IcebergHMSExternalCatalog.class, IcebergHMSExternalCatalog.class.getSimpleName())
.registerSubtype(IcebergRestExternalCatalog.class, IcebergRestExternalCatalog.class.getSimpleName());
private static RuntimeTypeAdapterFactory<DatabaseIf> dbTypeAdapterFactory = RuntimeTypeAdapterFactory.of(
DatabaseIf.class, "clazz")
.registerSubtype(ExternalDatabase.class, ExternalDatabase.class.getSimpleName())
.registerSubtype(EsExternalDatabase.class, EsExternalDatabase.class.getSimpleName())
.registerSubtype(HMSExternalDatabase.class, HMSExternalDatabase.class.getSimpleName())
.registerSubtype(JdbcExternalDatabase.class, JdbcExternalDatabase.class.getSimpleName());
.registerSubtype(JdbcExternalDatabase.class, JdbcExternalDatabase.class.getSimpleName())
.registerSubtype(IcebergExternalDatabase.class, IcebergExternalDatabase.class.getSimpleName());
private static RuntimeTypeAdapterFactory<TableIf> tblTypeAdapterFactory = RuntimeTypeAdapterFactory.of(
TableIf.class, "clazz")
.registerSubtype(ExternalTable.class, ExternalTable.class.getSimpleName())
.registerSubtype(EsExternalTable.class, EsExternalTable.class.getSimpleName())
.registerSubtype(HMSExternalTable.class, HMSExternalTable.class.getSimpleName())
.registerSubtype(JdbcExternalTable.class, JdbcExternalTable.class.getSimpleName());
.registerSubtype(JdbcExternalTable.class, JdbcExternalTable.class.getSimpleName())
.registerSubtype(IcebergExternalTable.class, IcebergExternalTable.class.getSimpleName());
// the builder of GSON instance.
// Add any other adapters if necessary.

10
fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java vendored
View File

@ -170,13 +170,13 @@ public class IcebergScanProvider extends QueryScanProvider {
List<InputSplit> splits = new ArrayList<>();
int formatVersion = ((BaseTable) table).operations().current().formatVersion();
for (FileScanTask task : scan.planFiles()) {
for (FileScanTask spitTask : task.split(128 * 1024 * 1024)) {
String dataFilePath = spitTask.file().path().toString();
IcebergSplit split = new IcebergSplit(new Path(dataFilePath), spitTask.start(),
spitTask.length(), new String[0]);
for (FileScanTask splitTask : task.split(128 * 1024 * 1024)) {
String dataFilePath = splitTask.file().path().toString();
IcebergSplit split = new IcebergSplit(new Path(dataFilePath), splitTask.start(),
splitTask.length(), new String[0]);
split.setFormatVersion(formatVersion);
if (formatVersion >= MIN_DELETE_FILE_SUPPORT_VERSION) {
split.setDeleteFileFilters(getDeleteFileFilters(spitTask));
split.setDeleteFileFilters(getDeleteFileFilters(splitTask));
}
split.setTableFormatType(TableFormatType.ICEBERG);
split.setAnalyzer(analyzer);