database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix] iceberg catalog to specify the version and time (#22209)

Browse Source 
problem:
1. create a iceberg_type catalog:
2. use iceberg catalog to specify verison
```
mysql> show catalog iceberg;
+----------------------+--------------------------+
| Key                  | Value                    |
+----------------------+--------------------------+
| type                 | iceberg                  |
| iceberg.catalog.type | hms                      |
| hive.metastore.uris  | thrift://127.0.0.1:9083 |
| hadoop.username      | hadoop                   |
| create_time          | 2023-07-25 16:51:00.522  |
+----------------------+--------------------------+
5 rows in set (0.02 sec)

mysql> select * from iceberg.iceberg_db.tb1 FOR VERSION AS OF 8783036402036752909;
ERROR 5090 (42000): errCode = 2, detailMessage = Only iceberg/hudi external table supports time travel in current version
```

change:
Add `ICEBERG_EXTERNAL_TABLE` type for specify the version and time
This commit is contained in:
wuwenchi
2023-07-27 12:04:41 +08:00
committed by GitHub
parent 31c856351a
commit 41a230b721
3 changed files with 68 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
View File

@ -534,32 +534,41 @@ public class TableRef implements ParseNode, Writable {
return;
}
TableIf.TableType tableType = this.getTable().getType();
if (tableType != TableIf.TableType.HMS_EXTERNAL_TABLE) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
}
HMSExternalTable extTable = (HMSExternalTable) this.getTable();
switch (extTable.getDlaType()) {
case ICEBERG:
if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
String asOfTime = tableSnapshot.getTime();
Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
if (!matcher.matches()) {
throw new AnalysisException("Invalid datetime string: " + asOfTime);
if (tableType == TableIf.TableType.HMS_EXTERNAL_TABLE) {
HMSExternalTable extTable = (HMSExternalTable) this.getTable();
switch (extTable.getDlaType()) {
case ICEBERG:
if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
String asOfTime = tableSnapshot.getTime();
Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
if (!matcher.matches()) {
throw new AnalysisException("Invalid datetime string: " + asOfTime);
}
}
break;
case HUDI:
if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) {
throw new AnalysisException("Hudi table only supports timestamp as snapshot ID");
}
try {
tableSnapshot.setTime(HudiUtils.formatQueryInstant(tableSnapshot.getTime()));
} catch (Exception e) {
throw new AnalysisException("Failed to parse hudi timestamp: " + e.getMessage(), e);
}
break;
default:
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
}
} else if (tableType == TableIf.TableType.ICEBERG_EXTERNAL_TABLE) {
if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
String asOfTime = tableSnapshot.getTime();
Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
if (!matcher.matches()) {
throw new AnalysisException("Invalid datetime string: " + asOfTime);
}
break;
case HUDI:
if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) {
throw new AnalysisException("Hudi table only supports timestamp as snapshot ID");
}
try {
tableSnapshot.setTime(HudiUtils.formatQueryInstant(tableSnapshot.getTime()));
} catch (Exception e) {
throw new AnalysisException("Failed to parse hudi timestamp: " + e.getMessage(), e);
}
break;
default:
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
}
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
}
}