From c7b9a32e3ea0f44673a12c8ad0e03512395b8879 Mon Sep 17 00:00:00 2001 From: HB Date: Mon, 27 Nov 2023 20:56:50 +0800 Subject: [PATCH] [improvement](show snapshot) show iceberg snapshot print summary info (#27448) Iceberg's snapshot has summary information, but Doris did not display it. This patch fixes this issue. --- .../table-functions/iceberg-meta.md | 24 +++++++++---------- .../table-functions/iceberg-meta.md | 24 +++++++++---------- .../IcebergTableValuedFunction.java | 3 ++- .../tablefunction/MetadataGenerator.java | 1 + 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md b/docs/en/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md index c63e5150b9..b9cc9d3a1f 100644 --- a/docs/en/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md @@ -81,13 +81,13 @@ Inspect the iceberg table snapshots : ```sql select * from iceberg_meta("table" = "iceberg_ctl.test_db.test_tbl", "query_type" = "snapshots"); -+------------------------+----------------+---------------+-----------+-------------------+ -| committed_at | snapshot_id | parent_id | operation | manifest_list | -+------------------------+----------------+---------------+-----------+-------------------+ -| 2022-09-20 11:14:29 | 64123452344 | -1 | append | hdfs:/path/to/m1 | -| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | -| 2022-09-21 21:44:11 | 51232845315 | 98865735822 | overwrite | hdfs:/path/to/m3 | -+------------------------+----------------+---------------+-----------+-------------------+ ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| committed_at | snapshot_id | parent_id | operation | manifest_list | summary | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| 2022-09-20 11:14:29 | 64123452344 | -1 | append | hdfs:/path/to/m1 | {"flink.job-id":"xxm1", ...} | +| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | {"flink.job-id":"xxm2", ...} | +| 2022-09-21 21:44:11 | 51232845315 | 98865735822 | overwrite | hdfs:/path/to/m3 | {"flink.job-id":"xxm3", ...} | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ ``` Filtered by snapshot_id : @@ -95,9 +95,9 @@ Filtered by snapshot_id : ```sql select * from iceberg_meta("table" = "iceberg_ctl.test_db.test_tbl", "query_type" = "snapshots") where snapshot_id = 98865735822; -+------------------------+----------------+---------------+-----------+-------------------+ -| committed_at | snapshot_id | parent_id | operation | manifest_list | -+------------------------+----------------+---------------+-----------+-------------------+ -| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | -+------------------------+----------------+---------------+-----------+-------------------+ ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| committed_at | snapshot_id | parent_id | operation | manifest_list | summary | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | {"flink.job-id":"xxm2", ...} | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ ``` diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md index d882ae996a..6ab59469a0 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/iceberg-meta.md @@ -79,13 +79,13 @@ desc function iceberg_meta("table" = "ctl.db.tbl", "query_type" = "snapshots"); ```sql select * from iceberg_meta("table" = "iceberg_ctl.test_db.test_tbl", "query_type" = "snapshots"); -+------------------------+----------------+---------------+-----------+-------------------+ -| committed_at | snapshot_id | parent_id | operation | manifest_list | -+------------------------+----------------+---------------+-----------+-------------------+ -| 2022-09-20 11:14:29 | 64123452344 | -1 | append | hdfs:/path/to/m1 | -| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | -| 2022-09-21 21:44:11 | 51232845315 | 98865735822 | overwrite | hdfs:/path/to/m3 | -+------------------------+----------------+---------------+-----------+-------------------+ ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| committed_at | snapshot_id | parent_id | operation | manifest_list | summary | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| 2022-09-20 11:14:29 | 64123452344 | -1 | append | hdfs:/path/to/m1 | {"flink.job-id":"xxm1", ...} | +| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | {"flink.job-id":"xxm2", ...} | +| 2022-09-21 21:44:11 | 51232845315 | 98865735822 | overwrite | hdfs:/path/to/m3 | {"flink.job-id":"xxm3", ...} | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ ``` 根据snapshot_id字段筛选 @@ -93,9 +93,9 @@ select * from iceberg_meta("table" = "iceberg_ctl.test_db.test_tbl", "query_type ```sql select * from iceberg_meta("table" = "iceberg_ctl.test_db.test_tbl", "query_type" = "snapshots") where snapshot_id = 98865735822; -+------------------------+----------------+---------------+-----------+-------------------+ -| committed_at | snapshot_id | parent_id | operation | manifest_list | -+------------------------+----------------+---------------+-----------+-------------------+ -| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | -+------------------------+----------------+---------------+-----------+-------------------+ ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| committed_at | snapshot_id | parent_id | operation | manifest_list | summary | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ +| 2022-09-21 10:36:35 | 98865735822 | 64123452344 | overwrite | hdfs:/path/to/m2 | {"flink.job-id":"xxm2", ...} | ++------------------------+----------------+---------------+-----------+-------------------+------------------------------+ ``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java index 9e4296809f..424b0387e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java @@ -58,7 +58,8 @@ public class IcebergTableValuedFunction extends MetadataTableValuedFunction { new Column("parent_id", PrimitiveType.BIGINT, false), new Column("operation", PrimitiveType.STRING, false), // todo: compress manifest_list string - new Column("manifest_list", PrimitiveType.STRING, false)); + new Column("manifest_list", PrimitiveType.STRING, false), + new Column("summary", PrimitiveType.STRING, false)); private static final ImmutableMap COLUMN_TO_INDEX; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 00b5180fdd..542ece27fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -156,6 +156,7 @@ public class MetadataGenerator { } trow.addToColumnValue(new TCell().setStringVal(snapshot.operation())); trow.addToColumnValue(new TCell().setStringVal(snapshot.manifestListLocation())); + trow.addToColumnValue(new TCell().setStringVal(new Gson().toJson(snapshot.summary()))); dataBatch.add(trow); }