From 1e70f992e7bbdd64b20ba67443fe502be23f42f9 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 8 Mar 2022 18:53:11 +0800 Subject: [PATCH] [improvement][fix](insert)(replay) support SHOW LAST INSERT stmt and fix json replay bug (#8355) 1. support SHOW LAST INSERT In the current implementation, the insert operation returns a json string to describe the result information of the insert. But this information is in the session track field of the mysql protocol, and it is difficult to obtain programmatically. Therefore, I provide a new syntax `show last insert` to explicitly obtain the result of the latest insert operation, and return a normal query result set to facilitate the user to obtain the result information of the insert. 2. the `ReturnRows` field in fe.audit.log of insert operation will be set to the loaded row num of the insert. 3. Fix a bug described in #8354 --- docs/.vuepress/sidebar/en.js | 1 + docs/.vuepress/sidebar/zh-CN.js | 1 + .../load-data/insert-into-manual.md | 22 ++++++ .../Data Manipulation/SHOW-LAST-INSERT.md | 67 ++++++++++++++++++ .../load-data/insert-into-manual.md | 23 ++++++- .../Data Manipulation/SHOW-LAST-INSERT.md | 68 +++++++++++++++++++ fe/fe-core/src/main/cup/sql_parser.cup | 5 ++ .../doris/analysis/ShowLastInsertStmt.java | 58 ++++++++++++++++ .../org/apache/doris/qe/ConnectContext.java | 17 +++++ .../org/apache/doris/qe/InsertResult.java | 66 ++++++++++++++++++ .../org/apache/doris/qe/SessionVariable.java | 6 +- .../org/apache/doris/qe/ShowExecutor.java | 18 ++++- .../org/apache/doris/qe/StmtExecutor.java | 7 ++ 13 files changed, 355 insertions(+), 4 deletions(-) create mode 100644 docs/en/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md create mode 100644 docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/qe/InsertResult.java diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index 41265027cd..8c050ca440 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -689,6 +689,7 @@ module.exports = [ "SHOW TABLE CREATION", "SHOW TABLET", "SHOW TRANSACTION", + "SHOW-LAST-INSERT", "STOP ROUTINE LOAD", "STOP SYNC JOB", "STREAM LOAD", diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index 37bd0b094d..a9154ad9ec 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -689,6 +689,7 @@ module.exports = [ "SHOW TABLE CREATION", "SHOW TABLET", "SHOW TRANSACTION", + "SHOW-LAST-INSERT", "SPARK LOAD", "STOP ROUTINE LOAD", "STOP SYNC JOB", diff --git a/docs/en/administrator-guide/load-data/insert-into-manual.md b/docs/en/administrator-guide/load-data/insert-into-manual.md index a67d4543dc..bdc85f439a 100644 --- a/docs/en/administrator-guide/load-data/insert-into-manual.md +++ b/docs/en/administrator-guide/load-data/insert-into-manual.md @@ -196,6 +196,28 @@ Insert Into itself is a SQL command, and the return result is divided into the f 2. If `status` is` visible`, the data is loaded successfully. 3. If `warnings` is greater than 0, it means that some data is filtered. You can get the url through the `show load` statement to see the filtered rows. +### SHOW LAST INSERT + +In the previous section, we described how to follow up on the results of insert operations. However, it is difficult to get the json string of the returned result in some mysql libraries. Therefore, Doris also provides the `SHOW LAST INSERT` command to explicitly retrieve the results of the last insert operation. + +After executing an insert operation, you can execute `SHOW LAST INSERT` on the same session connection. This command returns the result of the most recent insert operation, e.g. + +``` +mysql> show last insert\G +*************************** 1. row *************************** + TransactionId: 64067 + Label: insert_ba8f33aea9544866-8ed77e2844d0cc9b + Database: default_cluster:db1 + Table: t1 +TransactionStatus: VISIBLE + LoadedRows: 2 + FilteredRows: 0 +``` + +This command returns the insert results and the details of the corresponding transaction. Therefore, you can continue to execute the `show last insert` command after each insert operation to get the insert results. + +> Note: This command will only return the results of the last insert operation within the same session connection. If the connection is broken or replaced with a new one, the empty set will be returned. + ## Relevant System Configuration ### FE configuration diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md new file mode 100644 index 0000000000..5ea8da2db1 --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md @@ -0,0 +1,67 @@ +--- +{ + "title": "SHOW LAST INSERT", + "language": "en" +} +--- + + + +# SHOW LAST INSERT +## description + +This syntax is used to view the result of the last insert operation in the current session connection. + +Syntax: + +``` +SHOW LAST INSERT +``` + +Example of return result. + +``` + TransactionId: 64067 + Label: insert_ba8f33aea9544866-8ed77e2844d0cc9b + Database: default_cluster:db1 + Table: t1 +TransactionStatus: VISIBLE + LoadedRows: 2 + FilteredRows: 0 +``` + +* TransactionId: transaction id +* Label: label corresponding to the insert task +* Database: the database corresponding to the insert +* Table: the table corresponding to the insert +* TransactionStatus: the status of the transaction + * PREPARE: preparation phase + * PRECOMMITTED: Pre-committed stage + * COMMITTED: The transaction is successful, but the data is not visible + * VISIBLE: The transaction is successful and the data is visible + * ABORTED: The transaction failed. +* LoadedRows: Number of rows imported +* FilteredRows: number of rows filtered + +## example + +## keyword + + SHOW, LAST, INSERT diff --git a/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md b/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md index bd872f6867..9d66d1f07e 100644 --- a/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md +++ b/docs/zh-CN/administrator-guide/load-data/insert-into-manual.md @@ -184,7 +184,6 @@ Insert Into 本身就是一个 SQL 命令，其返回结果会根据执行结果 ``` 其中 `ERROR 1064 (HY000): all partitions have no load data` 显示失败原因。后面的 url 可以用于查询错误的数据，具体见后面 **查看错误行** 小结。 - **综上，对于 insert 操作返回结果的正确处理逻辑应为：** @@ -195,6 +194,28 @@ Insert Into 本身就是一个 SQL 命令，其返回结果会根据执行结果 1. 如果 `status` 为 `committed`，表示数据还不可见。需要通过 `show transaction` 语句查看状态直到 `visible` 2. 如果 `status` 为 `visible`，表示数据导入成功。 3. 如果 `warnings` 大于 0，表示有数据被过滤，可以通过 `show load` 语句获取 url 查看被过滤的行。 + +### SHOW LAST INSERT + +在上一小节中我们介绍了如何根据 insert 操作的返回结果进行后续处理。但一些语言的mysql类库中很难获取返回结果的中的 json 字符串。因此，Doris 还提供了 `SHOW LAST INSERT` 命令来显式的获取最近一次 insert 操作的结果。 + +当执行完一个 insert 操作后，可以在同一 session 连接中执行 `SHOW LAST INSERT`。该命令会返回最近一次insert 操作的结果，如： + +``` +mysql> show last insert\G +*************************** 1. row *************************** + TransactionId: 64067 + Label: insert_ba8f33aea9544866-8ed77e2844d0cc9b + Database: default_cluster:db1 + Table: t1 +TransactionStatus: VISIBLE + LoadedRows: 2 + FilteredRows: 0 +``` + +该命令会返回 insert 以及对应事务的详细信息。因此，用户可以在每次执行完 insert 操作后，继续执行 `show last insert` 命令来获取 insert 的结果。 + +> 注意：该命令只会返回在同一 session 连接中，最近一次 insert 操作的结果。如果连接断开或更换了新的连接，则将返回空集。 ## 相关系统配置 diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md new file mode 100644 index 0000000000..1a7f9f3abe --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW-LAST-INSERT.md @@ -0,0 +1,68 @@ +--- +{ + "title": "SHOW LAST INSERT", + "language": "zh-CN" +} +--- + + + +# SHOW LAST INSERT +## description + +该语法用于查看在当前session连接中，最近一次 insert 操作的结果 + +语法： + +``` +SHOW LAST INSERT +``` + +返回结果示例： + +``` + TransactionId: 64067 + Label: insert_ba8f33aea9544866-8ed77e2844d0cc9b + Database: default_cluster:db1 + Table: t1 +TransactionStatus: VISIBLE + LoadedRows: 2 + FilteredRows: 0 +``` + +* TransactionId：事务id +* Label：insert任务对应的 label +* Database：insert对应的数据库 +* Table：insert对应的表 +* TransactionStatus：事务状态 + * PREPARE：准备阶段 + * PRECOMMITTED：预提交阶段 + * COMMITTED：事务成功，但数据不可见 + * VISIBLE：事务成功且数据可见 + * ABORTED：事务失败 +* LoadedRows：导入的行数 +* FilteredRows：被过滤的行数 + +## example + +## keyword + + SHOW, LAST, INSERT + diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index dc2ee4183a..7c19093238 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -2876,6 +2876,11 @@ show_param ::= {: RESULT = new ShowTableCreationStmt(db, parser.wild); :} + /* show last insert */ + | KW_LAST KW_INSERT + {: + RESULT = new ShowLastInsertStmt(); + :} ; opt_tmp ::= diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java new file mode 100644 index 0000000000..9ed0ebc6cd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLastInsertStmt.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.qe.ShowResultSetMetaData; + +// SHOW LAST INSERT +public class ShowLastInsertStmt extends ShowStmt { + private static final ShowResultSetMetaData META_DATA = + ShowResultSetMetaData.builder() + .addColumn(new Column("TransactionId", ScalarType.createVarchar(128))) + .addColumn(new Column("Label", ScalarType.createVarchar(128))) + .addColumn(new Column("Database", ScalarType.createVarchar(128))) + .addColumn(new Column("Table", ScalarType.createVarchar(128))) + .addColumn(new Column("TransactionStatus", ScalarType.createVarchar(64))) + .addColumn(new Column("LoadedRows", ScalarType.createVarchar(128))) + .addColumn(new Column("FilteredRows", ScalarType.createVarchar(128))) + .build(); + + @Override + public void analyze(Analyzer analyzer) { + // No need to check priv here. Bacause `show last insert` can only view + // the insert result of current session. + // So if user does not have priv to insert, than there is no result to show. + } + + @Override + public String toSql() { + return "SHOW INSERT RESULT"; + } + + @Override + public String toString() { + return toSql(); + } + + @Override + public ShowResultSetMetaData getMetaData() { + return META_DATA; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index d24989c7f9..48b30906ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -33,6 +33,7 @@ import org.apache.doris.resource.Tag; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.TransactionEntry; +import org.apache.doris.transaction.TransactionStatus; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -134,6 +135,21 @@ public class ConnectContext { // The FE ip current connected private String currentConnectedFEIp = ""; + private InsertResult insertResult; + + public void setOrUpdateInsertResult(long txnId, String label, String db, String tbl, + TransactionStatus txnStatus, long loadedRows, int filteredRows) { + if (isTxnModel() && insertResult != null) { + insertResult.updateResult(txnStatus, loadedRows, filteredRows); + } else { + insertResult = new InsertResult(txnId, label, db, tbl, txnStatus, loadedRows, filteredRows); + } + } + + public InsertResult getInsertResult() { + return insertResult; + } + public static ConnectContext get() { return threadLocalInfo.get(); } @@ -544,4 +560,5 @@ public class ConnectContext { public String getQueryIdentifier() { return "stmt[" + stmtId + ", " + DebugUtil.printId(queryId) + "]"; } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/InsertResult.java b/fe/fe-core/src/main/java/org/apache/doris/qe/InsertResult.java new file mode 100644 index 0000000000..551bead627 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/InsertResult.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.transaction.TransactionStatus; + +import com.google.common.collect.Lists; + +import java.util.List; + +// Save the result of last insert operation. +// So that user can view it by executing SHOW LAST INSERT. +public class InsertResult { + public long txnId; + public String label; + public String db; + public String tbl; + public TransactionStatus txnStatus; + public long loadedRows; + public long filteredRows; + + public InsertResult(long txnId, String label, String db, String tbl, TransactionStatus txnStatus, + long loadedRows, long filteredRows) { + this.txnId = txnId; + this.label = label; + this.db = db; + this.tbl = tbl; + this.txnStatus = txnStatus; + this.loadedRows = loadedRows; + this.filteredRows = filteredRows; + } + + public void updateResult(TransactionStatus txnStatus, long loadedRows, int filteredRows) { + this.txnStatus = txnStatus; + this.loadedRows += loadedRows; + this.filteredRows += filteredRows; + } + + public List toRow() { + List row = Lists.newArrayList(); + row.add(String.valueOf(txnId)); + row.add(label); + row.add(db); + row.add(tbl); + row.add(txnStatus.name()); + row.add(String.valueOf(loadedRows)); + row.add(String.valueOf(filteredRows)); + return row; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index ea8f74a3ed..ab3bf079f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1000,10 +1000,11 @@ public class SessionVariable implements Serializable, Writable { field.set(this, root.get(attr.name())); break; case "int": - field.set(this, root.get(attr.name())); + // root.get(attr.name()) always return Long type, so need to convert it. + field.set(this, Integer.valueOf(root.get(attr.name()).toString())); break; case "long": - field.set(this, root.get(attr.name())); + field.set(this, (Long) root.get(attr.name())); break; case "float": field.set(this, root.get(attr.name())); @@ -1107,3 +1108,4 @@ public class SessionVariable implements Serializable, Writable { } } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 7559758832..5be7f1871f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -49,6 +49,7 @@ import org.apache.doris.analysis.ShowFrontendsStmt; import org.apache.doris.analysis.ShowFunctionsStmt; import org.apache.doris.analysis.ShowGrantsStmt; import org.apache.doris.analysis.ShowIndexStmt; +import org.apache.doris.analysis.ShowLastInsertStmt; import org.apache.doris.analysis.ShowLoadProfileStmt; import org.apache.doris.analysis.ShowLoadStmt; import org.apache.doris.analysis.ShowLoadWarningsStmt; @@ -157,6 +158,7 @@ import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TUnit; import org.apache.doris.transaction.GlobalTransactionMgr; +import org.apache.doris.transaction.TransactionStatus; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -165,7 +167,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.lang3.tuple.Triple; -import org.apache.doris.transaction.TransactionStatus; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -330,6 +331,8 @@ public class ShowExecutor { handleShowColumnStats(); } else if (stmt instanceof ShowTableCreationStmt) { handleShowTableCreation(); + } else if (stmt instanceof ShowLastInsertStmt) { + handleShowLastInsert(); } else { handleEmtpy(); } @@ -2108,4 +2111,17 @@ public class ShowExecutor { resultSet = new ShowResultSet(showMetaData, resultRowSet); } + private void handleShowLastInsert() { + ShowLastInsertStmt showStmt = (ShowLastInsertStmt) stmt; + List> resultRowSet = Lists.newArrayList(); + if (ConnectContext.get() != null) { + InsertResult insertResult = ConnectContext.get().getInsertResult(); + if (insertResult != null) { + resultRowSet.add(insertResult.toRow()); + } + } + ShowResultSetMetaData showMetaData = showStmt.getMetaData(); + resultSet = new ShowResultSet(showMetaData, resultRowSet); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index d5947caac1..702c3db6c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -1375,6 +1375,13 @@ public class StmtExecutor implements ProfileWriter { sb.append("}"); context.getState().setOk(loadedRows, filteredRows, sb.toString()); + + // set insert result in connection context, + // so that user can use `show insert result` to get info of the last insert operation. + context.setOrUpdateInsertResult(txnId, label, insertStmt.getDb(), insertStmt.getTbl(), + txnStatus, loadedRows, filteredRows); + // update it, so that user can get loaded rows in fe.audit.log + context.updateReturnRows((int) loadedRows); } private void handleUnsupportedStmt() {