database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature] add SHOW TABLET STORAGE FORMAT stmt (#9037)

Browse Source 
use this stmt to show tablets storage format in be, if verbose is set,
    will show detail message of tablet storage format.
    e.g.
    ```
    MySQL [(none)]> admin show tablet storage format;
    +-----------+---------+---------+
    | BackendId | V1Count | V2Count |
    +-----------+---------+---------+
    | 10002     | 0       | 2867    |
    +-----------+---------+---------+
    1 row in set (0.003 sec)
    MySQL [test_query_qa]> admin show tablet storage format verbose;
    +-----------+----------+---------------+
    | BackendId | TabletId | StorageFormat |
    +-----------+----------+---------------+
    | 10002     | 39227    | V2            |
    | 10002     | 39221    | V2            |
    | 10002     | 39215    | V2            |
    | 10002     | 39199    | V2            |
    +-----------+----------+---------------+
    4 rows in set (0.034 sec)
    ```
    add storage format infomation to show full table statment.
    ```
    MySQL [test_query_qa]> show full tables;
    +-------------------------+------------+---------------+
    | Tables_in_test_query_qa | Table_type | StorageFormat |
    +-------------------------+------------+---------------+
    | bigtable                | BASE TABLE | V2            |
    | test_dup                | BASE TABLE | V2            |
    | test                    | BASE TABLE | V2            |
    | baseall                 | BASE TABLE | V2            |
    | test_string             | BASE TABLE | V2            |
    +-------------------------+------------+---------------+
    5 rows in set (0.002 sec)
    ```
This commit is contained in:
Zhengguo Yang
2022-04-27 10:53:43 +08:00
committed by GitHub
parent c1ae1a0fa2
commit 597115c305
17 changed files with 381 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -5332,6 +5332,14 @@ admin_stmt ::=
{:
RESULT = new AdminDiagnoseTabletStmt(tabletId);
:}
| KW_ADMIN KW_TABLET KW_STORAGE KW_FORMAT
{:
RESULT = new AdminShowTabletStorageFormatStmt(false);
:}
| KW_ADMIN KW_TABLET KW_STORAGE KW_FORMAT KW_VERBOSE
{:
RESULT = new AdminShowTabletStorageFormatStmt(true);
:}
;
truncate_stmt ::=

83
fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowTabletStorageFormatStmt.java Normal file
View File

@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
public class AdminShowTabletStorageFormatStmt extends ShowStmt {
private boolean verbose;
@Override
public boolean isVerbose() {
return verbose;
}
public AdminShowTabletStorageFormatStmt(boolean verbose) {
this.verbose = verbose;
}
@Override
public void analyze(Analyzer analyzer) throws UserException {
// check access first
if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_ACCESS_DENIED_ERROR,
toSql(),
ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(), "ADMIN Privilege needed.");
}
super.analyze(analyzer);
}
@Override
public String toSql() {
StringBuilder sb = new StringBuilder("ADMIN SHOW TABLET STORAGE TYPE");
if (verbose) {
sb.append(" VERBOSE");
}
return sb.toString();
}
@Override
public String toString() {
return toSql();
}
@Override
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
if (verbose) {
builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30)))
.addColumn(new Column("TabletId", ScalarType.createVarchar(30)))
.addColumn(new Column("StorageFormat", ScalarType.createVarchar(30)));
} else {
builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30)))
.addColumn(new Column("V1Count", ScalarType.createVarchar(30)))
.addColumn(new Column("V2Count", ScalarType.createVarchar(30)));
}
return builder.build();
}
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java
View File

@ -37,6 +37,7 @@ public class ShowTableStmt extends ShowStmt {
private static final Logger LOG = LogManager.getLogger(ShowTableStmt.class);
private static final String NAME_COL_PREFIX = "Tables_in_";
private static final String TYPE_COL = "Table_type";
private static final String STORAGE_FORMAT_COL = "StorageFormat";
private static final TableName TABLE_NAME = new TableName(InfoSchemaDb.DATABASE_NAME, "tables");
private String db;
private boolean isVerbose;
@ -146,6 +147,7 @@ public class ShowTableStmt extends ShowStmt {
new Column(NAME_COL_PREFIX + ClusterNamespace.getNameFromFullName(db), ScalarType.createVarchar(20)));
if (isVerbose) {
builder.addColumn(new Column(TYPE_COL, ScalarType.createVarchar(20)));
builder.addColumn(new Column(STORAGE_FORMAT_COL, ScalarType.createVarchar(20)));
}
return builder.build();
}

57
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.AdminDiagnoseTabletStmt;
import org.apache.doris.analysis.AdminShowConfigStmt;
import org.apache.doris.analysis.AdminShowReplicaDistributionStmt;
import org.apache.doris.analysis.AdminShowReplicaStatusStmt;
import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt;
import org.apache.doris.analysis.DescribeStmt;
import org.apache.doris.analysis.HelpStmt;
import org.apache.doris.analysis.PartitionNames;
@ -158,6 +159,8 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.system.Backend;
import org.apache.doris.system.Diagnoser;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.task.AgentClient;
import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TUnit;
import org.apache.doris.transaction.GlobalTransactionMgr;
import org.apache.doris.transaction.TransactionStatus;
@ -165,7 +168,6 @@ import org.apache.doris.transaction.TransactionStatus;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.tuple.Triple;
@ -183,7 +185,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@ -335,6 +336,8 @@ public class ShowExecutor {
handleShowTableCreation();
} else if (stmt instanceof ShowLastInsertStmt) {
handleShowLastInsert();
} else if (stmt instanceof AdminShowTabletStorageFormatStmt) {
handleAdminShowTabletStorageFormat();
} else if (stmt instanceof AdminDiagnoseTabletStmt) {
handleAdminDiagnoseTablet();
} else {
@ -659,7 +662,6 @@ public class ShowExecutor {
ShowTableStmt showTableStmt = (ShowTableStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
Database db = ctx.getCatalog().getDbOrAnalysisException(showTableStmt.getDb());
Map<String, String> tableMap = Maps.newTreeMap();
PatternMatcher matcher = null;
if (showTableStmt.getPattern() != null) {
matcher = PatternMatcher.createMysqlPattern(showTableStmt.getPattern(),
@ -675,14 +677,14 @@ public class ShowExecutor {
PrivPredicate.SHOW)) {
continue;
}
tableMap.put(tbl.getName(), tbl.getMysqlType());
}
for (Map.Entry<String, String> entry : tableMap.entrySet()) {
if (showTableStmt.isVerbose()) {
rows.add(Lists.newArrayList(entry.getKey(), entry.getValue()));
String storageFormat = "NONE";
if (tbl instanceof OlapTable) {
storageFormat = ((OlapTable) tbl).getStorageFormat().toString();
}
rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(), storageFormat));
} else {
rows.add(Lists.newArrayList(entry.getKey()));
rows.add(Lists.newArrayList(tbl.getName()));
}
}
resultSet = new ShowResultSet(showTableStmt.getMetaData(), rows);
@ -2132,6 +2134,43 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showMetaData, resultRowSet);
}
private void handleAdminShowTabletStorageFormat() throws AnalysisException {
List<List<String>> resultRowSet = Lists.newArrayList();
for (Backend be : Catalog.getCurrentSystemInfo().getIdToBackend().values()) {
if (be.isQueryAvailable() && be.isLoadAvailable()) {
AgentClient client = new AgentClient(be.getHost(), be.getBePort());
TCheckStorageFormatResult result = client.checkStorageFormat();
if (result == null) {
throw new AnalysisException("get tablet data from backend: " + be.getId() + "error.");
}
if (stmt.isVerbose()) {
for (long tabletId : result.getV1Tablets()) {
List<String> row = new ArrayList<>();
row.add(String.valueOf(be.getId()));
row.add(String.valueOf(tabletId));
row.add("V1");
resultRowSet.add(row);
}
for (long tabletId : result.getV2Tablets()) {
List<String> row = new ArrayList<>();
row.add(String.valueOf(be.getId()));
row.add(String.valueOf(tabletId));
row.add("V2");
resultRowSet.add(row);
}
} else {
List<String> row = new ArrayList<>();
row.add(String.valueOf(be.getId()));
row.add(String.valueOf(result.getV1Tablets().size()));
row.add(String.valueOf(result.getV2Tablets().size()));
resultRowSet.add(row);
}
}
}
ShowResultSetMetaData showMetaData = stmt.getMetaData();
resultSet = new ShowResultSet(showMetaData, resultRowSet);
}
private void handleAdminDiagnoseTablet() {
AdminDiagnoseTabletStmt showStmt = (AdminDiagnoseTabletStmt) stmt;
List<List<String>> resultRowSet = Diagnoser.diagnoseTablet(showStmt.getTabletId());

17
fe/fe-core/src/main/java/org/apache/doris/task/AgentClient.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.thrift.BackendService;
import org.apache.doris.thrift.TAgentResult;
import org.apache.doris.thrift.TAgentServiceVersion;
import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TMiniLoadEtlStatusRequest;
import org.apache.doris.thrift.TMiniLoadEtlStatusResult;
import org.apache.doris.thrift.TMiniLoadEtlTaskRequest;
@ -167,6 +168,21 @@ public class AgentClient {
return result;
}
public TCheckStorageFormatResult checkStorageFormat() {
TCheckStorageFormatResult result = null;
LOG.debug("submit make snapshot task.");
try {
borrowClient();
result = client.checkStorageFormat();
ok = true;
} catch (Exception e) {
LOG.warn("checkStorageFormat error", e);
} finally {
returnClient();
}
return result;
}
public void deleteEtlFiles(long dbId, long jobId, String dbName, String label) {
TDeleteEtlFilesRequest request = new TDeleteEtlFilesRequest(TAgentServiceVersion.V1,
new TUniqueId(dbId, jobId), dbName, label);
@ -197,4 +213,5 @@ public class AgentClient {
ClientPool.backendPool.invalidateObject(address, client);
}
}
}

6
fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java
View File

@ -56,7 +56,7 @@ public class ShowTableStmtTest {
stmt = new ShowTableStmt("abc", true, null);
stmt.analyze(analyzer);
Assert.assertEquals("SHOW FULL TABLES FROM testCluster:abc", stmt.toString());
Assert.assertEquals(2, stmt.getMetaData().getColumnCount());
Assert.assertEquals(3, stmt.getMetaData().getColumnCount());
Assert.assertEquals("Tables_in_abc", stmt.getMetaData().getColumn(0).getName());
Assert.assertEquals("Table_type", stmt.getMetaData().getColumn(1).getName());
@ -64,7 +64,7 @@ public class ShowTableStmtTest {
stmt.analyze(analyzer);
Assert.assertEquals("bcd", stmt.getPattern());
Assert.assertEquals("SHOW FULL TABLES FROM testCluster:abc LIKE 'bcd'", stmt.toString());
Assert.assertEquals(2, stmt.getMetaData().getColumnCount());
Assert.assertEquals(3, stmt.getMetaData().getColumnCount());
Assert.assertEquals("Tables_in_abc", stmt.getMetaData().getColumn(0).getName());
Assert.assertEquals("Table_type", stmt.getMetaData().getColumn(1).getName());
}
@ -75,4 +75,4 @@ public class ShowTableStmtTest {
stmt.analyze(AccessTestUtil.fetchEmptyDbAnalyzer());
Assert.fail("No exception throws");
}
}
}

6
fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.thrift.TAgentResult;
import org.apache.doris.thrift.TAgentTaskRequest;
import org.apache.doris.thrift.TCancelPlanFragmentParams;
import org.apache.doris.thrift.TCancelPlanFragmentResult;
import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TDeleteEtlFilesRequest;
import org.apache.doris.thrift.TDiskTrashInfo;
import org.apache.doris.thrift.TExecPlanFragmentParams;
@ -242,6 +243,11 @@ public class GenericPoolTest {
public void cleanTrash() throws TException {
// TODO Auto-generated method stub
}
@Override
public TCheckStorageFormatResult checkStorageFormat() throws TException {
return new TCheckStorageFormatResult();
}
}
@Test

6
fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
View File

@ -32,6 +32,7 @@ import org.apache.doris.thrift.TBackend;
import org.apache.doris.thrift.TBackendInfo;
import org.apache.doris.thrift.TCancelPlanFragmentParams;
import org.apache.doris.thrift.TCancelPlanFragmentResult;
import org.apache.doris.thrift.TCheckStorageFormatResult;
import org.apache.doris.thrift.TCloneReq;
import org.apache.doris.thrift.TDeleteEtlFilesRequest;
import org.apache.doris.thrift.TDiskTrashInfo;
@ -319,6 +320,11 @@ public class MockedBackendFactory {
public void cleanTrash() throws TException {
return;
}
@Override
public TCheckStorageFormatResult checkStorageFormat() throws TException {
return new TCheckStorageFormatResult();
}
}
// The default Brpc service.