database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](statistics) add statistics module related syntax (#12766)

Browse Source 
This pull request includes some implementations of the statistics(#6370), it adds statistics module related syntax. The current syntax for collecting statistics will not collect statistics (It will collect statistics until test is stable). 

- `ANALYZE` syntax(collect statistics)

```SQL
ANALYZE [[ db_name.tb_name ] [( column_name [, ...] )], ...] [PARTITIONS(...)] [ PROPERTIES(...) ]
```
> db_name.tb_name: collect table and column statistics from tb_name.
> column_name: collect column statistics from column_name.
> properties: properties of statistics jobs.

example：
```SQL
ANALYZE;  -- collect statistics for all tables in the current database
ANALYZE table1(pv, citycode);  -- collect pv and citycode statistics for table1
ANALYZE test.table2 PARTITIONS(partition1); -- collect statistics for partition1 of table2
```

- `SHOW ANALYZE` syntax(show statistics job info)

```SQL
SHOW ANALYZE
    [TABLE | ID]
    [
        WHERE
        [STATE = ["PENDING"|"SCHEDULING"|"RUNNING"|"FINISHED"|"FAILED"|"CANCELLED"]]
    ]
    [ORDER BY ...]
    [LIMIT limit][OFFSET offset];
```

- `SHOW TABLE STATS`syntax(show table statistics)

```SQL
SHOW TABLE STATS [ db_name.tb_name ]  
```

- `SHOW COLUMN STATS` syntax(show column statistics)

```SQL
SHOW COLUMN STATS [ db_name.tb_name ] 
```
This commit is contained in:
ElvinWei
2022-09-22 11:15:00 +08:00
committed by GitHub
parent 3fa820ec50
commit 340784e294
4 changed files with 64 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -279,7 +279,8 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_A
KW_YEAR,
KW_NOT_NULL,
KW_CATALOG, KW_CATALOGS,
KW_SWITCH;
KW_SWITCH,
KW_ANALYZE;
terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
terminal BITAND, BITOR, BITXOR, BITNOT;
@ -304,7 +305,7 @@ nonterminal StatementBase stmt, show_stmt, show_param, help_stmt, load_stmt,
use_stmt, kill_stmt, drop_stmt, recover_stmt, grant_stmt, revoke_stmt, create_stmt, set_stmt, sync_stmt, cancel_stmt, cancel_param, delete_stmt,
link_stmt, migrate_stmt, switch_stmt, enter_stmt, transaction_stmt, unsupported_stmt, export_stmt, admin_stmt, truncate_stmt,
import_columns_stmt, import_delete_on_stmt, import_sequence_stmt, import_where_stmt, install_plugin_stmt, uninstall_plugin_stmt,
import_preceding_filter_stmt, unlock_tables_stmt, lock_tables_stmt, refresh_stmt, clean_stmt;
import_preceding_filter_stmt, unlock_tables_stmt, lock_tables_stmt, refresh_stmt, clean_stmt, analyze_stmt;
nonterminal String transaction_label;
nonterminal ImportColumnDesc import_column_desc;
@ -764,6 +765,8 @@ stmt ::=
{: RESULT = stmt; :}
| clean_stmt:stmt
{: RESULT = stmt; :}
| analyze_stmt:stmt
{: RESULT = stmt; :}
| /* empty: query only has comments */
{:
RESULT = new EmptyStmt();
@ -2079,6 +2082,14 @@ show_create_routine_load_stmt ::=
:}
;
// analyze statment
analyze_stmt ::=
KW_ANALYZE opt_table_name:tbl opt_col_list:cols opt_partition_names:partitionNames opt_properties:properties
{:
RESULT = new AnalyzeStmt(tbl, cols, partitionNames, properties);
:}
;
// Grant statement
grant_stmt ::=
KW_GRANT privilege_list:privs KW_ON tbl_pattern:tblPattern KW_TO user_identity:userId
@ -3093,14 +3104,14 @@ show_param ::=
RESULT = new ShowSyncJobStmt(dbName);
:}
/* show table stats */
| KW_TABLE KW_STATS opt_table_name:tbl
| KW_TABLE KW_STATS opt_table_name:tbl opt_partition_names:partitionNames
{:
RESULT = new ShowTableStatsStmt(tbl);
RESULT = new ShowTableStatsStmt(tbl, partitionNames);
:}
/* show column stats */
| KW_COLUMN KW_STATS table_name:tbl
| KW_COLUMN KW_STATS table_name:tbl opt_partition_names:partitionNames
{:
RESULT = new ShowColumnStatsStmt(tbl);
RESULT = new ShowColumnStatsStmt(tbl, partitionNames);
:}
/* show table creation statement */
| KW_TABLE KW_CREATION opt_db:db opt_wild_where
@ -3116,6 +3127,15 @@ show_param ::=
{:
RESULT = new ShowCreateMaterializedViewStmt(mvName, tableName);
:}
/* show analyze job */
| KW_ANALYZE integer_list:jobIds
{:
RESULT = new ShowAnalyzeStmt(jobIds);
:}
| KW_ANALYZE opt_table_name:tbl opt_wild_where order_by_clause:orderByClause limit_clause:limitClause
{:
RESULT = new ShowAnalyzeStmt(tbl, parser.where, orderByClause, limitClause);
:}
;
opt_tmp ::=

27
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
View File

@ -19,15 +19,14 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.qe.ShowResultSetMetaData;
import org.apache.doris.statistics.ColumnStats;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.Collections;
import java.util.List;
public class ShowColumnStatsStmt extends ShowStmt {
@ -43,22 +42,35 @@ public class ShowColumnStatsStmt extends ShowStmt {
.add(ColumnStats.MAX_VALUE.getValue())
.build();
private TableName tableName;
private final TableName tableName;
private final PartitionNames partitionNames;
public ShowColumnStatsStmt(TableName tableName) {
public ShowColumnStatsStmt(TableName tableName, PartitionNames partitionNames) {
this.tableName = tableName;
this.partitionNames = partitionNames;
}
public TableName getTableName() {
return tableName;
}
public List<String> getPartitionNames() {
if (partitionNames == null) {
return Collections.emptyList();
}
return partitionNames.getPartitionNames();
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
tableName.analyze(analyzer);
// disallow external catalog
Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName());
if (partitionNames != null) {
partitionNames.analyze(analyzer);
}
}
@Override
@ -70,9 +82,4 @@ public class ShowColumnStatsStmt extends ShowStmt {
}
return builder.build();
}
public List<String> getPartitionNames() {
// TODO(WZT): partition statistics
return Lists.newArrayList();
}
}

30
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
View File

@ -26,11 +26,11 @@ import org.apache.doris.common.util.Util;
import org.apache.doris.qe.ShowResultSetMetaData;
import org.apache.doris.statistics.TableStats;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.parquet.Preconditions;
import org.apache.parquet.Strings;
import java.util.Collections;
import java.util.List;
public class ShowTableStatsStmt extends ShowStmt {
@ -42,14 +42,17 @@ public class ShowTableStatsStmt extends ShowStmt {
.add(TableStats.DATA_SIZE.getValue())
.build();
private TableName tableName;
private final TableName tableName;
// after analyzed
// There is only on attribute for both @tableName and @dbName at the same time.
private String dbName;
public ShowTableStatsStmt(TableName tableName) {
private final PartitionNames partitionNames;
public ShowTableStatsStmt(TableName tableName, PartitionNames partitionNames) {
this.tableName = tableName;
this.partitionNames = partitionNames;
}
public String getTableName() {
@ -68,6 +71,13 @@ public class ShowTableStatsStmt extends ShowStmt {
return tableName.getDb();
}
public List<String> getPartitionNames() {
if (partitionNames == null) {
return Collections.emptyList();
}
return partitionNames.getPartitionNames();
}
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
@ -79,6 +89,11 @@ public class ShowTableStatsStmt extends ShowStmt {
return;
}
tableName.analyze(analyzer);
if (partitionNames != null) {
partitionNames.analyze(analyzer);
}
// disallow external catalog
Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName());
}
@ -92,9 +107,4 @@ public class ShowTableStatsStmt extends ShowStmt {
}
return builder.build();
}
public List<String> getPartitionNames() {
// TODO(WZT): partition statistics
return Lists.newArrayList();
}
}

1
fe/fe-core/src/main/jflex/sql_scanner.flex
View File

@ -100,6 +100,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("all", new Integer(SqlParserSymbols.KW_ALL));
keywordMap.put("alter", new Integer(SqlParserSymbols.KW_ALTER));
keywordMap.put("and", new Integer(SqlParserSymbols.KW_AND));
keywordMap.put("analyze", new Integer(SqlParserSymbols.KW_ANALYZE));
keywordMap.put("anti", new Integer(SqlParserSymbols.KW_ANTI));
keywordMap.put("append", new Integer(SqlParserSymbols.KW_APPEND));
keywordMap.put("array", new Integer(SqlParserSymbols.KW_ARRAY));