database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enchancement](statistics) manually inject table level statistics (#19495)

Browse Source 
supports users to manually inject table level statistics.

table stats type:
- row_count

Modify table or partition statistics:
```SQL
ALTER TABLE table_name SET STATS ('k1' = 'v1', ...) 
```

TODO：
- support other table stats type if necessary
- update statistics cache if necessary
This commit is contained in:
ElvinWei
2023-05-12 17:03:12 +08:00
committed by GitHub
parent 26a7f86b66
commit c37d781942
5 changed files with 230 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStatsStmt.java Normal file
View File

@ -0,0 +1,127 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.StatsType;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
/**
* Manually inject statistics for table.
* <p>
* Syntax:
* ALTER TABLE table_name SET STATS ('k1' = 'v1', ...);
* <p>
* e.g.
* ALTER TABLE stats_test.example_tbl SET STATS ('row_count'='6001215');
*/
public class AlterTableStatsStmt extends DdlStmt {
private static final ImmutableSet<StatsType> CONFIGURABLE_PROPERTIES_SET =
new ImmutableSet.Builder<StatsType>()
.add(StatsType.ROW_COUNT)
.build();
private final TableName tableName;
private final Map<String, String> properties;
private final Map<StatsType, String> statsTypeToValue = Maps.newHashMap();
// after analyzed
private long tableId;
public AlterTableStatsStmt(TableName tableName, Map<String, String> properties) {
this.tableName = tableName;
this.properties = properties == null ? Collections.emptyMap() : properties;
}
public TableName getTableName() {
return tableName;
}
@Override
public void analyze(Analyzer analyzer) throws UserException {
if (!Config.enable_stats) {
throw new UserException("Analyze function is forbidden, you should add `enable_stats=true`"
+ "in your FE conf file");
}
super.analyze(analyzer);
tableName.analyze(analyzer);
Optional<StatsType> optional = properties.keySet().stream().map(StatsType::fromString)
.filter(statsType -> !CONFIGURABLE_PROPERTIES_SET.contains(statsType))
.findFirst();
if (optional.isPresent()) {
throw new AnalysisException(optional.get() + " is invalid statistics");
}
if (!Env.getCurrentEnv().getAccessManager()
.checkTblPriv(ConnectContext.get(), tableName.getDb(), tableName.getTbl(), PrivPredicate.ALTER)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "ALTER COLUMN STATS",
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
tableName.getDb() + ": " + tableName.getTbl());
}
properties.forEach((key, value) -> {
StatsType statsType = StatsType.fromString(key);
statsTypeToValue.put(statsType, value);
});
Database db = analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(tableName.getDb());
Table table = db.getTableOrAnalysisException(tableName.getTbl());
tableId = table.getId();
}
public long getTableId() {
return tableId;
}
@Override
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("ALTER TABLE ");
sb.append(tableName.toSql());
sb.append(" SET STATS ");
sb.append("(");
sb.append(new PrintableMap<>(properties,
" = ", true, false));
sb.append(")");
return sb.toString();
}
public String getValue(StatsType statsType) {
return statsTypeToValue.get(statsType);
}
}

3
fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
View File

@ -39,6 +39,7 @@ import org.apache.doris.analysis.AlterResourceStmt;
import org.apache.doris.analysis.AlterRoutineLoadStmt;
import org.apache.doris.analysis.AlterSqlBlockRuleStmt;
import org.apache.doris.analysis.AlterSystemStmt;
import org.apache.doris.analysis.AlterTableStatsStmt;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.AlterUserStmt;
import org.apache.doris.analysis.AlterViewStmt;
@ -151,6 +152,8 @@ public class DdlExecutor {
env.createMaterializedView((CreateMaterializedViewStmt) ddlStmt);
} else if (ddlStmt instanceof AlterTableStmt) {
env.alterTable((AlterTableStmt) ddlStmt);
} else if (ddlStmt instanceof AlterTableStatsStmt) {
StatisticsRepository.alterTableStatistics((AlterTableStatsStmt) ddlStmt);
} else if (ddlStmt instanceof AlterColumnStatsStmt) {
StatisticsRepository.alterColumnStatistics((AlterColumnStatsStmt) ddlStmt);
} else if (ddlStmt instanceof AlterViewStmt) {

25
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AlterColumnStatsStmt;
import org.apache.doris.analysis.AlterTableStatsStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
@ -291,6 +292,30 @@ public class StatisticsRepository {
StatisticsUtil.execUpdate(PERSIST_TABLE_STATS_TEMPLATE, params);
}
public static void alterTableStatistics(AlterTableStatsStmt alterTableStatsStmt) throws Exception {
TableName tableName = alterTableStatsStmt.getTableName();
DBObjects objects = StatisticsUtil.convertTableNameToObjects(tableName);
String rowCount = alterTableStatsStmt.getValue(StatsType.ROW_COUNT);
TableStatisticBuilder builder = new TableStatisticBuilder();
builder.setRowCount(Long.parseLong(rowCount));
builder.setLastAnalyzeTimeInMs(0);
TableStatistic tableStatistic = builder.build();
Map<String, String> params = new HashMap<>();
String id = StatisticsUtil.constructId(objects.table.getId(), -1);
params.put("id", id);
params.put("catalogId", String.valueOf(objects.catalog.getId()));
params.put("dbId", String.valueOf(objects.db.getId()));
params.put("tblId", String.valueOf(objects.table.getId()));
params.put("indexId", "-1");
params.put("partId", "NULL");
params.put("rowCount", String.valueOf(tableStatistic.rowCount));
params.put("lastAnalyzeTimeInMs", "0");
StatisticsUtil.execUpdate(PERSIST_TABLE_STATS_TEMPLATE, params);
// TODO update statistics cache
// Env.getCurrentEnv().getStatisticsCache()
// .updateColStatsCache(objects.table.getId(), -1, builder.build());
}
public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsStmt) throws Exception {
TableName tableName = alterColumnStatsStmt.getTableName();
DBObjects objects = StatisticsUtil.convertTableNameToObjects(tableName);