diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 9ac1027efc..5bdf74f1fe 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1345,6 +1345,9 @@ public class Env { LOG.info(msg); // for master, there are some new thread pools need to register metric ThreadPoolManager.registerAllThreadPoolMetric(); + if (analysisManager != null) { + analysisManager.getStatisticsCache().preHeat(); + } } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 942c19a6ac..75b637890d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -90,9 +90,9 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.statistics.ColumnLevelStatisticCache; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.ColumnStatisticBuilder; +import org.apache.doris.statistics.Histogram; import org.apache.doris.statistics.StatisticRange; import org.apache.doris.statistics.Statistics; import org.apache.doris.statistics.StatisticsBuilder; @@ -177,7 +177,6 @@ public class StatsCalculator extends DefaultPlanVisitor { @Override public Statistics visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) { - olapScan.getExpressions(); return computeScan(olapScan); } @@ -431,17 +430,22 @@ public class StatsCalculator extends DefaultPlanVisitor { for (SlotReference slotReference : slotSet) { String colName = slotReference.getName(); if (colName == null) { - throw new RuntimeException(String.format("Column %s not found", colName)); + throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId())); } - ColumnLevelStatisticCache cache = - Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(), -1, colName); - if (cache == null || cache.columnStatistic == null) { - columnStatisticMap.put(slotReference, ColumnStatistic.UNKNOWN); + ColumnStatistic cache = + Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(), colName); + if (cache == ColumnStatistic.UNKNOWN) { + columnStatisticMap.put(slotReference, cache); continue; } - ColumnStatisticBuilder columnStatisticBuilder = - new ColumnStatisticBuilder(cache.columnStatistic).setHistogram(cache.getHistogram()); - columnStatisticMap.put(slotReference, columnStatisticBuilder.build()); + Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName); + if (histogram != null) { + ColumnStatisticBuilder columnStatisticBuilder = + new ColumnStatisticBuilder(cache).setHistogram(histogram); + columnStatisticMap.put(slotReference, columnStatisticBuilder.build()); + cache = columnStatisticBuilder.build(); + } + columnStatisticMap.put(slotReference, cache); } return new Statistics(rowCount, columnStatisticMap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 00fab11d31..848161bb31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -39,8 +39,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.text.SimpleDateFormat; import java.util.Date; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java index 3e2a7bf1c5..9ab062f541 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java @@ -17,15 +17,12 @@ package org.apache.doris.statistics; -import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; -import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.statistics.AnalysisTaskInfo.JobType; +import org.apache.doris.statistics.util.StatisticsUtil; -import com.google.common.base.Preconditions; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Comparator; import java.util.HashSet; @@ -47,27 +44,28 @@ public class AnalysisTaskScheduler { private final Set manualJobSet = new HashSet<>(); - public synchronized void schedule(AnalysisTaskInfo analysisJobInfo) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(analysisJobInfo.catalogName); - Preconditions.checkArgument(catalog != null); - DatabaseIf db = catalog.getDbNullable(analysisJobInfo.dbName); - Preconditions.checkArgument(db != null); - TableIf table = db.getTableNullable(analysisJobInfo.tblName); - Preconditions.checkArgument(table != null); - BaseAnalysisTask analysisTask = table.createAnalysisTask(this, analysisJobInfo); - addToManualJobQueue(analysisTask); - if (analysisJobInfo.jobType.equals(JobType.MANUAL)) { - return; + public synchronized void schedule(AnalysisTaskInfo analysisTaskInfo) { + try { + TableIf table = StatisticsUtil.findTable(analysisTaskInfo.catalogName, + analysisTaskInfo.dbName, analysisTaskInfo.tblName); + BaseAnalysisTask analysisTask = table.createAnalysisTask(this, analysisTaskInfo); + switch (analysisTaskInfo.jobType) { + case MANUAL: + addToManualJobQueue(analysisTask); + break; + case SYSTEM: + addToSystemQueue(analysisTask); + break; + default: + throw new IllegalArgumentException("Unknown job type: " + analysisTaskInfo.jobType); + } + } catch (Throwable t) { + Env.getCurrentEnv().getAnalysisManager().updateTaskStatus( + analysisTaskInfo, AnalysisState.FAILED, t.getMessage(), System.currentTimeMillis()); } - addToSystemQueue(analysisTask); } - private void removeFromSystemQueue(BaseAnalysisTask analysisJobInfo) { - if (manualJobSet.contains(analysisJobInfo)) { - systemJobQueue.remove(analysisJobInfo); - manualJobSet.remove(analysisJobInfo); - } - } + // Make sure invoker of this method is synchronized on object. private void addToSystemQueue(BaseAnalysisTask analysisJobInfo) { if (systemJobSet.contains(analysisJobInfo)) { @@ -78,6 +76,7 @@ public class AnalysisTaskScheduler { notify(); } + // Make sure invoker of this method is synchronized on object. private void addToManualJobQueue(BaseAnalysisTask analysisJobInfo) { if (manualJobSet.contains(analysisJobInfo)) { return; @@ -90,10 +89,10 @@ public class AnalysisTaskScheduler { public synchronized BaseAnalysisTask getPendingTasks() { while (true) { if (!manualJobQueue.isEmpty()) { - return manualJobQueue.poll(); + return pollAndRemove(manualJobQueue, manualJobSet); } if (!systemJobQueue.isEmpty()) { - return systemJobQueue.poll(); + return pollAndRemove(systemJobQueue, systemJobSet); } try { wait(); @@ -103,4 +102,11 @@ public class AnalysisTaskScheduler { } } } + + // Poll from queue, remove from set. Make sure invoker of this method is synchronized on object. + private BaseAnalysisTask pollAndRemove(Queue q, Set s) { + BaseAnalysisTask t = q.poll(); + s.remove(t); + return t; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index 54793881ab..ab518d66bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -20,6 +20,7 @@ package org.apache.doris.statistics; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.TableIf; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.qe.StmtExecutor; @@ -29,10 +30,18 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashSet; +import java.util.Set; + public abstract class BaseAnalysisTask { public static final Logger LOG = LogManager.getLogger(BaseAnalysisTask.class); + /** + * Stats stored in the column_statistics table basically has two types, `part_id` is null which means it is + * aggregate from partition level stats, `part_id` is not null which means it is partition level stats. + * For latter, it's id field contains part id, for previous doesn't. + */ protected static final String INSERT_PART_STATISTICS = "INSERT INTO " + "${internalDB}.${columnStatTbl}" + " SELECT " @@ -93,6 +102,8 @@ public abstract class BaseAnalysisTask { protected AnalysisState analysisState; + protected Set unsupportedType = new HashSet<>(); + @VisibleForTesting public BaseAnalysisTask() { @@ -104,7 +115,17 @@ public abstract class BaseAnalysisTask { init(info); } + protected void initUnsupportedType() { + unsupportedType.add(PrimitiveType.HLL); + unsupportedType.add(PrimitiveType.BITMAP); + unsupportedType.add(PrimitiveType.ARRAY); + unsupportedType.add(PrimitiveType.MAP); + unsupportedType.add(PrimitiveType.JSONB); + unsupportedType.add(PrimitiveType.STRUCT); + } + private void init(AnalysisTaskInfo info) { + initUnsupportedType(); catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.catalogName); if (catalog == null) { Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(info, AnalysisState.FAILED, @@ -127,9 +148,11 @@ public abstract class BaseAnalysisTask { || info.analysisType.equals(AnalysisType.HISTOGRAM))) { col = tbl.getColumn(info.colName); if (col == null) { - Env.getCurrentEnv().getAnalysisManager().updateTaskStatus( - info, AnalysisState.FAILED, String.format("Column with name %s not exists", info.tblName), - System.currentTimeMillis()); + throw new RuntimeException(String.format("Column with name %s not exists", info.tblName)); + } + if (isUnsupportedType(col.getType().getPrimitiveType())) { + throw new RuntimeException(String.format("Column with type %s is not supported", + col.getType().toString())); } } @@ -165,4 +188,8 @@ public abstract class BaseAnalysisTask { return "COUNT(1) * " + column.getType().getSlotSize(); } + private boolean isUnsupportedType(PrimitiveType type) { + return unsupportedType.contains(type); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnLevelStatisticCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnLevelStatisticCache.java deleted file mode 100644 index a01eb4678e..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnLevelStatisticCache.java +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.statistics; - -public class ColumnLevelStatisticCache { - - public Histogram histogram; - - public ColumnStatistic columnStatistic; - - public ColumnLevelStatisticCache() { - } - - public ColumnLevelStatisticCache(Histogram histogram, ColumnStatistic columnStatistic) { - this.histogram = histogram; - this.columnStatistic = columnStatistic; - } - - public Histogram getHistogram() { - return histogram; - } - - public void setHistogram(Histogram histogram) { - this.histogram = histogram; - } - - public ColumnStatistic getColumnStatistic() { - if (columnStatistic != null) { - return columnStatistic; - } - return ColumnStatistic.UNKNOWN; - } - - public void setColumnStatistic(ColumnStatistic columnStatistic) { - this.columnStatistic = columnStatistic; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java index eeb2b5dcac..5b58f5644a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java @@ -94,6 +94,7 @@ public class ColumnStatistic { public final LiteralExpr minExpr; public final LiteralExpr maxExpr; + // assign value when do stats estimation. public final Histogram histogram; public ColumnStatistic(double count, double ndv, double originalNdv, double avgSizeByte, @@ -154,7 +155,8 @@ public class ColumnStatistic { } columnStatisticBuilder.setSelectivity(1.0); columnStatisticBuilder.setOriginalNdv(ndv); - Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName); + Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName) + .orElse(null); columnStatisticBuilder.setHistogram(histogram); return columnStatisticBuilder.build(); } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java new file mode 100644 index 0000000000..c6ee9103cc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.common.FeConstants; +import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; +import org.apache.doris.statistics.util.StatisticsUtil; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.text.StringSubstitutor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletionException; + +public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader> { + + private static final Logger LOG = LogManager.getLogger(ColumnStatisticsCacheLoader.class); + + private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME + + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE " + + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')"; + + @Override + protected Optional doLoad(StatisticsCacheKey key) { + Map params = new HashMap<>(); + params.put("tblId", String.valueOf(key.tableId)); + params.put("idxId", String.valueOf(key.idxId)); + params.put("colId", String.valueOf(key.colName)); + + List columnStatistics; + List columnResult = + StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) + .replace(QUERY_COLUMN_STATISTICS)); + try { + columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResult); + } catch (Exception e) { + LOG.warn("Failed to deserialize column statistics", e); + throw new CompletionException(e); + } + if (CollectionUtils.isEmpty(columnStatistics)) { + return Optional.empty(); + } else { + return Optional.of(columnStatistics.get(0)); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java new file mode 100644 index 0000000000..0e07524092 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.common.FeConstants; +import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; +import org.apache.doris.statistics.util.StatisticsUtil; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.text.StringSubstitutor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletionException; + +public class HistogramCacheLoader extends StatisticsCacheLoader> { + + private static final Logger LOG = LogManager.getLogger(HistogramCacheLoader.class); + + private static final String QUERY_HISTOGRAM_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME + + "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE " + + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')"; + + @Override + protected Optional doLoad(StatisticsCacheKey key) { + List histogramStatistics; + Map params = new HashMap<>(); + params.put("tblId", String.valueOf(key.tableId)); + params.put("idxId", String.valueOf(key.idxId)); + params.put("colId", String.valueOf(key.colName)); + + List histogramResult = + StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) + .replace(QUERY_HISTOGRAM_STATISTICS)); + try { + histogramStatistics = StatisticsUtil.deserializeToHistogramStatistics(histogramResult); + } catch (Exception e) { + LOG.warn("Failed to deserialize histogram statistics", e); + throw new CompletionException(e); + } + if (!CollectionUtils.isEmpty(histogramStatistics)) { + return Optional.of(histogramStatistics.get(0)); + } + return Optional.empty(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index 5097a86767..f5fcd11ff9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -109,6 +109,6 @@ public class HistogramTask extends BaseAnalysisTask { this.stmtExecutor.execute(); } - Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName()); + Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync(tbl.getId(), -1, col.getName()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java index 200cd8559e..308e46c036 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java @@ -142,7 +142,7 @@ public class HiveAnalysisTask extends HMSAnalysisTask { this.stmtExecutor.execute(); } } - Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName()); + Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName()); } private void getStatData(ColumnStatisticsData data, Map params) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java index c91398a0a6..1194919606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java @@ -124,7 +124,7 @@ public class MVAnalysisTask extends BaseAnalysisTask { params.put("type", column.getType().toString()); StatisticsUtil.execUpdate(ANALYZE_MV_COL, params); Env.getCurrentEnv().getStatisticsCache() - .refreshSync(meta.getIndexId(), meta.getIndexId(), column.getName()); + .refreshColStatsSync(meta.getIndexId(), meta.getIndexId(), column.getName()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index 2509143c75..74f1d3cbad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -94,7 +94,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask { StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE); execSQL(sql); - Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName()); + Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName()); } @VisibleForTesting diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java index 0345c7930e..7feff48e26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java @@ -59,9 +59,8 @@ public class StatisticConstants { */ public static final long STATISTICS_TASKS_TIMEOUT_IN_MS = TimeUnit.MINUTES.toMillis(10); + public static final long PRELOAD_RETRY_TIMES = 5; - public static final int LOAD_TASK_LIMITS = 10; - - public static final double DEFAULT_INNER_JOIN_FACTOR = 0.1; + public static final long PRELOAD_RETRY_INTERVAL_IN_SECONDS = TimeUnit.SECONDS.toMillis(10); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index 9add3c1bc3..6ee30bb90a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -17,18 +17,21 @@ package org.apache.doris.statistics; -//import org.apache.doris.common.ThreadPoolManager; - import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.time.Duration; +import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -44,85 +47,159 @@ public class StatisticsCache { = ThreadPoolManager.newDaemonFixedThreadPool( 10, Integer.MAX_VALUE, "STATS_FETCH", true); - private final StatisticsCacheLoader cacheLoader = new StatisticsCacheLoader(); + private final ColumnStatisticsCacheLoader columnStatisticsCacheLoader = new ColumnStatisticsCacheLoader(); + private final HistogramCacheLoader histogramCacheLoader = new HistogramCacheLoader(); - private final AsyncLoadingCache cache = Caffeine.newBuilder() - .maximumSize(StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE) - .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS)) - .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL)) - .executor(threadPool) - .buildAsync(cacheLoader); + private final AsyncLoadingCache> columnStatisticsCache = + Caffeine.newBuilder() + .maximumSize(StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE) + .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS)) + .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL)) + .executor(threadPool) + .buildAsync(columnStatisticsCacheLoader); + + private final AsyncLoadingCache> histogramCache = + Caffeine.newBuilder() + .maximumSize(StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE) + .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS)) + .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL)) + .executor(threadPool) + .buildAsync(histogramCacheLoader); { threadPool.submit(() -> { while (true) { try { - cacheLoader.removeExpiredInProgressing(); - Thread.sleep(TimeUnit.MINUTES.toMillis(15)); + columnStatisticsCacheLoader.removeExpiredInProgressing(); + histogramCacheLoader.removeExpiredInProgressing(); } catch (Throwable t) { // IGNORE } + Thread.sleep(TimeUnit.MINUTES.toMillis(15)); } }); } public ColumnStatistic getColumnStatistics(long tblId, String colName) { - ColumnLevelStatisticCache columnLevelStatisticCache = getColumnStatistics(tblId, -1, colName); - if (columnLevelStatisticCache == null) { - return ColumnStatistic.UNKNOWN; - } - return columnLevelStatisticCache.columnStatistic; + return getColumnStatistics(tblId, -1, colName).orElse(ColumnStatistic.UNKNOWN); } - public ColumnLevelStatisticCache getColumnStatistics(long tblId, long idxId, String colName) { + public Optional getColumnStatistics(long tblId, long idxId, String colName) { ConnectContext ctx = ConnectContext.get(); if (ctx != null && ctx.getSessionVariable().internalSession) { - return null; + return Optional.empty(); } StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName); try { - CompletableFuture f = cache.get(k); + CompletableFuture> f = columnStatisticsCache.get(k); if (f.isDone() && f.get() != null) { return f.get(); } } catch (Exception e) { LOG.warn("Unexpected exception while returning ColumnStatistic", e); } - return null; + return Optional.empty(); } public Histogram getHistogram(long tblId, String colName) { - return getHistogram(tblId, -1, colName); + return getHistogram(tblId, -1, colName).orElse(null); } - public Histogram getHistogram(long tblId, long idxId, String colName) { + public Optional getHistogram(long tblId, long idxId, String colName) { ConnectContext ctx = ConnectContext.get(); if (ctx != null && ctx.getSessionVariable().internalSession) { - return null; + return Optional.empty(); } StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName); try { - CompletableFuture f = cache.get(k); + CompletableFuture> f = histogramCache.get(k); if (f.isDone() && f.get() != null) { - return f.get().getHistogram(); + return f.get(); } } catch (Exception e) { LOG.warn("Unexpected exception while returning Histogram", e); } - return null; + return Optional.empty(); } // TODO: finish this method. public void eraseExpiredCache(long tblId, long idxId, String colName) { - cache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName)); + columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName)); } - public void updateCache(long tblId, long idxId, String colName, ColumnLevelStatisticCache statistic) { - cache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), statistic); + public void updateColStatsCache(long tblId, long idxId, String colName, ColumnStatistic statistic) { + columnStatisticsCache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), Optional.of(statistic)); } - public void refreshSync(long tblId, long idxId, String colName) { - cache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); + public void refreshColStatsSync(long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); } + + public void refreshHistogramSync(long tblId, long idxId, String colName) { + histogramCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); + } + + public void preHeat() { + threadPool.submit(this::doPreHeat); + } + + private void doPreHeat() { + List recentStatsUpdatedCols = null; + long retryTimes = 0; + while (retryTimes < StatisticConstants.PRELOAD_RETRY_TIMES) { + try { + recentStatsUpdatedCols = StatisticsRepository.fetchRecentStatsUpdatedCol(); + break; + } catch (Throwable t) { + // IGNORE + } + retryTimes++; + try { + Thread.sleep(StatisticConstants.PRELOAD_RETRY_INTERVAL_IN_SECONDS); + } catch (Throwable t) { + // IGNORE + } + } + + if (CollectionUtils.isEmpty(recentStatsUpdatedCols)) { + return; + } + for (ResultRow r : recentStatsUpdatedCols) { + try { + String tblId = r.getColumnValue("tbl_id"); + String idxId = r.getColumnValue("idx_id"); + String colId = r.getColumnValue("col_id"); + final StatisticsCacheKey k = + new StatisticsCacheKey(Long.parseLong(tblId), Long.parseLong(idxId), colId); + final ColumnStatistic c = ColumnStatistic.fromResultRow(r); + CompletableFuture> f = new CompletableFuture>() { + + @Override + public Optional get() throws InterruptedException, ExecutionException { + return Optional.of(c); + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public boolean complete(Optional value) { + return true; + } + + @Override + public Optional join() { + return Optional.of(c); + } + }; + columnStatisticsCache.put(k, f); + } catch (Throwable t) { + LOG.warn("Error when preheating stats cache", t); + } + } + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java index 9b48876bc2..e31b545245 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java @@ -18,6 +18,7 @@ package org.apache.doris.statistics; import java.util.Objects; +import java.util.StringJoiner; public class StatisticsCacheKey { @@ -28,6 +29,8 @@ public class StatisticsCacheKey { public final long idxId; public final String colName; + private static final String DELIMITER = "-"; + public StatisticsCacheKey(long tableId, String colName) { this(tableId, -1, colName); } @@ -54,4 +57,13 @@ public class StatisticsCacheKey { StatisticsCacheKey k = (StatisticsCacheKey) obj; return this.tableId == k.tableId && this.idxId == k.idxId && this.colName.equals(k.colName); } + + @Override + public String toString() { + StringJoiner sj = new StringJoiner(DELIMITER); + sj.add(String.valueOf(tableId)); + sj.add(String.valueOf(idxId)); + sj.add(colName); + return sj.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java index 3417356b78..c212851a28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java @@ -17,90 +17,35 @@ package org.apache.doris.statistics; -import org.apache.doris.common.FeConstants; -import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; -import org.apache.doris.statistics.util.StatisticsUtil; - import com.github.benmanes.caffeine.cache.AsyncCacheLoader; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.NonNull; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; -public class StatisticsCacheLoader implements AsyncCacheLoader { +public abstract class StatisticsCacheLoader implements AsyncCacheLoader { private static final Logger LOG = LogManager.getLogger(StatisticsCacheLoader.class); - private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME - + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE " - + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')"; - - private static final String QUERY_HISTOGRAM_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME - + "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE " - + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')"; - - // TODO: Maybe we should trigger a analyze job when the required ColumnStatistic doesn't exists. - - private final Map - inProgressing = new HashMap<>(); + private final Map> inProgressing = new HashMap<>(); @Override - public @NonNull CompletableFuture asyncLoad(@NonNull StatisticsCacheKey key, + public @NonNull CompletableFuture asyncLoad( + @NonNull StatisticsCacheKey key, @NonNull Executor executor) { - CompletableFutureWithCreateTime cfWrapper = inProgressing.get(key); + CompletableFutureWithCreateTime cfWrapper = inProgressing.get(key); if (cfWrapper != null) { return cfWrapper.cf; } - CompletableFuture future = CompletableFuture.supplyAsync(() -> { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { long startTime = System.currentTimeMillis(); try { - LOG.info("Query BE for column stats:{}-{} start time:{}", key.tableId, key.colName, - startTime); - ColumnLevelStatisticCache statistic = new ColumnLevelStatisticCache(); - Map params = new HashMap<>(); - params.put("tblId", String.valueOf(key.tableId)); - params.put("idxId", String.valueOf(key.idxId)); - params.put("colId", String.valueOf(key.colName)); - - List columnStatistics; - List columnResult = - StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) - .replace(QUERY_COLUMN_STATISTICS)); - try { - columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResult); - } catch (Exception e) { - LOG.warn("Failed to deserialize column statistics", e); - throw new CompletionException(e); - } - if (CollectionUtils.isEmpty(columnStatistics)) { - statistic.setColumnStatistic(ColumnStatistic.UNKNOWN); - } else { - statistic.setColumnStatistic(columnStatistics.get(0)); - } - - List histogramStatistics; - List histogramResult = - StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) - .replace(QUERY_HISTOGRAM_STATISTICS)); - try { - histogramStatistics = StatisticsUtil.deserializeToHistogramStatistics(histogramResult); - } catch (Exception e) { - LOG.warn("Failed to deserialize histogram statistics", e); - throw new CompletionException(e); - } - if (!CollectionUtils.isEmpty(histogramStatistics)) { - statistic.setHistogram(histogramStatistics.get(0)); - } - return statistic; + return doLoad(key); } finally { long endTime = System.currentTimeMillis(); LOG.info("Query BE for column stats:{}-{} end time:{} cost time:{}", key.tableId, key.colName, @@ -108,11 +53,30 @@ public class StatisticsCacheLoader implements AsyncCacheLoader(System.currentTimeMillis(), future)); return future; } - private void putIntoIProgressing(StatisticsCacheKey k, CompletableFutureWithCreateTime v) { + protected abstract V doLoad(StatisticsCacheKey k); + + private static class CompletableFutureWithCreateTime extends CompletableFuture { + + public final long startTime; + public final CompletableFuture cf; + private final long expiredTimeMilli = TimeUnit.MINUTES.toMillis(30); + + public CompletableFutureWithCreateTime(long startTime, CompletableFuture cf) { + this.startTime = startTime; + this.cf = cf; + } + + public boolean isExpired() { + return System.currentTimeMillis() - startTime > expiredTimeMilli; + } + } + + private void putIntoIProgressing(StatisticsCacheKey k, CompletableFutureWithCreateTime v) { synchronized (inProgressing) { inProgressing.put(k, v); } @@ -131,24 +95,4 @@ public class StatisticsCacheLoader implements AsyncCacheLoader e.getValue().isExpired()); } } - - /** - * To make sure any item in the inProgressing would finally be removed to avoid potential mem leak. - */ - private static class CompletableFutureWithCreateTime extends CompletableFuture { - - private static final long EXPIRED_TIME_MILLI = TimeUnit.MINUTES.toMillis(30); - - public final long startTime; - public final CompletableFuture cf; - - public CompletableFutureWithCreateTime(long startTime, CompletableFuture cf) { - this.startTime = startTime; - this.cf = cf; - } - - public boolean isExpired() { - return System.currentTimeMillis() - startTime > EXPIRED_TIME_MILLI; - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index 8a5dab5b26..9a735cf98a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -83,6 +83,13 @@ public class StatisticsRepository { private static final String DROP_TABLE_HISTOGRAM_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE ${condition}"; + private static final String FETCH_RECENT_STATS_UPDATED_COL = + "SELECT * FROM " + + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME + + " WHERE part_id is NULL " + + " ORDER BY update_time DESC LIMIT " + + StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE; + public static ColumnStatistic queryColumnStatisticsByName(long tableId, String colName) { ResultRow resultRow = queryColumnStatisticById(tableId, colName); if (resultRow == null) { @@ -103,8 +110,8 @@ public class StatisticsRepository { partitionIds.add(partition.getId()); } return queryPartitionStatistics(dbObjects.table.getId(), - colName, partitionIds).stream().map(ColumnStatistic::fromResultRow).collect( - Collectors.toList()); + colName, partitionIds).stream().map(ColumnStatistic::fromResultRow).collect( + Collectors.toList()); } public static ResultRow queryColumnStatisticById(long tblId, String colName) { @@ -259,16 +266,8 @@ public class StatisticsRepository { params.put("max", max == null ? "NULL" : max); params.put("dataSize", String.valueOf(columnStatistic.dataSize)); StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, params); - - Histogram histogram = Env.getCurrentEnv().getStatisticsCache() - .getHistogram(objects.table.getId(), -1, colName); - - ColumnLevelStatisticCache statistic = new ColumnLevelStatisticCache(); - statistic.setHistogram(histogram); - statistic.setColumnStatistic(builder.build()); - Env.getCurrentEnv().getStatisticsCache() - .updateCache(objects.table.getId(), -1, colName, statistic); + .updateColStatsCache(objects.table.getId(), -1, colName, builder.build()); } public static void dropTableStatistics(DropTableStatsStmt dropTableStatsStmt) { @@ -279,4 +278,8 @@ public class StatisticsRepository { dropHistogram(dbId, tbIds, cols, partIds); dropStatistics(dbId, tbIds, cols, partIds); } + + public static List fetchRecentStatsUpdatedCol() { + return StatisticsUtil.execStatisticQuery(FETCH_RECENT_STATS_UPDATED_COL); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 2c2de6e3d8..19bcb6f2c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -284,6 +284,23 @@ public class StatisticsUtil { return tblIf.getColumn(columnName); } + @SuppressWarnings({"unchecked"}) + public static Column findColumn(String catalogName, String dbName, String tblName, String columnName) + throws Throwable { + TableIf tableIf = findTable(catalogName, dbName, tblName); + return tableIf.getColumn(columnName); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public static TableIf findTable(String catalogName, String dbName, String tblName) throws Throwable { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrException(catalogName, c -> new RuntimeException("Catalog: " + c + " not exists")); + DatabaseIf db = catalog.getDbOrException(dbName, + d -> new RuntimeException("DB: " + d + " not exists")); + return db.getTableOrException(tblName, + t -> new RuntimeException("Table: " + t + " not exists")); + } + public static boolean isNullOrEmpty(String str) { return Optional.ofNullable(str) .map(String::trim) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java index ea5ce02a96..a904197362 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.JobContext; @@ -53,8 +52,6 @@ public class DeriveStatsJobTest { @Mocked ConnectContext context; - @Mocked - Env env; SlotReference slot1; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TPCHStats.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TPCHStats.java deleted file mode 100644 index 7ce4e53694..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TPCHStats.java +++ /dev/null @@ -1,2094 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.stats; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.nereids.datasets.tpch.TPCHUtils; -import org.apache.doris.planner.PlanNodeId; -import org.apache.doris.statistics.ColumnLevelStatisticCache; -import org.apache.doris.statistics.ColumnStatistic; -import org.apache.doris.statistics.ColumnStatisticBuilder; -import org.apache.doris.statistics.Histogram; -import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; -import org.apache.doris.statistics.util.StatisticsUtil; - -import mockit.Mock; -import mockit.MockUp; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.HashMap; - -// CHECKSTYLE OFF -public class TPCHStats extends TestStats { - - @Test - public void testTPCHStats() throws Exception { - run(); - } - - @Override - protected void initEnv() throws Exception { - createDatabase("tpch"); - connectContext.setDatabase("default_cluster:tpch"); - TPCHUtils.createTables(this); - } - - @Override - protected void initQError() { - avgQError = 385.5745449282787; - } - - @Override - protected void initColNameToType() { - - colType.put("c_custkey", ScalarType.INT); - - colType.put("c_name", ScalarType.VARCHAR); - - colType.put("c_address", ScalarType.VARCHAR); - - colType.put("c_nationkey", ScalarType.INT); - - colType.put("c_phone", ScalarType.VARCHAR); - - colType.put("c_acctbal", ScalarType.createDecimalType(15,2)); - - colType.put("c_mktsegment", ScalarType.VARCHAR); - - colType.put("c_comment", ScalarType.VARCHAR); - - colType.put("l_shipdate", ScalarType.DATE); - - colType.put("l_orderkey", ScalarType.BIGINT); - - colType.put("l_linenumber", ScalarType.INT); - - colType.put("l_partkey", ScalarType.INT); - - colType.put("l_suppkey", ScalarType.INT); - - colType.put("l_quantity", ScalarType.createDecimalType(15,2)); - - colType.put("l_extendedprice", ScalarType.createDecimalType(15,2)); - - colType.put("l_discount", ScalarType.createDecimalType(15,2)); - - colType.put("l_tax", ScalarType.createDecimalType(15,2)); - - colType.put("l_returnflag", ScalarType.VARCHAR); - - colType.put("l_linestatus", ScalarType.VARCHAR); - - colType.put("l_commitdate", ScalarType.DATE); - - colType.put("l_receiptdate", ScalarType.DATE); - - colType.put("l_shipinstruct", ScalarType.VARCHAR); - - colType.put("l_shipmode", ScalarType.VARCHAR); - - colType.put("l_comment", ScalarType.VARCHAR); - - colType.put("n_nationkey", ScalarType.INT); - - colType.put("n_name", ScalarType.VARCHAR); - - colType.put("n_regionkey", ScalarType.INT); - - colType.put("n_comment", ScalarType.VARCHAR); - - colType.put("o_orderkey", ScalarType.BIGINT); - - colType.put("o_orderdate", ScalarType.DATE); - - colType.put("o_custkey", ScalarType.INT); - - colType.put("o_orderstatus", ScalarType.VARCHAR); - - colType.put("o_totalprice", ScalarType.createDecimalType(15,2)); - - colType.put("o_orderpriority", ScalarType.VARCHAR); - - colType.put("o_clerk", ScalarType.VARCHAR); - - colType.put("o_shippriority", ScalarType.INT); - - colType.put("o_comment", ScalarType.VARCHAR); - - colType.put("p_partkey", ScalarType.INT); - - colType.put("p_name", ScalarType.VARCHAR); - - colType.put("p_mfgr", ScalarType.VARCHAR); - - colType.put("p_brand", ScalarType.VARCHAR); - - colType.put("p_type", ScalarType.VARCHAR); - - colType.put("p_size", ScalarType.INT); - - colType.put("p_container", ScalarType.VARCHAR); - - colType.put("p_retailprice", ScalarType.createDecimalType(15,2)); - - colType.put("p_comment", ScalarType.VARCHAR); - - colType.put("ps_partkey", ScalarType.INT); - - colType.put("ps_suppkey", ScalarType.INT); - - colType.put("ps_availqty", ScalarType.INT); - - colType.put("ps_supplycost", ScalarType.createDecimalType(15,2)); - - colType.put("ps_comment", ScalarType.VARCHAR); - - colType.put("r_regionkey", ScalarType.INT); - - colType.put("r_name", ScalarType.VARCHAR); - - colType.put("r_comment", ScalarType.VARCHAR); - - colType.put("s_suppkey", ScalarType.INT); - - colType.put("s_name", ScalarType.VARCHAR); - - colType.put("s_address", ScalarType.VARCHAR); - - colType.put("s_nationkey", ScalarType.INT); - - colType.put("s_phone", ScalarType.VARCHAR); - - colType.put("s_acctbal", ScalarType.createDecimalType(15,2)); - - colType.put("s_comment", ScalarType.VARCHAR); - - } - - @Override - protected void initMockedReturnedRows() { - - queryIdToQError.put(1, 10.666666666666666); - - mockedExactReturnedRows.put(1, new HashMap<>()); - - mockedExactReturnedRows.get(1).put(new PlanNodeId(0), 5916591.0); - - mockedExactReturnedRows.get(1).put(new PlanNodeId(1), 64.0); - - mockedExactReturnedRows.get(1).put(new PlanNodeId(2), 64.0); - - mockedExactReturnedRows.get(1).put(new PlanNodeId(3), 4.0); - - mockedExactReturnedRows.get(1).put(new PlanNodeId(4), 4.0); - - mockedExactReturnedRows.get(1).put(new PlanNodeId(5), 4.0); - - queryIdToQError.put(2, 460.00000000000006); - - mockedExactReturnedRows.put(2, new HashMap<>()); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(0), 747.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(1), 800000.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(2), 2988.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(3), 2988.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(4), 10000.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(5), 2988.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(6), 2988.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(7), 25.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(8), 2988.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(9), 2988.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(10), 1.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(11), 642.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(12), 642.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(13), 642.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(14), 460.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(15), 100.0); - - mockedExactReturnedRows.get(2).put(new PlanNodeId(16), 100.0); - - queryIdToQError.put(3, 16.289786674344537); - - mockedExactReturnedRows.put(3, new HashMap<>()); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(0), 727305.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(1), 3241776.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(2), 151331.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(3), 151331.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(4), 30142.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(5), 30519.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(6), 30519.0); - - mockedExactReturnedRows.get(3).put(new PlanNodeId(9), 10.0); - - queryIdToQError.put(4, 7.612047232987107); - - mockedExactReturnedRows.put(4, new HashMap<>()); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(0), 3793296.0); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(1), 57218.0); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(2), 52523.0); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(3), 52523.0); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(4), 5.0); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(5), 5.0); - - mockedExactReturnedRows.get(4).put(new PlanNodeId(6), 5.0); - - queryIdToQError.put(5, 14.939600829885562); - - mockedExactReturnedRows.put(5, new HashMap<>()); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(0), 150000.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(1), 150000.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(2), 227597.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(3), 6001215.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(4), 910519.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(5), 910519.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(6), 10000.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(7), 910519.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(8), 910519.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(9), 25.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(10), 910519.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(11), 36236.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(12), 36236.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(13), 1.0); - - mockedExactReturnedRows.get(5).put(new PlanNodeId(14), 7243.0); - - queryIdToQError.put(6, 22.704312185172565); - - mockedExactReturnedRows.put(6, new HashMap<>()); - - mockedExactReturnedRows.get(6).put(new PlanNodeId(0), 114160.0); - - mockedExactReturnedRows.get(6).put(new PlanNodeId(1), 8.0); - - mockedExactReturnedRows.get(6).put(new PlanNodeId(2), 8.0); - - mockedExactReturnedRows.get(6).put(new PlanNodeId(3), 1.0); - - queryIdToQError.put(7, 16.98406908487828); - - mockedExactReturnedRows.put(7, new HashMap<>()); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(0), 1500000.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(1), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(2), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(3), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(4), 150000.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(5), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(6), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(7), 10000.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(8), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(9), 1828450.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(10), 2.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(11), 145703.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(12), 145703.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(13), 2.0); - - mockedExactReturnedRows.get(7).put(new PlanNodeId(14), 5924.0); - - queryIdToQError.put(8, 172.3936534440346); - - mockedExactReturnedRows.put(8, new HashMap<>()); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(0), 1.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(1), 1.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(2), 25.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(3), 25.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(4), 25.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(5), 25.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(6), 150000.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(7), 150000.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(8), 457263.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(9), 457263.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(10), 6001215.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(11), 6001215.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(12), 10000.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(13), 80000.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(14), 1451.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(15), 14510000.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(16), 43693.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(17), 13389.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(18), 13389.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(19), 13389.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(20), 13389.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(21), 2603.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(22), 2603.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(23), 2.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(24), 2.0); - - mockedExactReturnedRows.get(8).put(new PlanNodeId(25), 2.0); - - queryIdToQError.put(9, 24.523403087988246); - - mockedExactReturnedRows.put(9, new HashMap<>()); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(0), 800000.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(1), 800000.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(2), 6001215.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(3), 6001215.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(4), 1500000.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(5), 6001215.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(6), 6001215.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(7), 10664.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(8), 319404.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(9), 319404.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(10), 10000.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(11), 319404.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(12), 319404.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(13), 25.0); - - mockedExactReturnedRows.get(9).put(new PlanNodeId(14), 319404.0); - - queryIdToQError.put(10, 17.53057668876071); - - mockedExactReturnedRows.put(10, new HashMap<>()); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(0), 25.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(1), 25.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(2), 1478870.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(3), 1478870.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(4), 57069.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(5), 57069.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(6), 150000.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(7), 57069.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(8), 114705.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(9), 114705.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(10), 37967.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(11), 320.0); - - mockedExactReturnedRows.get(10).put(new PlanNodeId(12), 20.0); - - queryIdToQError.put(11, 1.1582088481002604); - - mockedExactReturnedRows.put(11, new HashMap<>()); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(0), 10000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(1), 10000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(2), 800000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(3), 800000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(4), 800000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(5), 1.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(6), 31680.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(7), 29818.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(8), 29818.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(9), 10000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(10), 10000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(11), 800000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(12), 800000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(13), 800000.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(14), 1.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(15), 31680.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(16), 1.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(17), 1.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(18), 27604.0); - - mockedExactReturnedRows.get(11).put(new PlanNodeId(19), 27604.0); - - queryIdToQError.put(12, 13.846666567293623); - - mockedExactReturnedRows.put(12, new HashMap<>()); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(0), 30988.0); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(1), 1500000.0); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(2), 30988.0); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(3), 30988.0); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(4), 2.0); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(5), 2.0); - - mockedExactReturnedRows.get(12).put(new PlanNodeId(6), 2.0); - - queryIdToQError.put(13, 590.0); - - mockedExactReturnedRows.put(13, new HashMap<>()); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(0), 1483918.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(1), 1483918.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(2), 150000.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(3), 1533923.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(4), 150000.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(5), 590.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(6), 590.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(7), 42.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(8), 42.0); - - mockedExactReturnedRows.get(13).put(new PlanNodeId(9), 42.0); - - queryIdToQError.put(14, 37.62936505020045); - - mockedExactReturnedRows.put(14, new HashMap<>()); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(0), 200000.0); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(1), 200000.0); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(2), 75983.0); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(3), 75983.0); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(4), 8.0); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(5), 8.0); - - mockedExactReturnedRows.get(14).put(new PlanNodeId(6), 1.0); - - queryIdToQError.put(15, 89.86896174525964); - - mockedExactReturnedRows.put(15, new HashMap<>()); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(0), 225954.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(1), 225954.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(2), 10000.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(3), 10000.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(4), 1.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(5), 1.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(6), 1.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(7), 225954.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(8), 225954.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(9), 10000.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(10), 10000.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(11), 10000.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(12), 10000.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(13), 1.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(14), 1.0); - - mockedExactReturnedRows.get(15).put(new PlanNodeId(15), 1.0); - - queryIdToQError.put(16, 1250.0); - - mockedExactReturnedRows.put(16, new HashMap<>()); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(0), 800000.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(1), 29581.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(2), 118324.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(3), 118324.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(4), 4.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(5), 118274.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(6), 118274.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(7), 118250.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(8), 18314.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(9), 18314.0); - - mockedExactReturnedRows.get(16).put(new PlanNodeId(10), 18314.0); - - queryIdToQError.put(17, 1.1230855208874255); - - mockedExactReturnedRows.put(17, new HashMap<>()); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(0), 204.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(1), 204.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(2), 6001215.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(3), 6088.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(4), 6088.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(5), 6088.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(6), 587.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(7), 587.0); - - mockedExactReturnedRows.get(17).put(new PlanNodeId(8), 1.0); - - queryIdToQError.put(18, 1052.8447368421052); - - mockedExactReturnedRows.put(18, new HashMap<>()); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(0), 6001215.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(1), 6001215.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(2), 6001215.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(3), 57.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(4), 1500000.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(5), 57.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(6), 57.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(7), 150000.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(8), 57.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(9), 399.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(10), 57.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(11), 57.0); - - mockedExactReturnedRows.get(18).put(new PlanNodeId(12), 57.0); - - queryIdToQError.put(19, 4.944574565005941); - - mockedExactReturnedRows.put(19, new HashMap<>()); - - mockedExactReturnedRows.get(19).put(new PlanNodeId(0), 485.0); - - mockedExactReturnedRows.get(19).put(new PlanNodeId(1), 485.0); - - mockedExactReturnedRows.get(19).put(new PlanNodeId(2), 128371.0); - - mockedExactReturnedRows.get(19).put(new PlanNodeId(3), 121.0); - - mockedExactReturnedRows.get(19).put(new PlanNodeId(4), 121.0); - - mockedExactReturnedRows.get(19).put(new PlanNodeId(5), 1.0); - - queryIdToQError.put(20, 47.94983607065881); - - mockedExactReturnedRows.put(20, new HashMap<>()); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(0), 1.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(1), 1.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(2), 909455.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(3), 909455.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(4), 543210.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(5), 543210.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(6), 2127.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(7), 800000.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(8), 8508.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(9), 5833.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(10), 5833.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(11), 10000.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(12), 4397.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(13), 186.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(14), 186.0); - - mockedExactReturnedRows.get(20).put(new PlanNodeId(15), 186.0); - - queryIdToQError.put(21, 24.328358395723587); - - mockedExactReturnedRows.put(21, new HashMap<>()); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(0), 3793296.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(1), 3793296.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(2), 1.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(3), 1.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(4), 729413.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(5), 6001215.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(6), 3793296.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(7), 3657708.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(8), 3657708.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(9), 10000.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(10), 3657708.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(11), 3657708.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(12), 1762253.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(14), 73089.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(15), 4141.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(16), 4141.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(17), 411.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(18), 411.0); - - mockedExactReturnedRows.get(21).put(new PlanNodeId(19), 100.0); - - queryIdToQError.put(22, 4604.465736181143); - - mockedExactReturnedRows.put(22, new HashMap<>()); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(0), 1500000.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(1), 1500000.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(2), 42015.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(3), 14086.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(4), 14086.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(5), 38120.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(6), 38120.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(7), 1.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(8), 1.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(9), 6384.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(10), 7.0); - - mockedExactReturnedRows.get(22).put(new PlanNodeId(11), 7.0); - - } - - @Override - protected void initMockedColumnsStats() { - - values = new ArrayList<>(); - values.add("10313--1-l_linenumber"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_linenumber"); - values.add("None"); - values.add("6001215"); - values.add("7"); - values.add("0"); - values.add("1"); - values.add("7"); - values.add("24004860"); - values.add("2023-03-26 22:38:18"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_linenumber", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_shipdate"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_shipdate"); - values.add("None"); - values.add("6001215"); - values.add("2549"); - values.add("0"); - values.add("1992-01-02"); - values.add("1998-12-01"); - values.add("96019440"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_shipdate", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_comment"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_comment"); - values.add("None"); - values.add("1500000"); - values.add("1465415"); - values.add("0"); - values.add(" Tiresias about the blithely ironic a"); - values.add("zzle? furiously ironic instructions among the unusual t"); - values.add("72770808"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10704--1-ps_partkey"); - values.add("0"); - values.add("10310"); - values.add("10704"); - values.add("-1"); - values.add("ps_partkey"); - values.add("None"); - values.add("800000"); - values.add("196099"); - values.add("0"); - values.add("1"); - values.add("200000"); - values.add("3200000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("ps_partkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_name"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_name"); - values.add("None"); - values.add("200000"); - values.add("200265"); - values.add("0"); - values.add("almond antique blue royal burnished"); - values.add("yellow white seashell lavender black"); - values.add("6550221"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_name", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_phone"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_phone"); - values.add("None"); - values.add("150000"); - values.add("152666"); - values.add("0"); - values.add("10-100-106-1617"); - values.add("34-999-618-6881"); - values.add("2250000"); - values.add("2023-03-26 22:38:16"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_phone", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_acctbal"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_acctbal"); - values.add("None"); - values.add("10000"); - values.add("9954"); - values.add("0"); - values.add("-998.220000000"); - values.add("9999.720000000"); - values.add("160000"); - values.add("2023-03-26 22:38:25"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_acctbal", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_linestatus"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_linestatus"); - values.add("None"); - values.add("6001215"); - values.add("2"); - values.add("0"); - values.add("F"); - values.add("O"); - values.add("6001215"); - values.add("2023-03-26 22:38:21"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_linestatus", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_orderkey"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_orderkey"); - values.add("None"); - values.add("6001215"); - values.add("1510112"); - values.add("0"); - values.add("1"); - values.add("6000000"); - values.add("48009720"); - values.add("2023-03-26 22:38:18"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_orderkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_partkey"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_partkey"); - values.add("None"); - values.add("6001215"); - values.add("196099"); - values.add("0"); - values.add("1"); - values.add("200000"); - values.add("24004860"); - values.add("2023-03-26 22:38:18"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_partkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_shipmode"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_shipmode"); - values.add("None"); - values.add("6001215"); - values.add("7"); - values.add("0"); - values.add("AIR"); - values.add("TRUCK"); - values.add("25717034"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_shipmode", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10704--1-ps_supplycost"); - values.add("0"); - values.add("10310"); - values.add("10704"); - values.add("-1"); - values.add("ps_supplycost"); - values.add("None"); - values.add("800000"); - values.add("100274"); - values.add("0"); - values.add("1.000000000"); - values.add("1000.000000000"); - values.add("12800000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("ps_supplycost", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_address"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_address"); - values.add("None"); - values.add("150000"); - values.add("150133"); - values.add("0"); - values.add(" 2uZwVhQvwA"); - values.add("zzxGktzXTMKS1BxZlgQ9nqQ"); - values.add("3758056"); - values.add("2023-03-26 22:38:16"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_address", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_address"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_address"); - values.add("None"); - values.add("10000"); - values.add("9888"); - values.add("0"); - values.add(" 9aW1wwnBJJPnCx,nox0MA48Y0zpI1IeVfYZ"); - values.add("zzfDhdtZcvmVzA8rNFU,Yctj1zBN"); - values.add("249771"); - values.add("2023-03-26 22:38:25"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_address", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_suppkey"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_suppkey"); - values.add("None"); - values.add("10000"); - values.add("10009"); - values.add("0"); - values.add("1"); - values.add("10000"); - values.add("40000"); - values.add("2023-03-26 22:38:25"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_suppkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_returnflag"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_returnflag"); - values.add("None"); - values.add("6001215"); - values.add("3"); - values.add("0"); - values.add("A"); - values.add("R"); - values.add("6001215"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_returnflag", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_shipinstruct"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_shipinstruct"); - values.add("None"); - values.add("6001215"); - values.add("4"); - values.add("0"); - values.add("COLLECT COD"); - values.add("TAKE BACK RETURN"); - values.add("72006409"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_shipinstruct", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_tax"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_tax"); - values.add("None"); - values.add("6001215"); - values.add("9"); - values.add("0"); - values.add("0.000000000"); - values.add("0.080000000"); - values.add("96019440"); - values.add("2023-03-26 22:38:19"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_tax", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_clerk"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_clerk"); - values.add("None"); - values.add("1500000"); - values.add("988"); - values.add("0"); - values.add("Clerk#000000001"); - values.add("Clerk#000001000"); - values.add("22500000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_clerk", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_shippriority"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_shippriority"); - values.add("None"); - values.add("1500000"); - values.add("1"); - values.add("0"); - values.add("0"); - values.add("0"); - values.add("6000000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_shippriority", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_totalprice"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_totalprice"); - values.add("None"); - values.add("1500000"); - values.add("1462416"); - values.add("0"); - values.add("857.710000000"); - values.add("555285.160000000"); - values.add("24000000"); - values.add("2023-03-26 22:38:21"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_totalprice", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10704--1-ps_comment"); - values.add("0"); - values.add("10310"); - values.add("10704"); - values.add("-1"); - values.add("ps_comment"); - values.add("None"); - values.add("800000"); - values.add("794782"); - values.add("0"); - values.add( - " Tiresias according to the quiet courts sleep against the ironic, final requests. carefully unusual requests affix fluffily quickly ironic packages. regular "); - values.add( - "zzle. unusual decoys detect slyly blithely express frays. furiously ironic packages about the bold accounts are close requests. slowly silent reque"); - values.add("98891983"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("ps_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10704--1-ps_suppkey"); - values.add("0"); - values.add("10310"); - values.add("10704"); - values.add("-1"); - values.add("ps_suppkey"); - values.add("None"); - values.add("800000"); - values.add("10009"); - values.add("0"); - values.add("1"); - values.add("10000"); - values.add("3200000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("ps_suppkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_mfgr"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_mfgr"); - values.add("None"); - values.add("200000"); - values.add("5"); - values.add("0"); - values.add("Manufacturer#1"); - values.add("Manufacturer#5"); - values.add("2800000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_mfgr", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_type"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_type"); - values.add("None"); - values.add("200000"); - values.add("150"); - values.add("0"); - values.add("ECONOMY ANODIZED BRASS"); - values.add("STANDARD POLISHED TIN"); - values.add("4119946"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_type", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_nationkey"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_nationkey"); - values.add("None"); - values.add("10000"); - values.add("25"); - values.add("0"); - values.add("0"); - values.add("24"); - values.add("40000"); - values.add("2023-03-26 22:38:24"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_nationkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10885--1-n_nationkey"); - values.add("0"); - values.add("10310"); - values.add("10885"); - values.add("-1"); - values.add("n_nationkey"); - values.add("None"); - values.add("25"); - values.add("25"); - values.add("0"); - values.add("0"); - values.add("24"); - values.add("100"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("n_nationkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10890--1-r_comment"); - values.add("0"); - values.add("10310"); - values.add("10890"); - values.add("-1"); - values.add("r_comment"); - values.add("None"); - values.add("5"); - values.add("5"); - values.add("0"); - values.add("ges. thinly even pinto beans ca"); - values.add( - "uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); - values.add("330"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("r_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_extendedprice"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_extendedprice"); - values.add("None"); - values.add("6001215"); - values.add("929697"); - values.add("0"); - values.add("901.000000000"); - values.add("104949.500000000"); - values.add("96019440"); - values.add("2023-03-26 22:38:21"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_extendedprice", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_receiptdate"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_receiptdate"); - values.add("None"); - values.add("6001215"); - values.add("2576"); - values.add("0"); - values.add("1992-01-04"); - values.add("1998-12-31"); - values.add("96019440"); - values.add("2023-03-26 22:38:18"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_receiptdate", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_orderstatus"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_orderstatus"); - values.add("None"); - values.add("1500000"); - values.add("3"); - values.add("0"); - values.add("F"); - values.add("P"); - values.add("1500000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_orderstatus", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10704--1-ps_availqty"); - values.add("0"); - values.add("10310"); - values.add("10704"); - values.add("-1"); - values.add("ps_availqty"); - values.add("None"); - values.add("800000"); - values.add("10008"); - values.add("0"); - values.add("1"); - values.add("9999"); - values.add("3200000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("ps_availqty", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_comment"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_comment"); - values.add("None"); - values.add("150000"); - values.add("147491"); - values.add("0"); - values.add( - " Tiresias according to the slyly blithe instructions detect quickly at the slyly express courts. express dinos wake "); - values.add("zzle. blithely regular instructions cajol"); - values.add("10876099"); - values.add("2023-03-26 22:38:15"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_mktsegment"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_mktsegment"); - values.add("None"); - values.add("150000"); - values.add("5"); - values.add("0"); - values.add("AUTOMOBILE"); - values.add("MACHINERY"); - values.add("1349610"); - values.add("2023-03-26 22:38:16"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_mktsegment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10890--1-r_name"); - values.add("0"); - values.add("10310"); - values.add("10890"); - values.add("-1"); - values.add("r_name"); - values.add("None"); - values.add("5"); - values.add("5"); - values.add("0"); - values.add("AFRICA"); - values.add("MIDDLE EAST"); - values.add("34"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("r_name", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_discount"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_discount"); - values.add("None"); - values.add("6001215"); - values.add("11"); - values.add("0"); - values.add("0.000000000"); - values.add("0.100000000"); - values.add("96019440"); - values.add("2023-03-26 22:38:19"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_discount", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_quantity"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_quantity"); - values.add("None"); - values.add("6001215"); - values.add("50"); - values.add("0"); - values.add("1.000000000"); - values.add("50.000000000"); - values.add("96019440"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_quantity", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_orderpriority"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_orderpriority"); - values.add("None"); - values.add("1500000"); - values.add("5"); - values.add("0"); - values.add("1-URGENT"); - values.add("5-LOW"); - values.add("12599829"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_orderpriority", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_brand"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_brand"); - values.add("None"); - values.add("200000"); - values.add("25"); - values.add("0"); - values.add("Brand#11"); - values.add("Brand#55"); - values.add("1600000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_brand", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_size"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_size"); - values.add("None"); - values.add("200000"); - values.add("50"); - values.add("0"); - values.add("1"); - values.add("50"); - values.add("800000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_size", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10890--1-r_regionkey"); - values.add("0"); - values.add("10310"); - values.add("10890"); - values.add("-1"); - values.add("r_regionkey"); - values.add("None"); - values.add("5"); - values.add("5"); - values.add("0"); - values.add("0"); - values.add("4"); - values.add("20"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("r_regionkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_comment"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_comment"); - values.add("None"); - values.add("6001215"); - values.add("4619207"); - values.add("0"); - values.add(" Tiresias "); - values.add("zzle? slyly final platelets sleep quickly. "); - values.add("158997209"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_commitdate"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_commitdate"); - values.add("None"); - values.add("6001215"); - values.add("2485"); - values.add("0"); - values.add("1992-01-31"); - values.add("1998-10-31"); - values.add("96019440"); - values.add("2023-03-26 22:38:18"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_commitdate", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10313--1-l_suppkey"); - values.add("0"); - values.add("10310"); - values.add("10313"); - values.add("-1"); - values.add("l_suppkey"); - values.add("None"); - values.add("6001215"); - values.add("10009"); - values.add("0"); - values.add("1"); - values.add("10000"); - values.add("24004860"); - values.add("2023-03-26 22:38:19"); - - resultRow = new ResultRow(cols, types, values); - stats.put("l_suppkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_custkey"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_custkey"); - values.add("None"); - values.add("1500000"); - values.add("99149"); - values.add("0"); - values.add("1"); - values.add("149999"); - values.add("6000000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_custkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_orderdate"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_orderdate"); - values.add("None"); - values.add("1500000"); - values.add("2428"); - values.add("0"); - values.add("1992-01-01"); - values.add("1998-08-02"); - values.add("24000000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_orderdate", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10509--1-o_orderkey"); - values.add("0"); - values.add("10310"); - values.add("10509"); - values.add("-1"); - values.add("o_orderkey"); - values.add("None"); - values.add("1500000"); - values.add("1510112"); - values.add("0"); - values.add("1"); - values.add("6000000"); - values.add("12000000"); - values.add("2023-03-26 22:38:21"); - - resultRow = new ResultRow(cols, types, values); - stats.put("o_orderkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_comment"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_comment"); - values.add("None"); - values.add("200000"); - values.add("133106"); - values.add("0"); - values.add(" Tire"); - values.add("zzle. quickly si"); - values.add("2702438"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_partkey"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_partkey"); - values.add("None"); - values.add("200000"); - values.add("196099"); - values.add("0"); - values.add("1"); - values.add("200000"); - values.add("800000"); - values.add("2023-03-26 22:38:23"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_partkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_retailprice"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_retailprice"); - values.add("None"); - values.add("200000"); - values.add("21096"); - values.add("0"); - values.add("901.000000000"); - values.add("2098.990000000"); - values.add("3200000"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_retailprice", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_name"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_name"); - values.add("None"); - values.add("150000"); - values.add("150431"); - values.add("0"); - values.add("Customer#000000001"); - values.add("Customer#000150000"); - values.add("2700000"); - values.add("2023-03-26 22:38:15"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_name", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10885--1-n_comment"); - values.add("0"); - values.add("10310"); - values.add("10885"); - values.add("-1"); - values.add("n_comment"); - values.add("None"); - values.add("25"); - values.add("25"); - values.add("0"); - values.add(" haggle. carefully final deposits detect slyly agai"); - values.add( - "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be"); - values.add("1857"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("n_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10885--1-n_name"); - values.add("0"); - values.add("10310"); - values.add("10885"); - values.add("-1"); - values.add("n_name"); - values.add("None"); - values.add("25"); - values.add("25"); - values.add("0"); - values.add("ALGERIA"); - values.add("VIETNAM"); - values.add("177"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("n_name", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10885--1-n_regionkey"); - values.add("0"); - values.add("10310"); - values.add("10885"); - values.add("-1"); - values.add("n_regionkey"); - values.add("None"); - values.add("25"); - values.add("5"); - values.add("0"); - values.add("0"); - values.add("4"); - values.add("100"); - values.add("2023-03-26 22:38:20"); - - resultRow = new ResultRow(cols, types, values); - stats.put("n_regionkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10756--1-p_container"); - values.add("0"); - values.add("10310"); - values.add("10756"); - values.add("-1"); - values.add("p_container"); - values.add("None"); - values.add("200000"); - values.add("40"); - values.add("0"); - values.add("JUMBO BAG"); - values.add("WRAP PKG"); - values.add("1514971"); - values.add("2023-03-26 22:38:22"); - - resultRow = new ResultRow(cols, types, values); - stats.put("p_container", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_acctbal"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_acctbal"); - values.add("None"); - values.add("150000"); - values.add("142496"); - values.add("0"); - values.add("-999.990000000"); - values.add("9999.990000000"); - values.add("2400000"); - values.add("2023-03-26 22:38:16"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_acctbal", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_custkey"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_custkey"); - values.add("None"); - values.add("150000"); - values.add("149087"); - values.add("0"); - values.add("1"); - values.add("150000"); - values.add("600000"); - values.add("2023-03-26 22:38:16"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_custkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10807--1-c_nationkey"); - values.add("0"); - values.add("10310"); - values.add("10807"); - values.add("-1"); - values.add("c_nationkey"); - values.add("None"); - values.add("150000"); - values.add("25"); - values.add("0"); - values.add("0"); - values.add("24"); - values.add("600000"); - values.add("2023-03-26 22:38:16"); - - resultRow = new ResultRow(cols, types, values); - stats.put("c_nationkey", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_comment"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_comment"); - values.add("None"); - values.add("10000"); - values.add("10039"); - values.add("0"); - values.add(" about the blithely express foxes. bli"); - values.add("zzle furiously. bold accounts haggle furiously ironic excuses. fur"); - values.add("625695"); - values.add("2023-03-26 22:38:24"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_comment", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_name"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_name"); - values.add("None"); - values.add("10000"); - values.add("10002"); - values.add("0"); - values.add("Supplier#000000001"); - values.add("Supplier#000010000"); - values.add("180000"); - values.add("2023-03-26 22:38:24"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_name", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - values = new ArrayList<>(); - values.add("10858--1-s_phone"); - values.add("0"); - values.add("10310"); - values.add("10858"); - values.add("-1"); - values.add("s_phone"); - values.add("None"); - values.add("10000"); - values.add("10021"); - values.add("0"); - values.add("10-102-116-6785"); - values.add("34-998-900-4911"); - values.add("150000"); - values.add("2023-03-26 22:38:24"); - - resultRow = new ResultRow(cols, types, values); - stats.put("s_phone", - new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow))); - - } - - public void t2est() { - new MockUp() { - - @Mock - public ColumnStatistic fromResultRow(ResultRow resultRow) { - try { - ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(); - double count = Double.parseDouble(resultRow.getColumnValue("count")); - columnStatisticBuilder.setCount(count); - double ndv = Double.parseDouble(resultRow.getColumnValue("ndv")); - if (0.99 * count < ndv && ndv < 1.01 * count) { - ndv = count; - } - columnStatisticBuilder.setNdv(ndv); - columnStatisticBuilder.setNumNulls(Double.parseDouble(resultRow.getColumnValue("null_count"))); - columnStatisticBuilder.setDataSize(Double - .parseDouble(resultRow.getColumnValue("data_size_in_bytes"))); - columnStatisticBuilder.setAvgSizeByte(columnStatisticBuilder.getDataSize() - / columnStatisticBuilder.getCount()); - long catalogId = Long.parseLong(resultRow.getColumnValue("catalog_id")); - long idxId = Long.parseLong(resultRow.getColumnValue("idx_id")); - long dbID = Long.parseLong(resultRow.getColumnValue("db_id")); - long tblId = Long.parseLong(resultRow.getColumnValue("tbl_id")); - String colName = resultRow.getColumnValue("col_id"); - Column col = new Column(colName, colType.get(colName)); - String min = resultRow.getColumnValue("min"); - String max = resultRow.getColumnValue("max"); - columnStatisticBuilder.setMinValue(StatisticsUtil.convertToDouble(col.getType(), min)); - columnStatisticBuilder.setMaxValue(StatisticsUtil.convertToDouble(col.getType(), max)); - columnStatisticBuilder.setMaxExpr(StatisticsUtil.readableValue(col.getType(), max)); - columnStatisticBuilder.setMinExpr(StatisticsUtil.readableValue(col.getType(), min)); - columnStatisticBuilder.setSelectivity(1.0); - Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName); - columnStatisticBuilder.setHistogram(histogram); - return columnStatisticBuilder.build(); - } catch (Exception e) { - return ColumnStatistic.UNKNOWN; - } - } - }; - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TestStats.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TestStats.java deleted file mode 100644 index 9058d89b3e..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TestStats.java +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.stats; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.Type; -import org.apache.doris.nereids.NereidsPlanner; -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.datasets.tpch.TPCHUtils; -import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.planner.PlanNodeId; -import org.apache.doris.qe.OriginStatement; -import org.apache.doris.statistics.ColumnLevelStatisticCache; -import org.apache.doris.statistics.StatisticsCache; -import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; -import org.apache.doris.statistics.util.StatisticsUtil; -import org.apache.doris.utframe.TestWithFeService; - -import mockit.Mock; -import mockit.MockUp; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -// Assume that all column name is unique in the tested database -// CHECKSTYLE OFF -public abstract class TestStats extends TestWithFeService { - - protected Map stats = new HashMap<>(); - - protected List cols = new ArrayList() {{ - add("id"); - add("catalog_id"); - add("db_id"); - add("tbl_id"); - add("idx_id"); - add("col_id"); - add("part_id"); - add("count"); - add("ndv"); - add("null_count"); - add("min"); - add("max"); - add("data_size_in_bytes"); - add("update_time"); - }}; - - protected List types = new ArrayList() {{ - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.BIGINT); - add(PrimitiveType.BIGINT); - add(PrimitiveType.BIGINT); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.VARCHAR); - add(PrimitiveType.BIGINT); - add(PrimitiveType.DATETIME); - }}; - - protected List values = new ArrayList<>(); - - protected ResultRow resultRow = null; - - protected final static Map colType = new HashMap<>(); - - protected abstract void initMockedColumnsStats(); - - protected abstract void initQError(); - - - protected abstract void initMockedReturnedRows(); - - protected abstract void initEnv() throws Exception; - - protected abstract void initColNameToType(); - - protected Map> mockedExactReturnedRows = new HashMap<>(); - protected Map queryIdToQError = new HashMap<>(); - - protected double avgQError; - - - public void run() throws Exception { - new MockUp() { - - @Mock - public Column findColumn(long catalogId, long dbId, long tblId, long idxId, String columnName) { - return new Column(columnName, colType.get(columnName)); - } - }; - initMockedReturnedRows(); - initColNameToType(); - initMockedColumnsStats(); - new MockUp() { - @Mock - public ColumnLevelStatisticCache getColumnStatistics(long tblId, long idxId, String colName) { - return stats.get(colName); - } - }; - - connectContext.getSessionVariable().setEnableNereidsPlanner(true); - connectContext.getSessionVariable().enableFallbackToOriginalPlanner = false; - StatsErrorEstimator statsErrorEstimator = new StatsErrorEstimator(); - connectContext.setStatsErrorEstimator(statsErrorEstimator); - List qErrorList = new ArrayList<>(); - initEnv(); - for (int i = 0; i < TPCHUtils.SQLS.size(); i++) { - String sql = TPCHUtils.SQLS.get(i); - int sqlNumber = i + 1; - NereidsPlanner nereidsPlanner = new NereidsPlanner( - new StatementContext(connectContext, new OriginStatement(sql, 0))); - NereidsParser nereidsParser = new NereidsParser(); - nereidsPlanner.plan(nereidsParser.parseSQL(sql).get(0)); - Map extractReturnedRows = mockedExactReturnedRows.get(sqlNumber); - for (Entry entry : extractReturnedRows.entrySet()) { - // statsErrorEstimator.setExactReturnedRow(entry.getKey(), entry.getValue()); - } - qErrorList.add(statsErrorEstimator.calculateQError()); - statsErrorEstimator = new StatsErrorEstimator(); - connectContext.setStatsErrorEstimator(statsErrorEstimator); - } - // Assert.assertTrue( - // qErrorList.stream() - // .mapToDouble(Double::doubleValue).average().orElseGet(() -> Double.POSITIVE_INFINITY) - // <= avgQError + 1); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java index a0cf4524b4..5a92bbdfbe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java @@ -19,10 +19,16 @@ package org.apache.doris.statistics; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Type; import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import mockit.Expectations; import mockit.Mock; import mockit.MockUp; @@ -40,7 +46,7 @@ import java.util.concurrent.Executor; public class CacheTest extends TestWithFeService { @Test - public void testColumn(@Mocked StatisticsCacheLoader cacheLoader) throws Exception { + public void testColumn(@Mocked ColumnStatisticsCacheLoader cacheLoader) throws Exception { new Expectations() { { cacheLoader.asyncLoad((StatisticsCacheKey) any, (Executor) any); @@ -130,6 +136,46 @@ public class CacheTest extends TestWithFeService { @Test public void testLoadHistogram() throws Exception { + new MockUp() { + + @Mock + public Histogram fromResultRow(ResultRow resultRow) { + try { + HistogramBuilder histogramBuilder = new HistogramBuilder(); + + Column col = new Column("abc", PrimitiveType.DATETIME); + + Type dataType = col.getType(); + histogramBuilder.setDataType(dataType); + + double sampleRate = Double.parseDouble(resultRow.getColumnValue("sample_rate")); + histogramBuilder.setSampleRate(sampleRate); + + String json = resultRow.getColumnValue("buckets"); + JsonObject jsonObj = JsonParser.parseString(json).getAsJsonObject(); + + int bucketNum = jsonObj.get("num_buckets").getAsInt(); + histogramBuilder.setNumBuckets(bucketNum); + + List buckets = Lists.newArrayList(); + JsonArray jsonArray = jsonObj.getAsJsonArray("buckets"); + for (JsonElement element : jsonArray) { + try { + String bucketJson = element.toString(); + buckets.add(Bucket.deserializeFromJson(dataType, bucketJson)); + } catch (Throwable t) { + t.printStackTrace(); + } + + } + histogramBuilder.setBuckets(buckets); + + return histogramBuilder.build(); + } catch (Exception e) { + return null; + } + } + }; new MockUp() { @Mock @@ -185,7 +231,9 @@ public class CacheTest extends TestWithFeService { }; StatisticsCache statisticsCache = new StatisticsCache(); + statisticsCache.refreshHistogramSync(0, -1, "col"); + Thread.sleep(10000); Histogram histogram = statisticsCache.getHistogram(0, "col"); - Assertions.assertEquals(null, histogram); + Assertions.assertNotNull(histogram); } }