diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 2385863a40..df013b7cc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -127,6 +127,8 @@ public interface TableIf { BaseAnalysisTask createAnalysisTask(AnalysisTaskScheduler scheduler, AnalysisTaskInfo info); + long estimatedRowCount(); + /** * Doris table type. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java index 8f111cf067..8d30eda46a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java @@ -305,6 +305,11 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { throw new NotImplementedException(); } + @Override + public long estimatedRowCount() { + return 1; + } + @Override public void write(DataOutput out) throws IOException { String json = GsonUtils.GSON.toJson(this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index b5db5df42f..29cff71cf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -194,6 +194,7 @@ public class HMSExternalTable extends ExternalTable { @Override public boolean isView() { + makeSureInitialized(); return remoteTable.isSetViewOriginalText() || remoteTable.isSetViewExpandedText(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java index b37b3ca2cf..51fb7076f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.properties.DistributionSpecReplicated; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; @@ -115,6 +116,12 @@ public class CostCalculator { costEstimate.getNetworkCost(), costEstimate.getPenalty()); } + @Override + public CostEstimate visitPhysicalFileScan(PhysicalFileScan physicalFileScan, PlanContext context) { + StatsDeriveResult statistics = context.getStatisticsWithCheck(); + return CostEstimate.ofCpu(statistics.getRowCount()); + } + @Override public CostEstimate visitPhysicalProject(PhysicalProject physicalProject, PlanContext context) { return CostEstimate.ofCpu(1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 7adedfcc1f..6ee4db75d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -34,6 +34,8 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.common.Pair; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.properties.DistributionSpecAny; @@ -65,6 +67,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; @@ -112,6 +115,7 @@ import org.apache.doris.planner.SetOperationNode; import org.apache.doris.planner.SortNode; import org.apache.doris.planner.TableFunctionNode; import org.apache.doris.planner.UnionNode; +import org.apache.doris.planner.external.ExternalFileScanNode; import org.apache.doris.tablefunction.TableValuedFunctionIf; import org.apache.doris.thrift.TPartitionType; import org.apache.doris.thrift.TPushAggOp; @@ -514,6 +518,28 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slotList = fileScan.getOutput(); + ExternalTable table = fileScan.getTable(); + TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); + tupleDescriptor.setTable(table); + ExternalFileScanNode fileScanNode = new ExternalFileScanNode(context.nextPlanNodeId(), tupleDescriptor); + TableName tableName = new TableName(null, "", ""); + TableRef ref = new TableRef(tableName, null, null); + BaseTableRef tableRef = new BaseTableRef(ref, table, tableName); + tupleDescriptor.setRef(tableRef); + + Utils.execWithUncheckedException(fileScanNode::init); + context.addScanNode(fileScanNode); + Utils.execWithUncheckedException(fileScanNode::finalizeForNerieds); + // Create PlanFragment + DataPartition dataPartition = DataPartition.RANDOM; + PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), fileScanNode, dataPartition); + context.addPlanFragment(planFragment); + return planFragment; + } + @Override public PlanFragment visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, PlanTranslatorContext context) { List slots = tvfRelation.getLogicalProperties().getOutput(); @@ -1406,7 +1432,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slotList, Table table, PlanTranslatorContext context) { + private TupleDescriptor generateTupleDesc(List slotList, TableIf table, PlanTranslatorContext context) { TupleDescriptor tupleDescriptor = context.generateTupleDesc(); tupleDescriptor.setTable(table); for (Slot slot : slotList) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 099c8f5ab1..ccdd299b9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFun import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; @@ -233,6 +234,11 @@ public class ChildOutputPropertyDeriver extends PlanVisitor db = null; + try { + db = (DatabaseIf) catalog.getDb(dbName) + .orElseThrow(() -> new RuntimeException("Database [" + dbName + "] does not exist.")); + } catch (Throwable e) { + throw new RuntimeException(e); + } + return db.getTable(tableName).orElseThrow(() -> new RuntimeException( + "Table [" + tableName + "] does not exist in database [" + dbName + "].")); + } + private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRelation unboundRelation) { String tableName = unboundRelation.getNameParts().get(0); // check if it is a CTE's name @@ -83,52 +109,64 @@ public class BindRelation extends OneAnalysisRuleFactory { } return new LogicalSubQueryAlias<>(tableName, ctePlan); } - + String catalogName = cascadesContext.getConnectContext().getCurrentCatalog().getName(); String dbName = cascadesContext.getConnectContext().getDatabase(); - Table table = cascadesContext.getTable(dbName, tableName, cascadesContext.getConnectContext().getEnv()); + TableIf table = getTable(catalogName, dbName, tableName, cascadesContext.getConnectContext().getEnv()); // TODO: should generate different Scan sub class according to table's type - List partIds = getPartitionIds(table, unboundRelation); - if (table.getType() == TableType.OLAP) { - if (!CollectionUtils.isEmpty(partIds)) { - return new LogicalOlapScan(RelationUtil.newRelationId(), - (OlapTable) table, ImmutableList.of(dbName), partIds); - } else { - return new LogicalOlapScan(RelationUtil.newRelationId(), - (OlapTable) table, ImmutableList.of(dbName)); - } - } else if (table.getType() == TableType.VIEW) { - Plan viewPlan = parseAndAnalyzeView(table.getDdlSql(), cascadesContext); - return new LogicalSubQueryAlias<>(table.getName(), viewPlan); - } else if (table.getType() == TableType.SCHEMA) { - return new LogicalSchemaScan(RelationUtil.newRelationId(), table, ImmutableList.of(dbName)); - } - throw new AnalysisException("Unsupported tableType:" + table.getType()); + return getLogicalPlan(table, unboundRelation, dbName, cascadesContext); } private LogicalPlan bindWithDbNameFromNamePart(CascadesContext cascadesContext, UnboundRelation unboundRelation) { List nameParts = unboundRelation.getNameParts(); ConnectContext connectContext = cascadesContext.getConnectContext(); + String catalogName = cascadesContext.getConnectContext().getCurrentCatalog().getName(); // if the relation is view, nameParts.get(0) is dbName. String dbName = nameParts.get(0); if (!dbName.equals(connectContext.getDatabase())) { dbName = connectContext.getClusterName() + ":" + dbName; } - Table table = cascadesContext.getTable(dbName, nameParts.get(1), connectContext.getEnv()); - List partIds = getPartitionIds(table, unboundRelation); - if (table.getType() == TableType.OLAP) { - if (!CollectionUtils.isEmpty(partIds)) { - return new LogicalOlapScan(RelationUtil.newRelationId(), (OlapTable) table, - ImmutableList.of(dbName), partIds); - } else { - return new LogicalOlapScan(RelationUtil.newRelationId(), (OlapTable) table, ImmutableList.of(dbName)); - } - } else if (table.getType() == TableType.VIEW) { - Plan viewPlan = parseAndAnalyzeView(table.getDdlSql(), cascadesContext); - return new LogicalSubQueryAlias<>(table.getName(), viewPlan); - } else if (table.getType() == TableType.SCHEMA) { - return new LogicalSchemaScan(RelationUtil.newRelationId(), table, ImmutableList.of(dbName)); + String tableName = nameParts.get(1); + TableIf table = getTable(catalogName, dbName, tableName, connectContext.getEnv()); + return getLogicalPlan(table, unboundRelation, dbName, cascadesContext); + } + + private LogicalPlan bindWithCatalogNameFromNamePart(CascadesContext cascadesContext, + UnboundRelation unboundRelation) { + List nameParts = unboundRelation.getNameParts(); + ConnectContext connectContext = cascadesContext.getConnectContext(); + String catalogName = nameParts.get(0); + String dbName = nameParts.get(1); + if (!dbName.equals(connectContext.getDatabase())) { + dbName = connectContext.getClusterName() + ":" + dbName; + } + String tableName = nameParts.get(2); + TableIf table = getTable(catalogName, dbName, tableName, connectContext.getEnv()); + return getLogicalPlan(table, unboundRelation, dbName, cascadesContext); + } + + private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, String dbName, + CascadesContext cascadesContext) { + switch (table.getType()) { + case OLAP: + List partIds = getPartitionIds(table, unboundRelation); + if (!CollectionUtils.isEmpty(partIds)) { + return new LogicalOlapScan(RelationUtil.newRelationId(), + (OlapTable) table, ImmutableList.of(dbName), partIds); + } else { + return new LogicalOlapScan(RelationUtil.newRelationId(), + (OlapTable) table, ImmutableList.of(dbName)); + } + case VIEW: + Plan viewPlan = parseAndAnalyzeView(((View) table).getDdlSql(), cascadesContext); + return new LogicalSubQueryAlias<>(table.getName(), viewPlan); + case HMS_EXTERNAL_TABLE: + return new LogicalFileScan(cascadesContext.getStatementContext().getNextRelationId(), + (HMSExternalTable) table, ImmutableList.of(dbName)); + case SCHEMA: + return new LogicalSchemaScan(RelationUtil.newRelationId(), (Table) table, ImmutableList.of(dbName)); + default: + throw new AnalysisException("Unsupported tableType:" + table.getType()); } - throw new AnalysisException("Unsupported tableType:" + table.getType()); } private Plan parseAndAnalyzeView(String viewSql, CascadesContext parentContext) { @@ -141,12 +179,12 @@ public class BindRelation extends OneAnalysisRuleFactory { return viewContext.getMemo().copyOut(false); } - private List getPartitionIds(Table t, UnboundRelation unboundRelation) { + private List getPartitionIds(TableIf t, UnboundRelation unboundRelation) { List parts = unboundRelation.getPartNames(); if (CollectionUtils.isEmpty(parts)) { return Collections.emptyList(); } - if (!t.getType().equals(TableType.OLAP)) { + if (!t.getType().equals(TableIf.TableType.OLAP)) { throw new IllegalStateException(String.format( "Only OLAP table is support select by partition for now," + "Table: %s is not OLAP table", t.getName())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java new file mode 100644 index 0000000000..e277c6b7b1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.properties.DistributionSpecAny; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; + +import java.util.Optional; + +/** + * Implementation rule that convert logical FileScan to physical FileScan. + */ +public class LogicalFileScanToPhysicalFileScan extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalFileScan().then(fileScan -> + new PhysicalFileScan( + fileScan.getId(), + fileScan.getTable(), + fileScan.getQualifier(), + DistributionSpecAny.INSTANCE, + Optional.empty(), + fileScan.getLogicalProperties()) + ).toRule(RuleType.LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 79b8b35d47..e464fd45ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.stats; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; @@ -42,6 +42,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; @@ -60,6 +61,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; @@ -171,6 +173,12 @@ public class StatsCalculator extends DefaultPlanVisitor return new StatsDeriveResult(1); } + @Override + public StatsDeriveResult visitLogicalFileScan(LogicalFileScan fileScan, Void context) { + fileScan.getExpressions(); + return computeScan(fileScan); + } + @Override public StatsDeriveResult visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) { return tvfRelation.getFunction().computeStats(tvfRelation.getOutput()); @@ -256,6 +264,11 @@ public class StatsCalculator extends DefaultPlanVisitor return new StatsDeriveResult(1); } + @Override + public StatsDeriveResult visitPhysicalFileScan(PhysicalFileScan fileScan, Void context) { + return computeScan(fileScan); + } + @Override public StatsDeriveResult visitPhysicalStorageLayerAggregate( PhysicalStorageLayerAggregate storageLayerAggregate, Void context) { @@ -359,7 +372,7 @@ public class StatsCalculator extends DefaultPlanVisitor Set slotSet = scan.getOutput().stream().filter(SlotReference.class::isInstance) .map(s -> (SlotReference) s).collect(Collectors.toSet()); Map columnStatisticMap = new HashMap<>(); - Table table = scan.getTable(); + TableIf table = scan.getTable(); double rowCount = scan.getTable().estimatedRowCount(); for (SlotReference slotReference : slotSet) { String colName = slotReference.getName(); @@ -370,7 +383,6 @@ public class StatsCalculator extends DefaultPlanVisitor Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(), colName); if (!colStats.isUnKnown) { rowCount = colStats.count; - } columnStatisticMap.put(slotReference.getExprId(), colStats); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 75933232db..ab355837fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -43,6 +43,7 @@ public enum PlanType { LOGICAL_LIMIT, LOGICAL_OLAP_SCAN, LOGICAL_SCHEMA_SCAN, + LOGICAL_FILE_SCAN, LOGICAL_APPLY, LOGICAL_SELECT_HINT, LOGICAL_ASSERT_NUM_ROWS, @@ -59,6 +60,7 @@ public enum PlanType { PHYSICAL_EMPTY_RELATION, PHYSICAL_ONE_ROW_RELATION, PHYSICAL_OLAP_SCAN, + PHYSICAL_FILE_SCAN, PHYSICAL_TVF_RELATION, PHYSICAL_SCHEMA_SCAN, PHYSICAL_PROJECT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java index c6a0b454b3..c221391e3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java @@ -17,12 +17,12 @@ package org.apache.doris.nereids.trees.plans.algebra; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.analyzer.Relation; /** * Common interface for logical/physical scan. */ public interface Scan extends Relation { - Table getTable(); + TableIf getTable(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java new file mode 100644 index 0000000000..43152b9fd6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.catalog.external.ExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +/** + * Logical file scan for external catalog. + */ +public class LogicalFileScan extends LogicalRelation { + + /** + * Constructor for LogicalFileScan. + */ + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + Optional groupExpression, + Optional logicalProperties) { + super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, + groupExpression, logicalProperties); + } + + public LogicalFileScan(RelationId id, ExternalTable table) { + this(id, table, ImmutableList.of()); + } + + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier) { + this(id, table, qualifier, Optional.empty(), Optional.empty()); + } + + @Override + public ExternalTable getTable() { + Preconditions.checkArgument(table instanceof ExternalTable); + return (ExternalTable) table; + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalFileScan", + "qualified", qualifiedName(), + "output", getOutput() + ); + } + + @Override + public LogicalFileScan withGroupExpression(Optional groupExpression) { + return new LogicalFileScan(id, (ExternalTable) table, qualifier, groupExpression, + Optional.of(getLogicalProperties())); + } + + @Override + public LogicalFileScan withLogicalProperties(Optional logicalProperties) { + return new LogicalFileScan(id, (ExternalTable) table, qualifier, groupExpression, + logicalProperties); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalFileScan(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 2d6e40b943..b11b65d0cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.RelationId; @@ -196,33 +195,33 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, } @Override - public Plan withGroupExpression(Optional groupExpression) { - return new LogicalOlapScan(id, table, qualifier, groupExpression, Optional.of(getLogicalProperties()), + public LogicalOlapScan withGroupExpression(Optional groupExpression) { + return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); } @Override public LogicalOlapScan withLogicalProperties(Optional logicalProperties) { - return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties, + return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { - return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, tabletPruned, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { - return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, indexId, true, preAgg, manuallySpecifiedPartitions); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { - return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, true, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java index f1c88adfad..f959ecca42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -42,12 +42,12 @@ import java.util.Optional; */ public abstract class LogicalRelation extends LogicalLeaf implements Scan { - protected final Table table; + protected final TableIf table; protected final ImmutableList qualifier; protected final RelationId id; - public LogicalRelation(RelationId id, PlanType type, Table table, List qualifier) { + public LogicalRelation(RelationId id, PlanType type, TableIf table, List qualifier) { this(id, type, table, qualifier, Optional.empty(), Optional.empty()); } @@ -63,7 +63,7 @@ public abstract class LogicalRelation extends LogicalLeaf implements Scan { * @param table Doris table * @param qualifier qualified relation name */ - public LogicalRelation(RelationId id, PlanType type, Table table, List qualifier, + public LogicalRelation(RelationId id, PlanType type, TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties) { super(type, groupExpression, logicalProperties); this.table = Objects.requireNonNull(table, "table can not be null"); @@ -72,7 +72,7 @@ public abstract class LogicalRelation extends LogicalLeaf implements Scan { } @Override - public Table getTable() { + public TableIf getTable() { return table; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java index 3641e36319..402da0da48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.SchemaTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; @@ -40,13 +40,13 @@ import java.util.Optional; */ public class LogicalSchemaScan extends LogicalRelation implements Scan { public LogicalSchemaScan(RelationId id, - Table table, + TableIf table, List qualifier) { super(id, PlanType.LOGICAL_SCHEMA_SCAN, table, qualifier); } public LogicalSchemaScan(RelationId id, - Table table, + TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties) { super(id, PlanType.LOGICAL_SCHEMA_SCAN, table, qualifier, groupExpression, logicalProperties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java new file mode 100644 index 0000000000..49091d229c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.external.ExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.DistributionSpec; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.StatsDeriveResult; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Physical file scan for external catalog. + */ +public class PhysicalFileScan extends PhysicalRelation { + + private final ExternalTable table; + private final DistributionSpec distributionSpec; + + /** + * Constructor for PhysicalFileScan. + */ + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties) { + super(id, PlanType.PHYSICAL_FILE_SCAN, qualifier, groupExpression, logicalProperties); + this.table = table; + this.distributionSpec = distributionSpec; + } + + /** + * Constructor for PhysicalFileScan. + */ + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + StatsDeriveResult statsDeriveResult) { + super(id, PlanType.PHYSICAL_FILE_SCAN, qualifier, groupExpression, logicalProperties, + physicalProperties, statsDeriveResult); + this.table = table; + this.distributionSpec = distributionSpec; + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalFileScan", + "qualified", Utils.qualifiedName(qualifier, table.getName()), + "output", getOutput(), + "stats", statsDeriveResult + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + PhysicalFileScan that = ((PhysicalFileScan) o); + return Objects.equals(table, that.table); + } + + @Override + public int hashCode() { + return Objects.hash(id, table); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalFileScan(this, context); + } + + @Override + public PhysicalFileScan withGroupExpression(Optional groupExpression) { + return new PhysicalFileScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties()); + } + + @Override + public PhysicalFileScan withLogicalProperties(Optional logicalProperties) { + return new PhysicalFileScan(id, table, qualifier, distributionSpec, groupExpression, logicalProperties.get()); + } + + @Override + public ExternalTable getTable() { + return table; + } + + @Override + public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, + StatsDeriveResult statsDeriveResult) { + return new PhysicalFileScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties(), + physicalProperties, statsDeriveResult); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 08eebb3e36..461266e440 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -69,7 +69,7 @@ public class PhysicalStorageLayerAggregate extends PhysicalRelation { @Override public Table getTable() { - return relation.getTable(); + return (Table) relation.getTable(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index 2af0bd3a3c..af464b0fb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; @@ -57,6 +58,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; @@ -168,6 +170,10 @@ public abstract class PlanVisitor { return visitLogicalRelation(schemaScan, context); } + public R visitLogicalFileScan(LogicalFileScan fileScan, C context) { + return visitLogicalRelation(fileScan, context); + } + public R visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, C context) { return visitLogicalRelation(tvfRelation, context); } @@ -261,6 +267,10 @@ public abstract class PlanVisitor { return visitPhysicalScan(schemaScan, context); } + public R visitPhysicalFileScan(PhysicalFileScan fileScan, C context) { + return visitPhysicalScan(fileScan, context); + } + public R visitPhysicalStorageLayerAggregate(PhysicalStorageLayerAggregate storageLayerAggregate, C context) { return storageLayerAggregate.getRelation().accept(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java index 4117f20539..37d7fd58f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java @@ -192,6 +192,43 @@ public class ExternalFileScanNode extends ExternalScanNode { initParamCreateContexts(analyzer); } + /** + * Init ExternalFileScanNode, ONLY used for Nereids. Should NOT use this function in anywhere else. + */ + public void init() throws UserException { + if (!Config.enable_vectorized_load) { + throw new UserException( + "Please set 'enable_vectorized_load=true' in fe.conf to enable external file scan node"); + } + + switch (type) { + case QUERY: + // prepare for partition prune + // computeColumnFilter(); + if (this.desc.getTable() instanceof HMSExternalTable) { + HMSExternalTable hmsTable = (HMSExternalTable) this.desc.getTable(); + initHMSExternalTable(hmsTable); + } else if (this.desc.getTable() instanceof FunctionGenTable) { + FunctionGenTable table = (FunctionGenTable) this.desc.getTable(); + initFunctionGenTable(table, (ExternalFileTableValuedFunction) table.getTvf()); + } + break; + default: + throw new UserException("Unknown type: " + type); + } + + backendPolicy.init(); + numNodes = backendPolicy.numBackends(); + for (FileScanProviderIf scanProvider : scanProviders) { + ParamCreateContext context = scanProvider.createContext(analyzer); + context.createDestSlotMap(); + initAndSetPrecedingFilter(context.fileGroup.getPrecedingFilterExpr(), context.srcTupleDescriptor, analyzer); + initAndSetWhereExpr(context.fileGroup.getWhereExpr(), context.destTupleDescriptor, analyzer); + context.conjuncts = conjuncts; + this.contexts.add(context); + } + } + private void initHMSExternalTable(HMSExternalTable hmsTable) throws UserException { Preconditions.checkNotNull(hmsTable); @@ -312,6 +349,25 @@ public class ExternalFileScanNode extends ExternalScanNode { } } + public void finalizeForNerieds() throws UserException { + Preconditions.checkState(contexts.size() == scanProviders.size(), + contexts.size() + " vs. " + scanProviders.size()); + for (int i = 0; i < contexts.size(); ++i) { + ParamCreateContext context = contexts.get(i); + FileScanProviderIf scanProvider = scanProviders.get(i); + setDefaultValueExprs(scanProvider, context); + setColumnPositionMappingForTextFile(scanProvider, context); + finalizeParamsForLoad(context, analyzer); + createScanRangeLocations(context, scanProvider); + this.inputSplitsNum += scanProvider.getInputSplitNum(); + this.totalFileSize += scanProvider.getInputFileSize(); + if (scanProvider instanceof HiveScanProvider) { + this.totalPartitionNum = ((HiveScanProvider) scanProvider).getTotalPartitionNum(); + this.readPartitionNum = ((HiveScanProvider) scanProvider).getReadPartitionNum(); + } + } + } + private void setColumnPositionMappingForTextFile(FileScanProviderIf scanProvider, ParamCreateContext context) throws UserException { if (type != Type.QUERY) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java index e993d764b6..b80d9af8d8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.jobs; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -103,11 +103,11 @@ public class RewriteTopDownJobTest { private static class LogicalBoundRelation extends LogicalRelation { - public LogicalBoundRelation(Table table, List qualifier) { + public LogicalBoundRelation(TableIf table, List qualifier) { super(RelationUtil.newRelationId(), PlanType.LOGICAL_BOUND_RELATION, table, qualifier); } - public LogicalBoundRelation(Table table, List qualifier, Optional groupExpression, + public LogicalBoundRelation(TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties) { super(RelationUtil.newRelationId(), PlanType.LOGICAL_BOUND_RELATION, table, qualifier, groupExpression, logicalProperties); diff --git a/regression-test/data/external_table_emr_p2/hive/test_external_yandex_nereids.out b/regression-test/data/external_table_emr_p2/hive/test_external_yandex_nereids.out new file mode 100644 index 0000000000..dc56cd16a2 --- /dev/null +++ b/regression-test/data/external_table_emr_p2/hive/test_external_yandex_nereids.out @@ -0,0 +1,296 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !01 -- +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-17 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-18 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-19 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-20 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-21 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-22 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 2014-03-23 http://auto.ru/chatay-barana.ru/traction.html#maybettaya +http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan 2014-03-17 http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan +http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan 2014-03-18 http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan +http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan 2014-03-19 http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan +http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan 2014-03-20 http://auto.ru/chatay-baranta_bound-in-tanks.ru/forumyazan +http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-17 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny +http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-18 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny +http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-19 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny +http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-20 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny +http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta 2014-03-17 http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta +http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta 2014-03-18 http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta +http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta 2014-03-19 http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta +http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta 2014-03-20 http://auto.ru/chatay-baranta_bound-in-the-asfalt.ru/guru.app4smarta +http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe 2014-03-17 http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe +http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe 2014-03-18 http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe +http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe 2014-03-19 http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe +http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe 2014-03-20 http://auto.ru/chatay-baranta_bound-in-trannie/spb_eco_classnoe +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-17 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-18 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-19 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-20 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-21 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-22 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi 2014-03-23 http://auto.ru/chatay-bibolge.ru/yandex.ua/obyaschenshchennoktalahi +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-17 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-18 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-19 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-20 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-21 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-22 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx 2014-03-23 http://auto.ru/chatay-bit-gd-geli-bolshe__298682.13.xls&nails.aspx +http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet 2014-03-17 http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet +http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet 2014-03-18 http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet +http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet 2014-03-19 http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet +http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet 2014-03-20 http://auto.ru/chatay-bit-yz-odezhda/velik.hurriyet +http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u 2014-03-17 http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u +http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u 2014-03-18 http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u +http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u 2014-03-19 http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u +http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u 2014-03-20 http://auto.ru/chatay-bit-yz-odezhda/vereya/dlike4u +http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod 2014-03-17 http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod +http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod 2014-03-18 http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod +http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod 2014-03-19 http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod +http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod 2014-03-20 http://auto.ru/chatay-bit-yz-odezhda/verhnya/bankovod +http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego 2014-03-17 http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego +http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego 2014-03-18 http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego +http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego 2014-03-19 http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego +http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego 2014-03-20 http://auto.ru/chatay-bit-yz-odezhda/versonal/5/sego +http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede 2014-03-17 http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede +http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede 2014-03-18 http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede +http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede 2014-03-19 http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede +http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede 2014-03-20 http://auto.ru/chatay-john-Den-You-TheAgence=&commemchurkiyede +http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 2014-03-17 http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 +http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 2014-03-18 http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 +http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 2014-03-19 http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 +http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 2014-03-20 http://auto.ru/chatay-john-Den-You-TheAgents/c1303954 +http://auto.ru/chatay-john-Den-Yunan-rasskRollback 2014-03-17 http://auto.ru/chatay-john-Den-Yunan-rasskRollback +http://auto.ru/chatay-john-Den-Yunan-rasskRollback 2014-03-18 http://auto.ru/chatay-john-Den-Yunan-rasskRollback +http://auto.ru/chatay-john-Den-Yunan-rasskRollback 2014-03-19 http://auto.ru/chatay-john-Den-Yunan-rasskRollback +http://auto.ru/chatay-john-Den-Yunan-rasskRollback 2014-03-20 http://auto.ru/chatay-john-Den-Yunan-rasskRollback +http://auto.ru/chatay-john-Den-Yunanija.com%2F20 2014-03-22 http://auto.ru/chatay-john-Den-Yunanija.com%2F20 +http://auto.ru/chatay-john-Den-Yunanija.com%2F20 2014-03-23 http://auto.ru/chatay-john-Den-Yunanija.com%2F20 +http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback +http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback +http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback +http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.com.tr&callback +http://auto.ru/chatay-john-Den-Yunanija.com.tr&user 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.com.tr&user +http://auto.ru/chatay-john-Den-Yunanija.com.tr&user 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.com.tr&user +http://auto.ru/chatay-john-Den-Yunanija.com.tr&user 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.com.tr&user +http://auto.ru/chatay-john-Den-Yunanija.com.tr&user 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.com.tr&user +http://auto.ru/chatay-john-Den-Yunanija.com.tr&useriya 2014-03-23 http://auto.ru/chatay-john-Den-Yunanija.com.tr&useriya +http://auto.ru/chatay-john-Den-Yunanija.com.tr/oauth 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.com.tr/oauth +http://auto.ru/chatay-john-Den-Yunanija.com.tr/oauth 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.com.tr/oauth +http://auto.ru/chatay-john-Den-Yunanija.com.tr/oauth 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.com.tr/oauth +http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina +http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina +http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina +http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.com.ua/ukraina +http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex +http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex +http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex +http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.com.ua/yandex +http://auto.ru/chatay-john-Den-Yunanija.com/commetrovies 2014-03-23 http://auto.ru/chatay-john-Den-Yunanija.com/commetrovies +http://auto.ru/chatay-john-Den-Yunanija.com/obyavlenie 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.com/obyavlenie +http://auto.ru/chatay-john-Den-Yunanija.com/obyavlenie 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.com/obyavlenie +http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom +http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom +http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom +http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.html#medictengliyskom +http://auto.ru/chatay-john-Den-Yunanija.html/photos 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.html/photos +http://auto.ru/chatay-john-Den-Yunanija.html/photos 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.html/photos +http://auto.ru/chatay-john-Den-Yunanija.html/photos 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.html/photos +http://auto.ru/chatay-john-Den-Yunanija.html/photos 2014-03-21 http://auto.ru/chatay-john-Den-Yunanija.html/photos +http://auto.ru/chatay-john-Den-Yunanija.html/photos 2014-03-22 http://auto.ru/chatay-john-Den-Yunanija.html/photos +http://auto.ru/chatay-john-Den-Yunanija.html?area 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.html?area +http://auto.ru/chatay-john-Den-Yunanija.html?area 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.html?area +http://auto.ru/chatay-john-Den-Yunanija.html?area 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.html?area +http://auto.ru/chatay-john-Den-Yunanija.html?area 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.html?area +http://auto.ru/chatay-john-Den-Yunanija.html?area 2014-03-21 http://auto.ru/chatay-john-Den-Yunanija.html?area +http://auto.ru/chatay-john-Den-Yunanija.html?area 2014-03-22 http://auto.ru/chatay-john-Den-Yunanija.html?area +http://auto.ru/chatay-john-Den-Yunanija.html?period 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.html?period +http://auto.ru/chatay-john-Den-Yunanija.html?period 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.html?period +http://auto.ru/chatay-john-Den-Yunanija.html?period 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.html?period +http://auto.ru/chatay-john-Den-Yunanija.html?period 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.html?period +http://auto.ru/chatay-john-Den-Yunanija.html?pid 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.html?pid +http://auto.ru/chatay-john-Den-Yunanija.html?pid 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.html?pid +http://auto.ru/chatay-john-Den-Yunanija.html?pid 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.html?pid +http://auto.ru/chatay-john-Den-Yunanija.html?pid 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.html?pid +http://auto.ru/chatay-john-Den-Yunanija.html?rtn 2014-03-17 http://auto.ru/chatay-john-Den-Yunanija.html?rtn +http://auto.ru/chatay-john-Den-Yunanija.html?rtn 2014-03-18 http://auto.ru/chatay-john-Den-Yunanija.html?rtn +http://auto.ru/chatay-john-Den-Yunanija.html?rtn 2014-03-19 http://auto.ru/chatay-john-Den-Yunanija.html?rtn +http://auto.ru/chatay-john-Den-Yunanija.html?rtn 2014-03-20 http://auto.ru/chatay-john-Den-Yunanija.html?rtn +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-17 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-18 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-19 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-20 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-21 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-22 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com 2014-03-23 http://auto.ru/chatay-uchits/sem-yzyil-sony-osta.coccoc.com +http://auto.ru/chatay-uchits/thegames/page=http: 2014-03-17 http://auto.ru/chatay-uchits/thegames/page=http: +http://auto.ru/chatay-uchits/thegames/page=http: 2014-03-18 http://auto.ru/chatay-uchits/thegames/page=http: +http://auto.ru/chatay-uchits/thegames/page=http: 2014-03-19 http://auto.ru/chatay-uchits/thegames/page=http: +http://auto.ru/chatay-uchits/thegames/page=http: 2014-03-20 http://auto.ru/chatay-uchits/thegames/page=http: +http://auto.ru/chatay-uchits/thegames/page=https 2014-03-17 http://auto.ru/chatay-uchits/thegames/page=https +http://auto.ru/chatay-uchits/thegames/page=https 2014-03-18 http://auto.ru/chatay-uchits/thegames/page=https +http://auto.ru/chatay-uchits/thegames/page=https 2014-03-19 http://auto.ru/chatay-uchits/thegames/page=https +http://auto.ru/chatay-uchits/thegames/page=https 2014-03-20 http://auto.ru/chatay-uchits/thegames/page=https +http://auto.ru/chatay-uchits/thegames/pass.com.ua 2014-03-18 http://auto.ru/chatay-uchits/thegames/pass.com.ua +http://auto.ru/chatay-uchits/thegames/pass.com.ua 2014-03-19 http://auto.ru/chatay-uchits/thegames/pass.com.ua +http://auto.ru/chatay-uchits/thegames/pass.com.ua 2014-03-20 http://auto.ru/chatay-uchits/thegames/pass.com.ua +http://auto.ru/chatay-uchits/thegames/pass.com/effers 2014-03-17 http://auto.ru/chatay-uchits/thegames/pass.com/effers +http://auto.ru/chatay-uchits/thegames/pass.com/effers 2014-03-18 http://auto.ru/chatay-uchits/thegames/pass.com/effers +http://auto.ru/chatay-uchits/thegames/pass.com/effers 2014-03-19 http://auto.ru/chatay-uchits/thegames/pass.com/effers +http://auto.ru/chatay-uchits/thegames/pass.com/effers 2014-03-20 http://auto.ru/chatay-uchits/thegames/pass.com/effers +http://auto.ru/chatay-uchits/thegames/pass.com/iframe 2014-03-18 http://auto.ru/chatay-uchits/thegames/pass.com/iframe +http://auto.ru/chatay-uchits/thegames/pass.com/iframe 2014-03-19 http://auto.ru/chatay-uchits/thegames/pass.com/iframe +http://auto.ru/chatay-uchits/thegames/pass.com/iframe 2014-03-20 http://auto.ru/chatay-uchits/thegames/pass.com/iframe +http://auto.ru/chatay-uchits/thegames/pass.com/iframe 2014-03-22 http://auto.ru/chatay-uchits/thegames/pass.com/iframe +http://auto.ru/chatay-uchits/thegames/pass.com/iframe 2014-03-23 http://auto.ru/chatay-uchits/thegames/pass.com/iframe +http://auto.ru/chatay-uchits/thegames/pass.com/istonomicha 2014-03-23 http://auto.ru/chatay-uchits/thegames/pass.com/istonomicha +http://auto.ru/chatay-uchits/thegames/pass.com/tovaja 2014-03-23 http://auto.ru/chatay-uchits/thegames/pass.com/tovaja +http://auto.ru/chatay-uchits/thegames/password/?type 2014-03-22 http://auto.ru/chatay-uchits/thegames/password/?type +http://auto.ru/chatay-uchits/thegames/password=&to 2014-03-21 http://auto.ru/chatay-uchits/thegames/password=&to +http://auto.ru/chatay-uchits/thegames/password=&to 2014-03-22 http://auto.ru/chatay-uchits/thegames/password=&to +http://auto.ru/chatay-uchits/thegames/passwords= 2014-03-22 http://auto.ru/chatay-uchits/thegames/passwords= +http://auto.ru/chatay-uchits/thegames/passwords=Поддержитель 2014-03-22 http://auto.ru/chatay-uchits/thegames/passwords=Поддержитель +http://auto.ru/chatay-uchits/thegames/passwordsPerPage 2014-03-22 http://auto.ru/chatay-uchits/thegames/passwordsPerPage +http://auto.ru/chatay-uchits/thegames/passwords[0 2014-03-22 http://auto.ru/chatay-uchits/thegames/passwords[0 +http://auto.ru/chatay-uchits/thegames/passwordstrelation 2014-03-22 http://auto.ru/chatay-uchits/thegames/passwordstrelation +http://auto.ru/chatay-uchits/thegames/public/?hash 2014-03-17 http://auto.ru/chatay-uchits/thegames/public/?hash +http://auto.ru/chatay-uchits/thegames/public/?hash 2014-03-18 http://auto.ru/chatay-uchits/thegames/public/?hash +http://auto.ru/chatay-uchits/thegames/public/?hash 2014-03-19 http://auto.ru/chatay-uchits/thegames/public/?hash +http://auto.ru/chatay-uchits/thegames/public/?hash 2014-03-20 http://auto.ru/chatay-uchits/thegames/public/?hash +http://auto.ru/chatay-uchits/thegames/puted_facet 2014-03-17 http://auto.ru/chatay-uchits/thegames/puted_facet +http://auto.ru/chatay-uchits/thegames/puted_facet 2014-03-18 http://auto.ru/chatay-uchits/thegames/puted_facet +http://auto.ru/chatay-uchits/thegames/puted_facet 2014-03-19 http://auto.ru/chatay-uchits/thegames/puted_facet +http://auto.ru/chatay-uchits/thegames/puted_facet 2014-03-20 http://auto.ru/chatay-uchits/thegames/puted_facet +http://auto.ru/chatay-uchits/thegames/puted_from 2014-03-17 http://auto.ru/chatay-uchits/thegames/puted_from +http://auto.ru/chatay-uchits/thegames/puted_from 2014-03-18 http://auto.ru/chatay-uchits/thegames/puted_from +http://auto.ru/chatay-uchits/thegames/puted_from 2014-03-19 http://auto.ru/chatay-uchits/thegames/puted_from +http://auto.ru/chatay-uchits/thegames/puted_from 2014-03-20 http://auto.ru/chatay-uchits/thegames/puted_from +http://auto.ru/chatay-uchits/thegames/putery/Roman 2014-03-17 http://auto.ru/chatay-uchits/thegames/putery/Roman +http://auto.ru/chatay-uchits/thegames/putery/Roman 2014-03-18 http://auto.ru/chatay-uchits/thegames/putery/Roman +http://auto.ru/chatay-uchits/thegames/putery/Roman 2014-03-19 http://auto.ru/chatay-uchits/thegames/putery/Roman +http://auto.ru/chatay-uchits/thegames/putery/Roman 2014-03-20 http://auto.ru/chatay-uchits/thegames/putery/Roman +http://auto.ru/chatay-uchits/thegames/putina/artira 2014-03-17 http://auto.ru/chatay-uchits/thegames/putina/artira +http://auto.ru/chatay-uchits/thegames/putina/artira 2014-03-18 http://auto.ru/chatay-uchits/thegames/putina/artira +http://auto.ru/chatay-uchits/thegames/putina/artira 2014-03-19 http://auto.ru/chatay-uchits/thegames/putina/artira +http://auto.ru/chatay-uchits/thegames/putina/artira 2014-03-20 http://auto.ru/chatay-uchits/thegames/putina/artira +http://auto.ru/chatay-uchits/thegames/putina_2008 2014-03-17 http://auto.ru/chatay-uchits/thegames/putina_2008 +http://auto.ru/chatay-uchits/thegames/putina_2008 2014-03-18 http://auto.ru/chatay-uchits/thegames/putina_2008 +http://auto.ru/chatay-uchits/thegames/putina_2008 2014-03-19 http://auto.ru/chatay-uchits/thegames/putina_2008 +http://auto.ru/chatay-uchits/thegames/putina_2008 2014-03-20 http://auto.ru/chatay-uchits/thegames/putina_2008 +http://auto.ru/chatay-uchits/thegames/putinatik_3 2014-03-17 http://auto.ru/chatay-uchits/thegames/putinatik_3 +http://auto.ru/chatay-uchits/thegames/putinatik_3 2014-03-18 http://auto.ru/chatay-uchits/thegames/putinatik_3 +http://auto.ru/chatay-uchits/thegames/putinatik_3 2014-03-19 http://auto.ru/chatay-uchits/thegames/putinatik_3 +http://auto.ru/chatay-uchits/thegames/putinatik_3 2014-03-20 http://auto.ru/chatay-uchits/thegames/putinatik_3 +http://auto.ru/chatay-uchits/thegames/putinato-po 2014-03-17 http://auto.ru/chatay-uchits/thegames/putinato-po +http://auto.ru/chatay-uchits/thegames/putinato-po 2014-03-18 http://auto.ru/chatay-uchits/thegames/putinato-po +http://auto.ru/chatay-uchits/thegames/putinato-po 2014-03-19 http://auto.ru/chatay-uchits/thegames/putinato-po +http://auto.ru/chatay-uchits/thegames/putinato-po 2014-03-20 http://auto.ru/chatay-uchits/thegames/putinato-po +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-17 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-18 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-19 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-20 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-21 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-22 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/chatayru.tsn.ua/artisemeyen 2014-03-23 http://auto.ru/chatayru.tsn.ua/artisemeyen +http://auto.ru/click.shortcut=true&groups[]=300&racelog&id=1872347416/view/48/r=a.me/istant-poshiny/9513701.ru/news.biz/sony 2014-03-21 http://auto.ru/click.shortcut=true&groups[]=300&racelog&id=1872347416/view/48/r=a.me/istant-poshiny/9513701.ru/news.biz/sony +http://auto.ru/click.shortcut=true&groups[]=300&racelog&id=1872347416/view/48/r=a.me/istant-poshiny/9513701.ru/news.biz/sony 2014-03-22 http://auto.ru/click.shortcut=true&groups[]=300&racelog&id=1872347416/view/48/r=a.me/istant-poshiny/9513701.ru/news.biz/sony +http://faber/frame/frm_index.ru 2014-03-17 http://faber/frame/frm_index.ru +http://faber/frame/frm_index.ru 2014-03-18 http://faber/frame/frm_index.ru +http://faber/frame/frm_index.ru 2014-03-19 http://faber/frame/frm_index.ru +http://faber/frame/frm_index.ru 2014-03-20 http://faber/frame/frm_index.ru +http://faber/frame/frm_index.ru 2014-03-21 http://faber/frame/frm_index.ru +http://faber/frame/frm_index.ru 2014-03-22 http://faber/frame/frm_index.ru +http://faber/frame/frm_index.ru 2014-03-23 http://faber/frame/frm_index.ru +http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr 2014-03-17 http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr +http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr 2014-03-18 http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr +http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr 2014-03-19 http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr +http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr 2014-03-20 http://minsk/odessages.yandex.ru/vorozhitelnichaet-risunk-v-kiev/школад&startv.com.tr +http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 2014-03-17 http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 +http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 2014-03-18 http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 +http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 2014-03-19 http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 +http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 2014-03-20 http://minsk/odessages.yandex.ru/vorozhitelniki.ru/?mode=on&order=02.03.1307%26lr%3D1%26sid%3D12499454a73bce1f196d88644194 +http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein 2014-03-19 http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein +http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein 2014-03-20 http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein +http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein 2014-03-22 http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein +http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein 2014-03-23 http://minsk/odessages.yandex.ru/vorozhitelniy-bolsha.info=ww-1425-wh-113467157_254222883_0&gearbox][2]=0&sort=mobilein +http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji 2014-03-17 http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji +http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji 2014-03-18 http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji +http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji 2014-03-19 http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji +http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji 2014-03-20 http://minsk/odessages.yandex.ru/vorozhitelno_otdyh/tube.com/Magazin%2Fttopic.ru/ulichnopoisk.ru/novosti/352662-25-teknoloji +http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 2014-03-17 http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 +http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 2014-03-18 http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 +http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 2014-03-19 http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 +http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 2014-03-20 http://minsk/odessages.yandex.ru/vorozhitelnosti/Armanada-yeni-boyiny_i_motot-birthday=1#body_key=50aa1f0bbce1fc00b6a54 + +-- !02 -- +7521 + +-- !03 -- +0 + +-- !04 -- +7521 + +-- !05 -- +51354 + +-- !06 -- +-10 5604 +0 1853 +10 4291 + +-- !07 -- +1704509 523264 نيك امريكي نيك افلام سكس جامد +732797 475698 نيك سكس سيحاق +598875 337212 سکس باصات +792887 252197 №2267 отзыв +3807842 196036 ярмаркетовара 200кг купить по неделю тебелье +25703952 147211 +716829 90109 яндекс повыш +59183 85379 франция машину угловы крузер из кофе +33010362 77807 ярмаркетовара 200кг купить по неделю тебелье +800784 77492 ярмаркур смерти теплицы из чего +20810645 73213 ярмаркетовара 200кг купить по неделю тебе перево метиков детский +25843850 68945 электросчет-фактура +23447120 67570 южная степанов +14739804 64174 штангал волк +32077710 60456 +22446879 58389 فیلم سكس امريكي نيك +170282 57017 ل افلام السكس +11482817 52345 я скачать игры +63469 52142 яндекс марте рокус надписями я любимому у полосы фото минск +29103473 47758 + +-- !08 -- +1704509 4611700827100483880 9223360787015464643 +732797 4611701940806302259 9223355550934604746 +598875 4611701407242345792 9223362250391155632 +792887 4611699550286611812 9223290551912005343 +3807842 4611710821592843606 9223326163906184987 +25703952 4611709443519524003 9223353913449113943 +716829 4611852156092872082 9223361623076951140 +59183 4611730685242027332 9223354909338698162 +33010362 4611704682869732882 9223268545373999677 +800784 4611752907938305166 9223340418389788041 +20810645 4611712185532639162 9223218900001937412 +25843850 4611690025407720929 9223346023778617822 +23447120 4611796031755620254 9223329309291309758 +14739804 4611692230555590277 9223313509005166531 +32077710 4611884228437061959 9223352444952988904 +22446879 4611846229717089436 9223124373140579096 +170282 4611833225706935900 9223371583739401906 +11482817 4611708000353743073 9223337838355779113 +63469 4611695097019173921 9223353530156141191 +29103473 4611744585914335132 9223333530281362537 + +-- !09 -- +0 + +-- !10 -- +1956422 + +-- !11 -- + +-- !12 -- + diff --git a/regression-test/suites/external_table_emr_p2/hive/test_external_yandex_nereids.groovy b/regression-test/suites/external_table_emr_p2/hive/test_external_yandex_nereids.groovy new file mode 100644 index 0000000000..9f06df3100 --- /dev/null +++ b/regression-test/suites/external_table_emr_p2/hive/test_external_yandex_nereids.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_yandex_nereids", "p2") { + + def formats = ["_parquet"] + def duplicateAggregationKeys = "SELECT URL, EventDate, max(URL) FROM hitsSUFFIX WHERE CounterID = 1704509 AND UserID = 4322253409885123546 GROUP BY URL, EventDate, EventDate ORDER BY URL, EventDate;" + def like1 = """SELECT count() FROM hitsSUFFIX WHERE URL LIKE '%/avtomobili_s_probegom/_%__%__%__%';""" + def like2 = """SELECT count() FROM hitsSUFFIX WHERE URL LIKE '/avtomobili_s_probegom/_%__%__%__%';""" + def like3 = """SELECT count() FROM hitsSUFFIX WHERE URL LIKE '%_/avtomobili_s_probegom/_%__%__%__%';""" + def like4 = """SELECT count() FROM hitsSUFFIX WHERE URL LIKE '%avtomobili%';""" + def loyalty = """SELECT loyalty, count() AS c + FROM + ( + SELECT UserID, CAST(((if(yandex > google, yandex / (yandex + google), 0 - google / (yandex + google))) * 10) AS TINYINT) AS loyalty + FROM + ( + SELECT UserID, sum(if(SearchEngineID = 2, 1, 0)) AS yandex, sum(if(SearchEngineID = 3, 1, 0)) AS google + FROM hitsSUFFIX + WHERE SearchEngineID = 2 OR SearchEngineID = 3 GROUP BY UserID HAVING yandex + google > 10 + ) t1 + ) t2 + GROUP BY loyalty + ORDER BY loyalty;""" + def maxStringIf = """SELECT CounterID, count(), max(if(SearchPhrase != "", SearchPhrase, "")) FROM hitsSUFFIX GROUP BY CounterID ORDER BY count() DESC LIMIT 20;""" + def minMax = """SELECT CounterID, min(WatchID), max(WatchID) FROM hitsSUFFIX GROUP BY CounterID ORDER BY count() DESC LIMIT 20;""" + def monotonicEvaluationSegfault = """SELECT max(0) FROM visitsSUFFIX WHERE (CAST(CAST(StartDate AS DATETIME) AS INT)) > 1000000000;""" + def subqueryInWhere = """SELECT count() FROM hitsSUFFIX WHERE UserID IN (SELECT UserID FROM hitsSUFFIX WHERE CounterID = 800784);""" + def where01 = """SELECT CounterID, count(distinct UserID) FROM hitsSUFFIX WHERE 0 != 0 GROUP BY CounterID;""" + def where02 = """SELECT CounterID, count(distinct UserID) FROM hitsSUFFIX WHERE CAST(0 AS BOOLEAN) AND CounterID = 1704509 GROUP BY CounterID;""" + + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String catalog_name = "external_yandex_nereids" + + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + sql """use multi_catalog;""" + logger.info("use multi_catalog") + sql """set enable_nereids_planner=true""" + + for (String format in formats) { + logger.info("Process format " + format) + qt_01 duplicateAggregationKeys.replace("SUFFIX", format) + qt_02 like1.replace("SUFFIX", format) + qt_03 like2.replace("SUFFIX", format) + qt_04 like3.replace("SUFFIX", format) + qt_05 like4.replace("SUFFIX", format) + qt_06 loyalty.replace("SUFFIX", format) + qt_07 maxStringIf.replace("SUFFIX", format) + qt_08 minMax.replace("SUFFIX", format) + qt_09 monotonicEvaluationSegfault.replace("SUFFIX", format) + qt_10 subqueryInWhere.replace("SUFFIX", format) + qt_11 where01.replace("SUFFIX", format) + qt_12 where02.replace("SUFFIX", format) + } + } +} +