database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](paimon)Remove the static attribute of the source for paimon (#29032)

Browse Source
This commit is contained in:
wuwenchi
2023-12-27 12:57:39 +08:00
committed by GitHub
parent 6f5672f318
commit e33395a0ee
6 changed files with 58 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java vendored
View File

@ -38,12 +38,6 @@ import org.apache.doris.datasource.hive.AcidInfo;
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.hudi.HudiSplit;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.iceberg.IcebergSplit;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.planner.external.paimon.PaimonSplit;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.spi.Split;
import org.apache.doris.statistics.StatisticalType;
@ -260,6 +254,10 @@ public abstract class FileQueryScanNode extends FileScanNode {
return params;
}
// Set some parameters of scan to support different types of file data sources
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
}
@Override
public void createScanRangeLocations() throws UserException {
long start = System.currentTimeMillis();
@ -353,17 +351,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
rangeDesc.setTableFormatParams(tableFormatFileDesc);
}
// external data lake table
if (fileSplit instanceof IcebergSplit) {
// TODO: extract all data lake split to factory
IcebergScanNode.setIcebergParams(rangeDesc, (IcebergSplit) fileSplit);
} else if (fileSplit instanceof PaimonSplit) {
PaimonScanNode.setPaimonParams(rangeDesc, (PaimonSplit) fileSplit);
} else if (fileSplit instanceof HudiSplit) {
HudiScanNode.setHudiParams(rangeDesc, (HudiSplit) fileSplit);
} else if (fileSplit instanceof MaxComputeSplit) {
MaxComputeScanNode.setScanParams(rangeDesc, (MaxComputeSplit) fileSplit);
}
setScanParams(rangeDesc, fileSplit);
curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc);
TScanRangeLocation location = new TScanRangeLocation();
@ -473,6 +461,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
protected abstract TFileFormatType getFileFormatType() throws UserException;
protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws UserException {
return Util.inferFileCompressTypeByPath(fileSplit.getPath().toString());
}

9
fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java vendored
View File

@ -63,7 +63,14 @@ public class MaxComputeScanNode extends FileQueryScanNode {
catalog = (MaxComputeExternalCatalog) table.getCatalog();
}
public static void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) {
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof MaxComputeSplit) {
setScanParams(rangeDesc, (MaxComputeSplit) split);
}
}
public void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(TableFormatType.MAX_COMPUTE.value());
TMaxComputeFileDesc fileDesc = new TMaxComputeFileDesc();

9
fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java vendored
View File

@ -132,7 +132,14 @@ public class HudiScanNode extends HiveScanNode {
}
}
public static void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof HudiSplit) {
setHudiParams(rangeDesc, (HudiSplit) split);
}
}
public void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(hudiSplit.getTableFormatType().value());
THudiFileDesc fileDesc = new THudiFileDesc();

9
fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java vendored
View File

@ -129,7 +129,14 @@ public class IcebergScanNode extends FileQueryScanNode {
super.doInitialize();
}
public static void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) {
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof IcebergSplit) {
setIcebergParams(rangeDesc, (IcebergSplit) split);
}
}
public void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value());
TIcebergFileDesc fileDesc = new TIcebergFileDesc();

13
fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java vendored
View File

@ -54,8 +54,8 @@ import java.util.Set;
import java.util.stream.Collectors;
public class PaimonScanNode extends FileQueryScanNode {
private static PaimonSource source = null;
private static List<Predicate> predicates;
private PaimonSource source = null;
private List<Predicate> predicates;
public PaimonScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv);
@ -91,7 +91,14 @@ public class PaimonScanNode extends FileQueryScanNode {
}
}
public static void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) {
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof PaimonSplit) {
setPaimonParams(rangeDesc, (PaimonSplit) split);
}
}
public void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(paimonSplit.getTableFormatType().value());
TPaimonFileDesc fileDesc = new TPaimonFileDesc();