database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](catalog) wrong required slot info causing BE crash (#21598)

Browse Source 
For file scan node, this is a special field `requiredSlot`, this field is set depends on the `isMaterialized` info of slot.
But `isMaterialized` info can be changed during the plan process, so we must update the `requiredSlot`
in `finalize` phase of scan node, otherwise, it may causing BE crash due to mismatching slot info.
This commit is contained in:
Mingyu Chen
2023-07-07 17:10:50 +08:00
committed by GitHub
parent 02149ff329
commit 0b7b5dc991
4 changed files with 15 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
View File

@ -389,7 +389,8 @@ public class HiveMetaStoreCache {
throw e;
}
}
result.setPartitionValues(partitionValues);
// Must copy the partitionValues to avoid concurrent modification of key and value
result.setPartitionValues(Lists.newArrayList(partitionValues));
return result;
}
@ -924,7 +925,7 @@ public class HiveMetaStoreCache {
return dummyKey.equals(((FileCacheKey) obj).dummyKey);
}
return location.equals(((FileCacheKey) obj).location)
&& partitionValues.equals(((FileCacheKey) obj).partitionValues);
&& Objects.equals(partitionValues, ((FileCacheKey) obj).partitionValues);
}
@Override

8
fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java vendored
View File

@ -81,7 +81,7 @@ import java.util.Set;
/**
* FileQueryScanNode for querying the file access type of catalog, now only support
* hive,hudi, iceberg and TVF.
* hive, hudi, iceberg and TVF.
*/
public abstract class FileQueryScanNode extends FileScanNode {
private static final Logger LOG = LogManager.getLogger(FileQueryScanNode.class);
@ -163,6 +163,10 @@ public abstract class FileQueryScanNode extends FileScanNode {
@Override
public void updateRequiredSlots(PlanTranslatorContext planTranslatorContext,
Set<SlotId> requiredByProjectSlotIdSet) throws UserException {
updateRequiredSlots();
}
private void updateRequiredSlots() throws UserException {
params.unsetRequiredSlots();
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
@ -196,6 +200,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
// Create scan range locations and the statistics.
protected void doFinalize() throws UserException {
createScanRangeLocations();
updateRequiredSlots();
}
private void setColumnPositionMapping()
@ -415,3 +420,4 @@ public abstract class FileQueryScanNode extends FileScanNode {
return Optional.empty();
}
}