From 72a27a093840ae8dec1843a271c18ca88ae70eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E5=B0=8F=E5=88=9A?= Date: Tue, 28 May 2024 16:47:51 +0800 Subject: [PATCH] [fix](paimon)fix paimon cache bug (#35309) Issue Number: close #35024 This bug is because the fe incorrectly sets the update time of paimon catalog, causing the be to be unable to update paimon's schema in time. ```c++ private void initTable() { PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId, paimonOptionParams, dbName, tblName); TableExt tableExt = PaimonTableCache.getTable(key); if (tableExt.getCreateTime() < lastUpdateTime) { LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}", key, tableExt.getCreateTime(), lastUpdateTime); PaimonTableCache.invalidateTableCache(key); tableExt = PaimonTableCache.getTable(key); } this.table = tableExt.getTable(); paimonAllFieldNames = PaimonScannerUtils.fieldNames(this.table.rowType()); if (LOG.isDebugEnabled()) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); } } ``` --- .../java/org/apache/doris/paimon/PaimonJniScanner.java | 9 ++++++++- .../src/main/java/org/apache/doris/catalog/TableIf.java | 4 ---- .../doris/datasource/paimon/source/PaimonScanNode.java | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index ad45c72919..9250761c00 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -107,6 +107,13 @@ public class PaimonJniScanner extends JniScanner { private void initReader() throws IOException { ReadBuilder readBuilder = table.newReadBuilder(); + if (this.fields.length != this.paimonAllFieldNames.size()) { + throw new IOException( + String.format( + "The jni reader fields' size {%s} is not matched with paimon fields' size {%s}." + + " Please refresh table and try again", + fields.length, paimonAllFieldNames.size())); + } readBuilder.withProjection(getProjected()); readBuilder.withFilter(getPredicates()); reader = readBuilder.newRead().createReader(getSplit()); @@ -182,7 +189,7 @@ public class PaimonJniScanner extends JniScanner { } catch (Exception e) { close(); LOG.warn("Failed to get the next batch of paimon. " - + "split: {}, requiredFieldNames: {}, paimonAllFieldNames: {}", + + "split: {}, requiredFieldNames: {}, paimonAllFieldNames: {}", getSplit(), params.get("required_fields"), paimonAllFieldNames, e); throw new IOException(e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index b3e27f286c..a1016cbd5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -539,10 +539,6 @@ public interface TableIf { return getType() == TableType.OLAP || getType() == TableType.MATERIALIZED_VIEW; } - default long getLastUpdateTime() { - return -1L; - } - default long getDataSize(boolean singleReplica) { // TODO: Each tableIf should impl it by itself. return 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 003ced7ead..7f0ea2e6d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -169,7 +169,7 @@ public class PaimonScanNode extends FileQueryScanNode { fileDesc.setCtlId(source.getCatalog().getId()); fileDesc.setDbId(((PaimonExternalTable) source.getTargetTable()).getDbId()); fileDesc.setTblId(source.getTargetTable().getId()); - fileDesc.setLastUpdateTime(source.getTargetTable().getLastUpdateTime()); + fileDesc.setLastUpdateTime(source.getTargetTable().getUpdateTime()); Optional optDeletionFile = paimonSplit.getDeletionFile(); if (optDeletionFile.isPresent()) { DeletionFile deletionFile = optDeletionFile.get();