database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](group commit) Pick add debug log show why group commit not work; delete wal when replay success (#38611) (#38659)

Browse Source 
Pick https://github.com/apache/doris/pull/38611
This commit is contained in:
meiyi
2024-08-01 16:59:54 +08:00
committed by GitHub
parent cafcf7acc1
commit e8690b62ee
5 changed files with 108 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
View File

@ -83,6 +83,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.BooleanSupplier;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@ -1200,28 +1202,53 @@ public class NativeInsertStmt extends InsertStmt {
LOG.warn("analyze group commit failed", e);
return;
}
boolean partialUpdate = ConnectContext.get().getSessionVariable().isEnableUniqueKeyPartialUpdate();
if (!isExplain() && !partialUpdate && ConnectContext.get().getSessionVariable().isEnableInsertGroupCommit()
&& ConnectContext.get().getSessionVariable().getSqlMode() != SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES
&& targetTable instanceof OlapTable
&& ((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange()
&& !targetTable.getQualifiedDbName().equalsIgnoreCase(FeConstants.INTERNAL_DB_NAME)
&& !ConnectContext.get().isTxnModel()
&& getQueryStmt() instanceof SelectStmt
&& ((SelectStmt) getQueryStmt()).getTableRefs().isEmpty() && targetPartitionNames == null
&& (label == null || Strings.isNullOrEmpty(label.getLabelName()))
&& (analyzer == null || analyzer != null && !analyzer.isReAnalyze())) {
ConnectContext ctx = ConnectContext.get();
List<Pair<BooleanSupplier, Supplier<String>>> conditions = new ArrayList<>();
conditions.add(Pair.of(() -> ctx.getSessionVariable().isEnableInsertGroupCommit(),
() -> "group_commit session variable: " + ctx.getSessionVariable().groupCommit));
conditions.add(Pair.of(() -> !isExplain(), () -> "isExplain"));
conditions.add(Pair.of(() -> !ctx.getSessionVariable().isEnableUniqueKeyPartialUpdate(),
() -> "enableUniqueKeyPartialUpdate"));
conditions.add(Pair.of(() -> !ctx.isTxnModel(), () -> "isTxnModel"));
conditions.add(Pair.of(() -> targetTable instanceof OlapTable,
() -> "not olapTable, class: " + targetTable.getClass().getName()));
conditions.add(Pair.of(() -> ((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange(),
() -> "notUseSchemaLightChange"));
conditions.add(Pair.of(() -> !targetTable.getQualifiedDbName().equalsIgnoreCase(FeConstants.INTERNAL_DB_NAME),
() -> "db is internal"));
conditions.add(
Pair.of(() -> targetPartitionNames == null, () -> "targetPartitionNames: " + targetPartitionNames));
conditions.add(Pair.of(() -> ctx.getSessionVariable().getSqlMode() != SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES,
() -> "sqlMode: " + ctx.getSessionVariable().getSqlMode()));
conditions.add(Pair.of(() -> queryStmt instanceof SelectStmt,
() -> "queryStmt is not SelectStmt, class: " + queryStmt.getClass().getName()));
conditions.add(Pair.of(() -> ((SelectStmt) queryStmt).getTableRefs().isEmpty(),
() -> "tableRefs is not empty: " + ((SelectStmt) queryStmt).getTableRefs()));
conditions.add(
Pair.of(() -> (label == null || Strings.isNullOrEmpty(label.getLabelName())), () -> "label: " + label));
conditions.add(
Pair.of(() -> (analyzer == null || analyzer != null && !analyzer.isReAnalyze()), () -> "analyzer"));
boolean match = conditions.stream().allMatch(p -> p.first.getAsBoolean());
if (match) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
for (List<Expr> row : selectStmt.getValueList().getRows()) {
for (Expr expr : row) {
if (!(expr instanceof LiteralExpr)) {
if (LOG.isDebugEnabled()) {
LOG.debug("group commit is off for table: {}, because not literal expr, "
+ "expr: {}, row: {}", targetTable.getName(), expr, row);
}
return;
}
}
}
// Does not support: insert into tbl values();
if (selectStmt.getValueList().getFirstRow().isEmpty() && CollectionUtils.isEmpty(targetColumnNames)) {
if (LOG.isDebugEnabled()) {
LOG.debug("group commit is off for table: {}, because first row: {}, target columns: {}",
targetTable.getName(), selectStmt.getValueList().getFirstRow(), targetColumnNames);
}
return;
}
} else {
@ -1231,6 +1258,10 @@ public class NativeInsertStmt extends InsertStmt {
if (items != null) {
for (SelectListItem item : items) {
if (item.getExpr() != null && !(item.getExpr() instanceof LiteralExpr)) {
if (LOG.isDebugEnabled()) {
LOG.debug("group commit is off for table: {}, because not literal expr, "
+ "expr: {}, row: {}", targetTable.getName(), item.getExpr(), item);
}
return;
}
}
@ -1238,6 +1269,16 @@ public class NativeInsertStmt extends InsertStmt {
}
}
isGroupCommit = true;
} else {
if (LOG.isDebugEnabled()) {
for (Pair<BooleanSupplier, Supplier<String>> pair : conditions) {
if (pair.first.getAsBoolean() == false) {
LOG.debug("group commit is off for table: {}, because: {}", targetTable.getName(),
pair.second.get());
break;
}
}
}
}
}