database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bug](group_commit) fix relay wal problem on materialized-view (#29848)

Browse Source
This commit is contained in:
huanghaibin
2024-01-11 19:56:47 +08:00
committed by yiguolei
parent a4f29193f6
commit cbffdbb8bf
4 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
be/src/olap/wal/wal_table.cpp
View File

@ -234,6 +234,7 @@ Status WalTable::_construct_sql_str(const std::string& wal, const std::string& l
auto it = column_info_map.find(column_id);
if (it != column_info_map.end()) {
ss_name << "`" << it->second << "`,";
column_info_map.erase(column_id);
}
} catch (const std::invalid_argument& e) {
return Status::InvalidArgument("Invalid format, {}", e.what());

4
be/src/runtime/group_commit_mgr.cpp
View File

@ -61,6 +61,8 @@ Status LoadBlockQueue::add_block(RuntimeState* runtime_state,
if (block->rows() > 0) {
if (!config::group_commit_wait_replay_wal_finish) {
_block_queue.push_back(block);
_data_bytes += block->bytes();
_all_block_queues_bytes->fetch_add(block->bytes(), std::memory_order_relaxed);
} else {
LOG(INFO) << "skip adding block to queue on txn " << txn_id;
}
@ -71,8 +73,6 @@ Status LoadBlockQueue::add_block(RuntimeState* runtime_state,
return st;
}
}
_data_bytes += block->bytes();
_all_block_queues_bytes->fetch_add(block->bytes(), std::memory_order_relaxed);
}
if (_data_bytes >= _group_commit_data_bytes) {
VLOG_DEBUG << "group commit meets commit condition for data size, label=" << label

2
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -607,7 +607,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
List<Long> aliveBeIds = Env.getCurrentSystemInfo().getAllBackendIds(true);
long expireTime = System.currentTimeMillis() + Config.check_wal_queue_timeout_threshold;
while (true) {
LOG.info("wai for wal queue size to be empty");
LOG.info("wait for wal queue size to be empty");
boolean walFinished = Env.getCurrentEnv().getGroupCommitManager()
.isPreviousWalFinished(tableId, aliveBeIds);
if (walFinished) {

7
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
View File

@ -333,7 +333,12 @@ public class LoadAction extends RestBaseController {
.setEnableRoundRobin(true)
.needLoadAvailable().build();
policy.nextRoundRobinIndex = getLastSelectedBackendIndexAndUpdate();
List<Long> backendIds = Env.getCurrentSystemInfo().selectBackendIdsByPolicy(policy, 1);
List<Long> backendIds;
if (groupCommit) {
backendIds = Env.getCurrentSystemInfo().selectBackendIdsByPolicy(policy, -1);
} else {
backendIds = Env.getCurrentSystemInfo().selectBackendIdsByPolicy(policy, 1);
}
if (backendIds.isEmpty()) {
throw new LoadException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG + ", policy: " + policy);
}