database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](routine-load) fix get kafka offset timeout may too long (#33502)

Browse Source
This commit is contained in:
HHoflittlefish777
2024-04-15 19:35:26 +08:00
committed by yiguolei
parent 341cb40693
commit e0ec2da29b
7 changed files with 37 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
be/src/runtime/routine_load/data_consumer.cpp
View File

@ -349,7 +349,7 @@ Status KafkaDataConsumer::get_partition_meta(std::vector<int32_t>* partition_ids
// corresponding partition.
// See librdkafka/rdkafkacpp.h##offsetsForTimes()
Status KafkaDataConsumer::get_offsets_for_times(const std::vector<PIntegerPair>& times,
std::vector<PIntegerPair>* offsets) {
std::vector<PIntegerPair>* offsets, int timeout) {
// create topic partition
std::vector<RdKafka::TopicPartition*> topic_partitions;
for (const auto& entry : times) {
@ -364,8 +364,8 @@ Status KafkaDataConsumer::get_offsets_for_times(const std::vector<PIntegerPair>&
}};
// get offsets for times
RdKafka::ErrorCode err = _k_consumer->offsetsForTimes(topic_partitions, 5000);
if (err != RdKafka::ERR_NO_ERROR) {
RdKafka::ErrorCode err = _k_consumer->offsetsForTimes(topic_partitions, timeout);
if (UNLIKELY(err != RdKafka::ERR_NO_ERROR)) {
std::stringstream ss;
ss << "failed to get offsets for times: " << RdKafka::err2str(err);
LOG(WARNING) << ss.str();
@ -384,13 +384,21 @@ Status KafkaDataConsumer::get_offsets_for_times(const std::vector<PIntegerPair>&
// get latest offsets for given partitions
Status KafkaDataConsumer::get_latest_offsets_for_partitions(
const std::vector<int32_t>& partition_ids, std::vector<PIntegerPair>* offsets) {
const std::vector<int32_t>& partition_ids, std::vector<PIntegerPair>* offsets,
int timeout) {
MonotonicStopWatch watch;
watch.start();
for (int32_t partition_id : partition_ids) {
int64_t low = 0;
int64_t high = 0;
auto timeout_ms = timeout - static_cast<int>(watch.elapsed_time() / 1000 / 1000);
if (UNLIKELY(timeout_ms <= 0)) {
return Status::InternalError("get kafka latest offsets for partitions timeout");
}
RdKafka::ErrorCode err =
_k_consumer->query_watermark_offsets(_topic, partition_id, &low, &high, 5000);
if (err != RdKafka::ERR_NO_ERROR) {
_k_consumer->query_watermark_offsets(_topic, partition_id, &low, &high, timeout_ms);
if (UNLIKELY(err != RdKafka::ERR_NO_ERROR)) {
std::stringstream ss;
ss << "failed to get latest offset for partition: " << partition_id
<< ", err: " << RdKafka::err2str(err);