database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

add read buffer when use s3 reader (#5791)

Browse Source
This commit is contained in:
Zhengguo Yang
2021-05-17 11:46:38 +08:00
committed by GitHub
parent e7a6d659a9
commit 01a45e8691
22 changed files with 180 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
be/src/exec/buffered_reader.cpp
View File

@ -45,21 +45,18 @@ Status BufferedReader::open() {
return Status::InternalError(ss.str());
}
RETURN_IF_ERROR(_reader->open());
RETURN_IF_ERROR(_fill());
return Status::OK();
}
//not support
Status BufferedReader::read_one_message(std::unique_ptr<uint8_t[]>* buf, size_t* length) {
Status BufferedReader::read_one_message(std::unique_ptr<uint8_t[]>* buf, int64_t* length) {
return Status::NotSupported("Not support");
}
Status BufferedReader::read(uint8_t* buf, size_t* buf_len, bool* eof) {
DCHECK_NE(*buf_len, 0);
int64_t bytes_read;
RETURN_IF_ERROR(readat(_cur_offset, (int64_t)*buf_len, &bytes_read, buf));
if (bytes_read == 0) {
Status BufferedReader::read(uint8_t* buf, int64_t buf_len, int64_t* bytes_read, bool* eof) {
DCHECK_NE(buf_len, 0);
RETURN_IF_ERROR(readat(_cur_offset, buf_len, bytes_read, buf));
if (*bytes_read == 0) {
*eof = true;
} else {
*eof = false;
@ -97,7 +94,11 @@ Status BufferedReader::_read_once(int64_t position, int64_t nbytes, int64_t* byt
// if requested length is larger than the capacity of buffer, do not
// need to copy the character into local buffer.
if (nbytes > _buffer_size) {
return _reader->readat(position, nbytes, bytes_read, out);
auto st = _reader->readat(position, nbytes, bytes_read, out);
if (st.ok()) {
_cur_offset = position + *bytes_read;
}
return st;
}
_buffer_offset = position;
RETURN_IF_ERROR(_fill());
@ -116,13 +117,8 @@ Status BufferedReader::_read_once(int64_t position, int64_t nbytes, int64_t* byt
Status BufferedReader::_fill() {
if (_buffer_offset >= 0) {
int64_t bytes_read;
// retry for new content
int retry_times = 1;
do {
// fill the buffer
RETURN_IF_ERROR(_reader->readat(_buffer_offset, _buffer_size, &bytes_read, _buffer));
} while (bytes_read == 0 && retry_times++ < 2);
int64_t bytes_read = 0;
RETURN_IF_ERROR(_reader->readat(_buffer_offset, _buffer_size, &bytes_read, _buffer));
_buffer_limit = _buffer_offset + bytes_read;
}
return Status::OK();