PalfBufferIterator support return padding log.

This commit is contained in:
HaHaJeff
2023-04-28 03:41:40 +00:00
committed by ob-robot
parent 64dc54c87e
commit 58bb3d34b7
21 changed files with 887 additions and 332 deletions

View File

@ -76,7 +76,6 @@ TEST(TestLogChecksum, test_log_checksum)
EXPECT_TRUE(parity_check(v1));
EXPECT_FALSE(parity_check(v2));
}
}
}

View File

@ -14,12 +14,16 @@
#include "lib/ob_errno.h"
#include "lib/net/ob_addr.h" // ObAddr
#include "logservice/palf/log_define.h"
#include "logservice/palf/log_group_entry.h"
#include "logservice/palf/log_writer_utils.h"
#include "lib/checksum/ob_crc64.h" // ob_crc64
#define private public
#include "logservice/palf/log_group_entry_header.h"
#include "logservice/palf/log_entry.h"
#include "share/scn.h"
#include "logservice/ob_log_base_header.h" // ObLogBaseHeader
#include "logservice/palf/log_group_buffer.h"
#include "logservice/palf/log_group_entry.h"
#include "logservice/palf/log_writer_utils.h"
#include "share/rc/ob_tenant_base.h"
#undef private
#include <gtest/gtest.h>
@ -220,14 +224,199 @@ TEST(TestLogGroupEntryHeader, test_log_group_entry_header)
EXPECT_TRUE(log_group_entry2.check_integrity());
}
TEST(TestPaddingLogEntry, test_invalid_padding_log_entry)
{
LogEntryHeader header;
char buf[1024];
EXPECT_EQ(OB_INVALID_ARGUMENT, header.generate_padding_header_(NULL, 1, 1, share::SCN::min_scn()));
EXPECT_EQ(OB_INVALID_ARGUMENT, header.generate_padding_header_(buf, 0, 1, share::SCN::min_scn()));
EXPECT_EQ(OB_INVALID_ARGUMENT, header.generate_padding_header_(buf, 1, 0, share::SCN::min_scn()));
EXPECT_EQ(OB_INVALID_ARGUMENT, header.generate_padding_header_(buf, 1, 1, share::SCN::invalid_scn()));
EXPECT_EQ(OB_SUCCESS, header.generate_padding_header_(buf, 1, 1, share::SCN::min_scn()));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(0, share::SCN::min_scn(), buf, 1));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(1, share::SCN::invalid_scn(), buf, 1));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(1, share::SCN::min_scn(), NULL, 1));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(1, share::SCN::min_scn(), buf, 0));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(1, share::SCN::min_scn(), buf, 2));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(2, share::SCN::min_scn(), buf, 1));
EXPECT_EQ(OB_INVALID_ARGUMENT, LogEntryHeader::generate_padding_log_buf(1, share::SCN::min_scn(), buf, 1));
logservice::ObLogBaseHeader base_header;
const int64_t min_padding_valid_data_len = header.get_serialize_size() + base_header.get_serialize_size();
EXPECT_EQ(OB_SUCCESS, LogEntryHeader::generate_padding_log_buf(1+min_padding_valid_data_len, share::SCN::min_scn(), buf, min_padding_valid_data_len));
}
TEST(TestPaddingLogEntry, test_padding_log_entry)
{
PALF_LOG(INFO, "test_padding_log_entry");
LogEntry padding_log_entry;
const int64_t padding_data_len = MAX_LOG_BODY_SIZE;
share::SCN padding_group_scn;
padding_group_scn.convert_for_logservice(ObTimeUtility::current_time_ns());
LogEntryHeader padding_log_entry_header;
char base_header_data[1024] = {'\0'};
logservice::ObLogBaseHeader base_header(logservice::ObLogBaseType::PADDING_LOG_BASE_TYPE,
logservice::ObReplayBarrierType::NO_NEED_BARRIER);
int64_t serialize_pos = 0;
EXPECT_EQ(OB_SUCCESS, base_header.serialize(base_header_data, 1024, serialize_pos));
// 生成padding log entry的有效数据
EXPECT_EQ(OB_SUCCESS, padding_log_entry_header.generate_padding_header_(
base_header_data, base_header.get_serialize_size(),
padding_data_len-LogEntryHeader::HEADER_SER_SIZE, padding_group_scn));
EXPECT_EQ(true, padding_log_entry_header.check_integrity(base_header_data, padding_data_len));
// padding group log format
// | GroupHeader | EntryHeader | BaseHeader | '\0' |
LogGroupEntry padding_group_entry;
LogGroupEntryHeader padding_group_entry_header;
const int64_t padding_buffer_len = MAX_LOG_BUFFER_SIZE;
char *padding_buffer = reinterpret_cast<char *>(ob_malloc(padding_buffer_len, "unittest"));
ASSERT_NE(nullptr, padding_buffer);
memset(padding_buffer, PADDING_LOG_CONTENT_CHAR, padding_buffer_len);
{
// 将base_data的数据拷贝到对应位置
memcpy(padding_buffer+padding_group_entry_header.get_serialize_size() + LogEntryHeader::HEADER_SER_SIZE, base_header_data, 1024);
padding_log_entry.header_ = padding_log_entry_header;
padding_log_entry.buf_ = padding_buffer+padding_group_entry_header.get_serialize_size() + LogEntryHeader::HEADER_SER_SIZE;
// 构造有效的padding_log_entry,用于后续的序列化操作
EXPECT_EQ(true, padding_log_entry.check_integrity());
EXPECT_EQ(true, padding_log_entry.header_.is_padding_log_());
}
{
LogEntry deserialize_padding_log_entry;
const int64_t tmp_padding_buffer_len = MAX_LOG_BUFFER_SIZE;
char *tmp_padding_buffer = reinterpret_cast<char *>(ob_malloc(tmp_padding_buffer_len, "unittest"));
ASSERT_NE(nullptr, tmp_padding_buffer);
int64_t pos = 0;
EXPECT_EQ(OB_SUCCESS, padding_log_entry.serialize(tmp_padding_buffer, tmp_padding_buffer_len, pos));
pos = 0;
EXPECT_EQ(OB_SUCCESS, deserialize_padding_log_entry.deserialize(tmp_padding_buffer, tmp_padding_buffer_len, pos));
EXPECT_EQ(true, deserialize_padding_log_entry.check_integrity());
EXPECT_EQ(padding_log_entry.header_.data_checksum_, deserialize_padding_log_entry.header_.data_checksum_);
ob_free(tmp_padding_buffer);
tmp_padding_buffer = nullptr;
}
serialize_pos = padding_group_entry_header.get_serialize_size();
// 拷贝LogEntry到padding_buffer指定位置
EXPECT_EQ(OB_SUCCESS, padding_log_entry.serialize(padding_buffer, padding_buffer_len, serialize_pos));
LogWriteBuf write_buf;
EXPECT_EQ(OB_SUCCESS, write_buf.push_back(padding_buffer, padding_buffer_len));
bool is_raw_write = false;
bool is_padding_log = true;
int64_t data_checksum = 0;
EXPECT_EQ(OB_SUCCESS, padding_group_entry_header.generate(is_raw_write, is_padding_log, write_buf, padding_data_len, padding_group_scn, 1, LSN(0), 1, data_checksum));
padding_group_entry_header.update_accumulated_checksum(0);
padding_group_entry_header.update_header_checksum();
padding_group_entry.header_ = padding_group_entry_header;
padding_group_entry.buf_ = padding_buffer + padding_group_entry_header.get_serialize_size();
EXPECT_EQ(true, padding_group_entry.check_integrity());
EXPECT_EQ(true, padding_group_entry.header_.is_padding_log());
// 验证反序列化LogEntry
{
int64_t pos = 0;
LogEntry tmp_padding_log_entry;
EXPECT_EQ(OB_SUCCESS, tmp_padding_log_entry.deserialize(padding_group_entry.buf_, padding_group_entry.get_data_len(), pos));
EXPECT_EQ(pos, padding_group_entry.get_data_len());
EXPECT_EQ(true, tmp_padding_log_entry.check_integrity());
EXPECT_EQ(true, tmp_padding_log_entry.header_.is_padding_log_());
logservice::ObLogBaseHeader tmp_base_header;
pos = 0;
EXPECT_EQ(OB_SUCCESS, tmp_base_header.deserialize(tmp_padding_log_entry.buf_, tmp_padding_log_entry.get_data_len(), pos));
EXPECT_EQ(base_header.log_type_, logservice::ObLogBaseType::PADDING_LOG_BASE_TYPE);
}
char *serialize_buffer = reinterpret_cast<char *>(ob_malloc(padding_buffer_len, "unittest"));
ASSERT_NE(nullptr, serialize_buffer);
memset(serialize_buffer, PADDING_LOG_CONTENT_CHAR, padding_buffer_len);
serialize_pos = 0;
// 验证序列化的数据是否符合预期
EXPECT_EQ(OB_SUCCESS, padding_group_entry.serialize(serialize_buffer, padding_buffer_len, serialize_pos));
EXPECT_EQ(serialize_pos, padding_data_len+padding_group_entry_header.get_serialize_size());
LogGroupEntry deserialize_group_entry;
serialize_pos = 0;
EXPECT_EQ(OB_SUCCESS, deserialize_group_entry.deserialize(serialize_buffer, padding_buffer_len, serialize_pos));
EXPECT_EQ(true, deserialize_group_entry.check_integrity());
EXPECT_EQ(true, deserialize_group_entry.header_.is_padding_log());
EXPECT_EQ(padding_group_entry.header_, deserialize_group_entry.header_);
// 验证反序列化LogEntry
{
int64_t pos = 0;
LogEntry tmp_padding_log_entry;
EXPECT_EQ(OB_SUCCESS, tmp_padding_log_entry.deserialize(deserialize_group_entry.buf_, padding_group_entry.get_data_len(), pos));
EXPECT_EQ(pos, deserialize_group_entry.get_data_len());
EXPECT_EQ(true, tmp_padding_log_entry.check_integrity());
EXPECT_EQ(true, tmp_padding_log_entry.header_.is_padding_log_());
logservice::ObLogBaseHeader tmp_base_header;
pos = 0;
EXPECT_EQ(OB_SUCCESS, tmp_base_header.deserialize(tmp_padding_log_entry.buf_, tmp_padding_log_entry.get_data_len(), pos));
EXPECT_EQ(base_header.log_type_, logservice::ObLogBaseType::PADDING_LOG_BASE_TYPE);
}
LogGroupBuffer group_buffer;
LSN start_lsn(0);
ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(1001);
// init MTL
share::ObTenantBase tbase(1001);
share::ObTenantEnv::set_tenant(&tbase);
EXPECT_EQ(OB_SUCCESS, group_buffer.init(start_lsn));
const int64_t padding_valid_data_len = deserialize_group_entry.get_header().get_serialize_size() + padding_log_entry_header.get_serialize_size() + base_header.get_serialize_size();
// 填充有效的padding日志到group buffer,验证数据是否相等
// padding_buffer包括LogGruopEntryHeader, LogEntryHeader, ObLogBaseHeader, padding log body(is filled with '\0')
EXPECT_EQ(OB_SUCCESS, group_buffer.fill_padding_body(start_lsn, serialize_buffer, padding_valid_data_len, padding_buffer_len));
EXPECT_EQ(0, memcmp(group_buffer.data_buf_, serialize_buffer, deserialize_group_entry.get_serialize_size()));
PALF_LOG(INFO, "runlin trace", K(group_buffer.data_buf_), K(serialize_buffer), K(padding_buffer_len), KP(group_buffer.data_buf_), KP(padding_buffer));
ob_free(padding_buffer);
padding_buffer = NULL;
ob_free(serialize_buffer);
serialize_buffer = NULL;
group_buffer.destroy();
ObMallocAllocator::get_instance()->recycle_tenant_allocator(1001);
}
TEST(TestPaddingLogEntry, test_generate_padding_log_entry)
{
PALF_LOG(INFO, "test_generate_padding_log_entry");
LogEntry padding_log_entry;
const int64_t padding_data_len = 1024;
const share::SCN padding_scn = share::SCN::min_scn();
const int64_t padding_log_entry_len = padding_data_len + LogEntryHeader::HEADER_SER_SIZE;
char *out_buf = reinterpret_cast<char*>(ob_malloc(padding_log_entry_len, "unittest"));
ASSERT_NE(nullptr, out_buf);
LogEntryHeader padding_header;
logservice::ObLogBaseHeader base_header(logservice::ObLogBaseType::PADDING_LOG_BASE_TYPE, logservice::ObReplayBarrierType::NO_NEED_BARRIER, 0);
char base_header_buf[1024];
memset(base_header_buf, 0, 1024);
int64_t serialize_base_header_pos = 0;
EXPECT_EQ(OB_SUCCESS, base_header.serialize(base_header_buf, 1024, serialize_base_header_pos));
EXPECT_EQ(OB_SUCCESS, padding_header.generate_padding_header_(base_header_buf, base_header.get_serialize_size(), padding_data_len, padding_scn));
EXPECT_EQ(true, padding_header.check_header_integrity());
EXPECT_EQ(OB_SUCCESS, LogEntryHeader::generate_padding_log_buf(padding_data_len, padding_scn, out_buf, LogEntryHeader::PADDING_LOG_ENTRY_SIZE));
int64_t pos = 0;
EXPECT_EQ(OB_SUCCESS, padding_log_entry.deserialize(out_buf, padding_log_entry_len, pos));
EXPECT_EQ(true, padding_log_entry.check_integrity());
EXPECT_EQ(true, padding_log_entry.header_.is_padding_log_());
EXPECT_EQ(padding_log_entry.header_.data_checksum_, padding_header.data_checksum_);
ob_free(out_buf);
out_buf = nullptr;
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_log_entry_and_group_entry.log");
OB_LOGGER.set_file_name("test_log_entry_and_group_entry.log", true);
OB_LOGGER.set_log_level("INFO");
PALF_LOG(INFO, "begin unittest::test_log_entry_and_grou_entry");
PALF_LOG(INFO, "begin unittest::test_log_entry_and_group_entry");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

View File

@ -18,6 +18,7 @@
#define private public
#include "logservice/palf/log_group_buffer.h"
#include "logservice/palf/log_writer_utils.h"
#include "logservice/palf/log_entry_header.h"
#undef private
#include "share/rc/ob_tenant_base.h"
@ -63,6 +64,7 @@ void TestLogGroupBuffer::TearDown()
{
PALF_LOG(INFO, "TestLogGroupBuffer has TearDown");
PALF_LOG(INFO, "TearDown success");
log_group_buffer_.destroy();
ObMallocAllocator::get_instance()->recycle_tenant_allocator(1001);
}
@ -170,35 +172,104 @@ TEST_F(TestLogGroupBuffer, test_fill)
TEST_F(TestLogGroupBuffer, test_fill_padding)
{
LSN lsn;
const int64_t padding_valid_data_len = LogEntryHeader::PADDING_LOG_ENTRY_SIZE;
char padding_valid_data[padding_valid_data_len];
int64_t len = 0;
LSN reuse_lsn(1024);
EXPECT_EQ(OB_NOT_INIT, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_NOT_INIT, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
LSN start_lsn(100);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.init(start_lsn));
EXPECT_EQ(OB_INVALID_ARGUMENT, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_INVALID_ARGUMENT, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
lsn = reuse_lsn;
EXPECT_EQ(OB_INVALID_ARGUMENT, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_INVALID_ARGUMENT, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_INVALID_ARGUMENT, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
len = 100;
lsn.val_ = start_lsn.val_ - 1;
EXPECT_EQ(OB_ERR_UNEXPECTED, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_ERR_UNEXPECTED, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
lsn.val_ = start_lsn.val_;
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.inc_update_reuse_lsn(reuse_lsn));
EXPECT_EQ(OB_ERR_UNEXPECTED, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_ERR_UNEXPECTED, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
lsn = reuse_lsn;
len = log_group_buffer_.get_available_buffer_size() + 1;
EXPECT_EQ(OB_EAGAIN, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_EAGAIN, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
len = 1024;
int64_t used_size = len;
const int64_t buf_size = log_group_buffer_.get_available_buffer_size();
LSN buf_end_lsn = reuse_lsn + (buf_size - (reuse_lsn.val_ - start_lsn.val_));
while (lsn + len < buf_end_lsn) {
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill_padding_body(lsn, len));
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill_padding_body(lsn, padding_valid_data, padding_valid_data_len, len));
lsn.val_ += len;
}
EXPECT_GT(lsn + len, buf_end_lsn);
}
TEST_F(TestLogGroupBuffer, test_fill_padding_cross_bround)
{
const int64_t padding_valid_data_len = LogEntryHeader::PADDING_LOG_ENTRY_SIZE;
char padding_valid_data[padding_valid_data_len];
memset(padding_valid_data, 'c', padding_valid_data_len);
int64_t len = 0;
LSN start_lsn(0);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.init(start_lsn));
const int64_t unittest_log_buffer_size = 4 * 1024 * 1024;
log_group_buffer_.available_buffer_size_ = log_group_buffer_.reserved_buffer_size_ = unittest_log_buffer_size;
// LSN为0,提交3条1M日志
const int64_t log_size = 1*1024*1024;
char *buf = reinterpret_cast<char*>(ob_malloc(1*1024*1024, "unittest"));
ASSERT_NE(nullptr, buf);
LSN lsn0(0);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill(lsn0, buf, log_size));
LSN lsn1(1*1024*1024);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill(lsn1, buf, log_size));
LSN lsn2(2*1024*1024);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill(lsn2, buf, log_size));
// 更新reuse lsn为1M, 继续提交2M的padding日志
LSN lsn3(3*1024*1024);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.inc_update_reuse_lsn(lsn1));
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill_padding_body(lsn3, padding_valid_data,
padding_valid_data_len, 2*log_size));
LSN lsn4(5*1024*1024);
// 预期已经没有可用空间
EXPECT_EQ(OB_EAGAIN, log_group_buffer_.fill(lsn4, buf, 1));
// |5.5|2|3|4|5.0|
// 判断5.0的数据是否是padding, 5号日志为2M, 其余日志为1M
EXPECT_EQ(0, memcmp(padding_valid_data, log_group_buffer_.data_buf_+3*1024*1024, padding_valid_data_len));
char *zero_data = reinterpret_cast<char*>(ob_malloc(2*log_size, "unittest"));
ASSERT_NE(nullptr, zero_data);
memset(zero_data, PADDING_LOG_CONTENT_CHAR, 2*1024*1024);
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_+3*1024*1024+1024, zero_data, 1*1024*1024-1*1024));
// 更新reuse_lsn为4M-LogEntryHeader::PADDING_LOG_ENTRY_SIZE,此时正好可以将padding日志的有效日志体放在group_buffer尾部
LSN reuse_lsn = LSN(unittest_log_buffer_size - LogEntryHeader::PADDING_LOG_ENTRY_SIZE);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.inc_update_reuse_lsn(reuse_lsn));
LSN lsn5(reuse_lsn);
// 提交2M的padding日志,有效日志长度为padding_valid_data_len
// 2.x表示2号日志被复写部分
// |5.x|2.x|3|4|5.0|, 5.0开始的地方为reuse_lsn
// 故意将5.x的一些数据搞坏,验证fill_padding_body是否会清0
memset(log_group_buffer_.data_buf_, 'x', 1*1024);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill_padding_body(lsn5, padding_valid_data, padding_valid_data_len, 2*log_size));
// 预期log_group_buffer的尾部数据恰好和padding_valid_data一样
// log_group_buffer的padding剩余数据和padding_valid_data一样
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_+lsn5.val_, padding_valid_data, LogEntryHeader::PADDING_LOG_ENTRY_SIZE));
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_, padding_valid_data+LogEntryHeader::PADDING_LOG_ENTRY_SIZE, padding_valid_data_len-LogEntryHeader::PADDING_LOG_ENTRY_SIZE));
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_+LogEntryHeader::PADDING_LOG_ENTRY_SIZE, zero_data, 2*log_size-padding_valid_data_len));
// 更新reuse_lsn为4M-LogEntryHeader::PADDING_LOG_ENTRY_SIZE+10,此时padding日志的有效日志体需要放在group_buffer的首尾
memset(log_group_buffer_.data_buf_, 'x', 1*1024);
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.inc_update_reuse_lsn(reuse_lsn+10));
LSN lsn6(reuse_lsn+10);
const int64_t tail_padding_log_size = LogEntryHeader::PADDING_LOG_ENTRY_SIZE - 10;
EXPECT_EQ(OB_SUCCESS, log_group_buffer_.fill_padding_body(lsn6, padding_valid_data, padding_valid_data_len, 2*log_size));
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_+lsn6.val_, padding_valid_data, tail_padding_log_size));
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_, padding_valid_data+tail_padding_log_size, padding_valid_data_len-tail_padding_log_size));
EXPECT_EQ(0, memcmp(log_group_buffer_.data_buf_+padding_valid_data_len-tail_padding_log_size, zero_data, 2*log_size-padding_valid_data_len));
ob_free(buf);
buf = nullptr;
ob_free(zero_data);
zero_data = nullptr;
}
TEST_F(TestLogGroupBuffer, test_check_log_buf_wrapped)
{
LSN lsn;