hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] add direct load function

Browse Source 
Co-authored-by: Monk-Liu <1152761042@qq.com>
Co-authored-by: saltonz <saltonzh@gmail.com>
Co-authored-by: yongshige <598633031@qq.com>
This commit is contained in:
obdev
2023-01-28 18:08:50 +08:00
committed by ob-robot
parent f27d2efc83
commit 81d28c0295
384 changed files with 55860 additions and 1239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
src/storage/blocksstable/ob_tmp_file.cpp
View File

@ -48,20 +48,20 @@ bool ObTmpFileIOInfo::is_valid() const
}
ObTmpFileIOHandle::ObTmpFileIOHandle()
: io_handles_(),
page_cache_handles_(),
block_cache_handles_(),
: is_read_(false),
has_wait_(false),
update_offset_in_file_(false),
fd_(OB_INVALID_FD),
dir_id_(OB_INVALID_ID),
tenant_id_(OB_INVALID_TENANT_ID),
buf_(NULL),
size_(0),
is_read_(false),
has_wait_(false),
expect_read_size_(0),
last_read_offset_(-1),
tenant_id_(OB_INVALID_TENANT_ID),
buf_(NULL),
io_flag_(),
update_offset_in_file_(false)
io_handles_(),
page_cache_handles_(),
block_cache_handles_()
{
}
@ -78,7 +78,7 @@ int ObTmpFileIOHandle::prepare_read(
ObTmpFile *file)
{
int ret = OB_SUCCESS;
if (NULL == read_buf || NULL == file) {
if (OB_ISNULL(read_buf) ||OB_ISNULL(file)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), KP_(buf));
} else {
@ -99,7 +99,7 @@ int ObTmpFileIOHandle::prepare_read(
int ObTmpFileIOHandle::prepare_write(char *write_buf, const int64_t write_size, ObTmpFile *file)
{
int ret = OB_SUCCESS;
if (NULL == write_buf || NULL == file) {
if (OB_ISNULL(write_buf) || OB_ISNULL(file)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), KP_(buf));
} else {
@ -353,9 +353,17 @@ void ObTmpFileExtent::get_global_offset(int64_t &g_offset_start, int64_t &g_offs
}
ObTmpFileExtent::ObTmpFileExtent(ObTmpFile *file)
: is_alloced_(false), fd_(file->get_fd()), g_offset_start_(0), g_offset_end_(0),
owner_(file), start_page_id_(-1), page_nums_(0), block_id_(-1), offset_(0),
lock_(common::ObLatchIds::TMP_FILE_EXTENT_LOCK), is_closed_(false)
: is_alloced_(false),
is_closed_(false),
start_page_id_(-1),
page_nums_(0),
offset_(0),
fd_(file->get_fd()),
g_offset_start_(0),
g_offset_end_(0),
owner_(file),
block_id_(-1),
lock_(common::ObLatchIds::TMP_FILE_EXTENT_LOCK)
{
}
@ -370,8 +378,8 @@ int ObTmpFileExtent::read(const ObTmpFileIOInfo &io_info, const int64_t offset,
if (OB_UNLIKELY(!is_alloced_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ObTmpFileExtent has not been allocated", K(ret));
} else if (offset < 0 || offset >= offset_ || size <= 0
|| offset + size > offset_ || NULL == buf) {
} else if (OB_UNLIKELY(offset < 0 || offset >= offset_ || size <= 0
|| offset + size > offset_) || OB_ISNULL(buf)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(offset), K(offset_), K(size), K(buf));
} else {
@ -397,7 +405,7 @@ int ObTmpFileExtent::write(const ObTmpFileIOInfo &io_info,int64_t &size, char *&
int write_size = 0;
int64_t remain = 0;
bool is_write = false;
if (size <= 0 || NULL == buf) {
if (OB_UNLIKELY(size <= 0) || OB_ISNULL(buf)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret));
} else if (OB_UNLIKELY(!is_alloced_)) {
@ -459,13 +467,14 @@ bool ObTmpFileExtent::is_valid()
bool ObTmpFileExtent::close(bool force)
{
int ret = OB_SUCCESS;
int32_t page_start_id = -1;
int32_t page_nums = 0;
uint8_t page_start_id = ObTmpFilePageBuddy::MAX_PAGE_NUMS;
uint8_t page_nums = 0;
if (!is_closed_) {
if (close(page_start_id, page_nums, force)) {
if (-1 == page_start_id && 0 == page_nums) {
if (ObTmpFilePageBuddy::MAX_PAGE_NUMS == page_start_id && 0 == page_nums) {
//nothing to do
} else if (page_start_id < 0 || page_nums < 0) {
} else if (OB_UNLIKELY(page_start_id > ObTmpFilePageBuddy::MAX_PAGE_NUMS - 1
|| page_nums > ObTmpFilePageBuddy::MAX_PAGE_NUMS)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "fail to close the extent", K(ret), K_(block_id), K(page_start_id),
K(page_nums), K_(offset));
@ -481,9 +490,9 @@ bool ObTmpFileExtent::close(bool force)
return is_closed_;
}
bool ObTmpFileExtent::close(int32_t &free_page_start_id, int32_t &free_page_nums, bool force)
bool ObTmpFileExtent::close(uint8_t &free_page_start_id, uint8_t &free_page_nums, bool force)
{
free_page_start_id = -1;
free_page_start_id = ObTmpFilePageBuddy::MAX_PAGE_NUMS;
free_page_nums = 0;
SpinWLockGuard guard(lock_);
if (!is_closed_) {
@ -491,7 +500,7 @@ bool ObTmpFileExtent::close(int32_t &free_page_start_id, int32_t &free_page_nums
// Nothing to do. This extent is alloced just now, so it cannot be closed.
} else {
if (offset_ != page_nums_ * ObTmpMacroBlock::get_default_page_size()) {
int32_t offset_page_id = common::upper_align(offset_, ObTmpMacroBlock::get_default_page_size())
uint8_t offset_page_id = common::upper_align(offset_, ObTmpMacroBlock::get_default_page_size())
/ ObTmpMacroBlock::get_default_page_size();
free_page_nums = page_nums_ - offset_page_id;
free_page_start_id = start_page_id_ + offset_page_id;
@ -520,7 +529,7 @@ ObTmpFileMeta::~ObTmpFileMeta()
int ObTmpFileMeta::init(const int64_t fd, const int64_t dir_id, common::ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (fd < 0 || dir_id < 0 || NULL == allocator) {
if (OB_UNLIKELY(fd < 0 || dir_id < 0) || OB_ISNULL(allocator)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(fd), K(dir_id));
} else {
@ -580,17 +589,17 @@ int ObTmpFileMeta::clear()
}
ObTmpFile::ObTmpFile()
: file_meta_(),
: is_inited_(false),
is_big_(false),
tenant_id_(-1),
offset_(0),
allocator_(NULL),
last_extent_id_(0),
last_extent_min_offset_(0),
last_extent_max_offset_(INT64_MAX),
offset_(0),
tenant_id_(-1),
extent_idx_lock_(common::ObLatchIds::TMP_FILE_LOCK),
lock_(common::ObLatchIds::TMP_FILE_LOCK),
is_inited_(false)
allocator_(NULL),
file_meta_()
{
}
@ -622,7 +631,7 @@ int ObTmpFile::clear()
int ObTmpFile::init(const int64_t fd, const int64_t dir_id, common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else if (fd < 0) {
@ -719,7 +728,7 @@ int ObTmpFile::once_aio_read_batch(
const int64_t remain_size = io_info.size_ - handle.get_data_size();
SpinWLockGuard guard(lock_);
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been initialized", K(ret));
} else if (OB_UNLIKELY(offset < 0 || remain_size < 0) || OB_ISNULL(io_info.buf_)) {
@ -794,7 +803,7 @@ int ObTmpFile::once_aio_read_batch_without_lock(
int ObTmpFile::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else {
@ -814,7 +823,7 @@ int ObTmpFile::aio_pread(const ObTmpFileIOInfo &io_info, const int64_t offset,
ObTmpFileIOHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else {
@ -833,7 +842,7 @@ int ObTmpFile::seek(const int64_t offset, const int whence)
{
int ret = OB_SUCCESS;
SpinWLockGuard guard(lock_);
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else {
@ -878,7 +887,7 @@ int ObTmpFile::pread(const ObTmpFileIOInfo &io_info, const int64_t offset, const
ObTmpFileIOHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else if (OB_FAIL(aio_pread(io_info, offset, handle))) {
@ -903,7 +912,7 @@ int ObTmpFile::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &hand
{
// only support append at present.
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -943,8 +952,8 @@ int ObTmpFile::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &hand
} else {
alloc_size = common::upper_align(size, big_file_prealloc_size());
}
} else if (size > OB_TMP_FILE_STORE.get_block_size()) {
alloc_size = OB_TMP_FILE_STORE.get_block_size();
} else if (size > ObTmpMacroBlock::get_block_size()) {
alloc_size = ObTmpMacroBlock::get_block_size();
} else {
alloc_size = size;
}
@ -1016,16 +1025,39 @@ int ObTmpFile::sync(const int64_t timeout_ms)
int ret = OB_SUCCESS;
SpinWLockGuard guard(lock_);
ObTmpFileExtent *tmp = file_meta_.get_last_extent();
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFile has not been inited", K(ret));
} else if (NULL == tmp) {
} else if (OB_ISNULL(tmp)) {
ret = OB_BAD_NULL_ERROR;
STORAGE_LOG(WARN, "the file does not have a extent", K(timeout_ms));
} else if (tmp->is_closed()) {
STORAGE_LOG(INFO, "the file has been closed", K(timeout_ms));
STORAGE_LOG(WARN, "the file does not have a extent", K(ret), K(timeout_ms));
} else {
// TODO : add timeout implementation
tmp->close(true/*force*/);
// all extents has been closed.
const ObIArray<ObTmpFileExtent *> &extents = file_meta_.get_extents();
common::hash::ObHashSet<int64_t> blk_id_set;
if (OB_FAIL(blk_id_set.create(extents.count()))){
STORAGE_LOG(WARN, "create block id set failed", K(ret), K(timeout_ms));
} else {
// get extents block id set.
for (int64_t i=0; OB_SUCC(ret) && i < extents.count(); ++i) {
const ObTmpFileExtent* e = extents.at(i);
const int64_t &blk_id = e->get_block_id();
if (OB_FAIL(blk_id_set. set_refactored(blk_id))) {
STORAGE_LOG(WARN, "add block id to set failed", K(ret), K(blk_id));
}
}
// iter all blocks, execute sync
common::hash::ObHashSet<int64_t>::const_iterator iter;
for (iter = blk_id_set.begin(); OB_SUCC(ret) && iter != blk_id_set.end(); ++iter) {
const int64_t &blk_id = iter->first;
if (OB_FAIL(OB_TMP_FILE_STORE.sync(tenant_id_, blk_id))) {
STORAGE_LOG(WARN, "sync block failed", K(ret), K(blk_id));
}
}
}
}
return ret;
}
@ -1068,7 +1100,7 @@ int ObTmpFile::write_file_extent(const ObTmpFileIOInfo &io_info, ObTmpFileExtent
int64_t &size, char *&buf)
{
int ret = OB_SUCCESS;
if (NULL == file_extent || size <= 0 || NULL == buf) {
if (OB_UNLIKELY(size <= 0)|| OB_ISNULL(file_extent) || OB_ISNULL(buf)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret));
} else if (OB_FAIL(file_extent->write(io_info, size, buf))) {
@ -1131,7 +1163,7 @@ ObTmpFileManager &ObTmpFileManager::get_instance()
int ObTmpFileManager::init()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(files_.init(DEFAULT_BUCKET_NUM, OB_SERVER_TENANT_ID,
@ -1153,7 +1185,7 @@ int ObTmpFileManager::init()
int ObTmpFileManager::alloc_dir(int64_t &dir)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(get_next_dir(dir))) {
@ -1189,7 +1221,7 @@ int ObTmpFileManager::open(int64_t &fd, int64_t &dir)
{
int ret = OB_SUCCESS;
ObTmpFile file;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(get_next_fd(fd))) {
@ -1228,7 +1260,7 @@ int ObTmpFileManager::aio_read(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
handle.reset();
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -1250,7 +1282,7 @@ int ObTmpFileManager::aio_pread(const ObTmpFileIOInfo &io_info, const int64_t of
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
handle.reset();
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -1272,7 +1304,7 @@ int ObTmpFileManager::read(const ObTmpFileIOInfo &io_info, const int64_t timeout
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
handle.reset();
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -1294,7 +1326,7 @@ int ObTmpFileManager::pread(const ObTmpFileIOInfo &io_info, const int64_t offset
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
handle.reset();
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -1315,7 +1347,7 @@ int ObTmpFileManager::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandl
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
handle.reset();
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -1333,7 +1365,7 @@ int ObTmpFileManager::write(const ObTmpFileIOInfo &io_info, const int64_t timeou
{
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (!io_info.is_valid()) {
@ -1351,7 +1383,7 @@ int ObTmpFileManager::seek(const int64_t fd, const int64_t offset, const int whe
{
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(files_.get(fd, file_handle))) {
@ -1377,7 +1409,7 @@ int ObTmpFileManager::get_tmp_file_handle(const int64_t fd, ObTmpFileHandle &han
int ObTmpFileManager::remove(const int64_t fd)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else{
@ -1405,11 +1437,11 @@ int ObTmpFileManager::remove_tenant_file(const uint64_t tenant_id)
int ret = OB_SUCCESS;
common::ObSEArray<int64_t, 32> fd_list;
RmTenantTmpFileOp rm_tenant_file_op(tenant_id, &fd_list);
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) {
ret = common::OB_INVALID_ARGUMENT;
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id));
} else if (OB_FAIL(files_.foreach(rm_tenant_file_op))) {
STORAGE_LOG(WARN, "fail to foreach files_", K(ret), K(tenant_id));
@ -1435,7 +1467,7 @@ int ObTmpFileManager::remove_tenant_file(const uint64_t tenant_id)
int ObTmpFileManager::get_all_tenant_id(common::ObIArray<uint64_t> &tenant_ids)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(OB_TMP_FILE_STORE.get_all_tenant_id(tenant_ids))) {
@ -1448,7 +1480,7 @@ int ObTmpFileManager::sync(const int64_t fd, const int64_t timeout_ms)
{
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(files_.get(fd, file_handle))) {
@ -1460,8 +1492,11 @@ int ObTmpFileManager::sync(const int64_t fd, const int64_t timeout_ms)
}
ObTmpFileManager::ObTmpFileManager()
: files_(), next_fd_(-1), next_dir_(-1),
rm_file_lock_(common::ObLatchIds::TMP_FILE_MGR_LOCK), is_inited_(false)
: is_inited_(false),
next_fd_(-1),
next_dir_(-1),
rm_file_lock_(common::ObLatchIds::TMP_FILE_MGR_LOCK),
files_()
{
}
@ -1482,7 +1517,7 @@ void ObTmpFileManager::destroy()
int ObTmpFileManager::dec_handle_ref(ObTmpFileHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (OB_FAIL(files_.dec_handle_ref(handle.ptr_))) {
@ -1495,7 +1530,7 @@ int ObTmpFileManager::get_tmp_file_size(const int64_t fd, int64_t &file_size)
{
int ret = OB_SUCCESS;
ObTmpFileHandle file_handle;
if (OB_UNLIKELY(!is_inited_)) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret));
} else if (fd < 0) {