hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

support strict replace

Browse Source
This commit is contained in:
yongshige
2023-08-08 05:12:15 +00:00
committed by ob-robot
parent 51bb12a3f6
commit a1fa978f84
45 changed files with 559 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
src/storage/direct_load/ob_direct_load_compare.cpp
View File

@ -28,8 +28,23 @@ int ObDirectLoadDatumRowkeyCompare::init(const ObStorageDatumUtils &datum_utils)
return ret;
}
bool ObDirectLoadDatumRowkeyCompare::operator()(const ObDatumRowkey *lhs,
const ObDatumRowkey *rhs)
int ObDirectLoadDatumRowkeyCompare::compare(const ObDatumRowkey *lhs, const ObDatumRowkey *rhs,
int &cmp_ret)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(datum_utils_) || OB_ISNULL(lhs) || OB_ISNULL(rhs) ||
OB_UNLIKELY(!lhs->is_valid() || !rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(datum_utils_), KP(lhs), KP(rhs));
} else {
if (OB_FAIL(lhs->compare(*rhs, *datum_utils_, cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(datum_utils_));
}
}
return ret;
}
bool ObDirectLoadDatumRowkeyCompare::operator()(const ObDatumRowkey *lhs, const ObDatumRowkey *rhs)
{
int ret = OB_SUCCESS;
int cmp_ret = 0;
@ -73,6 +88,33 @@ int ObDirectLoadDatumRowCompare::init(const ObStorageDatumUtils &datum_utils, in
return ret;
}
int ObDirectLoadDatumRowCompare::compare(const blocksstable::ObDatumRow *lhs,
const blocksstable::ObDatumRow *rhs, int &cmp_ret)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadDatumRowCompare not init", KR(ret), KP(this));
} else if (OB_ISNULL(lhs) || OB_ISNULL(rhs) ||
OB_UNLIKELY(!lhs->is_valid() || !rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else if (OB_UNLIKELY(lhs->get_column_count() < rowkey_size_ ||
rhs->get_column_count() < rowkey_size_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected row column cnt", KR(ret), K(lhs), K(rhs), K_(rowkey_size));
} else {
if (OB_FAIL(lhs_rowkey_.assign(lhs->storage_datums_, rowkey_size_))) {
LOG_WARN("Failed to assign datum rowkey", KR(ret), K(lhs), K_(rowkey_size));
} else if (OB_FAIL(rhs_rowkey_.assign(rhs->storage_datums_, rowkey_size_))) {
LOG_WARN("Failed to assign datum rowkey", KR(ret), K(rhs), K_(rowkey_size));
} else if (OB_FAIL(rowkey_compare_.compare(&lhs_rowkey_, &rhs_rowkey_, cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(cmp_ret));
}
}
return ret;
}
bool ObDirectLoadDatumRowCompare::operator()(const ObDatumRow *lhs, const ObDatumRow *rhs)
{
int ret = OB_SUCCESS;
@ -125,6 +167,29 @@ int ObDirectLoadDatumArrayCompare::init(const ObStorageDatumUtils &datum_utils)
return ret;
}
int ObDirectLoadDatumArrayCompare::compare(const ObDirectLoadDatumArray *lhs,
const ObDirectLoadDatumArray *rhs, int &cmp_ret)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadDatumArrayCompare not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == lhs || nullptr == rhs || !lhs->is_valid() || !rhs->is_valid() ||
lhs->count_ != rhs->count_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else if (lhs->count_ > 0) {
if (OB_FAIL(lhs_rowkey_.assign(lhs->datums_, lhs->count_))) {
LOG_WARN("fail to assign rowkey", KR(ret));
} else if (OB_FAIL(rhs_rowkey_.assign(rhs->datums_, rhs->count_))) {
LOG_WARN("fail to assign rowkey", KR(ret));
} else if (OB_FAIL(rowkey_compare_.compare(&lhs_rowkey_, &rhs_rowkey_, cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(cmp_ret));
}
}
return ret;
}
bool ObDirectLoadDatumArrayCompare::operator()(const ObDirectLoadDatumArray *lhs,
const ObDirectLoadDatumArray *rhs)
{
@ -154,6 +219,29 @@ bool ObDirectLoadDatumArrayCompare::operator()(const ObDirectLoadDatumArray *lhs
return bret;
}
int ObDirectLoadDatumArrayCompare::compare(const ObDirectLoadConstDatumArray *lhs,
const ObDirectLoadConstDatumArray *rhs, int &cmp_ret)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadDatumArrayCompare not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == lhs || nullptr == rhs || !lhs->is_valid() || !rhs->is_valid() ||
lhs->count_ != rhs->count_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else if (lhs->count_ > 0) {
if (OB_FAIL(lhs_rowkey_.assign(lhs->datums_, lhs->count_))) {
LOG_WARN("fail to assign rowkey", KR(ret));
} else if (OB_FAIL(rhs_rowkey_.assign(rhs->datums_, rhs->count_))) {
LOG_WARN("fail to assign rowkey", KR(ret));
} else if (OB_FAIL(rowkey_compare_.compare(&lhs_rowkey_, &rhs_rowkey_, cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(cmp_ret));
}
}
return ret;
}
bool ObDirectLoadDatumArrayCompare::operator()(const ObDirectLoadConstDatumArray *lhs,
const ObDirectLoadConstDatumArray *rhs)
{
@ -187,7 +275,8 @@ bool ObDirectLoadDatumArrayCompare::operator()(const ObDirectLoadConstDatumArray
* ObDirectLoadExternalRowCompare
*/
int ObDirectLoadExternalRowCompare::init(const ObStorageDatumUtils &datum_utils)
int ObDirectLoadExternalRowCompare::init(const ObStorageDatumUtils &datum_utils,
sql::ObLoadDupActionType dup_action)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
@ -197,6 +286,7 @@ int ObDirectLoadExternalRowCompare::init(const ObStorageDatumUtils &datum_utils)
if (OB_FAIL(datum_array_compare_.init(datum_utils))) {
LOG_WARN("fail to init datum array compare", KR(ret));
} else {
dup_action_ = dup_action;
is_inited_ = true;
}
}
@ -207,7 +297,7 @@ bool ObDirectLoadExternalRowCompare::operator()(const ObDirectLoadExternalRow *l
const ObDirectLoadExternalRow *rhs)
{
int ret = OB_SUCCESS;
bool bret = false;
int cmp_ret = 0;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadDatumRowCompare not init", KR(ret), KP(this));
@ -215,22 +305,57 @@ bool ObDirectLoadExternalRowCompare::operator()(const ObDirectLoadExternalRow *l
!rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else {
bret = datum_array_compare_.operator()(&lhs->rowkey_datum_array_, &rhs->rowkey_datum_array_);
} else if (OB_FAIL(compare(lhs, rhs, cmp_ret))) {
LOG_WARN("Fail to compare datum array", KR(ret), KP(lhs), KP(rhs));
}
if (OB_FAIL(ret)) {
result_code_ = ret;
} else if (OB_FAIL(datum_array_compare_.get_error_code())) {
result_code_ = datum_array_compare_.get_error_code();
}
return bret;
return cmp_ret < 0;
}
int ObDirectLoadExternalRowCompare::compare(const ObDirectLoadExternalRow *lhs,
const ObDirectLoadExternalRow *rhs, int &cmp_ret)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadDatumRowCompare not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == lhs || nullptr == rhs || !lhs->is_valid() ||
!rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else if (OB_FAIL(datum_array_compare_.compare(&lhs->rowkey_datum_array_,
&rhs->rowkey_datum_array_, cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(cmp_ret));
} else {
if (cmp_ret == 0) {
if (lhs->seq_no_ == rhs->seq_no_) {
cmp_ret = 0;
} else if (lhs->seq_no_ > rhs->seq_no_) {
if (dup_action_ == sql::ObLoadDupActionType::LOAD_REPLACE) {
cmp_ret = -1;
} else {
cmp_ret = 1;
}
} else {
if (dup_action_ == sql::ObLoadDupActionType::LOAD_REPLACE) {
cmp_ret = 1;
} else {
cmp_ret = -1;
}
}
}
}
return ret;
}
/**
* ObDirectLoadExternalMultiPartitionRowCompare
*/
int ObDirectLoadExternalMultiPartitionRowCompare::init(const ObStorageDatumUtils &datum_utils)
int ObDirectLoadExternalMultiPartitionRowCompare::init(const ObStorageDatumUtils &datum_utils,
sql::ObLoadDupActionType dup_action)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
@ -240,6 +365,7 @@ int ObDirectLoadExternalMultiPartitionRowCompare::init(const ObStorageDatumUtils
if (OB_FAIL(datum_array_compare_.init(datum_utils))) {
LOG_WARN("fail to init datum array compare", KR(ret));
} else {
dup_action_ = dup_action;
is_inited_ = true;
}
}
@ -251,7 +377,7 @@ bool ObDirectLoadExternalMultiPartitionRowCompare::operator()(
const ObDirectLoadExternalMultiPartitionRow *rhs)
{
int ret = OB_SUCCESS;
bool bret = false;
int cmp_ret = 0;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadExternalMultiPartitionRowCompare not init", KR(ret), KP(this));
@ -259,25 +385,40 @@ bool ObDirectLoadExternalMultiPartitionRowCompare::operator()(
!rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else {
if (lhs->tablet_id_ != rhs->tablet_id_) {
bret = lhs->tablet_id_ < rhs->tablet_id_;
} else {
bret = datum_array_compare_.operator()(&lhs->external_row_.rowkey_datum_array_,
&rhs->external_row_.rowkey_datum_array_);
}
} else if (OB_FAIL(compare(lhs, rhs, cmp_ret))) {
LOG_WARN("Fail to compare datum array", KR(ret), KP(lhs), KP(rhs));
}
if (OB_FAIL(ret)) {
result_code_ = ret;
} else if (OB_FAIL(datum_array_compare_.get_error_code())) {
result_code_ = datum_array_compare_.get_error_code();
}
return bret;
return cmp_ret < 0;
}
bool ObDirectLoadExternalMultiPartitionRowCompare::operator()(
const ObDirectLoadConstExternalMultiPartitionRow *lhs,
const ObDirectLoadConstExternalMultiPartitionRow *rhs)
{
int ret = OB_SUCCESS;
int cmp_ret = 0;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadExternalMultiPartitionRowCompare not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == lhs || nullptr == rhs || !lhs->is_valid() ||
!rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else if (OB_FAIL(compare(lhs, rhs, cmp_ret))) {
LOG_WARN("Fail to compare datum array", KR(ret), KP(lhs), KP(rhs));
}
if (OB_FAIL(ret)) {
result_code_ = ret;
}
return cmp_ret < 0;
}
int ObDirectLoadExternalMultiPartitionRowCompare::compare(
const ObDirectLoadExternalMultiPartitionRow *lhs,
const ObDirectLoadExternalMultiPartitionRow *rhs, int &cmp_ret)
{
int ret = OB_SUCCESS;
bool bret = false;
@ -290,18 +431,70 @@ bool ObDirectLoadExternalMultiPartitionRowCompare::operator()(
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else {
if (lhs->tablet_id_ != rhs->tablet_id_) {
bret = lhs->tablet_id_ < rhs->tablet_id_;
} else {
bret = datum_array_compare_.operator()(&lhs->rowkey_datum_array_, &rhs->rowkey_datum_array_);
cmp_ret = lhs->tablet_id_ < rhs->tablet_id_ ? -1 : 1;
} else if (OB_FAIL(datum_array_compare_.compare(&lhs->external_row_.rowkey_datum_array_,
&rhs->external_row_.rowkey_datum_array_,
cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(cmp_ret));
} else if (cmp_ret == 0) {
if (lhs->external_row_.seq_no_ == rhs->external_row_.seq_no_) {
cmp_ret = 0;
} else if (lhs->external_row_.seq_no_ > rhs->external_row_.seq_no_) {
if (dup_action_ == sql::ObLoadDupActionType::LOAD_REPLACE) {
cmp_ret = -1;
} else {
cmp_ret = 1;
}
} else {
if (dup_action_ == sql::ObLoadDupActionType::LOAD_REPLACE) {
cmp_ret = 1;
} else {
cmp_ret = -1;
}
}
}
}
if (OB_FAIL(ret)) {
result_code_ = ret;
} else if (OB_FAIL(datum_array_compare_.get_error_code())) {
result_code_ = datum_array_compare_.get_error_code();
}
return bret;
return ret;
}
} // namespace storage
} // namespace oceanbase
int ObDirectLoadExternalMultiPartitionRowCompare::compare(
const ObDirectLoadConstExternalMultiPartitionRow *lhs,
const ObDirectLoadConstExternalMultiPartitionRow *rhs, int &cmp_ret)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObDirectLoadExternalMultiPartitionRowCompare not init", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == lhs || nullptr == rhs || !lhs->is_valid() ||
!rhs->is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), KP(lhs), KP(rhs));
} else {
if (lhs->tablet_id_ != rhs->tablet_id_) {
cmp_ret = lhs->tablet_id_ < rhs->tablet_id_ ? -1 : 1;
} else if (OB_FAIL(datum_array_compare_.compare(&lhs->rowkey_datum_array_,
&rhs->rowkey_datum_array_, cmp_ret))) {
LOG_WARN("fail to compare rowkey", KR(ret), KP(lhs), K(rhs), K(cmp_ret));
} else if (cmp_ret == 0) {
if (lhs->seq_no_ == rhs->seq_no_) {
cmp_ret = 0;
} else if (lhs->seq_no_ > rhs->seq_no_) {
if (dup_action_ == sql::ObLoadDupActionType::LOAD_REPLACE) {
cmp_ret = -1;
} else {
cmp_ret = 1;
}
} else {
if (dup_action_ == sql::ObLoadDupActionType::LOAD_REPLACE) {
cmp_ret = 1;
} else {
cmp_ret = -1;
}
}
}
}
return ret;
}
} // namespace storage
} // namespace oceanbase

43
src/storage/direct_load/ob_direct_load_compare.h
View File

@ -6,6 +6,7 @@
#include "lib/ob_define.h"
#include "lib/ob_errno.h"
#include "sql/resolver/cmd/ob_load_data_stmt.h"
#include "storage/blocksstable/ob_datum_rowkey.h"
namespace oceanbase
@ -15,7 +16,7 @@ namespace blocksstable
class ObStorageDatumUtils;
class ObDatumRowkey;
class ObDatumRow;
} // namespace blocksstable
} // namespace blocksstable
namespace storage
{
class ObDirectLoadDatumArray;
@ -29,8 +30,11 @@ class ObDirectLoadDatumRowkeyCompare
public:
ObDirectLoadDatumRowkeyCompare() : datum_utils_(nullptr), result_code_(common::OB_SUCCESS) {}
int init(const blocksstable::ObStorageDatumUtils &datum_utils);
int compare(const blocksstable::ObDatumRowkey *lhs, const blocksstable::ObDatumRowkey *rhs,
int &cmp_ret);
bool operator()(const blocksstable::ObDatumRowkey *lhs, const blocksstable::ObDatumRowkey *rhs);
int get_error_code() const { return result_code_; }
public:
const blocksstable::ObStorageDatumUtils *datum_utils_;
int result_code_;
@ -40,10 +44,15 @@ class ObDirectLoadDatumRowCompare
{
public:
ObDirectLoadDatumRowCompare()
: rowkey_size_(0), result_code_(common::OB_SUCCESS), is_inited_(false) {}
: rowkey_size_(0), result_code_(common::OB_SUCCESS), is_inited_(false)
{
}
int init(const blocksstable::ObStorageDatumUtils &datum_utils, int64_t rowkey_size);
int compare(const blocksstable::ObDatumRow *lhs, const blocksstable::ObDatumRow *rhs,
int &cmp_ret);
bool operator()(const blocksstable::ObDatumRow *lhs, const blocksstable::ObDatumRow *rhs);
int get_error_code() const { return result_code_; }
public:
blocksstable::ObDatumRowkey lhs_rowkey_;
blocksstable::ObDatumRowkey rhs_rowkey_;
@ -56,12 +65,15 @@ public:
class ObDirectLoadDatumArrayCompare
{
public:
ObDirectLoadDatumArrayCompare()
: result_code_(common::OB_SUCCESS), is_inited_(false) {}
ObDirectLoadDatumArrayCompare() : result_code_(common::OB_SUCCESS), is_inited_(false) {}
int init(const blocksstable::ObStorageDatumUtils &datum_utils);
bool operator()(const ObDirectLoadDatumArray *lhs, const ObDirectLoadDatumArray *rhs);
int compare(const ObDirectLoadDatumArray *lhs, const ObDirectLoadDatumArray *rhs, int &cmp_ret);
bool operator()(const ObDirectLoadConstDatumArray *lhs, const ObDirectLoadConstDatumArray *rhs);
int compare(const ObDirectLoadConstDatumArray *lhs, const ObDirectLoadConstDatumArray *rhs,
int &cmp_ret);
int get_error_code() const { return result_code_; }
public:
blocksstable::ObDatumRowkey lhs_rowkey_;
blocksstable::ObDatumRowkey rhs_rowkey_;
@ -74,11 +86,15 @@ class ObDirectLoadExternalRowCompare
{
public:
ObDirectLoadExternalRowCompare() : result_code_(common::OB_SUCCESS), is_inited_(false) {}
int init(const blocksstable::ObStorageDatumUtils &datum_utils);
int init(const blocksstable::ObStorageDatumUtils &datum_utils,
sql::ObLoadDupActionType dup_action);
int compare(const ObDirectLoadExternalRow *lhs, const ObDirectLoadExternalRow *rhs, int &cmp_ret);
bool operator()(const ObDirectLoadExternalRow *lhs, const ObDirectLoadExternalRow *rhs);
int get_error_code() const { return result_code_; }
public:
ObDirectLoadDatumArrayCompare datum_array_compare_;
sql::ObLoadDupActionType dup_action_;
int result_code_;
bool is_inited_;
};
@ -87,18 +103,27 @@ class ObDirectLoadExternalMultiPartitionRowCompare
{
public:
ObDirectLoadExternalMultiPartitionRowCompare()
: result_code_(common::OB_SUCCESS), is_inited_(false) {}
int init(const blocksstable::ObStorageDatumUtils &datum_utils);
: result_code_(common::OB_SUCCESS), is_inited_(false)
{
}
int init(const blocksstable::ObStorageDatumUtils &datum_utils,
sql::ObLoadDupActionType dup_action);
bool operator()(const ObDirectLoadExternalMultiPartitionRow *lhs,
const ObDirectLoadExternalMultiPartitionRow *rhs);
bool operator()(const ObDirectLoadConstExternalMultiPartitionRow *lhs,
const ObDirectLoadConstExternalMultiPartitionRow *rhs);
int compare(const ObDirectLoadExternalMultiPartitionRow *lhs,
const ObDirectLoadExternalMultiPartitionRow *rhs, int &cmp_ret);
int compare(const ObDirectLoadConstExternalMultiPartitionRow *lhs,
const ObDirectLoadConstExternalMultiPartitionRow *rhs, int &cmp_ret);
int get_error_code() const { return result_code_; }
public:
ObDirectLoadDatumArrayCompare datum_array_compare_;
sql::ObLoadDupActionType dup_action_;
int result_code_;
bool is_inited_;
};
} // namespace storage
} // namespace oceanbase
} // namespace storage
} // namespace oceanbase

4
src/storage/direct_load/ob_direct_load_external_multi_partition_row.cpp
View File

@ -95,6 +95,7 @@ void ObDirectLoadConstExternalMultiPartitionRow::reset()
{
tablet_id_.reset();
rowkey_datum_array_.reset();
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
}
@ -107,6 +108,7 @@ ObDirectLoadConstExternalMultiPartitionRow &ObDirectLoadConstExternalMultiPartit
tablet_id_ = other.tablet_id_;
rowkey_datum_array_ = other.rowkey_datum_array_;
buf_size_ = other.buf_size_;
seq_no_ = other.seq_no_;
buf_ = other.buf_;
}
return *this;
@ -118,6 +120,7 @@ ObDirectLoadConstExternalMultiPartitionRow &ObDirectLoadConstExternalMultiPartit
tablet_id_ = other.tablet_id_;
rowkey_datum_array_ = other.external_row_.rowkey_datum_array_;
buf_size_ = other.external_row_.buf_size_;
seq_no_ = other.external_row_.seq_no_;
buf_ = other.external_row_.buf_;
return *this;
}
@ -147,6 +150,7 @@ int ObDirectLoadConstExternalMultiPartitionRow::deep_copy(
LOG_WARN("fail to deep copy datum array", KR(ret));
} else {
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;

8
src/storage/direct_load/ob_direct_load_external_multi_partition_row.h
View File

@ -5,6 +5,7 @@
#pragma once
#include "storage/direct_load/ob_direct_load_external_row.h"
#include "share/table/ob_table_load_define.h"
namespace oceanbase
{
@ -47,13 +48,14 @@ public:
int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const;
bool is_valid() const
{
return tablet_id_.is_valid() && rowkey_datum_array_.is_valid() && buf_size_ > 0 &&
nullptr != buf_;
return tablet_id_.is_valid() && rowkey_datum_array_.is_valid() && seq_no_.is_valid() &&
buf_size_ > 0 && nullptr != buf_;
}
TO_STRING_KV(K_(tablet_id), K_(rowkey_datum_array), K_(buf_size), KP_(buf));
TO_STRING_KV(K_(tablet_id), K_(rowkey_datum_array), K_(seq_no), K_(buf_size), KP_(buf));
public:
common::ObTabletID tablet_id_;
ObDirectLoadConstDatumArray rowkey_datum_array_;
table::ObTableLoadSequenceNo seq_no_;
int64_t buf_size_;
const char *buf_;
};

4
src/storage/direct_load/ob_direct_load_external_multi_partition_table.cpp
View File

@ -7,6 +7,7 @@
#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "storage/direct_load/ob_direct_load_external_table.h"
#include "share/table/ob_table_load_define.h"
namespace oceanbase
{
@ -84,6 +85,7 @@ int ObDirectLoadExternalMultiPartitionTableBuilder::init(
}
int ObDirectLoadExternalMultiPartitionTableBuilder::append_row(const ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const ObDatumRow &datum_row)
{
int ret = OB_SUCCESS;
@ -101,7 +103,7 @@ int ObDirectLoadExternalMultiPartitionTableBuilder::append_row(const ObTabletID
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_append_row_time_us);
row_.tablet_id_ = tablet_id;
if (OB_FAIL(row_.external_row_.from_datums(datum_row.storage_datums_, datum_row.count_,
param_.table_data_desc_.rowkey_column_num_))) {
param_.table_data_desc_.rowkey_column_num_, seq_no))) {
LOG_WARN("fail to from datums", KR(ret));
} else if (OB_FAIL(external_writer_.write_item(row_))) {
LOG_WARN("fail to write item", KR(ret));

1
src/storage/direct_load/ob_direct_load_external_multi_partition_table.h
View File

@ -41,6 +41,7 @@ public:
virtual ~ObDirectLoadExternalMultiPartitionTableBuilder();
int init(const ObDirectLoadExternalMultiPartitionTableBuildParam &param);
int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) override;
int close() override;
int64_t get_row_count() const override { return total_row_count_; }

13
src/storage/direct_load/ob_direct_load_external_row.cpp
View File

@ -14,6 +14,7 @@ namespace storage
{
using namespace common;
using namespace blocksstable;
using namespace table;
ObDirectLoadExternalRow::ObDirectLoadExternalRow()
: allocator_("TLD_ext_row"), buf_size_(0), buf_(nullptr)
@ -24,6 +25,7 @@ ObDirectLoadExternalRow::ObDirectLoadExternalRow()
void ObDirectLoadExternalRow::reset()
{
rowkey_datum_array_.reset();
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
allocator_.reset();
@ -32,6 +34,7 @@ void ObDirectLoadExternalRow::reset()
void ObDirectLoadExternalRow::reuse()
{
rowkey_datum_array_.reuse();
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
allocator_.reuse();
@ -62,6 +65,7 @@ int ObDirectLoadExternalRow::deep_copy(const ObDirectLoadExternalRow &src, char
LOG_WARN("fail to deep copy datum array", KR(ret));
} else {
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;
@ -71,7 +75,7 @@ int ObDirectLoadExternalRow::deep_copy(const ObDirectLoadExternalRow &src, char
}
int ObDirectLoadExternalRow::from_datums(ObStorageDatum *datums, int64_t column_count,
int64_t rowkey_column_count)
int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us);
int ret = OB_SUCCESS;
@ -98,6 +102,7 @@ int ObDirectLoadExternalRow::from_datums(ObStorageDatum *datums, int64_t column_
} else {
buf_ = buf;
buf_size_ = buf_size;
seq_no_ = seq_no;
}
}
}
@ -153,7 +158,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadExternalRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_datum_array_, buf_size_);
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_datum_array_, seq_no_, buf_size_);
if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) {
MEMCPY(buf + pos, buf_, buf_size_);
pos += buf_size_;
@ -166,7 +171,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadExternalRow)
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_deserialize_time_us);
int ret = OB_SUCCESS;
reuse();
LST_DO_CODE(OB_UNIS_DECODE, rowkey_datum_array_, buf_size_);
LST_DO_CODE(OB_UNIS_DECODE, rowkey_datum_array_, seq_no_, buf_size_);
if (OB_SUCC(ret)) {
buf_ = buf + pos;
pos += buf_size_;
@ -178,7 +183,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadExternalRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int64_t len = 0;
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_datum_array_, buf_size_);
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_datum_array_, seq_no_, buf_size_);
len += buf_size_;
return len;
}

9
src/storage/direct_load/ob_direct_load_external_row.h
View File

@ -5,6 +5,7 @@
#pragma once
#include "lib/allocator/page_arena.h"
#include "share/table/ob_table_load_define.h"
#include "storage/blocksstable/ob_datum_rowkey.h"
#include "storage/direct_load/ob_direct_load_datum.h"
@ -24,18 +25,20 @@ public:
int deep_copy(const ObDirectLoadExternalRow &src, char *buf, const int64_t len, int64_t &pos);
// not deep copy
int from_datums(blocksstable::ObStorageDatum *datums, int64_t column_count,
int64_t rowkey_column_count);
int64_t rowkey_column_count, const table::ObTableLoadSequenceNo &seq_no);
int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const;
int get_rowkey(blocksstable::ObDatumRowkey &rowkey) const;
bool is_valid() const
{
return rowkey_datum_array_.is_valid() && buf_size_ > 0 && nullptr != buf_;
return rowkey_datum_array_.is_valid() && seq_no_.is_valid() && buf_size_ > 0 && nullptr != buf_;
}
OB_INLINE int64_t get_raw_size() const { return buf_size_; }
TO_STRING_KV(K_(rowkey_datum_array), K_(buf_size), KP_(buf));
TO_STRING_KV(K_(rowkey_datum_array), K_(seq_no), K_(buf_size), KP_(buf));
public:
common::ObArenaAllocator allocator_;
ObDirectLoadDatumArray rowkey_datum_array_;
table::ObTableLoadSequenceNo seq_no_;
int64_t buf_size_;
const char *buf_;
};

3
src/storage/direct_load/ob_direct_load_external_table_builder.cpp
View File

@ -79,6 +79,7 @@ int ObDirectLoadExternalTableBuilder::init(const ObDirectLoadExternalTableBuildP
}
int ObDirectLoadExternalTableBuilder::append_row(const ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const ObDatumRow &datum_row)
{
int ret = OB_SUCCESS;
@ -96,7 +97,7 @@ int ObDirectLoadExternalTableBuilder::append_row(const ObTabletID &tablet_id,
} else {
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_append_row_time_us);
if (OB_FAIL(external_row_.from_datums(datum_row.storage_datums_, datum_row.count_,
build_param_.table_data_desc_.rowkey_column_num_))) {
build_param_.table_data_desc_.rowkey_column_num_, seq_no))) {
LOG_WARN("fail to from datums", KR(ret));
} else if (OB_FAIL(external_writer_.write_item(external_row_))) {
LOG_WARN("fail to write item", KR(ret));

1
src/storage/direct_load/ob_direct_load_external_table_builder.h
View File

@ -42,6 +42,7 @@ public:
virtual ~ObDirectLoadExternalTableBuilder();
int init(const ObDirectLoadExternalTableBuildParam &build_param);
int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) override;
int close() override;
int64_t get_row_count() const override { return row_count_; }

2
src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp
View File

@ -191,9 +191,11 @@ int ObDirectLoadFastHeapTableBuilder::switch_sstable_slice()
}
int ObDirectLoadFastHeapTableBuilder::append_row(const ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const ObDatumRow &datum_row)
{
UNUSED(tablet_id);
UNUSED(seq_no);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;

1
src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h
View File

@ -56,6 +56,7 @@ public:
virtual ~ObDirectLoadFastHeapTableBuilder();
int init(const ObDirectLoadFastHeapTableBuildParam &param);
int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) override;
int close() override;
int64_t get_row_count() const override { return row_count_; }

2
src/storage/direct_load/ob_direct_load_i_table.h
View File

@ -5,6 +5,7 @@
#pragma once
#include "storage/blocksstable/ob_datum_row.h"
#include "share/table/ob_table_load_define.h"
namespace oceanbase
{
@ -28,6 +29,7 @@ public:
ObIDirectLoadPartitionTableBuilder() = default;
virtual ~ObIDirectLoadPartitionTableBuilder() = default;
virtual int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) = 0;
virtual int close() = 0;
virtual int64_t get_row_count() const = 0;

1
src/storage/direct_load/ob_direct_load_mem_context.h
View File

@ -78,6 +78,7 @@ public:
int32_t column_count_;
ObDirectLoadDMLRowHandler *dml_row_handler_;
ObDirectLoadTmpFileManager *file_mgr_;
sql::ObLoadDupActionType dup_action_;
ObDirectLoadEasyQueue<storage::ObDirectLoadExternalMultiPartitionRowChunk *> mem_chunk_queue_;
int64_t fly_mem_chunk_count_;

4
src/storage/direct_load/ob_direct_load_mem_dump.cpp
View File

@ -215,7 +215,7 @@ int ObDirectLoadMemDump::dump_tables()
if (OB_ISNULL(extra_buf_ = static_cast<char *>(allocator_.alloc(extra_buf_size_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory", KR(ret));
} else if (OB_FAIL(compare.init(*(mem_ctx_->datum_utils_)))) {
} else if (OB_FAIL(compare.init(*(mem_ctx_->datum_utils_), mem_ctx_->dup_action_))) {
LOG_WARN("fail to init compare", KR(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < context_ptr_->mem_chunk_array_.count(); i++) {
@ -263,7 +263,7 @@ int ObDirectLoadMemDump::dump_tables()
if (OB_SUCC(ret)) {
if (OB_FAIL(external_row->to_datums(datum_row.storage_datums_, datum_row.count_))) {
LOG_WARN("fail to transfer dataum row", KR(ret));
} else if (OB_FAIL(table_builder->append_row(external_row->tablet_id_, datum_row))) {
} else if (OB_FAIL(table_builder->append_row(external_row->tablet_id_, external_row->seq_no_, datum_row))) {
if (OB_LIKELY(OB_ERR_PRIMARY_KEY_DUPLICATE == ret)) {
if (OB_FAIL(mem_ctx_->dml_row_handler_->handle_update_row(datum_row))) {
LOG_WARN("fail to handle update row", KR(ret), K(datum_row));

2
src/storage/direct_load/ob_direct_load_mem_loader.cpp
View File

@ -142,7 +142,7 @@ int ObDirectLoadMemLoader::close_chunk(ChunkType *&chunk)
{
int ret = OB_SUCCESS;
CompareType compare;
if (OB_FAIL(compare.init(*(mem_ctx_->datum_utils_)))) {
if (OB_FAIL(compare.init(*(mem_ctx_->datum_utils_), mem_ctx_->dup_action_))) {
LOG_WARN("fail to init compare", KR(ret));
} else if (OB_FAIL(chunk->sort(compare))) {
LOG_WARN("fail to sort chunk", KR(ret));

2
src/storage/direct_load/ob_direct_load_mem_sample.cpp
View File

@ -38,7 +38,7 @@ int ObDirectLoadMemSample::gen_ranges(ObIArray<ChunkType *> &chunks, ObIArray<Ra
}
if (OB_SUCC(ret)) {
CompareType compare;
if (OB_FAIL(compare.init(*(mem_ctx_->datum_utils_)))) {
if (OB_FAIL(compare.init(*(mem_ctx_->datum_utils_), mem_ctx_->dup_action_))) {
LOG_WARN("fail to init compare", KR(ret));
} else {
std::sort(sample_rows.begin(), sample_rows.end(), compare);

13
src/storage/direct_load/ob_direct_load_multiple_datum_row.cpp
View File

@ -14,6 +14,7 @@ namespace storage
{
using namespace common;
using namespace blocksstable;
using namespace table;
ObDirectLoadMultipleDatumRow::ObDirectLoadMultipleDatumRow()
: allocator_("TLD_MultiRow"), buf_size_(0), buf_(nullptr)
@ -28,6 +29,7 @@ ObDirectLoadMultipleDatumRow::~ObDirectLoadMultipleDatumRow()
void ObDirectLoadMultipleDatumRow::reset()
{
rowkey_.reset();
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
allocator_.reset();
@ -36,6 +38,7 @@ void ObDirectLoadMultipleDatumRow::reset()
void ObDirectLoadMultipleDatumRow::reuse()
{
rowkey_.reuse();
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
allocator_.reuse();
@ -66,6 +69,7 @@ int ObDirectLoadMultipleDatumRow::deep_copy(const ObDirectLoadMultipleDatumRow &
LOG_WARN("fail to deep copy rowkey", KR(ret));
} else {
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;
@ -75,7 +79,7 @@ int ObDirectLoadMultipleDatumRow::deep_copy(const ObDirectLoadMultipleDatumRow &
}
int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObStorageDatum *datums,
int64_t column_count, int64_t rowkey_column_count)
int64_t column_count, int64_t rowkey_column_count, const ObTableLoadSequenceNo &seq_no)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us);
int ret = OB_SUCCESS;
@ -104,6 +108,7 @@ int ObDirectLoadMultipleDatumRow::from_datums(const ObTabletID &tablet_id, ObSto
} else {
buf_ = buf;
buf_size_ = buf_size;
seq_no_ = seq_no;
}
}
}
@ -147,7 +152,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadMultipleDatumRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_, buf_size_);
LST_DO_CODE(OB_UNIS_ENCODE, rowkey_, seq_no_, buf_size_);
if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) {
MEMCPY(buf + pos, buf_, buf_size_);
pos += buf_size_;
@ -160,7 +165,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadMultipleDatumRow)
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_deserialize_time_us);
int ret = OB_SUCCESS;
reuse();
LST_DO_CODE(OB_UNIS_DECODE, rowkey_, buf_size_);
LST_DO_CODE(OB_UNIS_DECODE, rowkey_, seq_no_, buf_size_);
if (OB_SUCC(ret)) {
buf_ = buf + pos;
pos += buf_size_;
@ -172,7 +177,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadMultipleDatumRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int64_t len = 0;
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_, buf_size_);
LST_DO_CODE(OB_UNIS_ADD_LEN, rowkey_, seq_no_, buf_size_);
len += buf_size_;
return len;
}

13
src/storage/direct_load/ob_direct_load_multiple_datum_row.h
View File

@ -4,6 +4,7 @@
#pragma once
#include "share/table/ob_table_load_define.h"
#include "storage/direct_load/ob_direct_load_multiple_datum_rowkey.h"
namespace oceanbase
@ -24,14 +25,20 @@ public:
int64_t &pos);
// not deep copy
int from_datums(const common::ObTabletID &tablet_id, blocksstable::ObStorageDatum *datums,
int64_t column_count, int64_t rowkey_column_count);
int64_t column_count, int64_t rowkey_column_count,
const table::ObTableLoadSequenceNo &seq_no);
int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const;
OB_INLINE bool is_valid() const { return rowkey_.is_valid() && buf_size_ > 0 && nullptr != buf_; }
OB_INLINE bool is_valid() const
{
return rowkey_.is_valid() && seq_no_.is_valid() && buf_size_ > 0 && nullptr != buf_;
}
OB_INLINE int64_t get_raw_size() const { return buf_size_; }
TO_STRING_KV(K_(rowkey), K_(buf_size), KP_(buf));
TO_STRING_KV(K_(rowkey), K_(seq_no), K_(buf_size), KP_(buf));
public:
common::ObArenaAllocator allocator_;
ObDirectLoadMultipleDatumRowkey rowkey_;
table::ObTableLoadSequenceNo seq_no_;
int64_t buf_size_;
const char *buf_;
};

13
src/storage/direct_load/ob_direct_load_multiple_external_row.cpp
View File

@ -14,6 +14,7 @@ namespace storage
{
using namespace common;
using namespace blocksstable;
using namespace table;
ObDirectLoadMultipleExternalRow::ObDirectLoadMultipleExternalRow()
: allocator_("TLD_ME_Row"), buf_size_(0), buf_(nullptr)
@ -23,6 +24,7 @@ ObDirectLoadMultipleExternalRow::ObDirectLoadMultipleExternalRow()
void ObDirectLoadMultipleExternalRow::reset()
{
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
allocator_.reset();
@ -30,6 +32,7 @@ void ObDirectLoadMultipleExternalRow::reset()
void ObDirectLoadMultipleExternalRow::reuse()
{
seq_no_.reset();
buf_size_ = 0;
buf_ = nullptr;
allocator_.reuse();
@ -54,6 +57,7 @@ int ObDirectLoadMultipleExternalRow::deep_copy(const ObDirectLoadMultipleExterna
} else {
reuse();
buf_size_ = src.buf_size_;
seq_no_ = src.seq_no_;
buf_ = buf + pos;
MEMCPY(buf + pos, src.buf_, buf_size_);
pos += buf_size_;
@ -61,7 +65,7 @@ int ObDirectLoadMultipleExternalRow::deep_copy(const ObDirectLoadMultipleExterna
return ret;
}
int ObDirectLoadMultipleExternalRow::from_datums(ObStorageDatum *datums, int64_t column_count)
int ObDirectLoadMultipleExternalRow::from_datums(ObStorageDatum *datums, int64_t column_count, const ObTableLoadSequenceNo &seq_no)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, transfer_external_row_time_us);
int ret = OB_SUCCESS;
@ -84,6 +88,7 @@ int ObDirectLoadMultipleExternalRow::from_datums(ObStorageDatum *datums, int64_t
LOG_WARN("fail to serialize datum array", KR(ret));
} else {
buf_ = buf;
seq_no_ = seq_no;
buf_size_ = buf_size;
}
}
@ -121,7 +126,7 @@ OB_DEF_SERIALIZE_SIMPLE(ObDirectLoadMultipleExternalRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE, tablet_id_.id(), buf_size_);
LST_DO_CODE(OB_UNIS_ENCODE, tablet_id_.id(), seq_no_, buf_size_);
if (OB_SUCC(ret) && OB_NOT_NULL(buf_)) {
MEMCPY(buf + pos, buf_, buf_size_);
pos += buf_size_;
@ -135,7 +140,7 @@ OB_DEF_DESERIALIZE_SIMPLE(ObDirectLoadMultipleExternalRow)
int ret = OB_SUCCESS;
reset();
uint64_t id = 0;
LST_DO_CODE(OB_UNIS_DECODE, id, buf_size_);
LST_DO_CODE(OB_UNIS_DECODE, id, seq_no_, buf_size_);
if (OB_SUCC(ret)) {
tablet_id_ = id;
buf_ = buf + pos;
@ -148,7 +153,7 @@ OB_DEF_SERIALIZE_SIZE_SIMPLE(ObDirectLoadMultipleExternalRow)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, external_row_serialize_time_us);
int64_t len = 0;
LST_DO_CODE(OB_UNIS_ADD_LEN, tablet_id_.id(), buf_size_);
LST_DO_CODE(OB_UNIS_ADD_LEN, tablet_id_.id(), seq_no_, buf_size_);
len += buf_size_;
return len;
}

11
src/storage/direct_load/ob_direct_load_multiple_external_row.h
View File

@ -5,6 +5,7 @@
#pragma once
#include "lib/allocator/page_arena.h"
#include "share/table/ob_table_load_define.h"
#include "storage/blocksstable/ob_datum_rowkey.h"
#include "storage/direct_load/ob_direct_load_datum.h"
@ -12,7 +13,6 @@ namespace oceanbase
{
namespace storage
{
class ObDirectLoadMultipleExternalRow
{
OB_UNIS_VERSION(1);
@ -23,16 +23,19 @@ public:
int64_t get_deep_copy_size() const;
int deep_copy(const ObDirectLoadMultipleExternalRow &src, char *buf, const int64_t len,
int64_t &pos);
int from_datums(blocksstable::ObStorageDatum *datums, int64_t column_count);
int from_datums(blocksstable::ObStorageDatum *datums, int64_t column_count,
const table::ObTableLoadSequenceNo &seq_no);
int to_datums(blocksstable::ObStorageDatum *datums, int64_t column_count) const;
OB_INLINE bool is_valid() const
{
return tablet_id_.is_valid() && buf_size_ > 0 && nullptr != buf_;
return tablet_id_.is_valid() && seq_no_.is_valid() && buf_size_ > 0 && nullptr != buf_;
}
TO_STRING_KV(K_(tablet_id), K_(buf_size), KP_(buf));
TO_STRING_KV(K_(tablet_id), K_(seq_no), K_(buf_size), KP_(buf));
public:
common::ObArenaAllocator allocator_;
common::ObTabletID tablet_id_;
table::ObTableLoadSequenceNo seq_no_;
int64_t buf_size_;
const char *buf_;
};

3
src/storage/direct_load/ob_direct_load_multiple_heap_table_builder.cpp
View File

@ -81,6 +81,7 @@ int ObDirectLoadMultipleHeapTableBuilder::init(const ObDirectLoadMultipleHeapTab
}
int ObDirectLoadMultipleHeapTableBuilder::append_row(const ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const ObDatumRow &datum_row)
{
int ret = OB_SUCCESS;
@ -96,7 +97,7 @@ int ObDirectLoadMultipleHeapTableBuilder::append_row(const ObTabletID &tablet_id
LOG_WARN("invalid args", KR(ret), K(param_), K(datum_row));
} else {
row_.tablet_id_ = tablet_id;
if (OB_FAIL(row_.from_datums(datum_row.storage_datums_, datum_row.count_))) {
if (OB_FAIL(row_.from_datums(datum_row.storage_datums_, datum_row.count_, seq_no))) {
LOG_WARN("fail to from datum row", KR(ret));
} else if (OB_FAIL(append_row(row_))) {
LOG_WARN("fail to append row", KR(ret), K(row_));

1
src/storage/direct_load/ob_direct_load_multiple_heap_table_builder.h
View File

@ -46,6 +46,7 @@ public:
virtual ~ObDirectLoadMultipleHeapTableBuilder();
int init(const ObDirectLoadMultipleHeapTableBuildParam &param);
int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) override;
int append_row(const RowType &row);
int close() override;

2
src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp
View File

@ -102,7 +102,7 @@ int ObDirectLoadMultipleHeapTableSorter::close_chunk(ObDirectLoadMultipleHeapTab
for (int64_t j = 0; OB_SUCC(ret) && j < bag.count(); j ++) {
if (OB_FAIL(bag.at(j)->to_datums(datum_row.storage_datums_, datum_row.count_))) {
LOG_WARN("fail to transfer dataum row", KR(ret));
} else if (OB_FAIL(table_builder.append_row(keys.at(i), datum_row))) {
} else if (OB_FAIL(table_builder.append_row(keys.at(i), bag.at(j)->seq_no_, datum_row))) {
LOG_WARN("fail to append row", KR(ret));
}
}

3
src/storage/direct_load/ob_direct_load_multiple_sstable_builder.cpp
View File

@ -142,6 +142,7 @@ int ObDirectLoadMultipleSSTableBuilder::init(const ObDirectLoadMultipleSSTableBu
}
int ObDirectLoadMultipleSSTableBuilder::append_row(const ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const ObDatumRow &datum_row)
{
int ret = OB_SUCCESS;
@ -157,7 +158,7 @@ int ObDirectLoadMultipleSSTableBuilder::append_row(const ObTabletID &tablet_id,
LOG_WARN("invalid args", KR(ret), K(param_), K(datum_row));
} else {
if (OB_FAIL(row_.from_datums(tablet_id, datum_row.storage_datums_, datum_row.count_,
param_.table_data_desc_.rowkey_column_num_))) {
param_.table_data_desc_.rowkey_column_num_, seq_no))) {
LOG_WARN("fail to from datum row", KR(ret));
} else if (OB_FAIL(append_row(row_))) {
LOG_WARN("fail to append row", KR(ret), K(row_));

1
src/storage/direct_load/ob_direct_load_multiple_sstable_builder.h
View File

@ -41,6 +41,7 @@ public:
virtual ~ObDirectLoadMultipleSSTableBuilder();
int init(const ObDirectLoadMultipleSSTableBuildParam &param);
int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) override;
int append_row(const RowType &row);
int close() override;

4
src/storage/direct_load/ob_direct_load_sstable_builder.cpp
View File

@ -77,7 +77,7 @@ int ObDirectLoadSSTableBuilder::init(const ObDirectLoadSSTableBuildParam &param)
return ret;
}
int ObDirectLoadSSTableBuilder::append_row(const ObTabletID &tablet_id, const ObDatumRow &datum_row)
int ObDirectLoadSSTableBuilder::append_row(const ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, const ObDatumRow &datum_row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -97,7 +97,7 @@ int ObDirectLoadSSTableBuilder::append_row(const ObTabletID &tablet_id, const Ob
if (OB_FAIL(check_rowkey_order(key))) {
LOG_WARN("fail to check rowkey order", KR(ret), K(datum_row));
} else if (OB_FAIL(external_row.from_datums(datum_row.storage_datums_, datum_row.count_,
param_.table_data_desc_.rowkey_column_num_))) {
param_.table_data_desc_.rowkey_column_num_, seq_no))) {
LOG_WARN("fail to from datum row", KR(ret));
} else if (OB_FAIL(data_block_writer_.append_row(external_row))) {
LOG_WARN("fail to append row to data block writer", KR(ret), K(external_row));

1
src/storage/direct_load/ob_direct_load_sstable_builder.h
View File

@ -161,6 +161,7 @@ public:
virtual ~ObDirectLoadSSTableBuilder() = default;
int init(const ObDirectLoadSSTableBuildParam &param);
int append_row(const common::ObTabletID &tablet_id,
const table::ObTableLoadSequenceNo &seq_no,
const blocksstable::ObDatumRow &datum_row) override;
int append_row(const ObDirectLoadExternalRow &external_row);
int close() override;

7
src/storage/direct_load/ob_direct_load_table_store.cpp
View File

@ -148,6 +148,7 @@ int ObDirectLoadTableStoreBucket::init(const ObDirectLoadTableStoreParam &param,
}
int ObDirectLoadTableStoreBucket::append_row(const ObTabletID &tablet_id,
const ObTableLoadSequenceNo &seq_no,
const ObDatumRow &datum_row)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, table_store_bucket_append_row);
@ -160,7 +161,7 @@ int ObDirectLoadTableStoreBucket::append_row(const ObTabletID &tablet_id,
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(tablet_id), K(datum_row), KPC(param_));
} else {
if (OB_FAIL(table_builder_->append_row(tablet_id, datum_row))) {
if (OB_FAIL(table_builder_->append_row(tablet_id, seq_no, datum_row))) {
LOG_WARN("fail to append row", KR(ret));
}
}
@ -291,7 +292,7 @@ int ObDirectLoadTableStore::get_bucket(const ObTabletID &tablet_id,
return ret;
}
int ObDirectLoadTableStore::append_row(const ObTabletID &tablet_id, const ObDatumRow &datum_row)
int ObDirectLoadTableStore::append_row(const ObTabletID &tablet_id, const ObTableLoadSequenceNo &seq_no, const ObDatumRow &datum_row)
{
OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, table_store_append_row);
OB_TABLE_LOAD_STATISTICS_COUNTER(table_store_row_count);
@ -306,7 +307,7 @@ int ObDirectLoadTableStore::append_row(const ObTabletID &tablet_id, const ObDatu
ObDirectLoadTableStoreBucket *bucket = nullptr;
if (OB_FAIL(get_bucket(tablet_id, bucket))) {
LOG_WARN("fail to get bucket", KR(ret), K(tablet_id));
} else if (OB_FAIL(bucket->append_row(tablet_id, datum_row))) {
} else if (OB_FAIL(bucket->append_row(tablet_id, seq_no, datum_row))) {
LOG_WARN("fail to append row to bucket", KR(ret), K(tablet_id), K(datum_row));
}
}

4
src/storage/direct_load/ob_direct_load_table_store.h
View File

@ -54,7 +54,7 @@ public:
ObDirectLoadTableStoreBucket();
~ObDirectLoadTableStoreBucket();
int init(const ObDirectLoadTableStoreParam &param, const common::ObTabletID &tablet_id);
int append_row(const common::ObTabletID &tablet_id, const blocksstable::ObDatumRow &datum_row);
int append_row(const common::ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, const blocksstable::ObDatumRow &datum_row);
int close();
int get_tables(common::ObIArray<ObIDirectLoadPartitionTable *> &table_array,
common::ObIAllocator &allocator);
@ -74,7 +74,7 @@ public:
ObDirectLoadTableStore() : allocator_("TLD_TSBucket"), is_inited_(false) {}
~ObDirectLoadTableStore();
int init(const ObDirectLoadTableStoreParam &param);
int append_row(const common::ObTabletID &tablet_id, const blocksstable::ObDatumRow &datum_row);
int append_row(const common::ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, const blocksstable::ObDatumRow &datum_row);
int close();
void clean_up();
int get_tables(common::ObIArray<ObIDirectLoadPartitionTable *> &table_array,