hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Use the corresponding column fill cg

Browse Source 
Co-authored-by: Charles0429 <xiezhenjiang@gmail.com>
Co-authored-by: Monk-Liu <1152761042@qq.com>
This commit is contained in:
renju96
2024-01-23 03:12:29 +00:00
committed by ob-robot
parent dc76ae43b8
commit 27c9a5278f
15 changed files with 355 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/sql/engine/basic/chunk_store/ob_block_iwriter.h
View File

@ -47,7 +47,7 @@ public:
virtual int add_row(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx, ObChunkDatumStore::StoredRow **stored_row = nullptr) = 0;
virtual int add_row(const ObChunkDatumStore::StoredRow &src_sr, ObChunkDatumStore::StoredRow **dst_sr = nullptr) = 0;
virtual int add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
virtual int add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row) = 0;
virtual int add_batch(const common::ObDatum **datums, const common::ObIArray<ObExpr *> &exprs,

2
src/sql/engine/basic/chunk_store/ob_compact_block_reader.cpp
View File

@ -62,7 +62,7 @@ int ObCompactBlockReader::get_row(const ObChunkDatumStore::StoredRow *&sr)
ret = OB_ITER_END;
} else if (cur_pos_in_blk_ > cur_blk_->raw_size_ - sizeof(ObTempBlockStore::Block)) {
ret = OB_INDEX_OUT_OF_RANGE;
LOG_WARN("invalid index", K(ret), K(cur_pos_in_blk_), KP(cur_blk_));
LOG_WARN("invalid index", K(ret), K(cur_pos_in_blk_), KP(cur_blk_), K(cur_row_in_blk_), K(cur_blk_->cnt_), K(row_meta_->column_offset_));
} else {
int64_t size = 0;
sr = nullptr;

2
src/sql/engine/basic/chunk_store/ob_compact_block_reader.h
View File

@ -112,6 +112,7 @@ public:
cur_row_in_blk_ = 0;
cur_row_offset_width_ = 0;
row_meta_ = nullptr;
cur_blk_ = nullptr;
}
void reuse()
@ -123,6 +124,7 @@ public:
cur_row_in_blk_ = 0;
cur_row_offset_width_ = 0;
row_info_.reset();
cur_blk_ = nullptr;
}
int get_row(const ObChunkDatumStore::StoredRow *&sr) override;

44
src/sql/engine/basic/chunk_store/ob_compact_block_writer.cpp
View File

@ -101,29 +101,31 @@ int ObCompactBlockWriter::add_row(const ObChunkDatumStore::StoredRow &src_sr, Ob
}
int ObCompactBlockWriter::add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row)
int ObCompactBlockWriter::add_row(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ensure_init())) {
LOG_WARN("fail to ensure init", K(ret));
} else if (OB_FAIL(ensure_write(storage_datums, cnt, extra_size))) {
} else if (OB_FAIL(ensure_write(storage_datums, cg_schema, extra_size))) {
LOG_WARN("fail to ensure write", K(ret));
} else {
if ((cur_row_offset_width_ == BASE_OFFSET_SIZE) &&
OB_FAIL(inner_add_row<uint16_t>(storage_datums, cnt, extra_size, stored_row))) {
LOG_WARN("add row to block failed", K(ret), K(storage_datums), K(cnt), K(extra_size));
OB_FAIL(inner_add_row<uint16_t>(storage_datums, cg_schema, extra_size, stored_row))) {
LOG_WARN("add row to block failed", K(ret), K(storage_datums), K(cg_schema), K(extra_size));
} else if (cur_row_offset_width_ == EXTENDED_OFFSET_SIZE &&
OB_FAIL(inner_add_row<uint32_t>(storage_datums, cnt, extra_size, stored_row))) {
LOG_WARN("add row to block failed", K(ret), K(storage_datums), K(cnt), K(extra_size));
OB_FAIL(inner_add_row<uint32_t>(storage_datums, cg_schema, extra_size, stored_row))) {
LOG_WARN("add row to block failed", K(ret), K(storage_datums), K(cg_schema), K(extra_size));
}
}
return ret;
}
template <typename T>
int ObCompactBlockWriter::inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row)
int ObCompactBlockWriter::inner_add_row(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(row_meta_)) {
@ -136,8 +138,9 @@ int ObCompactBlockWriter::inner_add_row(const blocksstable::ObStorageDatum *stor
LOG_WARN("buf is null", K(ret));
} else {
T *var_offset_array = reinterpret_cast<T*>(row_info_.buf_ + HEAD_SIZE + row_info_.bitmap_size_);
for (int64_t i = 0; OB_SUCC(ret) && i < cnt; i++) {
if (OB_FAIL(inner_process_datum<T>(storage_datums[i], i, *row_meta_, row_info_))) {
for (int64_t i = 0; OB_SUCC(ret) && i < cg_schema.column_cnt_; i++) {
int64_t column_idx = cg_schema.column_idxs_ ? cg_schema.column_idxs_[i] : i;
if (OB_FAIL(inner_process_datum<T>(storage_datums[column_idx], i, *row_meta_, row_info_))) {
LOG_WARN("fail to process datum", K(ret));
}
}
@ -368,22 +371,24 @@ int ObCompactBlockWriter::get_row_stored_size(const ObChunkDatumStore::StoredRow
}
int ObCompactBlockWriter::get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObCompactBlockWriter::get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, uint64_t &size)
{
int ret = OB_SUCCESS;
int64_t head_size = sizeof(ObChunkDatumStore::StoredRow);
const ChunkRowMeta &row_meta = *get_meta();
int64_t bit_map_size = sql::ObBitVector::memory_size(row_meta.col_cnt_);
int64_t datum_size = sizeof(ObDatum) * cnt;
int64_t datum_size = sizeof(ObDatum) * cg_schema.column_cnt_;
int64_t data_size = 0;
int64_t offset_size = 0;
for (int64_t i = 0; i < cnt; ++i) {
for (int64_t i = 0; i < cg_schema.column_cnt_; ++i) {
if (row_meta.column_length_[i] != 0) {
data_size += row_meta.column_length_[i];
} else {
if (!storage_datums[i].is_null()) {
data_size += storage_datums[i].len_;
int64_t column_idx = cg_schema.column_idxs_ ? cg_schema.column_idxs_[i] : i;
if (!storage_datums[column_idx].is_null()) {
data_size += storage_datums[column_idx].len_;
}
}
}
@ -413,13 +418,14 @@ int ObCompactBlockWriter::ensure_write(const common::ObIArray<ObExpr*> &exprs, O
return ret;
}
int ObCompactBlockWriter::ensure_write(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObCompactBlockWriter::ensure_write(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size)
{
int ret = OB_SUCCESS;
uint64_t row_size = 0;
if (OB_FAIL(get_row_stored_size(storage_datums, cnt, extra_size, row_size))) {
LOG_WARN("fail to get row_size", K(cnt), K(extra_size), K(row_size), K(ret));
if (OB_FAIL(get_row_stored_size(storage_datums, cg_schema, extra_size, row_size))) {
LOG_WARN("fail to get row_size", K(cg_schema), K(extra_size), K(row_size), K(ret));
} else if (OB_FAIL(ensure_write(row_size))) {
LOG_WARN("fail to call inner ensure write", K(ret));
}

8
src/sql/engine/basic/chunk_store/ob_compact_block_writer.h
View File

@ -104,7 +104,7 @@ public:
// if full, construct the block and use the block's block_mgr. return block
virtual int add_row(const ObChunkDatumStore::StoredRow &src_sr, ObChunkDatumStore::StoredRow **dst_sr = nullptr) override;
//virtual int try_add_row(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx);
virtual int add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
virtual int add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row) override;
virtual int add_batch(const common::ObDatum **datums, const common::ObIArray<ObExpr *> &exprs,
@ -127,14 +127,14 @@ protected:
*/
int ensure_write(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx);
int ensure_write(const ObChunkDatumStore::StoredRow &stored_row);
int ensure_write(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ensure_write(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size);
int ensure_write(const int64_t size);
// get the stored size in writer buffer for a row.
int get_row_stored_size(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx, uint64_t &size);
int get_row_stored_size(const ObChunkDatumStore::StoredRow &sr, uint64_t &size);
int get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, uint64_t &size);
private:
@ -149,7 +149,7 @@ private:
template <typename T>
int inner_add_row(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx);
template <typename T>
int inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row);
inline int ensure_init()

10
src/sql/engine/basic/chunk_store/ob_compact_store.cpp
View File

@ -72,7 +72,7 @@ int ObCompactStore::inner_get_next_row(const ObChunkDatumStore::StoredRow *&sr)
ret = OB_ITER_END;
} else if (OB_FAIL(block_reader_.get_block(cur_blk_id_, tmp_blk))) {
if (ret != OB_ITER_END) {
LOG_WARN("fail to get block", K(ret));
LOG_WARN("fail to get block", K(ret), K(cur_blk_id_));
}
} else {
start_iter_ = true;
@ -83,12 +83,12 @@ int ObCompactStore::inner_get_next_row(const ObChunkDatumStore::StoredRow *&sr)
if (OB_FAIL(ret)) {
} else if (OB_FAIL(reader_->get_row(sr))) {
if (ret != OB_ITER_END) {
LOG_WARN("fail to get row", K(ret));
LOG_WARN("fail to get row", K(ret), K(cur_blk_id_));
} else if (cur_blk_id_ >= get_block_id_cnt()) {
ret = OB_ITER_END;
} else if (OB_FAIL(block_reader_.get_block(cur_blk_id_, tmp_blk))) {
if (ret != OB_ITER_END) {
LOG_WARN("fail to get block", K(ret));
LOG_WARN("fail to get block", K(ret), K(cur_blk_id_));
}
} else {
reader_->reuse();
@ -258,12 +258,12 @@ int ObCompactStore::add_row(const common::ObIArray<ObExpr *> &exprs, ObEvalCtx &
return ret;
}
int ObCompactStore::add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObCompactStore::add_row(const blocksstable::ObDatumRow &datum_row, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row)
{
int ret = OB_SUCCESS;
if (inited_) {
if (OB_FAIL(writer_->add_row(storage_datums, cnt, extra_size, stored_row))) {
if (OB_FAIL(writer_->add_row(datum_row.storage_datums_, cg_schema, extra_size, stored_row))) {
LOG_WARN("fail to add row", K(ret));
} else {
row_cnt_++;

2
src/sql/engine/basic/chunk_store/ob_compact_store.h
View File

@ -76,7 +76,7 @@ public:
int add_row(const common::ObIArray<ObExpr *> &exprs, ObEvalCtx &ctx, ObChunkDatumStore::StoredRow **stored_row = nullptr);
int add_row(const ObChunkDatumStore::StoredRow &src_sr, ObChunkDatumStore::StoredRow **dst_sr = nullptr);
// for chunkslicestore.
int add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int add_row(const blocksstable::ObDatumRow &datum_row, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row = nullptr);
int get_next_row(const ObChunkDatumStore::StoredRow *&sr);

47
src/sql/engine/basic/chunk_store/ob_default_block_writer.cpp
View File

@ -65,16 +65,16 @@ int ObDefaultBlockWriter::add_row(const ObChunkDatumStore::StoredRow &src_sr, Ob
return ret;
}
int ObDefaultBlockWriter::add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObDefaultBlockWriter::add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ensure_init())) {
LOG_WARN("fail to ensure init", K(ret));
} else if (OB_FAIL(ensure_write(storage_datums, cnt, extra_size))) {
} else if (OB_FAIL(ensure_write(storage_datums, cg_schema, extra_size))) {
LOG_WARN("fail to ensure write", K(ret));
} else if (OB_FAIL(inner_add_row(storage_datums, cnt, extra_size, stored_row))) {
LOG_WARN("add row to block failed", K(ret), K(storage_datums), K(cnt), K(extra_size));
} else if (OB_FAIL(inner_add_row(storage_datums, cg_schema, extra_size, stored_row))) {
LOG_WARN("add row to block failed", K(ret), K(storage_datums), K(cg_schema), K(extra_size));
}
return ret;
}
@ -210,30 +210,32 @@ int ObDefaultBlockWriter::add_batch(const common::ObDatum **datums, const common
return ret;
}
int ObDefaultBlockWriter::inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObDefaultBlockWriter::inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **dst_sr)
{
int ret = OB_SUCCESS;
int64_t head_size = sizeof(ObChunkDatumStore::StoredRow);
int64_t datum_size = sizeof(ObDatum) * cnt;
int64_t datum_size = sizeof(ObDatum) * cg_schema.column_cnt_;
int64_t row_size = head_size + datum_size + extra_size;
ObChunkDatumStore::StoredRow *sr = static_cast<ObChunkDatumStore::StoredRow *>((void*)get_cur_buf());
if (OB_ISNULL(sr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get buffer", K(ret));
} else {
sr->cnt_ = cnt;
for (int64_t i = 0; i < cnt; ++i) {
const ObDatum *tmp_datum = static_cast<const ObDatum *>(&storage_datums[i]);
sr->cnt_ = cg_schema.column_cnt_;
for (int64_t i = 0; i < cg_schema.column_cnt_; ++i) {
int64_t column_idx = cg_schema.column_idxs_ ? cg_schema.column_idxs_[i] : i;
const ObDatum *tmp_datum = static_cast<const ObDatum *>(&storage_datums[column_idx]);
MEMCPY(sr->payload_ + i * sizeof(ObDatum), tmp_datum, sizeof(ObDatum));
}
char* data_start = sr->payload_ + datum_size + extra_size;
int64_t pos = 0;
for (int64_t i = 0; i < cnt; ++i) {
MEMCPY(data_start + pos, storage_datums[i].ptr_, storage_datums[i].len_);
for (int64_t i = 0; i < cg_schema.column_cnt_; ++i) {
int64_t column_idx = cg_schema.column_idxs_ ? cg_schema.column_idxs_[i] : i;
MEMCPY(data_start + pos, storage_datums[column_idx].ptr_, storage_datums[column_idx].len_);
sr->cells()[i].ptr_ = data_start + pos;
pos += storage_datums[i].len_;
row_size += storage_datums[i].len_;
pos += storage_datums[column_idx].len_;
row_size += storage_datums[column_idx].len_;
}
sr->row_size_ = row_size;
if (OB_FAIL(advance(row_size))) {
@ -364,13 +366,14 @@ int ObDefaultBlockWriter::prepare_blk_for_write(ObTempBlockStore::Block *blk)
return ret;
}
int ObDefaultBlockWriter::ensure_write(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObDefaultBlockWriter::ensure_write(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size)
{
int ret = OB_SUCCESS;
uint64_t row_size;
if (OB_FAIL(get_row_stored_size(storage_datums, cnt, extra_size, row_size))) {
LOG_WARN("fail to get row_size", K(cnt), K(extra_size), K(row_size), K(ret));
if (OB_FAIL(get_row_stored_size(storage_datums, cg_schema, extra_size, row_size))) {
LOG_WARN("fail to get row_size", K(cg_schema), K(extra_size), K(row_size), K(ret));
} else if (OB_FAIL(ensure_write(row_size))) {
LOG_WARN("fail to call inner ensure write", K(ret));
}
@ -378,15 +381,17 @@ int ObDefaultBlockWriter::ensure_write(const blocksstable::ObStorageDatum *stora
return ret;
}
int ObDefaultBlockWriter::get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int ObDefaultBlockWriter::get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, uint64_t &size)
{
int ret = OB_SUCCESS;
int64_t head_size = sizeof(ObChunkDatumStore::StoredRow);
int64_t datum_size = sizeof(ObDatum) * cnt;
int64_t datum_size = sizeof(ObDatum) * cg_schema.column_cnt_;
int64_t data_size = 0;
for (int64_t i = 0; i < cnt; ++i) {
data_size += storage_datums[i].len_;
for (int64_t i = 0; i < cg_schema.column_cnt_; ++i) {
int64_t column_idx = cg_schema.column_idxs_ ? cg_schema.column_idxs_[i] : i;
data_size += storage_datums[column_idx].len_;
}
size = head_size + datum_size + extra_size + data_size;
return ret;
@ -413,4 +418,4 @@ int ObDefaultBlockWriter::ensure_write(const int64_t size)
}
}
}
}

12
src/sql/engine/basic/chunk_store/ob_default_block_writer.h
View File

@ -39,7 +39,7 @@ public:
void reset() {}
int add_row(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx, ObChunkDatumStore::StoredRow **stored_row = nullptr) override;
int add_row(const ObChunkDatumStore::StoredRow &src_sr, ObChunkDatumStore::StoredRow **dst_sr = nullptr) override;
int add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **stored_row) override;
int close() override;
int add_batch(const common::ObDatum **datums, const common::ObIArray<ObExpr *> &exprs,
@ -64,8 +64,9 @@ private:
*/
int ensure_write(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx);
int ensure_write(const ObChunkDatumStore::StoredRow &stored_row);
int ensure_write(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
const int64_t extra_size);
int ensure_write(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size);
// before dump the block we need to unswizzling each row;
int block_unswizzling(ObTempBlockStore::Block *blk);
@ -73,7 +74,8 @@ private:
int ensure_write(const int64_t size);
// get the stored size in writer buffer for a row.
int get_row_stored_size(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx, uint64_t &size);
int get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int get_row_stored_size(const blocksstable::ObStorageDatum *storage_datums,
const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, uint64_t &size);
inline int ensure_init()
{
@ -84,7 +86,7 @@ private:
return ret;
}
int inner_add_row(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx, ObChunkDatumStore::StoredRow **stored_row = nullptr);
int inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const int64_t cnt,
int inner_add_row(const blocksstable::ObStorageDatum *storage_datums, const ObStorageColumnGroupSchema &cg_schema,
const int64_t extra_size, ObChunkDatumStore::StoredRow **dst_sr);
};

3
src/sql/engine/basic/ob_temp_block_store.cpp
View File

@ -1023,6 +1023,9 @@ int ObTempBlockStore::ensure_reader_buffer(BlockReader &reader, ShrinkBuffer &bu
free_blk_mem(try_reuse_blk);
try_reuse_blk = NULL;
}
} else if (try_reuse_blk != NULL) {
free_blk_mem(try_reuse_blk);
try_reuse_blk = NULL;
}
}
return ret;