hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'column_store'

Browse Source 
Co-authored-by: wangt1xiuyi <13547954130@163.com>
Co-authored-by: yangqise7en <877793735@qq.com>
Co-authored-by: Zach41 <zach_41@163.com>
This commit is contained in:
chaser-ch
2023-10-31 15:39:20 +00:00
committed by ob-robot
parent 4057fbc4ae
commit 566e920620
1375 changed files with 239147 additions and 56014 deletions
Download Patch File Download Diff File Expand all files Collapse all files

697
src/storage/ob_storage_schema.cpp
View File

@ -118,13 +118,244 @@ int ObStorageColumnSchema::deep_copy_default_val(ObIAllocator &allocator, const
return ret;
}
OB_SERIALIZE_MEMBER_SIMPLE(
OB_SERIALIZE_MEMBER(
ObStorageColumnSchema,
info_,
default_checksum_,
meta_type_,
orig_default_value_);
int ObStorageColumnSchema::legacy_deserialize(const char *buf, const int64_t data_len, int64_t &pos)
{
// For schema version before 4_2_0_0
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_DECODE,
info_,
default_checksum_,
meta_type_,
orig_default_value_);
return ret;
}
int ObStorageColumnSchema::legacy_serialize(char *buf, const int64_t buf_len, int64_t &pos) const
{
// For schema version before 4_2_0_0
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_ENCODE,
info_,
default_checksum_,
meta_type_,
orig_default_value_);
return ret;
}
/*
* ObStorageColumnGroupSchema
*/
void ObStorageColumnGroupSchema::destroy(ObIAllocator &allocator)
{
if (nullptr != column_idxs_) {
allocator.free(column_idxs_);
}
reset();
}
int ObStorageColumnGroupSchema::serialize(char *buf, const int64_t buf_len, int64_t &pos) const
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "Unexpected column group schema to serialize", K(ret), K_(version), K_(type), K_(column_cnt));
} else {
LST_DO_CODE(OB_UNIS_ENCODE,
version_,
type_,
compressor_type_,
row_store_type_,
block_size_,
schema_column_cnt_,
rowkey_column_cnt_,
schema_rowkey_column_cnt_);
if (is_all_column_group()) {
OB_UNIS_ENCODE(column_cnt_);
} else {
OB_UNIS_ENCODE_ARRAY(column_idxs_, column_cnt_);
}
}
return ret;
}
int ObStorageColumnGroupSchema::deserialize(ObIAllocator &allocator, const char *buf, const int64_t data_len, int64_t &pos)
{
int ret = OB_SUCCESS;
LST_DO_CODE(OB_UNIS_DECODE,
version_,
type_,
compressor_type_,
row_store_type_,
block_size_,
schema_column_cnt_,
rowkey_column_cnt_,
schema_rowkey_column_cnt_,
column_cnt_);
if (OB_SUCC(ret)) {
if (column_cnt_ > 0) {
if (is_all_column_group()) {
column_idxs_ = nullptr;
} else if (OB_ISNULL(column_idxs_ = reinterpret_cast<uint16_t *> (allocator.alloc(sizeof(uint16_t) * column_cnt_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(column_cnt_));
} else {
OB_UNIS_DECODE_ARRAY(column_idxs_, column_cnt_);
if (OB_FAIL(ret)) {
allocator.free(column_idxs_);
reset();
}
}
} else {
column_idxs_ = nullptr;
}
}
return ret;
}
int64_t ObStorageColumnGroupSchema::get_serialize_size() const
{
int64_t len = 0;
if (OB_UNLIKELY(!is_valid())) {
STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "Unexpected column group schema to serialize", K_(version), K_(type), K_(column_cnt));
} else {
LST_DO_CODE(OB_UNIS_ADD_LEN,
version_,
type_,
compressor_type_,
row_store_type_,
block_size_,
schema_column_cnt_,
rowkey_column_cnt_,
schema_rowkey_column_cnt_);
if (is_all_column_group()) {
OB_UNIS_ADD_LEN(column_cnt_);
} else {
OB_UNIS_ADD_LEN_ARRAY(column_idxs_, column_cnt_);
}
}
return len;
}
int ObStorageColumnGroupSchema::deep_copy(const ObStorageColumnGroupSchema &other, common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
reset();
if (OB_UNLIKELY(!other.is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to deep copy column group schema", K(ret), K(other));
} else {
version_ = COLUMN_GRUOP_SCHEMA_VERSION;
type_ = other.type_;
compressor_type_ = other.compressor_type_;
row_store_type_ = other.row_store_type_;
block_size_ = other.block_size_;
schema_column_cnt_ = other.schema_column_cnt_;
rowkey_column_cnt_ = other.rowkey_column_cnt_;
schema_rowkey_column_cnt_ = other.schema_rowkey_column_cnt_;
column_cnt_ = other.column_cnt_;
if (column_cnt_ == 0 ||is_all_column_group()) {
column_idxs_ = nullptr;
} else if (OB_ISNULL(column_idxs_ = reinterpret_cast<uint16_t *> (allocator.alloc(sizeof(uint16_t) * column_cnt_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(column_cnt_));
} else {
MEMCPY(column_idxs_, other.column_idxs_, sizeof(uint16_t) * column_cnt_);
}
}
return ret;
}
int ObStorageColumnGroupSchema::copy_from(ObIArray<ObColDesc> &column_ids,
const int64_t schema_rowkey_cnt,
const ObColumnGroupSchema &cg_schema,
ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!cg_schema.is_valid() || column_ids.empty() || schema_rowkey_cnt <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to copy column group schema", K(ret), K(cg_schema), K(column_ids), K(schema_rowkey_cnt));
} else if (OB_UNLIKELY(is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected error for copy to valid cg schema", K(ret), K(*this), K(cg_schema));
} else if (OB_UNLIKELY(cg_schema.get_column_id_count() > column_ids.count())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected error for cg schema", K(ret), K(column_ids), K(cg_schema));
} else {
const bool is_normal_cg_schema = cg_schema.is_normal_column_group();
version_ = COLUMN_GRUOP_SCHEMA_VERSION;
block_size_ = cg_schema.get_block_size();
type_ = cg_schema.get_column_group_type();
compressor_type_ = cg_schema.get_compressor_type();
row_store_type_ = cg_schema.get_row_store_type();
schema_column_cnt_ = cg_schema.get_column_id_count();
schema_rowkey_column_cnt_ = is_normal_cg_schema ? 0 : schema_rowkey_cnt;
rowkey_column_cnt_ = has_multi_version_column() ? schema_rowkey_column_cnt_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()
: schema_rowkey_column_cnt_;
column_cnt_ = has_multi_version_column() ? schema_column_cnt_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()
: schema_column_cnt_;
const uint64_t *column_ids_arr = nullptr;
const int64_t mv_rowkey_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
if (is_all_column_group()) {
column_idxs_ = nullptr;
} else if (column_cnt_ == 0) {
if (!is_default_column_group()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected column group schma", K(ret), K(cg_schema));
} else {
column_idxs_ = nullptr;
}
} else if (OB_UNLIKELY(is_rowkey_column_group() && schema_column_cnt_ != schema_rowkey_cnt)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get unexpected column cnt for rowkey cg", K(ret));
} else if (OB_ISNULL(column_idxs_ = reinterpret_cast<uint16_t *> (allocator.alloc(sizeof(uint16_t) * column_cnt_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(column_cnt_));
} else if (OB_ISNULL(column_ids_arr = cg_schema.get_column_ids())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected error for null column ids", K(ret), K(cg_schema));
} else if (is_rowkey_column_group()) {
//generate storage rowkey column group
for (int64_t i = 0; i < schema_rowkey_cnt + mv_rowkey_cnt; ++i) {
column_idxs_[i] = i;
}
} else { // deal with normal cg
for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt_; i++) {
column_idxs_[i] = UINT16_MAX;
bool find_column_id = false;
for (int64_t j = 0; j < column_ids.count(); j++) {
if (column_ids_arr[i] == column_ids.at(j).col_id_) {
column_idxs_[i] = j + (j >= schema_rowkey_cnt ? mv_rowkey_cnt : 0);
find_column_id = true;
break;
}
}
if (OB_UNLIKELY(!find_column_id)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected column id in schema",
K(ret), K(i), K(column_ids_arr[i]), K(column_ids), K(cg_schema), K(i), K(column_ids_arr[i]));
}
}
}
if (OB_FAIL(ret)) {
destroy(allocator);
}
}
return ret;
}
/*
* ObStorageSchema
* */
@ -151,7 +382,10 @@ ObStorageSchema::ObStorageSchema()
encrypt_key_(),
rowkey_array_(),
column_array_(),
column_group_array_(),
skip_idx_attr_array_(),
store_column_cnt_(0),
has_all_column_group_(false),
is_inited_(false)
{
}
@ -166,7 +400,7 @@ int ObStorageSchema::init(
const ObTableSchema &input_schema,
const lib::Worker::CompatMode compat_mode,
const bool skip_column_info/* = false*/,
const int64_t compat_version/* = STORAGE_SCHEMA_VERSION_V2*/)
const int64_t compat_version/* = STORAGE_SCHEMA_VERSION_V3*/)
{
int ret = OB_SUCCESS;
@ -180,6 +414,8 @@ int ObStorageSchema::init(
allocator_ = &allocator;
rowkey_array_.set_allocator(&allocator);
column_array_.set_allocator(&allocator);
column_group_array_.set_allocator(&allocator);
skip_idx_attr_array_.set_allocator(&allocator);
storage_schema_version_ = compat_version;
copy_from(input_schema);
@ -192,9 +428,11 @@ int ObStorageSchema::init(
} else if (FALSE_IT(column_info_simplified_ = skip_column_info)) {
} else if (OB_FAIL(generate_column_array(input_schema))) {
STORAGE_LOG(WARN, "failed to generate column array", K(ret), K(input_schema));
} else if (OB_FAIL(generate_column_group_array(input_schema, allocator))) {
STORAGE_LOG(WARN, "Failed to generate column group array", K(ret));
} else if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "storage schema is invalid", K(ret));
STORAGE_LOG(ERROR, "storage schema is invalid", K(ret));
} else {
is_inited_ = true;
}
@ -209,7 +447,8 @@ int ObStorageSchema::init(
int ObStorageSchema::init(
common::ObIAllocator &allocator,
const ObStorageSchema &old_schema,
const bool skip_column_info/* = false*/)
const bool skip_column_info/* = false*/,
const ObStorageSchema *column_group_schema)
{
int ret = OB_SUCCESS;
@ -223,35 +462,41 @@ int ObStorageSchema::init(
allocator_ = &allocator;
rowkey_array_.set_allocator(&allocator);
column_array_.set_allocator(&allocator);
column_group_array_.set_allocator(&allocator);
skip_idx_attr_array_.set_allocator(&allocator);
storage_schema_version_ = STORAGE_SCHEMA_VERSION_V2;;
storage_schema_version_ = STORAGE_SCHEMA_VERSION_V3;
copy_from(old_schema);
compat_mode_ = old_schema.compat_mode_;
compressor_type_ = old_schema.compressor_type_;
column_cnt_ = old_schema.column_cnt_;
store_column_cnt_ = old_schema.store_column_cnt_;
column_info_simplified_ = (skip_column_info || old_schema.column_info_simplified_);
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(deep_copy_str(old_schema.encryption_, encryption_))) {
STORAGE_LOG(WARN, "failed to deep copy encryption", K(ret), K(old_schema));
} else if (OB_FAIL(deep_copy_str(old_schema.encrypt_key_, encrypt_key_))) {
STORAGE_LOG(WARN, "failed to deep copy encryption key", K(ret), K(old_schema));
} else if (OB_FAIL(rowkey_array_.reserve(old_schema.rowkey_array_.count()))) {
STORAGE_LOG(WARN, "failed to reserve for rowkey array", K(ret), K(old_schema));
} else if (OB_FAIL(rowkey_array_.assign(old_schema.rowkey_array_))) {
STORAGE_LOG(WARN, "failed to copy row key array", K(ret), K(old_schema));
} else if (column_info_simplified_) {
// do nothing
} else if (OB_FAIL(deep_copy_column_array(allocator, old_schema, old_schema.column_array_.count()))) {
STORAGE_LOG(WARN, "failed to deep copy column array", K(ret), K(old_schema));
}
if (OB_FAIL(ret)) {
} else if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "storage schema is invalid", K(ret));
} else {
is_inited_ = true;
if (OB_FAIL(deep_copy_str(old_schema.encryption_, encryption_))) {
STORAGE_LOG(WARN, "failed to deep copy encryption", K(ret), K(old_schema));
} else if (OB_FAIL(deep_copy_str(old_schema.encrypt_key_, encrypt_key_))) {
STORAGE_LOG(WARN, "failed to deep copy encryption key", K(ret), K(old_schema));
} else if (OB_FAIL(rowkey_array_.reserve(old_schema.rowkey_array_.count()))) {
STORAGE_LOG(WARN, "failed to reserve for rowkey array", K(ret), K(old_schema));
} else if (OB_FAIL(rowkey_array_.assign(old_schema.rowkey_array_))) {
STORAGE_LOG(WARN, "failed to copy row key array", K(ret), K(old_schema));
} else if (OB_FAIL(skip_idx_attr_array_.reserve(old_schema.skip_idx_attr_array_.count()))) {
STORAGE_LOG(WARN, "failed to reserve for skip idx attr array", K(ret), K(old_schema));
} else if (OB_FAIL(skip_idx_attr_array_.assign(old_schema.skip_idx_attr_array_))) {
STORAGE_LOG(WARN, "failed to copy skip idx attr array", K(ret), K(old_schema));
} else if (!column_info_simplified_ && OB_FAIL(deep_copy_column_array(allocator, old_schema, old_schema.column_array_.count()))) {
STORAGE_LOG(WARN, "failed to deep copy column array", K(ret), K(old_schema));
} else if (NULL != column_group_schema && OB_FAIL(deep_copy_column_group_array(allocator, *column_group_schema))) {
STORAGE_LOG(WARN, "failed to deep copy column array from column group schema", K(ret), K(old_schema), KPC(column_group_schema));
} else if (NULL == column_group_schema && OB_FAIL(deep_copy_column_group_array(allocator, old_schema))) {
STORAGE_LOG(WARN, "failed to deep copy column array", K(ret), K(old_schema));
} else if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "storage schema is invalid", K(ret));
} else {
is_inited_ = true;
}
}
if (OB_UNLIKELY(!is_inited_)) {
@ -290,6 +535,47 @@ int ObStorageSchema::deep_copy_column_array(
return ret;
}
int ObStorageSchema::deep_copy_column_group_array(common::ObIAllocator &allocator, const ObStorageSchema &src_schema)
{
int ret = OB_SUCCESS;
if (src_schema.get_column_groups().empty()) {
ObStorageColumnGroupSchema column_group;
if (OB_FAIL(generate_all_column_group_schema(column_group, row_store_type_))) {
STORAGE_LOG(WARN, "Failed to mock column group for compat", K(ret));
} else if (OB_FAIL(column_group_array_.reserve(1))) {
STORAGE_LOG(WARN, "failed to reserve for column group array", K(ret), K(src_schema));
} else if (OB_FAIL(add_column_group(column_group))) {
STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group));
}
} else if (OB_FAIL(column_group_array_.reserve(src_schema.column_group_array_.count()))) {
STORAGE_LOG(WARN, "failed to reserve for column group array", K(ret), K(src_schema));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < src_schema.column_group_array_.count(); i++) {
ObStorageColumnGroupSchema column_group_schema;
if (OB_FAIL(column_group_schema.deep_copy(src_schema.column_group_array_.at(i), allocator))) {
STORAGE_LOG(WARN, "Failed to deep copy column group schema", K(ret));
} else if (OB_FAIL(add_column_group(column_group_schema))) {
STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group_schema));
column_group_schema.destroy(allocator);
}
}
}
return ret;
}
int ObStorageSchema::add_column_group(const ObStorageColumnGroupSchema &column_group)
{
int ret = OB_SUCCESS;
if (OB_FAIL(column_group_array_.push_back(column_group))) {
STORAGE_LOG(WARN, "Failed to push back column group", K(ret), K(column_group));
} else if (column_group.is_all_column_group()) {
has_all_column_group_ = true;
}
return ret;
}
void ObStorageSchema::reset()
{
storage_schema_version_ = 0;
@ -317,11 +603,22 @@ void ObStorageSchema::reset()
column_array_.at(i).destroy(*allocator_);
}
column_array_.reset();
(void) reset_column_group_array();
skip_idx_attr_array_.reset();
has_all_column_group_ = false;
allocator_ = nullptr;
}
is_inited_ = false;
}
void ObStorageSchema::reset_column_group_array()
{
for (int i = 0; i < column_group_array_.count(); ++i) {
column_group_array_.at(i).destroy(*allocator_);
}
column_group_array_.reset();
}
bool ObStorageSchema::is_valid() const
{
bool valid_ret = true;
@ -334,7 +631,9 @@ bool ObStorageSchema::is_valid() const
|| !table_mode_.is_valid()
|| index_type_ >= INDEX_TYPE_MAX
|| !check_column_array_valid(rowkey_array_)
|| !check_column_array_valid(column_array_)) {
|| !check_column_array_valid(column_array_)
|| !check_column_array_valid(column_group_array_)
|| !check_column_array_valid(skip_idx_attr_array_)) {
valid_ret = false;
STORAGE_LOG_RET(WARN, OB_INVALID_ERROR, "invalid", K_(is_inited), KP_(allocator), K_(schema_version), K_(column_cnt),
K_(tablet_size), K_(pctfree), K_(table_type), K_(table_mode), K_(index_type));
@ -371,8 +670,8 @@ int ObStorageSchema::serialize(char *buf, const int64_t buf_len, int64_t &pos) c
|| OB_UNLIKELY(pos < 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid args", K(ret), K(buf), K(buf_len), K(pos));
} else if (STORAGE_SCHEMA_VERSION == storage_schema_version_
|| STORAGE_SCHEMA_VERSION_V2 == storage_schema_version_) {
} else if (STORAGE_SCHEMA_VERSION <= storage_schema_version_
&& STORAGE_SCHEMA_VERSION_V3 >= storage_schema_version_) {
LST_DO_CODE(OB_UNIS_ENCODE,
storage_schema_version_,
info_,
@ -392,14 +691,20 @@ int ObStorageSchema::serialize(char *buf, const int64_t buf_len, int64_t &pos) c
encryption_,
encrypt_key_);
if (OB_FAIL(ret)) {
} else if (OB_FAIL(serialize_column_array(buf, buf_len, pos, rowkey_array_))){
} else if (OB_FAIL(serialize_schema_array(buf, buf_len, pos, rowkey_array_))){
STORAGE_LOG(WARN, "failed to serialize rowkey columns", K_(rowkey_array));
} else if (!column_info_simplified_
&& OB_FAIL(serialize_column_array(buf, buf_len, pos, column_array_))){
} else if (!column_info_simplified_ && OB_FAIL(serialize_column_array(buf, buf_len, pos))){
STORAGE_LOG(WARN, "failed to serialize columns", K_(column_array));
} else if (STORAGE_SCHEMA_VERSION_V2 == storage_schema_version_
&& OB_FAIL(serialization::encode_i64(buf, buf_len, pos, store_column_cnt_))) {
STORAGE_LOG(WARN, "failed to serialize store_column_cnt", K(ret), K(store_column_cnt_));
} else if (storage_schema_version_ >= STORAGE_SCHEMA_VERSION_V2) {
if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, store_column_cnt_))) {
STORAGE_LOG(WARN, "failed to serialize store_column_cnt", K(ret), K(store_column_cnt_));
} else if (storage_schema_version_ == STORAGE_SCHEMA_VERSION_V2) {
// v2 do not need column group
} else if (OB_FAIL(serialize_schema_array(buf, buf_len, pos, column_group_array_))){
STORAGE_LOG(WARN, "failed to serialize column grups", K_(column_group_array));
} else if (OB_FAIL(serialize_schema_array(buf, buf_len, pos, skip_idx_attr_array_))){
STORAGE_LOG(WARN, "failed to serialize skip idx attr array", K_(skip_idx_attr_array));
}
}
} else {
ret = OB_ERR_UNEXPECTED;
@ -409,6 +714,30 @@ int ObStorageSchema::serialize(char *buf, const int64_t buf_len, int64_t &pos) c
return ret;
}
int ObStorageSchema::serialize_column_array(char *buf, const int64_t data_len, int64_t &pos) const
{
int ret = OB_SUCCESS;
if (OB_FAIL(serialization::encode_vi64(buf, data_len, pos, column_array_.count()))) {
STORAGE_LOG(WARN, "Fail to encode column count", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < column_array_.count(); ++i) {
if (STORAGE_SCHEMA_VERSION_V3 > storage_schema_version_) {
if (OB_FAIL(column_array_.at(i).legacy_serialize(buf, data_len, pos))) {
STORAGE_LOG(WARN, "Fail to serialize column schema for legacy version", K(ret), K(i), K_(column_array));
}
} else if (STORAGE_SCHEMA_VERSION_V3 == storage_schema_version_) {
if (OB_FAIL(column_array_.at(i).serialize(buf, data_len, pos))) {
STORAGE_LOG(WARN, "Fail to serialize column schema", K(ret), K(i), K_(column_array));
}
} else {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected storage schema version", K(ret), K_(storage_schema_version));
}
}
return ret;
}
int ObStorageSchema::deserialize(
common::ObIAllocator &allocator,
const char *buf,
@ -430,12 +759,15 @@ int ObStorageSchema::deserialize(
allocator_ = &allocator;
rowkey_array_.set_allocator(&allocator);
column_array_.set_allocator(&allocator);
column_group_array_.set_allocator(&allocator);
skip_idx_attr_array_.set_allocator(&allocator);
}
if (OB_FAIL(serialization::decode(buf, data_len, pos, storage_schema_version_))) {
if (OB_FAIL(ret)) {
} else if (OB_FAIL(serialization::decode(buf, data_len, pos, storage_schema_version_))) {
STORAGE_LOG(WARN, "failed to deserialize version", K(ret), K(data_len), K(pos));
} else if (STORAGE_SCHEMA_VERSION == storage_schema_version_
|| STORAGE_SCHEMA_VERSION_V2 == storage_schema_version_) {
} else if (STORAGE_SCHEMA_VERSION <= storage_schema_version_
&& STORAGE_SCHEMA_VERSION_V3 >= storage_schema_version_) {
ObString tmp_encryption;
ObString tmp_encrypt_key;
LST_DO_CODE(OB_UNIS_DECODE,
@ -464,15 +796,34 @@ int ObStorageSchema::deserialize(
STORAGE_LOG(WARN, "failed to deserialize rowkey columns", K(ret), K_(rowkey_array));
} else if (!column_info_simplified_ && OB_FAIL(deserialize_column_array(allocator, buf, data_len, pos))){
STORAGE_LOG(WARN, "failed to deserialize columns", K(ret), K_(column_array));
} else if (STORAGE_SCHEMA_VERSION == storage_schema_version_) {
store_column_cnt_ = get_store_column_count_by_column_array();
// TODO(lixia.yq) delete column array or later?
} else if (storage_schema_version_ < STORAGE_SCHEMA_VERSION_V3) {
if (STORAGE_SCHEMA_VERSION_V2 == storage_schema_version_) {
if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &store_column_cnt_))) {
STORAGE_LOG(WARN, "failed to deserialize store_column_cnt", K(ret), K_(store_column_cnt));
}
} else {
store_column_cnt_ = get_store_column_count_by_column_array();
}
ObStorageColumnGroupSchema column_group;
if (FAILEDx(generate_all_column_group_schema(column_group, row_store_type_))) {
STORAGE_LOG(WARN, "Failed to mock column group for compat", K(ret));
} else if (OB_FAIL(column_group_array_.reserve(1))) {
STORAGE_LOG(WARN, "failed to reserve for column group array", K(ret));
} else if (OB_FAIL(add_column_group(column_group))) {
STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group));
} else {
storage_schema_version_ = STORAGE_SCHEMA_VERSION_V3;
}
} else if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &store_column_cnt_))) {
STORAGE_LOG(WARN, "failed to deserialize store_column_cnt", K(ret), K_(store_column_cnt));
}
} else if (OB_FAIL(deserialize_column_group_array(allocator, buf, data_len, pos))) {
STORAGE_LOG(WARN, "Failed to deserialize column groups", K(ret));
} else if (OB_FAIL(deserialize_skip_idx_attr_array(buf, data_len, pos))) {
STORAGE_LOG(WARN, "failed to deserialize skip idx attr array", K(ret));
} // TODO(@lixia.yq) need to add compat log for column_group after transfer refresh
if (OB_SUCC(ret)) {
is_inited_ = true;
storage_schema_version_ = STORAGE_SCHEMA_VERSION_V2;
}
} else {
ret = OB_ERR_UNEXPECTED;
@ -537,9 +888,20 @@ int ObStorageSchema::deserialize_column_array(
ObStorageColumnSchema column;
for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) {
column.reset();
if (OB_FAIL(column.deserialize(buf, data_len, pos))) {
STORAGE_LOG(WARN,"Fail to deserialize column", K(ret));
} else if (column.orig_default_value_.get_deep_copy_size() > 0) {
if (STORAGE_SCHEMA_VERSION_V3 > storage_schema_version_) {
if (OB_FAIL(column.legacy_deserialize(buf, data_len, pos))) {
STORAGE_LOG(WARN, "Fail to deserialize column schema for legacy version", K(ret));
}
} else if (STORAGE_SCHEMA_VERSION_V3 == storage_schema_version_) {
if (OB_FAIL(column.deserialize(buf, data_len, pos))) {
STORAGE_LOG(WARN, "Fail to deserialize column schema", K(ret));
}
} else {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected storage schema version", K(ret), K_(storage_schema_version));
}
if (OB_SUCC(ret) && column.orig_default_value_.get_deep_copy_size() > 0) {
ObStorageColumnSchema deep_copy_column;
if (OB_FAIL(deep_copy_column.deep_copy_default_val(allocator, column.get_orig_default_value()))) {
STORAGE_LOG(WARN, "failed to deep copy default val", K(ret), K(column));
@ -557,6 +919,141 @@ int ObStorageSchema::deserialize_column_array(
return ret;
}
int ObStorageSchema::generate_column_group_array(const ObTableSchema &input_schema, ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
ObStorageColumnGroupSchema column_group;
int64_t column_group_cnt = 0;
// temporarily we use cnt to decide the row store or column store
if (OB_FAIL(input_schema.get_store_column_group_count(column_group_cnt))) {
STORAGE_LOG(WARN, "failed to get column group count", K(ret), K(input_schema));
} else if (OB_FAIL(column_group_array_.reserve(MAX(column_group_cnt, 1)))) {
STORAGE_LOG(WARN, "failed to reserve for column group array", K(ret), K(input_schema));
} else if (column_group_cnt <= 1) {
if (OB_FAIL(generate_all_column_group_schema(column_group, row_store_type_))) {
STORAGE_LOG(WARN, "Failed to generate column array group for row store", K(ret));
} else if (OB_FAIL(add_column_group(column_group))) {
STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group));
}
} else {
ObSEArray<ObColDesc, 32> column_ids;
ObSEArray<const ObColumnGroupSchema *, 8> column_group_metas;
if (OB_FAIL(input_schema.get_store_column_ids(column_ids))) {
STORAGE_LOG(WARN, "Failed to get column_ids", K(ret));
} else if (OB_FAIL(input_schema.get_store_column_groups(column_group_metas))) { // get cg metas without empty default cg
STORAGE_LOG(WARN, "failed to get column group metas", K(ret), K(input_schema));
} else {
const int64_t schema_rowkey_cnt = input_schema.get_rowkey_column_num();
const ObColumnGroupSchema *cg_schema = nullptr;
for (int64_t idx = 0; OB_SUCC(ret) && idx < column_group_metas.count(); ++idx) {
column_group.reset();
if (OB_ISNULL(cg_schema = column_group_metas.at(idx))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected null cg_schema", K(ret));
} else if (cg_schema->get_column_group_type() == ALL_COLUMN_GROUP) {
if (OB_FAIL(generate_all_column_group_schema(column_group, cg_schema->get_row_store_type()))) {
STORAGE_LOG(WARN, "Failed to generage all column group schema", K(ret));
}
} else if (OB_FAIL(column_group.copy_from(column_ids, schema_rowkey_cnt, *cg_schema, allocator))) {
STORAGE_LOG(WARN, "Failed to copy column group schema", K(ret));
}
if (FAILEDx(add_column_group(column_group))) {
STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group));
column_group.destroy(allocator);
}
}
}
}
return ret;
}
int ObStorageSchema::generate_all_column_group_schema(ObStorageColumnGroupSchema &column_group, const ObRowStoreType row_store_type)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(store_column_cnt_ <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to mock column group", K(ret), K_(store_column_cnt), K(column_array_));
} else {
// all the row-oriented tables should only consist of one default column group
column_group.reset();
column_group.version_ = ObStorageColumnGroupSchema::COLUMN_GRUOP_SCHEMA_VERSION;
column_group.type_ = ALL_COLUMN_GROUP;
column_group.schema_column_cnt_ = store_column_cnt_;
column_group.rowkey_column_cnt_ = get_rowkey_column_num();
column_group.schema_rowkey_column_cnt_ = column_group.rowkey_column_cnt_ - ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
column_group.column_cnt_ = column_group.schema_column_cnt_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
column_group.column_idxs_ = nullptr;
column_group.block_size_ = block_size_;
column_group.compressor_type_ = compressor_type_;
column_group.row_store_type_ = row_store_type;
}
return ret;
}
int ObStorageSchema::deserialize_column_group_array(ObIAllocator &allocator,
const char *buf,
const int64_t data_len,
int64_t &pos)
{
int ret = OB_SUCCESS;
int64_t count = 0;
if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0) || OB_UNLIKELY(pos > data_len)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(buf), K(data_len), K(pos), K(ret));
} else if (pos == data_len) {
//do nothing
} else if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) {
STORAGE_LOG(WARN, "Fail to decode column count", K(ret));
} else if (OB_FAIL(column_group_array_.reserve(count))) {
STORAGE_LOG(WARN, "Fail to reserve column array", K(ret), K(count));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) {
ObStorageColumnGroupSchema column_group;
if (OB_FAIL(column_group.deserialize(allocator, buf, data_len, pos))) {
STORAGE_LOG(WARN, "Failed to deserialize column group", K(ret));
} else if (OB_FAIL(add_column_group(column_group))) {
STORAGE_LOG(WARN, "failed to add column group", K(ret), K(column_group));
column_group.destroy(allocator);
}
}
}
return ret;
}
int ObStorageSchema::deserialize_skip_idx_attr_array(const char *buf,
const int64_t data_len,
int64_t &pos)
{
int ret = OB_SUCCESS;
int64_t count = 0;
if (OB_ISNULL(buf) || OB_UNLIKELY(data_len <= 0) || OB_UNLIKELY(pos > data_len)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(buf), K(data_len), K(pos), K(ret));
} else if (pos == data_len) {
//do nothing
} else if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) {
STORAGE_LOG(WARN, "Fail to decode column count", K(ret));
} else if (OB_FAIL(skip_idx_attr_array_.reserve(count))) {
STORAGE_LOG(WARN, "Fail to reserve skip idx array", K(ret), K(count));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) {
ObSkipIndexAttrWithId skip_attr_with_id;
if (OB_FAIL(skip_attr_with_id.deserialize(buf, data_len, pos))) {
STORAGE_LOG(WARN, "Failed to deserialize skip idx attr with col id", K(ret));
} else if (OB_FAIL(skip_idx_attr_array_.push_back(skip_attr_with_id))) {
STORAGE_LOG(WARN, "Fail to add skip attr with col id", K(ret), K(skip_attr_with_id));
}
}
}
return ret;
}
int64_t ObStorageSchema::get_serialize_size() const
{
int64_t len = 0;
@ -584,9 +1081,13 @@ int64_t ObStorageSchema::get_serialize_size() const
if (!column_info_simplified_) {
len += get_column_array_serialize_length(column_array_);
}
if (STORAGE_SCHEMA_VERSION_V2 == storage_schema_version_) {
if (storage_schema_version_ >= STORAGE_SCHEMA_VERSION_V2) {
len += serialization::encoded_length_i64(store_column_cnt_);
}
if (storage_schema_version_ >= STORAGE_SCHEMA_VERSION_V3) {
len += get_column_array_serialize_length(column_group_array_);
len += get_column_array_serialize_length(skip_idx_attr_array_);
}
return len;
}
@ -620,7 +1121,9 @@ int ObStorageSchema::generate_column_array(const ObTableSchema &input_schema)
}
int64_t col_idx = 0;
int64_t col_cnt_in_sstable = 0;
int64_t has_skip_index_cnt = 0;
blocksstable::ObStorageDatum datum;
ObSEArray<share::schema::ObSkipIndexAttrWithId, 16> tmp_skip_array;
for ( ; OB_SUCC(ret) && iter != input_schema.column_end(); iter++) {
if (OB_ISNULL(col = *iter)) {
ret = OB_ERR_UNEXPECTED;
@ -632,7 +1135,12 @@ int ObStorageSchema::generate_column_array(const ObTableSchema &input_schema)
col_schema.is_rowkey_column_ = col->is_rowkey_column();
col_schema.is_column_stored_in_sstable_ = col->is_column_stored_in_sstable();
col_schema.is_generated_column_ = col->is_generated_column();
col_schema.meta_type_ = col->get_meta_type();
ObObjMeta meta_type = col->get_meta_type();
if (meta_type.is_decimal_int()) {
meta_type.set_stored_precision(col->get_accuracy().get_precision());
meta_type.set_scale(col->get_accuracy().get_scale());
}
col_schema.meta_type_ = meta_type;
if (ob_is_large_text(col->get_data_type())) {
col_schema.default_checksum_ = 0;
} else if (OB_FAIL(datum.from_obj_enhance(col->get_orig_default_value()))) {
@ -647,17 +1155,30 @@ int ObStorageSchema::generate_column_array(const ObTableSchema &input_schema)
col_schema.destroy(*allocator_);
}
}
const share::schema::ObSkipIndexColumnAttr &skip_idx_attr = col->get_skip_index_attr();
ObSkipIndexAttrWithId skip_attr_with_id;
skip_attr_with_id.col_idx_ = col_idx;
skip_attr_with_id.skip_idx_attr_ = skip_idx_attr;
if (FAILEDx(tmp_map.set_refactored(col->get_column_id(), col_idx))) {
STORAGE_LOG(WARN, "failed to set column map", K(ret), "col_id", col->get_column_id(), K(col_idx));
} else if (skip_idx_attr.has_skip_index() && col->is_column_stored_in_sstable() &&
OB_FAIL(tmp_skip_array.push_back(skip_attr_with_id))) {
STORAGE_LOG(WARN, "fail to push into skip idx attr with col id array",
K(ret), K(col_idx), K(skip_attr_with_id));
} else {
col_idx++;
}
} // end of for
if (OB_SUCC(ret)) {
if (OB_FAIL(ret)) {
} else if (OB_FAIL(skip_idx_attr_array_.assign(tmp_skip_array))){
STORAGE_LOG(WARN, "Fail to reserve skip idx attr with col id array", K(ret), K(tmp_skip_array));
} else {
store_column_cnt_ = is_storage_index_table() ? input_schema.get_column_count() : col_cnt_in_sstable;
}
// add rowkey columns
ObStorageRowkeyColumnSchema rowkey_schema;
const ObColumnSchemaV2 *rowkey_col_schema = nullptr;
const common::ObRowkeyInfo &rowkey_info = input_schema.get_rowkey_info();
const ObRowkeyColumn *rowkey_column = NULL;
if (OB_FAIL(ret)) {
@ -671,10 +1192,19 @@ int ObStorageSchema::generate_column_array(const ObTableSchema &input_schema)
STORAGE_LOG(WARN, "The rowkey column is NULL", K(i));
} else if (OB_FAIL(tmp_map.get_refactored(rowkey_column->column_id_, find_idx))) {
STORAGE_LOG(WARN, "failed to get column idx from tmp_map", K(ret), "column_id", rowkey_column->column_id_);
} else if (OB_ISNULL(rowkey_col_schema =
input_schema.get_column_schema(rowkey_column->column_id_))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "failed to get rowkey column schema", K(ret));
} else {
rowkey_schema.reset();
rowkey_schema.column_idx_ = common::OB_APP_MIN_COLUMN_ID + find_idx;
rowkey_schema.meta_type_ = rowkey_column->type_;
ObObjMeta meta_type = rowkey_column->type_;
if (meta_type.is_decimal_int()) {
meta_type.set_stored_precision(rowkey_col_schema->get_accuracy().get_precision());
meta_type.set_scale(rowkey_col_schema->get_accuracy().get_scale());
}
rowkey_schema.meta_type_ = meta_type;
rowkey_schema.order_ = rowkey_column->order_;
if (OB_FAIL(rowkey_array_.push_back(rowkey_schema))) {
STORAGE_LOG(WARN, "Fail to add rowkey column id to rowkey array", K(ret));
@ -742,6 +1272,73 @@ int ObStorageSchema::get_rowkey_column_ids(common::ObIArray<ObColDesc> &column_i
return ret;
}
int ObStorageSchema::get_skip_index_col_attr(
ObIArray<share::schema::ObSkipIndexColumnAttr> &skip_idx_attrs) const
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret), K_(is_inited));
} else {
skip_idx_attrs.reset();
// add rowkey columns
share::schema::ObSkipIndexColumnAttr rowkey_skip_idx_attr;
for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_array_.count(); ++i) {
rowkey_skip_idx_attr.reset();
int64_t rowkey_col_idx = rowkey_array_[i].column_idx_ - common::OB_APP_MIN_COLUMN_ID;
for (int64_t skip_col_id = 0; OB_SUCC(ret) && skip_col_id < skip_idx_attr_array_.count(); ++skip_col_id) {
if (rowkey_col_idx == skip_idx_attr_array_.at(skip_col_id).col_idx_) {
rowkey_skip_idx_attr = skip_idx_attr_array_.at(skip_col_id).skip_idx_attr_;
break;
}
}
if (FAILEDx(skip_idx_attrs.push_back(rowkey_skip_idx_attr))) {
STORAGE_LOG(WARN, "fail to append rowkey skip index attr to array",
K(ret), K(i), K(rowkey_skip_idx_attr));
}
}
// add dummy idx for stored multi-version columns
if (OB_SUCC(ret)) {
ObSkipIndexColumnAttr dummy_multi_version_col_attr;
if (OB_FAIL(skip_idx_attrs.push_back(dummy_multi_version_col_attr))) {
STORAGE_LOG(WARN, "failed to push dummy multi version column skip index attr", K(ret));
} else if (OB_FAIL(skip_idx_attrs.push_back(dummy_multi_version_col_attr))) {
STORAGE_LOG(WARN, "failed to push dummy multi version column skip index attr", K(ret));
}
}
// add non-rowkey columns
share::schema::ObSkipIndexColumnAttr no_rowkey_skip_idx_attr;
for (int64_t col_idx = 0; OB_SUCC(ret) && col_idx < column_cnt_; ++col_idx) {
no_rowkey_skip_idx_attr.reset();
bool is_rowkey = false;
for (int64_t j = 0; OB_SUCC(ret) && j < rowkey_array_.count(); ++j) {
int64_t rowkey_col_idx = rowkey_array_[j].column_idx_ - common::OB_APP_MIN_COLUMN_ID;
if (rowkey_col_idx == col_idx) {
is_rowkey = true;
break;
}
}
if (OB_FAIL(ret)) {
} else if (is_rowkey) {
//skip
} else {
for (int64_t skip_col_id = 0; OB_SUCC(ret) && skip_col_id < skip_idx_attr_array_.count(); ++skip_col_id) {
if (col_idx == skip_idx_attr_array_.at(skip_col_id).col_idx_) {
no_rowkey_skip_idx_attr = skip_idx_attr_array_.at(skip_col_id).skip_idx_attr_;
break;
}
}
if (FAILEDx(skip_idx_attrs.push_back(no_rowkey_skip_idx_attr))) {
STORAGE_LOG(WARN, "fail to append no rowkey skip index attr to array",
K(ret), K(col_idx), K(rowkey_skip_idx_attr));
}
}
}
}
return ret;
}
int ObStorageSchema::get_encryption_id(int64_t &encrypt_id) const
{
int ret = OB_SUCCESS;