From 22a9b2252c43ebf9e00d8f1597aa89816a3f0aa2 Mon Sep 17 00:00:00 2001 From: jingtaoye35 <1255153887@qq.com> Date: Thu, 24 Aug 2023 04:10:29 +0000 Subject: [PATCH] [FEAT MERGE] support aead in tde Co-authored-by: yinyj17 --- mittest/mtlenv/storage/test_memtable_v2.cpp | 35 +- .../storage/test_physical_copy_task.cpp | 2 +- src/share/config/ob_config.cpp | 44 ++- src/share/config/ob_config.h | 6 +- src/share/config/ob_config_helper.cpp | 2 +- .../ob_inner_table_schema.12101_12150.cpp | 8 +- .../ob_inner_table_schema.15101_15150.cpp | 2 +- .../ob_inner_table_schema.15151_15200.cpp | 2 +- .../ob_inner_table_schema.201_250.cpp | 8 +- .../inner_table/ob_inner_table_schema_def.py | 4 +- src/share/ob_encryption_struct.cpp | 39 +- src/share/ob_encryption_struct.h | 11 +- src/share/ob_encryption_util.h | 159 +++++--- src/share/ob_encryption_util_os.cpp | 362 ++++++++++-------- src/sql/code_generator/ob_dml_cg_service.cpp | 19 +- .../code_generator/ob_static_engine_cg.cpp | 19 +- src/sql/engine/expr/ob_expr_aes_encrypt.cpp | 62 ++- .../resolver/cmd/ob_alter_system_resolver.cpp | 16 +- src/sql/resolver/ob_resolver_utils.cpp | 49 +++ src/sql/resolver/ob_resolver_utils.h | 2 + .../blocksstable/ob_index_block_builder.cpp | 14 +- src/storage/blocksstable/ob_macro_block.h | 1 + .../blocksstable/ob_macro_block_reader.cpp | 6 +- .../blocksstable/ob_macro_block_writer.cpp | 15 +- .../blocksstable/ob_macro_block_writer.h | 2 +- .../blocksstable/ob_micro_block_encryption.h | 4 + src/storage/memtable/ob_memtable_mutator.cpp | 12 +- src/storage/tx/ob_clog_encrypt_info.h | 5 +- src/storage/tx/ob_clog_encrypt_info_os.cpp | 4 +- unittest/share/test_ob_log_archive_config.cpp | 10 +- .../test_micro_block_encryption.cpp | 8 +- .../blocksstable/test_sstable_meta.cpp | 2 +- 32 files changed, 540 insertions(+), 394 deletions(-) diff --git a/mittest/mtlenv/storage/test_memtable_v2.cpp b/mittest/mtlenv/storage/test_memtable_v2.cpp index 16f815c567..82c19616fd 100644 --- a/mittest/mtlenv/storage/test_memtable_v2.cpp +++ b/mittest/mtlenv/storage/test_memtable_v2.cpp @@ -818,16 +818,15 @@ public: OB_MAX_MASTER_KEY_LENGTH, master_key_len)); meta.master_key_.get_content().set_length(master_key_len); - EXPECT_EQ(OB_SUCCESS, ObAesEncryption::aes_decrypt(meta.master_key_.ptr(), - meta.master_key_.size(), - meta.encrypted_table_key_.ptr(), - meta.encrypted_table_key_.size(), - OB_ENCRYPTED_TABLE_KEY_LEN, - NULL, - 0, - static_cast(meta.encrypt_algorithm_), - decrypted_table_key, - out_len)); + EXPECT_EQ(OB_SUCCESS, ObBlockCipher::decrypt(meta.master_key_.ptr(), + meta.master_key_.size(), + meta.encrypted_table_key_.ptr(), + meta.encrypted_table_key_.size(), + OB_ENCRYPTED_TABLE_KEY_LEN, + NULL, 0, NULL, 0, NULL, 0, + static_cast(meta.encrypt_algorithm_), + decrypted_table_key, + out_len)); meta.table_key_.set_content(decrypted_table_key, out_len); EXPECT_EQ(true, meta.is_valid()); @@ -1114,13 +1113,13 @@ public: char encrypt_table_key[OB_ENCRYPTED_TABLE_KEY_LEN] = {0}; int64_t encrypt_out_len = 0; - int algorithm = ObAesOpMode::ob_invalid_mode + 3; + int algorithm = ObCipherOpMode::ob_invalid_mode + 3; EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN)); - EXPECT_EQ(OB_SUCCESS, ObAesEncryption::aes_encrypt(cur_master_key, strlen(cur_master_key), - origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN, - OB_ENCRYPTED_TABLE_KEY_LEN, NULL, 0, - static_cast(algorithm), - encrypt_table_key, encrypt_out_len)); + EXPECT_EQ(OB_SUCCESS, ObBlockCipher::encrypt(cur_master_key, strlen(cur_master_key), + origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN, + OB_ENCRYPTED_TABLE_KEY_LEN, NULL, 0, NULL, 0, 0, + static_cast(algorithm), + encrypt_table_key, encrypt_out_len, NULL)); encrypt_table_key[encrypt_out_len] = '\0'; EXPECT_STRNE(origin_table_key, encrypt_table_key); @@ -1132,13 +1131,9 @@ public: EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(tenant_id, master_key_id, master_key, OB_MAX_MASTER_KEY_LENGTH, master_key_len)); EXPECT_STREQ(master_key, cur_master_key); - char random_string[OB_CLOG_ENCRYPT_RANDOM_LEN] = {0}; - EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(random_string, OB_CLOG_ENCRYPT_RANDOM_LEN)); - encrypt_meta.master_key_.set_content(master_key, master_key_len); encrypt_meta.table_key_.set_content(origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN); encrypt_meta.encrypted_table_key_.set_content(encrypt_table_key, OB_ENCRYPTED_TABLE_KEY_LEN); - encrypt_meta.random_.set_content(random_string, OB_CLOG_ENCRYPT_RANDOM_LEN); encrypt_meta.tenant_id_ = tenant_id; encrypt_meta.master_key_version_ = master_key_id; encrypt_meta.encrypt_algorithm_ = algorithm; diff --git a/mittest/mtlenv/storage/test_physical_copy_task.cpp b/mittest/mtlenv/storage/test_physical_copy_task.cpp index 78cccffdb9..0e48dd074e 100644 --- a/mittest/mtlenv/storage/test_physical_copy_task.cpp +++ b/mittest/mtlenv/storage/test_physical_copy_task.cpp @@ -73,7 +73,7 @@ void TestRootBlockInfo::SetUp() { TestDataFilePrepare::SetUp(); prepare_tablet_read_info(); - des_meta_.encrypt_id_ = ObAesOpMode::ob_invalid_mode; + des_meta_.encrypt_id_ = ObCipherOpMode::ob_invalid_mode; des_meta_.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; ASSERT_TRUE(!root_info_.is_valid()); root_info_.reset(); diff --git a/src/share/config/ob_config.cpp b/src/share/config/ob_config.cpp index d0548adc08..558ba34870 100644 --- a/src/share/config/ob_config.cpp +++ b/src/share/config/ob_config.cpp @@ -1014,7 +1014,7 @@ int ObConfigLogArchiveOptionsItem::ObInnerConfigLogArchiveOptionsItem::set_defau if (ObBackupEncryptionMode::NONE == encryption_mode_) { //do nothing } else if (ObBackupEncryptionMode::TRANSPARENT_ENCRYPTION == encryption_mode_) { - encryption_algorithm_ = ObAesOpMode::ob_aes_128_ecb; + encryption_algorithm_ = ObCipherOpMode::ob_aes_128_ecb; } else { ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid mode for log_archive", K(encryption_mode_)); @@ -1028,10 +1028,44 @@ bool ObConfigLogArchiveOptionsItem::ObInnerConfigLogArchiveOptionsItem::is_encry if (ObBackupEncryptionMode::NONE == encryption_mode_) { //do nothing } else if (ObBackupEncryptionMode::TRANSPARENT_ENCRYPTION == encryption_mode_) { - is_valid = (ObAesOpMode::ob_aes_128_ecb == encryption_algorithm_ - || ObAesOpMode::ob_aes_192_ecb == encryption_algorithm_ - || ObAesOpMode::ob_aes_256_ecb == encryption_algorithm_ - || ObAesOpMode::ob_sm4_cbc_mode == encryption_algorithm_); + switch (encryption_algorithm_) { + case ObCipherOpMode::ob_aes_128_ecb: + case ObCipherOpMode::ob_aes_192_ecb: + case ObCipherOpMode::ob_aes_256_ecb: + case ObCipherOpMode::ob_aes_128_gcm: + case ObCipherOpMode::ob_aes_192_gcm: + case ObCipherOpMode::ob_aes_256_gcm: + case ObCipherOpMode::ob_sm4_cbc_mode: + case ObCipherOpMode::ob_sm4_gcm: + is_valid = true; + break; + case ObCipherOpMode::ob_aes_128_cbc: + case ObCipherOpMode::ob_aes_192_cbc: + case ObCipherOpMode::ob_aes_256_cbc: + case ObCipherOpMode::ob_aes_128_cfb1: + case ObCipherOpMode::ob_aes_192_cfb1: + case ObCipherOpMode::ob_aes_256_cfb1: + case ObCipherOpMode::ob_aes_128_cfb8: + case ObCipherOpMode::ob_aes_192_cfb8: + case ObCipherOpMode::ob_aes_256_cfb8: + case ObCipherOpMode::ob_aes_128_cfb128: + case ObCipherOpMode::ob_aes_192_cfb128: + case ObCipherOpMode::ob_aes_256_cfb128: + case ObCipherOpMode::ob_aes_128_ofb: + case ObCipherOpMode::ob_aes_192_ofb: + case ObCipherOpMode::ob_aes_256_ofb: + case ObCipherOpMode::ob_sm4_mode: + case ObCipherOpMode::ob_sm4_cbc: + case ObCipherOpMode::ob_sm4_ecb: + case ObCipherOpMode::ob_sm4_ofb: + case ObCipherOpMode::ob_sm4_ctr: + case ObCipherOpMode::ob_sm4_cfb128: + is_valid = false; + break; + default: + is_valid = false; + break; + } } else { is_valid = false; } diff --git a/src/share/config/ob_config.h b/src/share/config/ob_config.h index 71f0328784..cb09d7190c 100644 --- a/src/share/config/ob_config.h +++ b/src/share/config/ob_config.h @@ -994,7 +994,7 @@ public: share::ObBackupEncryptionMode::EncryptionMode get_encryption_mode() const {return value_.encryption_mode_;} - share::ObAesOpMode get_encryption_algorithm() const + share::ObCipherOpMode get_encryption_algorithm() const {return value_.encryption_algorithm_;} virtual ObConfigItemType get_config_item_type() const { return ObConfigItemType::OB_CONF_ITEM_TYPE_LOGARCHIVEOPT; @@ -1007,7 +1007,7 @@ public: is_compress_enabled_(false), compressor_type_(common::INVALID_COMPRESSOR), encryption_mode_(share::ObBackupEncryptionMode::NONE), - encryption_algorithm_(share::ObAesOpMode::ob_invalid_mode) + encryption_algorithm_(share::ObCipherOpMode::ob_invalid_mode) {} ~ObInnerConfigLogArchiveOptionsItem() {} @@ -1052,7 +1052,7 @@ public: bool is_compress_enabled_; common::ObCompressorType compressor_type_; share::ObBackupEncryptionMode::EncryptionMode encryption_mode_; - share::ObAesOpMode encryption_algorithm_; + share::ObCipherOpMode encryption_algorithm_; }; public: diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index 0ea661973e..f1edaa7c23 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -341,7 +341,7 @@ bool ObConfigLogArchiveOptionsChecker::check(const ObConfigItem &t) const } key_idx = -1; } else if (LOG_ARCHIVE_ENCRYPTION_ALGORITHM_IDX == key_idx) { - share::ObAesOpMode encryption_algorithm; + share::ObCipherOpMode encryption_algorithm; if (OB_FAIL(ObEncryptionUtil::parse_encryption_algorithm(s, encryption_algorithm))) { bret = false; OB_LOG(WARN, "invalid encrytion algorithm", K(s)); diff --git a/src/share/inner_table/ob_inner_table_schema.12101_12150.cpp b/src/share/inner_table/ob_inner_table_schema.12101_12150.cpp index b59a706996..ea6f3a5736 100644 --- a/src/share/inner_table/ob_inner_table_schema.12101_12150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12101_12150.cpp @@ -7233,7 +7233,7 @@ int ObInnerTableSchema::all_virtual_tenant_keystore_schema(ObTableSchema &table_ 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_INVALID, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_INTERNAL_MASTER_KEY_LENGTH, //column_length -1, //column_precision -1, //column_scale false, //is_nullable @@ -7467,7 +7467,7 @@ int ObInnerTableSchema::all_virtual_tenant_keystore_history_schema(ObTableSchema 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_INVALID, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_INTERNAL_MASTER_KEY_LENGTH, //column_length -1, //column_precision -1, //column_scale true, //is_nullable @@ -9205,7 +9205,7 @@ int ObInnerTableSchema::all_virtual_tenant_tablespace_schema(ObTableSchema &tabl 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_BINARY, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_ENCRYPTED_TABLE_KEY_LEN, //column_length -1, //column_precision -1, //column_scale false, //is_nullable @@ -9401,7 +9401,7 @@ int ObInnerTableSchema::all_virtual_tenant_tablespace_history_schema(ObTableSche 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_BINARY, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_ENCRYPTED_TABLE_KEY_LEN, //column_length -1, //column_precision -1, //column_scale true, //is_nullable diff --git a/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp b/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp index 26071b81c3..726aa959e7 100644 --- a/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp @@ -12670,7 +12670,7 @@ int ObInnerTableSchema::all_virtual_tenant_keystore_real_agent_ora_schema(ObTabl 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_UTF8MB4_BIN, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_INTERNAL_MASTER_KEY_LENGTH, //column_length 2, //column_precision -1, //column_scale false, //is_nullable diff --git a/src/share/inner_table/ob_inner_table_schema.15151_15200.cpp b/src/share/inner_table/ob_inner_table_schema.15151_15200.cpp index f7b1d91871..86470c61b4 100644 --- a/src/share/inner_table/ob_inner_table_schema.15151_15200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15151_15200.cpp @@ -127,7 +127,7 @@ int ObInnerTableSchema::all_virtual_tenant_tablespace_real_agent_ora_schema(ObTa 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_BINARY, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_ENCRYPTED_TABLE_KEY_LEN, //column_length -1, //column_precision -1, //column_scale false, //is_nullable diff --git a/src/share/inner_table/ob_inner_table_schema.201_250.cpp b/src/share/inner_table/ob_inner_table_schema.201_250.cpp index dbbf183f4f..0b0a098b24 100644 --- a/src/share/inner_table/ob_inner_table_schema.201_250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.201_250.cpp @@ -6202,7 +6202,7 @@ int ObInnerTableSchema::all_tenant_keystore_schema(ObTableSchema &table_schema) 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_INVALID, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_INTERNAL_MASTER_KEY_LENGTH, //column_length -1, //column_precision -1, //column_scale false, //is_nullable @@ -6452,7 +6452,7 @@ int ObInnerTableSchema::all_tenant_keystore_history_schema(ObTableSchema &table_ 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_INVALID, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_INTERNAL_MASTER_KEY_LENGTH, //column_length -1, //column_precision -1, //column_scale true, //is_nullable @@ -8334,7 +8334,7 @@ int ObInnerTableSchema::all_tenant_tablespace_schema(ObTableSchema &table_schema 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_BINARY, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_ENCRYPTED_TABLE_KEY_LEN, //column_length -1, //column_precision -1, //column_scale false, //is_nullable @@ -8546,7 +8546,7 @@ int ObInnerTableSchema::all_tenant_tablespace_history_schema(ObTableSchema &tabl 0, //part_key_pos ObVarcharType, //column_type CS_TYPE_BINARY, //column_collation_type - OB_MAX_MASTER_KEY_LENGTH, //column_length + OB_ENCRYPTED_TABLE_KEY_LEN, //column_length -1, //column_precision -1, //column_scale true, //is_nullable diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index a03d27c976..d1e1c6c232 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -2021,7 +2021,7 @@ all_tenant_keystore_def = dict( ('password', 'varchar:OB_MAX_PASSWORD_LENGTH'), ('status', 'int', 'false', 0), ('master_key_id', 'int'), - ('master_key', 'varchar:OB_MAX_MASTER_KEY_LENGTH'), + ('master_key', 'varchar:OB_INTERNAL_MASTER_KEY_LENGTH'), ('encrypted_key', 'varbinary:OB_MAX_ENCRYPTED_KEY_LENGTH', 'false', ''), ], ) @@ -2143,7 +2143,7 @@ all_tenant_tablespace_def = dict( normal_columns = [ ('tablespace_name', 'varchar:MAX_ORACLE_NAME_LENGTH'), ('encryption_name', 'varchar:MAX_ORACLE_NAME_LENGTH'), - ('encrypt_key', 'varbinary:OB_MAX_MASTER_KEY_LENGTH'), + ('encrypt_key', 'varbinary:OB_ENCRYPTED_TABLE_KEY_LEN'), ('master_key_id', 'uint'), ], ) diff --git a/src/share/ob_encryption_struct.cpp b/src/share/ob_encryption_struct.cpp index c11fc68fa5..90a08f15d1 100644 --- a/src/share/ob_encryption_struct.cpp +++ b/src/share/ob_encryption_struct.cpp @@ -31,8 +31,6 @@ OB_DEF_SERIALIZE(ObEncryptMeta) SHARE_LOG(WARN, "serialize master key verison failed", K(ret), K(buf_len), K(pos)); } else if (OB_FAIL(serialization::encode(buf, buf_len, pos, encrypt_algorithm_))) { SHARE_LOG(WARN, "serialize encrypt algorithm failed", K(ret), K(buf_len), K(pos)); - } else if (OB_FAIL(random_.serialize(buf, buf_len, pos))) { - SHARE_LOG(WARN, "serialize random failed", K(ret), K(buf_len), K(pos)); } else if (OB_FAIL(encrypted_table_key_.serialize(buf, buf_len, pos))) { SHARE_LOG(WARN, "serialize table key failed", K(ret), K(buf_len), K(pos)); } @@ -49,8 +47,6 @@ OB_DEF_DESERIALIZE(ObEncryptMeta) SHARE_LOG(WARN, "deserialize master key version failed", K(ret), K(data_len), K(pos)); } else if (OB_FAIL(serialization::decode(buf, data_len, pos, encrypt_algorithm_))) { SHARE_LOG(WARN, "deserialize encrypt algorithm failed", K(ret), K(data_len), K(pos)); - } else if (OB_FAIL(random_.deserialize(buf, data_len, pos))) { - SHARE_LOG(WARN, "deserialize random failed", K(ret), K(data_len), K(pos)); } else if (OB_FAIL(encrypted_table_key_.deserialize(buf, data_len, pos))) { SHARE_LOG(WARN, "deserialize table key failed", K(ret), K(data_len), K(pos)); } @@ -62,7 +58,6 @@ OB_DEF_SERIALIZE_SIZE(ObEncryptMeta) int64_t size = 0; size += serialization::encoded_length(master_key_version_); size += serialization::encoded_length(encrypt_algorithm_); - size += random_.get_serialize_size(); size += encrypted_table_key_.get_serialize_size(); return size; } @@ -70,9 +65,7 @@ OB_DEF_SERIALIZE_SIZE(ObEncryptMeta) int ObEncryptMeta::assign(const ObEncryptMeta &other) { int ret = OB_SUCCESS; - if (OB_FAIL(random_.assign(other.random_))) { - SHARE_LOG(WARN, "failed to assign random_", KR(ret), K(other)); - } else if (OB_FAIL(master_key_.assign(other.master_key_))) { + if (OB_FAIL(master_key_.assign(other.master_key_))) { SHARE_LOG(WARN, "failed to assign master_key_", KR(ret), K(other)); } else if (OB_FAIL(table_key_.assign(other.table_key_))) { SHARE_LOG(WARN, "failed to assign table_key_", KR(ret), K(other)); @@ -89,7 +82,6 @@ void ObEncryptMeta::reset() { master_key_version_ = -1; encrypt_algorithm_ = -1; - random_.reset(); master_key_.reset(); table_key_.reset(); encrypted_table_key_.reset(); @@ -114,8 +106,6 @@ OB_DEF_SERIALIZE(ObZoneEncryptMeta) SHARE_LOG(WARN, "serialize master key verison failed", K(ret), K(buf_len), K(pos)); } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, encrypt_algorithm_))) { SHARE_LOG(WARN, "serialize encrypt algorithm failed", K(ret), K(buf_len), K(pos)); - } else if (OB_FAIL(random_.serialize(buf, buf_len, pos))) { - SHARE_LOG(WARN, "serialize random failed", K(ret), K(buf_len), K(pos)); } return ret; } @@ -132,8 +122,6 @@ OB_DEF_DESERIALIZE(ObZoneEncryptMeta) SHARE_LOG(WARN, "deserialize master key version failed", K(ret), K(data_len), K(pos)); } else if (OB_FAIL(serialization::decode_i64(buf, data_len, pos, &encrypt_algorithm_))) { SHARE_LOG(WARN, "deserialize encrypt algorithm failed", K(ret), K(data_len), K(pos)); - } else if (OB_FAIL(random_.deserialize(buf, data_len, pos))) { - SHARE_LOG(WARN, "deserialize random failed", K(ret), K(data_len), K(pos)); } return ret; } @@ -144,7 +132,6 @@ OB_DEF_SERIALIZE_SIZE(ObZoneEncryptMeta) size += serialization::encoded_length_i16(version_); size += serialization::encoded_length_i64(master_key_version_); size += serialization::encoded_length_i64(encrypt_algorithm_); - size += random_.get_serialize_size(); return size; } @@ -153,23 +140,11 @@ void ObZoneEncryptMeta::reset() version_ = 0; master_key_version_ = -1; encrypt_algorithm_ = -1; - random_.reset(); } bool ObZoneEncryptMeta::is_valid_except_master_key_version() const { - bool b_ret = false; - int ret = OB_SUCCESS; - bool need_iv = false; - if (OB_UNLIKELY(encrypt_algorithm_ <= 0)) { - b_ret = false; - } else if (OB_FAIL(ObAesEncryption::aes_needs_iv(static_cast(encrypt_algorithm_), need_iv))) { - b_ret = false; - SHARE_LOG(WARN, "aes_needs_iv failed", K(ret), K(encrypt_algorithm_)); - } else { - b_ret = (!need_iv) || (random_.size() > 0); - } - return b_ret; + return ObBlockCipher::is_valid_cipher_opmode(static_cast(encrypt_algorithm_)); } bool ObZoneEncryptMeta::is_valid() const @@ -180,13 +155,9 @@ bool ObZoneEncryptMeta::is_valid() const int ObZoneEncryptMeta::assign(const ObZoneEncryptMeta &other) { int ret = OB_SUCCESS; - if (OB_FAIL(random_.assign(other.random_))) { - SHARE_LOG(WARN, "failed to assign random", K(ret), K(other)); - } else { - version_ = other.version_; - master_key_version_ = other.master_key_version_; - encrypt_algorithm_ = other.encrypt_algorithm_; - } + version_ = other.version_; + master_key_version_ = other.master_key_version_; + encrypt_algorithm_ = other.encrypt_algorithm_; return ret; } diff --git a/src/share/ob_encryption_struct.h b/src/share/ob_encryption_struct.h index e6999ead29..acce422a10 100644 --- a/src/share/ob_encryption_struct.h +++ b/src/share/ob_encryption_struct.h @@ -109,7 +109,6 @@ int ObEncryptKey::set_content(const char *ptr, const int64_t len) return ret; } -typedef ObEncryptKey ObEncryptRandomString; typedef ObEncryptKey ObEncryptMasterKeyString; typedef ObEncryptKey ObEncryptTableKeyString; @@ -120,13 +119,12 @@ struct ObEncryptMeta //The actual length of table_key is 15, and OB_ENCRYPTED_TABLE_KEY_LEN is used here, in order to avoid defining one more String ObEncryptTableKeyString table_key_; ObEncryptTableKeyString encrypted_table_key_; - ObEncryptRandomString random_; uint64_t tenant_id_; // no need to serialize int64_t master_key_version_; int64_t encrypt_algorithm_; ObEncryptMeta() : tenant_id_(OB_INVALID_ID), master_key_version_(-1), encrypt_algorithm_(-1) {} - TO_STRING_KV(K_(tenant_id), K_(master_key_version), K_(encrypt_algorithm), K_(random), - K_(master_key), K_(table_key)); + TO_STRING_KV(K_(tenant_id), K_(master_key_version), K_(encrypt_algorithm), K_(master_key), + K_(table_key)); int assign(const ObEncryptMeta &other); void reset(); int replace_tenant_id(const uint64_t real_tenant_id); @@ -134,14 +132,12 @@ struct ObEncryptMeta { return encrypt_algorithm_ > 0 && master_key_version_ > 0 - && random_.size() > 0 && encrypted_table_key_.size() > 0; } bool is_valid() const { return encrypt_algorithm_ > 0 && master_key_version_ > 0 - && random_.size() > 0 && master_key_.size() > 0 && table_key_.size() > 0 && encrypted_table_key_.size() > 0; @@ -159,14 +155,13 @@ public: bool is_valid() const; int assign(const ObZoneEncryptMeta &other); void set_master_key_version(int64_t master_key_version) {master_key_version_ = master_key_version;} - TO_STRING_KV(K_(master_key_version), K_(encrypt_algorithm), K_(random)); + TO_STRING_KV(K_(master_key_version), K_(encrypt_algorithm)); OB_UNIS_VERSION(1); public: static const int64_t VERSION = 1; int16_t version_; int64_t master_key_version_; int64_t encrypt_algorithm_; - ObEncryptRandomString random_; }; enum CLogEncryptStatBitIndex diff --git a/src/share/ob_encryption_util.h b/src/share/ob_encryption_util.h index f7334b5e19..4c0e447b25 100644 --- a/src/share/ob_encryption_util.h +++ b/src/share/ob_encryption_util.h @@ -34,7 +34,7 @@ public: static int generate_encrypt_key(char *buf, int64_t len); }; -enum ObAesOpMode { +enum ObCipherOpMode { ob_invalid_mode = 0, ob_aes_128_ecb = 1, ob_aes_192_ecb = 2, @@ -54,57 +54,70 @@ enum ObAesOpMode { ob_aes_128_ofb = 16, ob_aes_192_ofb = 17, ob_aes_256_ofb = 18, - ob_sm4_mode = 19, - ob_sm4_cbc_mode = 20, - //attention:remember to modify compare_aes_mod_safety when add new mode + ob_sm4_mode = 19, // old sm4_ctr using NULL as IV which is wrong, should not be used + ob_sm4_cbc_mode = 20, // sm4_cbc, use NULL as IV + ob_aes_128_gcm = 21, + ob_aes_192_gcm = 22, + ob_aes_256_gcm = 23, + ob_sm4_cbc = 24, // sm4_cbc, use random iv + ob_sm4_ecb = 25, + ob_sm4_ofb = 26, + ob_sm4_cfb128 = 27, + ob_sm4_ctr = 28, // sm4_ctr using unique iv + ob_sm4_gcm = 29, + /* attention: + 1.remember to modify compare_aes_mod_safety when add new mode + 2.considering compatibility of parse_encryption_id, only add modes sequentially + */ ob_max_mode }; -class ObAesEncryption +class ObBlockCipher { public: -//高级对称加密算法AES - static int aes_encrypt(const char *key, const int64_t &key_len, const char *data, - const int64_t data_len, const int64_t &buf_len, const char *iv, - const int64_t iv_len, enum ObAesOpMode mode, char *buf, int64_t &out_len); - static int aes_decrypt(const char *key, const int64_t &key_len, const char *data, - const int64_t data_len, const int64_t &buf_len, const char *iv, - const int64_t iv_len, enum ObAesOpMode mode, char *buf, int64_t &out_len); - static int aes_needs_iv(ObAesOpMode opmode, bool &need_iv); - static int64_t aes_encrypted_length(const int64_t data_len) - { - return (data_len / OB_AES_BLOCK_SIZE + 1) * OB_AES_BLOCK_SIZE; - } - static int64_t aes_max_decrypted_length(const int64_t encrypt_len) - { - /*encrypt_len is supposed >= OB_AES_BLOCK_SIZE*/ - return encrypt_len - 1 - (encrypt_len % OB_AES_BLOCK_SIZE); - } - static int64_t aes_safe_buffer_length(const int64_t data_len) - { - return data_len + OB_AES_BLOCK_SIZE; - } + static int encrypt(const char *key, const int64_t key_len, + const char *data, const int64_t data_len, const int64_t buf_len, + const char *iv, const int64_t iv_len, const char *aad, const int64_t aad_len, + const int64_t tag_len, const ObCipherOpMode mode, char *buf, int64_t &out_len, + char *tag); + static int decrypt(const char *key, const int64_t key_len, + const char *data, const int64_t data_len, const int64_t buf_len, + const char *iv, const int64_t iv_len, const char *aad, const int64_t aad_len, + const char *tag, const int64_t tag_len, const ObCipherOpMode mode, char *buf, + int64_t &out_len); + static bool is_valid_cipher_opmode(const ObCipherOpMode opmode); + static bool is_need_iv(const ObCipherOpMode opmode); + static bool is_need_aead(const ObCipherOpMode opmode); + static int get_key_length(const ObCipherOpMode opmode); + static int64_t get_iv_length(const ObCipherOpMode opmode); + static int64_t get_aead_tag_length(const ObCipherOpMode opmode); + static int64_t get_ciphertext_length(const ObCipherOpMode opmode, const int64_t plaintext_len); + static int64_t get_max_plaintext_length(const ObCipherOpMode opmode, + const int64_t ciphertext_len); //return true if right is more safe then left - static bool compare_aes_mod_safety(ObAesOpMode left, ObAesOpMode right); + static bool compare_aes_mod_safety(ObCipherOpMode left, ObCipherOpMode right); private: - static void aes_create_key(const unsigned char *key, int key_length, char *rkey, enum ObAesOpMode opmode); + static bool is_need_padding(const ObCipherOpMode opmode); + static void create_key(const unsigned char *key, int key_length, char *rkey, + enum ObCipherOpMode opmode); public: - static const int OB_MAX_AES_KEY_LENGTH = 256; - static const int OB_AES_IV_SIZE = 16; - static const int OB_AES_BLOCK_SIZE = 16; + static const int OB_MAX_CIPHER_KEY_LENGTH = 256; // max aes key bit length + static const int OB_DEFAULT_IV_LENGTH = 16; + static const int OB_CIPHER_BLOCK_LENGTH = 16; + static const int OB_DEFAULT_AEAD_AAD_LENGTH = 16; + static const int OB_DEFAULT_AEAD_TAG_LENGTH = 16; }; const int64_t OB_ROOT_KEY_LEN = 16; const int64_t OB_ORIGINAL_TABLE_KEY_LEN = 15; const int64_t OB_ENCRYPTED_TABLE_KEY_LEN = 16; -const int64_t OB_MAX_MASTER_KEY_LENGTH = 16; +const int64_t OB_INTERNAL_MASTER_KEY_LENGTH = 16; +const int64_t OB_MAX_MASTER_KEY_LENGTH = 32; const int64_t OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH = 16; const int64_t OB_ENCRYPT_KEY_SALT_LEN = 8; -const int64_t OB_SALTED_MASTER_KEY_LEN = OB_MAX_MASTER_KEY_LENGTH + OB_ENCRYPT_KEY_SALT_LEN; -const int64_t OB_MAX_ENCRYPTED_KEY_LENGTH = 80; +const int64_t OB_SALTED_MASTER_KEY_LEN = OB_INTERNAL_MASTER_KEY_LENGTH + OB_ENCRYPT_KEY_SALT_LEN; +const int64_t OB_MAX_ENCRYPTED_KEY_LENGTH = 256; const int64_t OB_INTERNAL_ENCRYPTED_KEY_LENGTH = 32; - -const int64_t OB_CLOG_ENCRYPT_RANDOM_LEN = 16; const int64_t OB_CLOG_ENCRYPT_MASTER_KEY_LEN = 32; const int64_t OB_CLOG_ENCRYPT_TABLE_KEY_LEN = 32; @@ -114,47 +127,61 @@ public: static int init_ssl_malloc(); #ifdef OB_BUILD_TDE_SECURITY - static bool need_encrypt(int64_t encrypt_id); + static bool need_encrypt(const ObCipherOpMode opmode); static int get_tde_method(int64_t tenant_id, common::ObString &tde_method); static int get_tde_kms_info(int64_t tenant_id, common::ObString &kms_info); #endif - static int parse_encryption_algorithm(const common::ObString &str, ObAesOpMode &encryption_algorithm); - static int parse_encryption_algorithm(const char *str, ObAesOpMode &encryption_algorithm); + static int parse_encryption_algorithm(const common::ObString &str, + ObCipherOpMode &encryption_algorithm); + static int parse_encryption_algorithm(const char *str, + ObCipherOpMode &encryption_algorithm); static int parse_encryption_id(const char *str, int64_t &encrypt_id); static int parse_encryption_id(const common::ObString &str, int64_t &encrypt_id); + static bool is_aes_encryption(const ObCipherOpMode opmode); + static bool is_sm4_encryption(const ObCipherOpMode opmode); #ifdef OB_BUILD_TDE_SECURITY + static const ObCipherOpMode DEFAULT_TABLE_KEY_AES_ENCRYPT_ALGORITHM = + share::ObCipherOpMode::ob_aes_128_ecb; + static const ObCipherOpMode DEFAULT_TABLE_KEY_SM4_ENCRYPT_ALGORITHM = + share::ObCipherOpMode::ob_sm4_cbc_mode; + static const ObCipherOpMode DEFAULT_TABLE_KEY_AES256_ENCRYPT_ALGORITHM = + share::ObCipherOpMode::ob_aes_256_ecb; + static const ObCipherOpMode SYS_DATA_ENCRYPT_ALGORITHM = share::ObCipherOpMode::ob_aes_128_cbc; static int64_t sys_encrypted_length(int64_t data_len); - - static int encrypt_data(const char *key, const int64_t key_len, enum ObAesOpMode mode, + // encrypt data with create a random iv and store iv in buf if needed. + static int encrypt_data(const char *key, const int64_t key_len, enum ObCipherOpMode mode, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); - static int decrypt_data(const char *key, const int64_t key_len, enum ObAesOpMode mode, + // using to decrypt data which encrypted by encrypt_data + static int decrypt_data(const char *key, const int64_t key_len, enum ObCipherOpMode mode, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); + static int encrypt_data(const char *key, const int64_t key_len, enum ObCipherOpMode mode, + const char *data, const int64_t data_len, const char *iv, + const int64_t iv_len, char *buf, const int64_t buf_len, int64_t &out_len); static int encrypt_data(const share::ObEncryptMeta &meta, - const char *from_buf, const int64_t from_len, - char *to_buf, int64_t to_buf_size, int64_t &to_len); + const char *data, const int64_t data_len, + char *buf, int64_t const buf_len, int64_t &out_len); static int decrypt_data(const share::ObEncryptMeta &meta, - const char *from_buf, const int64_t from_len, - char *to_buf, int64_t to_buf_size, int64_t &to_len); - static int encrypt_table_key(const share::ObEncryptMeta &meta, + const char *data, const int64_t data_len, + char *buf, int64_t const buf_len, int64_t &out_len); + static int create_encrypted_table_key(const uint64_t tenant_id, uint64_t &master_key_id, + char *out_buf, const int64_t out_buf_len, int64_t &out_len, + int retry_times = 3); + static int decrypt_table_key(const uint64_t tenant_id, + const char *master_key, const int64_t master_key_len, + const char *encrypt_key, const int64_t encrypt_key_len, char *out_buf, const int64_t out_buf_len, int64_t &out_len); - static int decrypt_table_key(share::ObEncryptMeta &meta, - const char *in_buf, const int64_t in_buf_len); + static int decrypt_table_key(share::ObEncryptMeta &meta); static int encrypt_master_key(const uint64_t tenant_id, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); static int decrypt_master_key(const uint64_t tenant_id, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); static int encrypt_sys_data(const uint64_t tenant_id, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); - static int encrypt_sys_data_default(const char *data, const int64_t data_len, - char *buf, const int64_t buf_len, int64_t &out_len); - static int decrypt_sys_data(const uint64_t tenant_id, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); - static int decrypt_sys_data_default(const char *data, const int64_t data_len, - char *buf, const int64_t buf_len, int64_t &out_len); static int encrypt_zone_data(share::ObZoneEncryptMeta &meta, const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); @@ -162,13 +189,19 @@ public: const char *data, const int64_t data_len, char *buf, const int64_t buf_len, int64_t &out_len); #endif - // return the max length after encryption - static int64_t encrypted_length(const int64_t data_len); + +#ifdef OB_BUILD_TDE_SECURITY + // return the max buf length after encryption + static int64_t encrypted_length(const ObCipherOpMode mode, const int64_t data_len); // return an unencrypted data length whose length after encryption is always less than data_len - static int64_t decrypted_length(const int64_t data_len); + static int64_t decrypted_length(const ObCipherOpMode mode, const int64_t data_len); // return the max length after decryption static int64_t safe_buffer_length(const int64_t data_len); -#ifdef OB_BUILD_TDE_SECURITY +private: + static int encrypt_sys_data_default(const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int decrypt_sys_data_default(const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); private: static const char* system_encrypt_key_; static const char* system_encrypt_iv_; @@ -201,7 +234,15 @@ public: static bool is_internal(const common::ObString &tde_method); static bool is_kms(const common::ObString &tde_method); static bool is_sm_algorithm(const common::ObString &tde_method); + static bool is_aes256_algorithm(const common::ObString &tde_method); static bool use_external_key_id(const common::ObString &tde_method); + static ObString extract_kms_name(const common::ObString &tde_method); + static ObString extract_mode_name(const common::ObString &tde_method); +private: + static bool inner_is_internal(const common::ObString &kms_name); + static bool inner_is_kms(const common::ObString &kms_name); + static bool is_valid_kms(const common::ObString &kms_name); + static bool is_valid_mode(const common::ObString &mode_name); }; #endif @@ -249,7 +290,7 @@ public: void destroy(); int load(const common::ObString& engine); ObEncryptEngineType get_engine_type(const common::ObString& engine); - ENGINE* get_tde_engine(ObAesOpMode &mode) const; + ENGINE* get_tde_engine(ObCipherOpMode &mode) const; int reload_config(); private: bool is_inited_; diff --git a/src/share/ob_encryption_util_os.cpp b/src/share/ob_encryption_util_os.cpp index 56870626b9..94bb674ac2 100644 --- a/src/share/ob_encryption_util_os.cpp +++ b/src/share/ob_encryption_util_os.cpp @@ -39,52 +39,14 @@ int ObKeyGenerator::generate_encrypt_key(char *buf, int64_t len) if (len <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("the buf of len is invalid", K(ret)); - } else { - int i; - for (i = 0; i < len; ++i) { - switch (common::ObRandom::rand(0, 2)) { - case 1: - buf[i] = 'A' + common::ObRandom::rand(0, 25); - break; - case 2: - buf[i] = 'a' + common::ObRandom::rand(0, 25); - break; - default: - buf[i] = '0' + common::ObRandom::rand(0, 9); - break; - } - } + } else if (!RAND_bytes((unsigned char*)buf, (int)len)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("failed to get a rand bytes"); } return ret; } -static int ob_aes_opmode_key_sizes_impl[]= -{ - 0 /* invlid_mode*/, - 128 /* aes-128-ecb */, - 192 /* aes-192-ecb */, - 256 /* aes-256-ecb */, - 128 /* aes-128-cbc */, - 192 /* aes-192-cbc */, - 256 /* aes-256-cbc */, - 128 /* aes-128-cfb1 */, - 192 /* aes-192-cfb1 */, - 256 /* aes-256-cfb1 */, - 128 /* aes-128-cfb8 */, - 192 /* aes-192-cfb8 */, - 256 /* aes-256-cfb8 */, - 128 /* aes-128-cfb128 */, - 192 /* aes-192-cfb128 */, - 256 /* aes-256-cfb128 */, - 128 /* aes-128-ofb */, - 192 /* aes-192-ofb */, - 256 /* aes-256-ofb */, - 128 /* sm4-mode*/, - 128 /* sm4-cbc-mode*/ -}; -STATIC_ASSERT(ob_max_mode == ARRAYSIZEOF(ob_aes_opmode_key_sizes_impl), "key size count mismatch"); - -static const EVP_CIPHER *aes_evp_type(const ObAesOpMode mode) +static const EVP_CIPHER *get_evp_cipher(const ObCipherOpMode mode) { switch (mode) { @@ -106,35 +68,97 @@ static const EVP_CIPHER *aes_evp_type(const ObAesOpMode mode) case ob_aes_256_cfb8: return EVP_aes_256_cfb8(); case ob_aes_256_cfb128: return EVP_aes_256_cfb128(); case ob_aes_256_ofb: return EVP_aes_256_ofb(); + case ob_aes_128_gcm: return EVP_aes_128_gcm(); + case ob_aes_192_gcm: return EVP_aes_192_gcm(); + case ob_aes_256_gcm: return EVP_aes_256_gcm(); #ifdef OB_USE_BABASSL case ob_sm4_mode: return EVP_sm4_ctr(); case ob_sm4_cbc_mode: return EVP_sm4_cbc(); + case ob_sm4_cbc: return EVP_sm4_cbc(); + case ob_sm4_ecb: return EVP_sm4_ecb(); + case ob_sm4_ofb: return EVP_sm4_ofb(); + case ob_sm4_cfb128: return EVP_sm4_cfb128(); + case ob_sm4_ctr: return EVP_sm4_ctr(); + case ob_sm4_gcm: return EVP_sm4_gcm(); #endif default: return NULL; } } -/** - 判断当前加密算法是否需要一个初始向量,只有ecb模式无需初始化向量,其他则需要. -*/ - -int ObAesEncryption::aes_needs_iv(ObAesOpMode opmode, bool &need_iv) +int ObBlockCipher::get_key_length(const ObCipherOpMode opmode) { - int ret = OB_SUCCESS; - const EVP_CIPHER *cipher= aes_evp_type(opmode); - int iv_length = 0; - iv_length= EVP_CIPHER_iv_length(cipher); - if (iv_length != 0 && iv_length != OB_AES_IV_SIZE) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid iv length", K(ret)); - } else { - need_iv = (iv_length != 0 ? true : false); + int length = 0; + const EVP_CIPHER *cipher = get_evp_cipher(opmode); + if (NULL != cipher) { + length = EVP_CIPHER_key_length(cipher); } - return ret; + return length; } -void ObAesEncryption::aes_create_key(const unsigned char *key, int key_length, char *rkey, enum ObAesOpMode opmode) + +bool ObBlockCipher::is_need_iv(const ObCipherOpMode opmode) { - const int key_size= ob_aes_opmode_key_sizes_impl[opmode] / 8; + bool need_iv = false; + const EVP_CIPHER *cipher = get_evp_cipher(opmode); + if (NULL != cipher) { + int default_iv_length = EVP_CIPHER_iv_length(cipher); + need_iv = default_iv_length == 0 ? false : true; + } + return need_iv; +} + +bool ObBlockCipher::is_need_aead(const ObCipherOpMode opmode) +{ + bool need_aead = false; + const EVP_CIPHER *cipher = get_evp_cipher(opmode); + if (NULL != cipher) { + need_aead = EVP_CIPHER_mode(cipher) == EVP_CIPH_GCM_MODE; + } + return need_aead; +} + +bool ObBlockCipher::is_need_padding(const ObCipherOpMode opmode) +{ + bool need_padding = false; + const EVP_CIPHER *cipher = get_evp_cipher(opmode); + if (NULL != cipher) { + need_padding = EVP_CIPH_ECB_MODE == EVP_CIPHER_mode(cipher) || + EVP_CIPH_CBC_MODE == EVP_CIPHER_mode(cipher); + } + return need_padding; +} + +int64_t ObBlockCipher::get_iv_length(const ObCipherOpMode opmode) +{ + int64_t iv_length = 0; + const EVP_CIPHER *cipher = get_evp_cipher(opmode); + if (NULL != cipher) { + iv_length = EVP_CIPHER_iv_length(cipher); + } + return iv_length; +} + +int64_t ObBlockCipher::get_aead_tag_length(const ObCipherOpMode opmode) +{ + return is_need_aead(opmode) ? OB_DEFAULT_AEAD_TAG_LENGTH : 0; +} + +int64_t ObBlockCipher::get_ciphertext_length(const ObCipherOpMode opmode, const int64_t data_len) +{ + return is_need_padding(opmode) ? + (data_len / OB_CIPHER_BLOCK_LENGTH + 1) * OB_CIPHER_BLOCK_LENGTH : data_len; +} + +int64_t ObBlockCipher::get_max_plaintext_length(const ObCipherOpMode opmode, + const int64_t encrypt_len) +{ + return is_need_padding(opmode) ? + encrypt_len - 1 - (encrypt_len % OB_CIPHER_BLOCK_LENGTH) : encrypt_len; +} + +void ObBlockCipher::create_key(const unsigned char *key, int key_length, char *rkey, + enum ObCipherOpMode opmode) +{ + const int key_size= get_key_length(opmode); char *rkey_end = NULL; char *ptr = NULL; /* 真正的key的起始位置 */ char *sptr = NULL; @@ -149,59 +173,80 @@ void ObAesEncryption::aes_create_key(const unsigned char *key, int key_length, c } } -int ObAesEncryption::aes_encrypt(const char *key, const int64_t &key_len, const char *data, - const int64_t data_len, const int64_t &buf_len, const char *iv, - const int64_t iv_len, enum ObAesOpMode mode, char *buf, int64_t &out_len) +bool ObBlockCipher::is_valid_cipher_opmode(const ObCipherOpMode opmode) +{ + return opmode > ObCipherOpMode::ob_invalid_mode && opmode < ObCipherOpMode::ob_max_mode && + opmode != ObCipherOpMode::ob_sm4_mode; +} + +int ObBlockCipher::encrypt(const char *key, const int64_t key_len, + const char *data, const int64_t data_len, const int64_t buf_len, + const char *iv, const int64_t iv_len, + const char *aad, const int64_t aad_len, const int64_t tag_len, + enum ObCipherOpMode mode, char *buf, int64_t &out_len, char *tag) { int ret = OB_SUCCESS; - int64_t expect_buf_len = (data_len / OB_AES_BLOCK_SIZE + 1) * OB_AES_BLOCK_SIZE; - bool need_iv = false; - ENGINE *engine = NULL; - common::ObString err_reason; - if (OB_ISNULL(buf) || buf_len < expect_buf_len || key_len < 0 || - ObAesOpMode::ob_invalid_mode == mode || - (iv_len != 0 && iv_len != OB_AES_IV_SIZE) || - (OB_ISNULL(data) && data_len != 0) || - (OB_ISNULL(key) && key_len != 0)) { + bool need_iv = is_need_iv(mode); + bool need_aead = is_need_aead(mode); + bool need_padding = is_need_padding(mode); + if (!is_valid_cipher_opmode(mode) || + (OB_ISNULL(key) && key_len != 0) || key_len < 0 || // allow NULL key + (OB_ISNULL(data) && data_len != 0) || data_len < 0 || // allow NULL data + OB_ISNULL(buf) || buf_len < get_ciphertext_length(mode, data_len)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(data_len), K(buf_len), K(iv_len), K(expect_buf_len), K(data), K(key)); - } else if (OB_FAIL(aes_needs_iv(mode, need_iv))) { - LOG_WARN("fail to check whether need iv", K(ret)); + LOG_WARN("invalid argument", K(ret), K(data_len), K(key_len), K(buf_len), KP(buf), KP(data), + KP(key), K(mode)); + } else if (need_iv && ((iv_len != 0 && OB_ISNULL(iv)) || + (iv_len != 0 && iv_len != get_iv_length(mode)))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid initial vector argument", K(ret), K(iv_len), KP(iv), K(mode)); + } else if (need_aead && ((aad_len != 0 && OB_ISNULL(aad)) || + (aad_len != 0 && aad_len != OB_DEFAULT_AEAD_AAD_LENGTH) || + tag_len != OB_DEFAULT_AEAD_TAG_LENGTH || OB_ISNULL(tag))){ + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid aead argument", K(ret), K(aad_len), K(tag_len), KP(aad), K(mode)); } else { int u_len=0, f_len=0; - unsigned char rkey[OB_MAX_AES_KEY_LENGTH / 8]; - unsigned char *iv_encrypt = need_iv ? (unsigned char *)iv : NULL; - engine = ObTdeEncryptEngineLoader::get_instance().get_tde_engine(mode); + unsigned char rkey[OB_MAX_CIPHER_KEY_LENGTH / 8]; + unsigned char *iv_encrypt = (!need_iv || iv_len == 0) ? NULL : (unsigned char *)iv; + unsigned char *add_encrypt = (!need_aead || aad_len == 0) ? NULL : (unsigned char *)aad; + ENGINE *engine = ObTdeEncryptEngineLoader::get_instance().get_tde_engine(mode); + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + const EVP_CIPHER *cipher = get_evp_cipher(mode); if (NULL != engine) { if (EXECUTE_COUNT_PER_SEC(10)) { LOG_INFO("tde use engine to encrypt data", K(mode)); } } - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (OB_ISNULL(ctx)) { + if (OB_ISNULL(ctx) ||OB_ISNULL(cipher)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("fail to to init cipher ctx in aes_encrypt", K(ret)); + LOG_WARN("get EVP_ctx or EVP_cipher is NULL", K(ret), KP(ctx), KP(cipher)); + } else if (FALSE_IT(create_key((const unsigned char *)key, (int)key_len, (char *)rkey, mode))) { } else if (FALSE_IT(EVP_CIPHER_CTX_init(ctx))) { - } else if (FALSE_IT(aes_create_key((const unsigned char *)key, (int)key_len, - (char *)rkey, mode))) { - } else if (!EVP_EncryptInit_ex(ctx, aes_evp_type(mode), engine, rkey, iv_encrypt)) { + } else if (!EVP_EncryptInit_ex(ctx, cipher, engine, rkey, (unsigned char*)iv_encrypt)) { ret = OB_ERR_AES_ENCRYPT; - LOG_WARN("fail to init evp ctx in aes_encrypt", K(ret)); - } else if (!EVP_CIPHER_CTX_set_padding(ctx, true)) { + LOG_WARN("fail to init evp encrytion cipher and engine in encrypt", K(ret)); + } else if (need_padding && !EVP_CIPHER_CTX_set_padding(ctx, true)) { ret = OB_ERR_AES_ENCRYPT; - LOG_WARN("fail to set padding in aes_encrypt", K(ret)); + LOG_WARN("fail to set padding in encrypt", K(ret)); + } else if (need_aead && NULL != add_encrypt && !EVP_EncryptUpdate(ctx, NULL, &u_len, (unsigned char *)add_encrypt, (int)aad_len)) { + ret = OB_ERR_AES_ENCRYPT; + LOG_WARN("fail to set AAD in encrypt", K(ret)); } else if (!EVP_EncryptUpdate(ctx, (unsigned char *)buf, &u_len, (unsigned char *)data, (int)data_len)) { ret = OB_ERR_AES_ENCRYPT; - LOG_WARN("fail to encrypt data in aes_encrypt", K(ret)); + LOG_WARN("fail to encrypt data in encrypt", K(ret)); } else if (!EVP_EncryptFinal(ctx, (unsigned char*)buf + u_len, &f_len)) { ret = OB_ERR_AES_ENCRYPT; - LOG_WARN("fail to encrypt final frame data in aes_encrypt", K(ret)); + LOG_WARN("fail to encrypt final frame data in encrypt", K(ret)); + } else if (need_aead && !EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, (int)tag_len, (unsigned char*)tag)) { + ret = OB_ERR_AES_ENCRYPT; + LOG_WARN("fail to encrypt final frame data in encrypt", K(ret)); } else { out_len = u_len + f_len; } - if (OB_FAIL(ret)) { - err_reason = common::ObString::make_string(ERR_reason_error_string(ERR_get_error())); - LOG_WARN("tde encrypt failed", K(err_reason)); + if (OB_UNLIKELY(OB_ERR_AES_ENCRYPT == ret)) { + common::ObString err_reason = common::ObString::make_string(ERR_reason_error_string(ERR_get_error())); + LOG_WARN("encrypt failed", K(err_reason)); } if (OB_NOT_NULL(ctx)) { EVP_CIPHER_CTX_free(ctx); @@ -210,58 +255,73 @@ int ObAesEncryption::aes_encrypt(const char *key, const int64_t &key_len, const return ret; } -int ObAesEncryption::aes_decrypt(const char *key, const int64_t &key_len, const char *data, - const int64_t data_len, const int64_t &buf_len, const char *iv, - const int64_t iv_len, enum ObAesOpMode mode, char *buf, int64_t &out_len) +int ObBlockCipher::decrypt(const char *key, const int64_t key_len, + const char *data, const int64_t data_len, const int64_t buf_len, + const char *iv, const int64_t iv_len, const char *aad, + const int64_t aad_len, const char *tag, const int64_t tag_len, + enum ObCipherOpMode mode, char *buf, int64_t &out_len) { int ret = OB_SUCCESS; - bool need_iv = false; - ENGINE *engine = NULL; - common::ObString err_reason; - if (OB_ISNULL(buf) || key_len < 0 || buf_len < data_len || - ObAesOpMode::ob_invalid_mode == mode || - (iv_len != 0 && iv_len != OB_AES_IV_SIZE) || - (OB_ISNULL(key) && key_len != 0) || - (OB_ISNULL(data) && data_len != 0)) { + bool need_iv = is_need_iv(mode); + bool need_aead = is_need_aead(mode); + bool need_padding = is_need_padding(mode); + if (!is_valid_cipher_opmode(mode) || + (OB_ISNULL(key) && key_len != 0) || key_len < 0 || + (OB_ISNULL(data) && data_len != 0) || data_len < 0 || + OB_ISNULL(buf) || buf_len < data_len) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(data_len), K(data), K(key)); - } else if (OB_FAIL(aes_needs_iv(mode, need_iv))) { - LOG_WARN("fail to check whether need iv", K(ret)); + LOG_WARN("invalid argument", K(ret), K(data_len), K(key_len), K(buf_len), KP(buf), KP(data), + KP(key), K(mode)); + } else if (need_iv && ((iv_len != 0 && OB_ISNULL(iv)) || + (iv_len != 0 && iv_len != get_iv_length(mode)))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid initial vector argument", K(ret), K(iv_len), KP(iv), K(mode)); + } else if (need_aead && ((aad_len != 0 && OB_ISNULL(aad)) || + (aad_len != 0 && aad_len != OB_DEFAULT_AEAD_AAD_LENGTH) || + tag_len != OB_DEFAULT_AEAD_TAG_LENGTH || OB_ISNULL(tag))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid aead argument", K(ret), K(aad_len), K(tag_len), KP(aad), KP(tag), K(mode)); } else { int u_len=0, f_len=0; - unsigned char rkey[OB_MAX_AES_KEY_LENGTH / 8]; - unsigned char *iv_encrypt = need_iv ? (unsigned char *)iv : NULL; - engine = ObTdeEncryptEngineLoader::get_instance().get_tde_engine(mode); + unsigned char rkey[OB_MAX_CIPHER_KEY_LENGTH / 8]; + unsigned char *iv_decrypt = (!need_iv || iv_len == 0) ? NULL : (unsigned char *)iv; + unsigned char *add_decrypt = (!need_aead || aad_len == 0) ? NULL : (unsigned char *)aad; + ENGINE *engine = ObTdeEncryptEngineLoader::get_instance().get_tde_engine(mode); + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + const EVP_CIPHER *cipher = get_evp_cipher(mode); if (NULL != engine) { if (EXECUTE_COUNT_PER_SEC(10)) { - LOG_INFO("tde use engine to decrypt data", K(mode)); + LOG_INFO("use engine to decrypt data", K(mode)); } } - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (OB_ISNULL(ctx)) { + if (OB_ISNULL(ctx) || OB_ISNULL(cipher)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("fail to to init cipher ctx in aes_decrypt", K(ret)); + LOG_WARN("fail to to init cipher ctx in decrypt", K(ret), KP(ctx), KP(cipher)); + } else if (FALSE_IT(create_key((const unsigned char *)key, (int)key_len, (char *)rkey, mode))) { } else if (FALSE_IT(EVP_CIPHER_CTX_init(ctx))) { - } else if (FALSE_IT(aes_create_key((const unsigned char *)key, (int)key_len, - (char *)rkey, mode))) { - } else if (!EVP_DecryptInit_ex(ctx, aes_evp_type(mode), engine, rkey, iv_encrypt)) { + } else if (!EVP_DecryptInit_ex(ctx, cipher, engine, rkey, (unsigned char*)iv_decrypt)) { ret = OB_ERR_AES_DECRYPT; - LOG_WARN("fail to init evp ctx in aes_decrypt", K(ret)); - } else if (!EVP_CIPHER_CTX_set_padding(ctx, true)) { + LOG_WARN("fail to init evp ctx in decrypt", K(ret)); + } else if (need_padding && !EVP_CIPHER_CTX_set_padding(ctx, true)) { ret = OB_ERR_AES_DECRYPT; - LOG_WARN("fail to set padding in aes_decrypt", K(ret)); - } else if (!EVP_DecryptUpdate(ctx, (unsigned char *)buf, &u_len, (unsigned char *)data, (int)data_len)) { + LOG_WARN("fail to set padding in decrypt", K(ret)); + } else if (need_aead && NULL != add_decrypt && !EVP_DecryptUpdate(ctx, NULL, &u_len, (const unsigned char *)add_decrypt, (int)aad_len)) { + ret = OB_ERR_AES_DECRYPT; + LOG_WARN("fail to set AAD in decrypt", K(ret)); + } else if (!EVP_DecryptUpdate(ctx, (unsigned char *)buf, &u_len, (const unsigned char *)data, (int)data_len)) { + ret = OB_ERR_AES_DECRYPT; + LOG_WARN("fail to decrypt data in decrypt", K(ret)); + } else if (need_aead && !EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, (int)tag_len, (unsigned char *)tag)) { ret = OB_ERR_AES_DECRYPT; - LOG_WARN("fail to decrypt data in aes_decrypt", K(ret)); } else if (!EVP_DecryptFinal(ctx, (unsigned char*)buf + u_len, &f_len)) { ret = OB_ERR_AES_DECRYPT; - LOG_WARN("fail to decrypt final frame data in aes_decrypt", K(ret), K(u_len), K(f_len)); + LOG_WARN("fail to decrypt final frame data in decrypt", K(ret), K(u_len), K(f_len)); } else { out_len = u_len + f_len; } - if (OB_FAIL(ret)) { - err_reason = common::ObString::make_string(ERR_reason_error_string(ERR_get_error())); - LOG_WARN("tde decrypt failed", K(err_reason)); + if (OB_UNLIKELY(OB_ERR_AES_DECRYPT == ret)) { + common::ObString err_reason = common::ObString::make_string(ERR_reason_error_string(ERR_get_error())); + LOG_WARN("decrypt failed", K(err_reason)); } if (OB_NOT_NULL(ctx)) { EVP_CIPHER_CTX_free(ctx); @@ -270,6 +330,7 @@ int ObAesEncryption::aes_decrypt(const char *key, const int64_t &key_len, const return ret; } + #ifndef OB_USE_BABASSL static void* ob_malloc_openssl(size_t nbytes) { @@ -324,25 +385,29 @@ int ObEncryptionUtil::init_ssl_malloc() } int ObEncryptionUtil::parse_encryption_algorithm(const char *str, - ObAesOpMode &encryption_algorithm) + ObCipherOpMode &encryption_algorithm) { int ret = OB_SUCCESS; - encryption_algorithm = ObAesOpMode::ob_invalid_mode; + encryption_algorithm = ObCipherOpMode::ob_invalid_mode; return ret; } int ObEncryptionUtil::parse_encryption_id(const ObString &str, int64_t &encrypt_id) { int ret = OB_SUCCESS; - ObAesOpMode encryption_algorithm = ObAesOpMode::ob_invalid_mode; + ObCipherOpMode encryption_algorithm = ObCipherOpMode::ob_invalid_mode; return ret; } -int64_t ObEncryptionUtil::encrypted_length(const int64_t data_len) -{ - return ObAesEncryption::aes_encrypted_length(data_len); +bool ObEncryptionUtil::is_aes_encryption(const ObCipherOpMode opmode) { + return (opmode >= ObCipherOpMode::ob_aes_128_ecb && opmode <= ObCipherOpMode::ob_aes_256_ofb) || + (opmode >= ObCipherOpMode::ob_aes_128_gcm && opmode <= ObCipherOpMode::ob_aes_256_gcm); } +bool ObEncryptionUtil::is_sm4_encryption(const ObCipherOpMode opmode) { + return (opmode >= ObCipherOpMode::ob_sm4_mode && opmode <= ObCipherOpMode::ob_sm4_cbc_mode) || + (opmode >= ObCipherOpMode::ob_sm4_cbc && opmode <= ObCipherOpMode::ob_sm4_gcm); +} bool ObBackupEncryptionMode::is_valid(const EncryptionMode &mode) { @@ -587,38 +652,15 @@ void ObTdeEncryptEngineLoader::destroy() } } -ENGINE* ObTdeEncryptEngineLoader::get_tde_engine(ObAesOpMode &mode) const +ENGINE* ObTdeEncryptEngineLoader::get_tde_engine(ObCipherOpMode &mode) const { ObEncryptEngineType type = OB_INVALID_ENGINE; - switch(mode) { - case ob_sm4_mode: - case ob_sm4_cbc_mode: - type = OB_SM4_ENGINE; - break; - case ob_aes_128_ecb: - case ob_aes_128_cbc: - case ob_aes_128_cfb1: - case ob_aes_128_cfb8: - case ob_aes_128_cfb128: - case ob_aes_128_ofb: - case ob_aes_192_ecb: - case ob_aes_192_cbc: - case ob_aes_192_cfb1: - case ob_aes_192_cfb8: - case ob_aes_192_cfb128: - case ob_aes_192_ofb: - case ob_aes_256_ecb: - case ob_aes_256_cbc: - case ob_aes_256_cfb1: - case ob_aes_256_cfb8: - case ob_aes_256_cfb128: - case ob_aes_256_ofb: - type = OB_AES_ENGINE; - break; - case ob_invalid_mode: - case ob_max_mode: - type = OB_INVALID_ENGINE; - break; + if (ObEncryptionUtil::is_aes_encryption(mode)) { + type = OB_AES_ENGINE; + } else if (ObEncryptionUtil::is_sm4_encryption(mode)) { + type = OB_SM4_ENGINE; + } else { + type = OB_INVALID_ENGINE; } return tde_engine_[type]; } diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index 087717fffc..d59569dba9 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -2877,7 +2877,6 @@ int ObDmlCgService::init_encrypt_table_meta_( int ret = OB_SUCCESS; transaction::ObEncryptMetaCache meta_cache; char master_key[OB_MAX_MASTER_KEY_LENGTH]; - char random_string[OB_CLOG_ENCRYPT_RANDOM_LEN]; int64_t master_key_length = 0; if (OB_ISNULL(table_schema) || OB_ISNULL(guard)) { ret = OB_INVALID_ARGUMENT; @@ -2899,12 +2898,6 @@ int ObDmlCgService::init_encrypt_table_meta_( if (OB_FAIL(meta_cache.meta_.encrypted_table_key_.set_content( table_schema->get_encrypt_key()))) { LOG_WARN("fail to assign encrypt key", K(ret)); - } else if (OB_FAIL(share::ObKeyGenerator::generate_encrypt_key( - random_string, OB_CLOG_ENCRYPT_RANDOM_LEN))) { - LOG_WARN("fail to generate random string", K(ret)); - } else if (OB_FAIL(meta_cache.meta_.random_.set_content(ObString( - OB_CLOG_ENCRYPT_RANDOM_LEN, random_string)))) { - LOG_WARN("fail to assign random string", K(ret)); } #ifdef ERRSIM else if (OB_FAIL(OB_E(EventTable::EN_ENCRYPT_GET_MASTER_KEY_FAILED) OB_SUCCESS)) { @@ -2912,8 +2905,8 @@ int ObDmlCgService::init_encrypt_table_meta_( } #endif else if (OB_FAIL(share::ObMasterKeyGetter::get_master_key(table_schema->get_tenant_id(), - table_schema->get_master_key_id(), - master_key, OB_MAX_MASTER_KEY_LENGTH, master_key_length))) { + table_schema->get_master_key_id(), master_key, OB_MAX_MASTER_KEY_LENGTH, + master_key_length))) { LOG_WARN("fail to get master key", K(ret)); // 如果在cg阶段获取主密钥失败了, 有可能是因为RS执行内部sql没有租户资源引起的. // 在没有租户资源的情况下获取主密钥, 获取加密租户配置项时会失败 @@ -2922,12 +2915,10 @@ int ObDmlCgService::init_encrypt_table_meta_( // 兜底是执行期再次获取, 再次获取成功了则继续往下走, 失败了则报错出来. // 见bug ret = OB_SUCCESS; - } else if (OB_FAIL(meta_cache.meta_.master_key_.set_content(ObString( - master_key_length, master_key)))) { + } else if (OB_FAIL(meta_cache.meta_.master_key_.set_content( + ObString(master_key_length, master_key)))) { LOG_WARN("fail to assign master_key", K(ret)); - } else if (OB_FAIL(ObEncryptionUtil::decrypt_table_key(meta_cache.meta_, - meta_cache.meta_.encrypted_table_key_.ptr(), - meta_cache.meta_.encrypted_table_key_.size()))) { + } else if (OB_FAIL(ObEncryptionUtil::decrypt_table_key(meta_cache.meta_))) { LOG_WARN("failed to decrypt_table_key", K(ret)); } else {/*do nothing*/} diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 249097293e..f2461908ba 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -7292,7 +7292,6 @@ int ObStaticEngineCG::init_encrypt_table_meta( int ret = OB_SUCCESS; transaction::ObEncryptMetaCache meta_cache; char master_key[OB_MAX_MASTER_KEY_LENGTH]; - char random_string[OB_CLOG_ENCRYPT_RANDOM_LEN]; int64_t master_key_length = 0; if (OB_ISNULL(table_schema) || OB_ISNULL(guard)) { ret = OB_INVALID_ARGUMENT; @@ -7314,12 +7313,6 @@ int ObStaticEngineCG::init_encrypt_table_meta( if (OB_FAIL(meta_cache.meta_.encrypted_table_key_.set_content( table_schema->get_encrypt_key()))) { LOG_WARN("fail to assign encrypt key", K(ret)); - } else if (OB_FAIL(share::ObKeyGenerator::generate_encrypt_key( - random_string, OB_CLOG_ENCRYPT_RANDOM_LEN))) { - LOG_WARN("fail to generate random string", K(ret)); - } else if (OB_FAIL(meta_cache.meta_.random_.set_content(ObString( - OB_CLOG_ENCRYPT_RANDOM_LEN, random_string)))) { - LOG_WARN("fail to assign random string", K(ret)); } #ifdef ERRSIM else if (OB_FAIL(OB_E(EventTable::EN_ENCRYPT_GET_MASTER_KEY_FAILED) OB_SUCCESS)) { @@ -7327,8 +7320,8 @@ int ObStaticEngineCG::init_encrypt_table_meta( } #endif else if (OB_FAIL(share::ObMasterKeyGetter::get_master_key(table_schema->get_tenant_id(), - table_schema->get_master_key_id(), - master_key, OB_MAX_MASTER_KEY_LENGTH, master_key_length))) { + table_schema->get_master_key_id(), master_key, OB_MAX_MASTER_KEY_LENGTH, + master_key_length))) { LOG_WARN("fail to get master key", K(ret)); // 如果在cg阶段获取主密钥失败了, 有可能是因为RS执行内部sql没有租户资源引起的. // 在没有租户资源的情况下获取主密钥, 获取加密租户配置项时会失败 @@ -7337,12 +7330,10 @@ int ObStaticEngineCG::init_encrypt_table_meta( // 兜底是执行期再次获取, 再次获取成功了则继续往下走, 失败了则报错出来. // 见bug ret = OB_SUCCESS; - } else if (OB_FAIL(meta_cache.meta_.master_key_.set_content(ObString( - master_key_length, master_key)))) { + } else if (OB_FAIL(meta_cache.meta_.master_key_.set_content( + ObString(master_key_length, master_key)))) { LOG_WARN("fail to assign master_key", K(ret)); - } else if (OB_FAIL(ObEncryptionUtil::decrypt_table_key(meta_cache.meta_, - meta_cache.meta_.encrypted_table_key_.ptr(), - meta_cache.meta_.encrypted_table_key_.size()))) { + } else if (OB_FAIL(ObEncryptionUtil::decrypt_table_key(meta_cache.meta_))) { LOG_WARN("failed to decrypt_table_key", K(ret)); } else {/*do nothing*/} diff --git a/src/sql/engine/expr/ob_expr_aes_encrypt.cpp b/src/sql/engine/expr/ob_expr_aes_encrypt.cpp index 4ab21b2c42..a8e110a35c 100644 --- a/src/sql/engine/expr/ob_expr_aes_encrypt.cpp +++ b/src/sql/engine/expr/ob_expr_aes_encrypt.cpp @@ -71,8 +71,8 @@ int ObExprAesEncrypt::calc_result_typeN(ObExprResType& type, types_stack[i].set_calc_collation_level(types_stack[i].get_collation_level()); } type.set_varbinary(); - type.set_length((types_stack[0].get_length() * 3 / ObAesEncryption::OB_AES_BLOCK_SIZE + 1) * - ObAesEncryption::OB_AES_BLOCK_SIZE); + type.set_length((types_stack[0].get_length() * 3 / ObBlockCipher::OB_CIPHER_BLOCK_LENGTH + 1) * + ObBlockCipher::OB_CIPHER_BLOCK_LENGTH); type.set_collation_level(CS_LEVEL_COERCIBLE); } return ret; @@ -94,16 +94,15 @@ int ObExprAesEncrypt::eval_aes_encrypt(const ObExpr &expr, ObEvalCtx &ctx, CK(2 == expr.arg_cnt_ || 3 == expr.arg_cnt_); if (OB_SUCC(ret)) { const ObString &src_str = expr.locate_param_datum(ctx, 0).get_string(); - ObString key_str = expr.locate_param_datum(ctx, 1).get_string(); - char *buf = NULL; + const ObString &key_str = expr.locate_param_datum(ctx, 1).get_string(); int64_t out_len = 0; - int64_t block_size = ObAesEncryption::OB_AES_BLOCK_SIZE; ObEvalCtx::TempAllocGuard alloc_guard(ctx); ObIAllocator &calc_alloc = alloc_guard.get_allocator(); - int buf_length = (src_str.length() / block_size + 1) * block_size; - buf = static_cast(calc_alloc.alloc(buf_length)); - bool is_ecb = encryption <= static_cast(ObAesOpMode::ob_aes_256_ecb) && - encryption >= static_cast(ObAesOpMode::ob_aes_128_ecb); + ObCipherOpMode opmode = static_cast(encryption); + int buf_length = (src_str.length() / ObBlockCipher::OB_CIPHER_BLOCK_LENGTH + 1) * ObBlockCipher::OB_CIPHER_BLOCK_LENGTH; + char *buf = static_cast(calc_alloc.alloc(buf_length)); + bool is_ecb = opmode <= ObCipherOpMode::ob_aes_256_ecb && + opmode >= ObCipherOpMode::ob_aes_128_ecb; if (OB_ISNULL(buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed", K(ret)); @@ -116,17 +115,16 @@ int ObExprAesEncrypt::eval_aes_encrypt(const ObExpr &expr, ObEvalCtx &ctx, } if (OB_SUCC(ret)) { if (2 == expr.arg_cnt_ || is_ecb) { - OZ(ObAesEncryption::aes_encrypt(key_str.ptr(), key_str.length(), src_str.ptr(), - src_str.length(), buf_length, NULL, 0, - static_cast(encryption), buf, out_len)); + OZ(ObBlockCipher::encrypt(key_str.ptr(), key_str.length(), src_str.ptr(), + src_str.length(), buf_length, NULL, 0, NULL, 0, 0, opmode, buf, + out_len, NULL)); } else { ObString iv_str = expr.locate_param_datum(ctx, 2).get_string(); - OV(iv_str.length() >= ObAesEncryption::OB_AES_IV_SIZE, OB_ERR_AES_IV_LENGTH); - OX(iv_str.assign(iv_str.ptr(), (int32_t)ObAesEncryption::OB_AES_IV_SIZE)); - OZ(ObAesEncryption::aes_encrypt(key_str.ptr(), key_str.length(), src_str.ptr(), - src_str.length(), buf_length, iv_str.ptr(), - iv_str.length(), static_cast(encryption), - buf, out_len)); + OV(iv_str.length() >= ObBlockCipher::OB_DEFAULT_IV_LENGTH, OB_ERR_AES_IV_LENGTH); + OX(iv_str.assign(iv_str.ptr(), ObBlockCipher::OB_DEFAULT_IV_LENGTH)); + OZ(ObBlockCipher::encrypt(key_str.ptr(), key_str.length(), src_str.ptr(), src_str.length(), + buf_length, iv_str.ptr(), iv_str.length(), NULL, 0, 0, opmode, + buf, out_len, NULL)); } } if (OB_SUCC(ret)) { @@ -205,14 +203,15 @@ int ObExprAesDecrypt::eval_aes_decrypt(const ObExpr &expr, ObEvalCtx &ctx, bool is_null = false; if (OB_SUCC(ret)) { const ObString &src_str = expr.locate_param_datum(ctx, 0).get_string(); - ObString key_str = expr.locate_param_datum(ctx, 1).get_string(); - char *buf = NULL; + const ObString &key_str = expr.locate_param_datum(ctx, 1).get_string(); int64_t out_len = 0; ObEvalCtx::TempAllocGuard alloc_guard(ctx); ObIAllocator &calc_alloc = alloc_guard.get_allocator(); - buf = static_cast(calc_alloc.alloc((src_str.length() + 1))); - bool is_ecb = encryption <= static_cast(ObAesOpMode::ob_aes_256_ecb) && - encryption >= static_cast(ObAesOpMode::ob_aes_128_ecb); + ObCipherOpMode opmode = static_cast(encryption); + const int64_t buf_len = src_str.length() + 1; + char *buf = static_cast(calc_alloc.alloc(buf_len)); + bool is_ecb = opmode <= ObCipherOpMode::ob_aes_256_ecb && + opmode >= ObCipherOpMode::ob_aes_128_ecb; if (OB_ISNULL(buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc mem failed", K(ret)); @@ -225,17 +224,16 @@ int ObExprAesDecrypt::eval_aes_decrypt(const ObExpr &expr, ObEvalCtx &ctx, } if (OB_SUCC(ret)) { if (2 == expr.arg_cnt_ || is_ecb) { - OZ(ObAesEncryption::aes_decrypt(key_str.ptr(), key_str.length(), src_str.ptr(), - src_str.length(), src_str.length(), NULL, 0, - static_cast(encryption), buf, out_len)); + OZ(ObBlockCipher::decrypt(key_str.ptr(), key_str.length(), src_str.ptr(), + src_str.length(), src_str.length(), NULL, 0, NULL, 0, NULL, 0, + opmode, buf, out_len)); } else { ObString iv_str = expr.locate_param_datum(ctx, 2).get_string(); - OV(iv_str.length() >= ObAesEncryption::OB_AES_IV_SIZE, OB_ERR_AES_IV_LENGTH); - OX(iv_str.assign(iv_str.ptr(), (int32_t)ObAesEncryption::OB_AES_IV_SIZE)); - OZ(ObAesEncryption::aes_decrypt(key_str.ptr(), key_str.length(), src_str.ptr(), - src_str.length(), src_str.length(), iv_str.ptr(), - iv_str.length(), static_cast(encryption), - buf, out_len)); + OV(iv_str.length() >= ObBlockCipher::OB_DEFAULT_IV_LENGTH, OB_ERR_AES_IV_LENGTH); + OX(iv_str.assign(iv_str.ptr(), ObBlockCipher::OB_DEFAULT_IV_LENGTH)); + OZ(ObBlockCipher::decrypt(key_str.ptr(), key_str.length(), src_str.ptr(), src_str.length(), + src_str.length(), iv_str.ptr(), iv_str.length(), NULL, 0, NULL, 0, + opmode, buf, out_len)); } if (OB_ERR_AES_DECRYPT == ret) { //按照mysql兼容的做法,如果解密失败,则将结果设置为null diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 9fbadca346..418927e154 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -2114,10 +2114,17 @@ int ObSetConfigResolver::check_param_valid(int64_t tenant_id , if (OB_SUCC(ret)) { if (0 == name.case_compare("tde_method")) { ObString tde_method; + uint64_t compat_version = 0; if (!ObTdeMethodUtil::is_valid(value)) { ret = OB_NOT_SUPPORTED; LOG_WARN("not supported other method", K(value), K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter invalid tde_method"); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); + } else if (compat_version < DATA_VERSION_4_2_1_0 + && ObTdeMethodUtil::is_aes256_algorithm(value)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("encrypt table key with aes256 is not supported", K(ret), K(value)); } else if (OB_FAIL(share::ObEncryptionUtil::get_tde_method(tenant_id, tde_method))) { LOG_WARN("fail to check tenant is method internal", K(ret)); } else if (0 != tde_method.case_compare("none") && 0 != value.case_compare(tde_method)) { @@ -3634,10 +3641,17 @@ int ObAlterSystemSetResolver::check_param_valid(int64_t tenant_id , #ifdef OB_BUILD_TDE_SECURITY if (0 == name.case_compare("tde_method")) { ObString tde_method; - if (!ObTdeMethodUtil::is_valid(value)) { + uint64_t compat_version = 0; + if (!ObTdeMethodUtil::is_valid(value)) { ret = OB_NOT_SUPPORTED; LOG_WARN("not supported other method", K(value), K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter invalid tde_method"); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); + } else if (compat_version < DATA_VERSION_4_2_1_0 + && ObTdeMethodUtil::is_aes256_algorithm(value)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("encrypt table key with aes256 is not supported", K(ret), K(value)); } else if (OB_FAIL(share::ObEncryptionUtil::get_tde_method(tenant_id, tde_method))) { LOG_WARN("fail to check tenant is method internal", K(ret)); } else if (0 != tde_method.case_compare("none") && 0 != value.case_compare(tde_method)) { diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 2c3d511842..f137bd2434 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -8000,5 +8000,54 @@ int ObResolverUtils::resolve_file_format_string_value(const ParseNode *node, return ret; } +int ObResolverUtils::check_keystore_status(const uint64_t tenant_id, + ObSchemaChecker &schema_checker) +{ + int ret = OB_SUCCESS; + const ObKeystoreSchema *keystore_schema = NULL; + if (OB_FAIL(schema_checker.get_keystore_schema(tenant_id, keystore_schema))) { + LOG_WARN("fail to get keystore schema", K(ret)); + } else if (OB_ISNULL(keystore_schema)) { + ret = OB_KEYSTORE_NOT_EXIST; + LOG_WARN("the keystore is not exist", K(ret)); + } else if (0 == keystore_schema->get_status()) { + ret = OB_KEYSTORE_NOT_OPEN; + LOG_WARN("the keystore is not open", K(ret)); + } else if (2 == keystore_schema->get_status()) { + ret = OB_KEYSTORE_OPEN_NO_MASTER_KEY; + LOG_WARN("the keystore dont have any master key", K(ret)); + } + return ret; +} + +int ObResolverUtils::check_encryption_name(ObString &encryption_name, bool &need_encrypt) +{ + int ret = OB_SUCCESS; + bool is_oracle = lib::is_oracle_mode(); + if (0 == encryption_name.length()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("encryption name cannot be empty", K(ret)); + } else if (0 == encryption_name.case_compare("aes-128") || + 0 == encryption_name.case_compare("aes-192") || + 0 == encryption_name.case_compare("aes-256") || + 0 == encryption_name.case_compare("aes-128-gcm") || + 0 == encryption_name.case_compare("aes-192-gcm") || + 0 == encryption_name.case_compare("aes-256-gcm") || + 0 == encryption_name.case_compare("sm4-cbc") || + 0 == encryption_name.case_compare("sm4-gcm")) { + need_encrypt = true; + } else if (!is_oracle && 0 == encryption_name.case_compare("y")) { + need_encrypt = true; + encryption_name = common::ObString::make_string("aes-256"); + } else if (!is_oracle && 0 == encryption_name.case_compare("n")) { + need_encrypt = false; + } else if (is_oracle && 0 == encryption_name.case_compare("none")){ + need_encrypt = false; + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the encryption name is invalid", K(ret), K(encryption_name)); + } + return ret; +} } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/ob_resolver_utils.h b/src/sql/resolver/ob_resolver_utils.h index 61c3ed6df6..2691cf2224 100644 --- a/src/sql/resolver/ob_resolver_utils.h +++ b/src/sql/resolver/ob_resolver_utils.h @@ -754,6 +754,8 @@ public: ObSQLSessionInfo &session_info, ObDataTypeCastParams &dtc_params); static bool is_expr_can_be_used_in_table_function(const ObRawExpr &expr); + static int check_keystore_status(const uint64_t tenant_id, ObSchemaChecker &schema_checker); + static int check_encryption_name(common::ObString &encryption_name, bool &need_encrypt); private: static int try_convert_to_unsiged(const ObExprResType restype, ObRawExpr& src_expr, diff --git a/src/storage/blocksstable/ob_index_block_builder.cpp b/src/storage/blocksstable/ob_index_block_builder.cpp index bff6c0464a..55cf55df52 100644 --- a/src/storage/blocksstable/ob_index_block_builder.cpp +++ b/src/storage/blocksstable/ob_index_block_builder.cpp @@ -1663,8 +1663,12 @@ int ObDataIndexBlockBuilder::cal_macro_meta_block_size( STORAGE_LOG(WARN, "unexpected meta block desc", K(ret), K(meta_block_desc)); } else { estimate_meta_block_size = meta_block_desc.buf_size_ + meta_block_desc.header_->header_size_; - const int64_t encrypted_size = share::ObEncryptionUtil::encrypted_length(estimate_meta_block_size); +#ifdef OB_BUILD_TDE_SECURITY + const int64_t encrypted_size = share::ObEncryptionUtil::encrypted_length( + static_cast(index_store_desc_->encrypt_id_), + estimate_meta_block_size); estimate_meta_block_size = max(estimate_meta_block_size, encrypted_size); +#endif } } } @@ -1785,7 +1789,9 @@ int ObDataIndexBlockBuilder::write_meta_block( } else if (OB_FAIL(meta_block_writer_->build_micro_block_desc(meta_block_desc))) { STORAGE_LOG(WARN, "fail to build meta block", K(ret)); } else if (FALSE_IT(meta_block_desc.last_rowkey_ = last_rowkey_)) { - } else if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(meta_block_desc))) { + } else if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(meta_block_desc, + macro_block.get_current_macro_seq(), + macro_block.get_data_size()))) { STORAGE_LOG(WARN, "fail to compress and encrypt micro block", K(ret)); } else if (OB_FAIL(macro_block.write_index_micro_block(meta_block_desc, false, data_offset))) { STORAGE_LOG(WARN, "fail to write meta index block", K(ret), K(meta_block_desc)); @@ -1822,7 +1828,9 @@ int ObDataIndexBlockBuilder::append_index_micro_block(ObMacroBlock ¯o_block, STORAGE_LOG(WARN, "Fail to reserve index block value", K(tmp_ret)); } } - if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(leaf_block_desc))) { + if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(leaf_block_desc, + macro_block.get_current_macro_seq(), + macro_block.get_data_size()))) { STORAGE_LOG(WARN, "fail to compress and encrypt micro block", K(ret)); } else if (OB_FAIL(macro_block.write_index_micro_block(leaf_block_desc, true, data_offset))) { STORAGE_LOG(WARN, "fail to write n-1 level index block", K(ret), K(leaf_block_desc)); diff --git a/src/storage/blocksstable/ob_macro_block.h b/src/storage/blocksstable/ob_macro_block.h index 6c6c73ec9d..d83338d59f 100644 --- a/src/storage/blocksstable/ob_macro_block.h +++ b/src/storage/blocksstable/ob_macro_block.h @@ -227,6 +227,7 @@ public: OB_INLINE bool is_dirty() const { return is_dirty_; } int64_t get_data_size() const; int64_t get_remain_size() const; + int64_t get_current_macro_seq() const {return cur_macro_seq_; } OB_INLINE char *get_data_buf() { return data_.data(); } OB_INLINE int32_t get_row_count() const { return macro_header_.fixed_header_.row_count_; } OB_INLINE int32_t get_micro_block_count() const { return macro_header_.fixed_header_.micro_block_count_; } diff --git a/src/storage/blocksstable/ob_macro_block_reader.cpp b/src/storage/blocksstable/ob_macro_block_reader.cpp index 91027f8fea..372e7bd6ae 100644 --- a/src/storage/blocksstable/ob_macro_block_reader.cpp +++ b/src/storage/blocksstable/ob_macro_block_reader.cpp @@ -338,7 +338,8 @@ int ObMacroBlockReader::decrypt_and_decompress_data( #ifndef OB_BUILD_TDE_SECURITY is_compressed = header.is_compressed_data(); #else - if (OB_UNLIKELY(share::ObEncryptionUtil::need_encrypt(deserialize_meta.encrypt_id_))) { + if (OB_UNLIKELY(ObEncryptionUtil::need_encrypt( + static_cast(deserialize_meta.encrypt_id_)))) { LOG_DEBUG("Macro data need decrypt", K(deserialize_meta.encrypt_id_), K(data_buf_size)); const char *decrypt_buf = NULL; int64_t decrypt_size = 0; @@ -412,7 +413,8 @@ int ObMacroBlockReader::decrypt_buf( if (OB_ISNULL(buf)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument of input data", K(ret), KP(buf)); - } else if (OB_UNLIKELY(!share::ObEncryptionUtil::need_encrypt(deserialize_meta.encrypt_id_))) { + } else if (OB_UNLIKELY(!ObEncryptionUtil::need_encrypt( + static_cast(deserialize_meta.encrypt_id_)))) { decrypt_buf = buf; decrypt_size = size; } else if (OB_FAIL(init_encrypter_if_needed())) { diff --git a/src/storage/blocksstable/ob_macro_block_writer.cpp b/src/storage/blocksstable/ob_macro_block_writer.cpp index 60382b0f37..a1426ef61a 100644 --- a/src/storage/blocksstable/ob_macro_block_writer.cpp +++ b/src/storage/blocksstable/ob_macro_block_writer.cpp @@ -91,7 +91,9 @@ void ObMicroBlockBufferHelper::reset() check_datum_row_.reset(); } -int ObMicroBlockBufferHelper::compress_encrypt_micro_block(ObMicroBlockDesc µ_block_desc) +int ObMicroBlockBufferHelper::compress_encrypt_micro_block(ObMicroBlockDesc µ_block_desc, + const int64_t seq, + const int64_t offset) { int ret = OB_SUCCESS; const char *block_buffer = micro_block_desc.buf_; @@ -114,6 +116,8 @@ int ObMicroBlockBufferHelper::compress_encrypt_micro_block(ObMicroBlockDesc &mic micro_block_desc.buf_ = compress_buf; micro_block_desc.buf_size_ = compress_buf_size; #else + } else if (OB_FAIL(encryption_.generate_iv(seq, offset))) { + STORAGE_LOG(WARN, "failed to generate iv", K(ret)); } else if (OB_FAIL(encryption_.encrypt(compress_buf, compress_buf_size, micro_block_desc.buf_, micro_block_desc.buf_size_))) { STORAGE_LOG(WARN, "fail to encrypt micro block", K(ret)); } else { @@ -904,7 +908,9 @@ int ObMacroBlockWriter::append_index_micro_block(ObMicroBlockDesc µ_block_d } else if (OB_UNLIKELY(nullptr != builder_)) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "expect null builder for index macro writer", K(ret), K_(builder)); - } else if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(micro_block_desc))) { + } else if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(micro_block_desc, + macro_blocks_[current_index_].get_current_macro_seq(), + macro_blocks_[current_index_].get_data_size()))) { // do not dump micro_writer_ here STORAGE_LOG(WARN, "failed to compress and encrypt micro block", K(ret), K(micro_block_desc)); } else if (OB_FAIL(write_micro_block(micro_block_desc))) { @@ -936,8 +942,9 @@ int ObMacroBlockWriter::build_micro_block() STORAGE_LOG(WARN, "Fail to reserve data block cache value", K(tmp_ret)); } } - - if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(micro_block_desc))) { + if (OB_FAIL(micro_helper_.compress_encrypt_micro_block(micro_block_desc, + macro_blocks_[current_index_].get_current_macro_seq(), + macro_blocks_[current_index_].get_data_size()))) { micro_writer_->dump_diagnose_info(); // ignore dump error STORAGE_LOG(WARN, "failed to compress and encrypt micro block", K(ret), K(micro_block_desc)); } else { diff --git a/src/storage/blocksstable/ob_macro_block_writer.h b/src/storage/blocksstable/ob_macro_block_writer.h index 2972a99f03..adf29210f3 100644 --- a/src/storage/blocksstable/ob_macro_block_writer.h +++ b/src/storage/blocksstable/ob_macro_block_writer.h @@ -59,7 +59,7 @@ public: ObDataStoreDesc &data_store_desc, const ObITableReadInfo &read_info, common::ObIAllocator &allocator); - int compress_encrypt_micro_block(ObMicroBlockDesc µ_block_desc); + int compress_encrypt_micro_block(ObMicroBlockDesc µ_block_desc, const int64_t macro_seq, const int64_t micro_offset); int dump_micro_block_writer_buffer(const char *buf, const int64_t size); void reset(); private: diff --git a/src/storage/blocksstable/ob_micro_block_encryption.h b/src/storage/blocksstable/ob_micro_block_encryption.h index d37a89747c..55edbf9554 100644 --- a/src/storage/blocksstable/ob_micro_block_encryption.h +++ b/src/storage/blocksstable/ob_micro_block_encryption.h @@ -13,6 +13,7 @@ #ifndef OB_MICRO_BLOCK_ENCRYPTION_H_ #define OB_MICRO_BLOCK_ENCRYPTION_H_ #include "ob_data_buffer.h" +#include "share/ob_encryption_util.h" namespace oceanbase { @@ -32,6 +33,7 @@ public: const int64_t master_key_id = 0, const char *encrypt_key = NULL, const int64_t encrypt_key_len = 0); int encrypt(const char *in, const int64_t in_size, const char *&out, int64_t &out_size); int decrypt(const char *in, const int64_t in_size, const char *&out, int64_t &out_size); + int generate_iv(const int64_t macro_seq, const int64_t block_offset); private: //此密钥用于mysql模式下表加解密 int generate_key(); @@ -49,8 +51,10 @@ private: int64_t raw_key_len_; char raw_key_[common::OB_MAX_ENCRYPTION_KEY_NAME_LENGTH]; // 真正用于数据加解密的密钥 char encrypt_key_[common::OB_MAX_ENCRYPTION_KEY_NAME_LENGTH]; // 密钥的密文 + char iv_[share::ObBlockCipher::OB_DEFAULT_IV_LENGTH]; int64_t encrypt_key_len_; // 密钥的密文的长度 int64_t master_key_id_; // 主密钥版本 + int64_t iv_len_; #endif DISALLOW_COPY_AND_ASSIGN(ObMicroBlockEncryption); }; diff --git a/src/storage/memtable/ob_memtable_mutator.cpp b/src/storage/memtable/ob_memtable_mutator.cpp index 32038e02dc..7d983001df 100644 --- a/src/storage/memtable/ob_memtable_mutator.cpp +++ b/src/storage/memtable/ob_memtable_mutator.cpp @@ -405,7 +405,8 @@ int ObMemtableMutatorRow::handle_encrypt_row_(char *buf, const int64_t buf_len, { int ret = OB_SUCCESS; int64_t encrypted_len = 0; - const int64_t expected_encrypted_len = ObEncryptionUtil::encrypted_length(end_pos - start_pos); + const int64_t expected_encrypted_len = ObEncryptionUtil::encrypted_length( + static_cast(meta.encrypt_algorithm_), end_pos - start_pos); char *encrypted_buf = nullptr; int64_t encrypted_buf_size = 0; ObEncryptRowBuf row_buf; @@ -624,9 +625,9 @@ int ObMemtableMutatorRow::deserialize(const char *buf, const int64_t buf_len, in new_pos = 0; if (need_extract_encrypt_meta) { encrypt_stat_map.set_map(CLOG_CONTAIN_ENCRYPTED_ROW); - ObAesOpMode final_mode = static_cast(final_encrypt_meta.encrypt_algorithm_); - ObAesOpMode cur_mode = static_cast(encrypt_meta->meta_.encrypt_algorithm_); - if (ObAesEncryption::compare_aes_mod_safety(final_mode, cur_mode)) { + ObCipherOpMode final_mode = static_cast(final_encrypt_meta.encrypt_algorithm_); + ObCipherOpMode cur_mode = static_cast(encrypt_meta->meta_.encrypt_algorithm_); + if (ObBlockCipher::compare_aes_mod_safety(final_mode, cur_mode)) { if (OB_FAIL(final_encrypt_meta.assign(encrypt_meta->meta_))) { TRANS_LOG(WARN, "failed to assign encrypt_meta", K(ret), K(table_id_)); } @@ -1297,7 +1298,8 @@ int ObMutatorWriter::encrypt_big_row_data( TRANS_LOG(WARN, "failed to encode table id", K(ret)); } else { int64_t encrypted_len = 0; - int64_t data_len = ObEncryptionUtil::decrypted_length(out_buf_len - out_buf_pos); + ObCipherOpMode opmode = static_cast(encrypt_meta->meta_.encrypt_algorithm_); + int64_t data_len = ObEncryptionUtil::decrypted_length(opmode, out_buf_len - out_buf_pos); data_len = min(data_len, in_buf_len - in_buf_pos); if (OB_UNLIKELY(data_len < 0)) { TRANS_LOG(WARN, "buf to short to hold encrypted data", diff --git a/src/storage/tx/ob_clog_encrypt_info.h b/src/storage/tx/ob_clog_encrypt_info.h index d1813a60d3..773c4ac817 100644 --- a/src/storage/tx/ob_clog_encrypt_info.h +++ b/src/storage/tx/ob_clog_encrypt_info.h @@ -56,9 +56,8 @@ struct ObTxEncryptMeta //for memtable compare bool is_memtable_equal(const share::ObEncryptMeta &other) { - //now, we only take random_、table_key_、master_key_version_、encrypt_algorithm_ into consideration. - return (meta_.random_.get_content() == other.random_.get_content() && - meta_.table_key_.get_content() == other.table_key_.get_content() && + //now, we only take table_key_、master_key_version_、encrypt_algorithm_ into consideration. + return (meta_.table_key_.get_content() == other.table_key_.get_content() && meta_.master_key_version_ == other.master_key_version_ && meta_.encrypt_algorithm_ == other.encrypt_algorithm_); } diff --git a/src/storage/tx/ob_clog_encrypt_info_os.cpp b/src/storage/tx/ob_clog_encrypt_info_os.cpp index 504a0cc73a..11714a248b 100644 --- a/src/storage/tx/ob_clog_encrypt_info_os.cpp +++ b/src/storage/tx/ob_clog_encrypt_info_os.cpp @@ -129,8 +129,8 @@ OB_DEF_SERIALIZE_SIZE(ObCLogEncryptInfo) return size; } -OB_SERIALIZE_MEMBER(ObSerializeEncryptMeta, master_key_version_, encrypt_algorithm_, - random_, master_key_, table_key_, encrypted_table_key_); +OB_SERIALIZE_MEMBER(ObSerializeEncryptMeta, master_key_version_, encrypt_algorithm_, master_key_, + table_key_, encrypted_table_key_); OB_SERIALIZE_MEMBER(ObEncryptMetaCache, table_id_, local_index_id_, meta_); }//transaction diff --git a/unittest/share/test_ob_log_archive_config.cpp b/unittest/share/test_ob_log_archive_config.cpp index c0bcfd43b2..e56b6e63bd 100644 --- a/unittest/share/test_ob_log_archive_config.cpp +++ b/unittest/share/test_ob_log_archive_config.cpp @@ -77,29 +77,29 @@ TEST_F(TestLogArchiveConfig, log_archive) // ASSERT_EQ(true, log_archive_item.value_.is_mandatory_); // ASSERT_EQ(false, log_archive_item.value_.is_compress_enabled_); // ASSERT_EQ(ObBackupEncryptionMode::TRANSPARENT_ENCRYPTION, log_archive_item.value_.encryption_mode_); -// ASSERT_EQ(share::ObAesOpMode::ob_aes_128_ecb, log_archive_item.value_.encryption_algorithm_); +// ASSERT_EQ(share::ObCipherOpMode::ob_aes_128_ecb, log_archive_item.value_.encryption_algorithm_); // // ret = log_archive_item.set_value("MANDATORY encryption_mode= none ENCRYPTION_ALGORITHM = aes-128"); // ASSERT_EQ(true, log_archive_item.value_.valid_); // ASSERT_EQ(true, log_archive_item.value_.is_mandatory_); // ASSERT_EQ(false, log_archive_item.value_.is_compress_enabled_); // ASSERT_EQ(ObBackupEncryptionMode::NONE, log_archive_item.value_.encryption_mode_); -// ASSERT_EQ(share::ObAesOpMode::ob_aes_128_ecb, log_archive_item.value_.encryption_algorithm_); +// ASSERT_EQ(share::ObCipherOpMode::ob_aes_128_ecb, log_archive_item.value_.encryption_algorithm_); // // ret = log_archive_item.set_value("MANDATORY encryption_mode= none ENCRYPTION_ALGORITHM = aes-192"); // ASSERT_EQ(true, log_archive_item.value_.valid_); // ASSERT_EQ(ObBackupEncryptionMode::NONE, log_archive_item.value_.encryption_mode_); -// ASSERT_EQ(share::ObAesOpMode::ob_aes_192_ecb, log_archive_item.value_.encryption_algorithm_); +// ASSERT_EQ(share::ObCipherOpMode::ob_aes_192_ecb, log_archive_item.value_.encryption_algorithm_); // // ret = log_archive_item.set_value("MANDATORY encryption_mode= none ENCRYPTION_ALGORITHM = aes-256"); // ASSERT_EQ(true, log_archive_item.value_.valid_); // ASSERT_EQ(ObBackupEncryptionMode::NONE, log_archive_item.value_.encryption_mode_); -// ASSERT_EQ(share::ObAesOpMode::ob_aes_256_ecb, log_archive_item.value_.encryption_algorithm_); +// ASSERT_EQ(share::ObCipherOpMode::ob_aes_256_ecb, log_archive_item.value_.encryption_algorithm_); // // ret = log_archive_item.set_value("MANDATORY encryption_mode= none ENCRYPTION_ALGORITHM = sm4"); // ASSERT_EQ(true, log_archive_item.value_.valid_); // ASSERT_EQ(ObBackupEncryptionMode::NONE, log_archive_item.value_.encryption_mode_); -// ASSERT_EQ(share::ObAesOpMode::ob_sm4_mode, log_archive_item.value_.encryption_algorithm_); +// ASSERT_EQ(share::ObCipherOpMode::ob_sm4_mode, log_archive_item.value_.encryption_algorithm_); // // ret = log_archive_item.set_value("encryption_mode= none ENCRYPTION_ALGORITHM = aes-120"); // ASSERT_EQ(false, ret); diff --git a/unittest/storage/blocksstable/test_micro_block_encryption.cpp b/unittest/storage/blocksstable/test_micro_block_encryption.cpp index e938f2f976..419f1ab46f 100644 --- a/unittest/storage/blocksstable/test_micro_block_encryption.cpp +++ b/unittest/storage/blocksstable/test_micro_block_encryption.cpp @@ -49,11 +49,11 @@ void TestMicroBlockEncryption::SetUp() ASSERT_EQ(OB_SUCCESS, share::ObMasterKeyGetter::instance().init(nullptr)); ASSERT_EQ(OB_SUCCESS, share::ObMasterKeyGetter::instance().set_master_key( tenant_id_, master_key_id_, master_key_, strlen(master_key_))); - ObAesOpMode mode = ObAesOpMode::ob_invalid_mode; + ObCipherOpMode mode = ObCipherOpMode::ob_invalid_mode; ASSERT_EQ(OB_SUCCESS, share::ObMasterKeyGetter::get_table_key_algorithm(tenant_id_, mode)); - ASSERT_EQ(OB_SUCCESS, share::ObAesEncryption::aes_encrypt(master_key_, strlen(master_key_), - raw_key_, strlen(raw_key_), OB_MAX_ENCRYPTION_KEY_NAME_LENGTH, nullptr, 0, mode, - encrypt_key_, encrypt_key_len_)); + ASSERT_EQ(OB_SUCCESS, share::ObBlockCipher::encrypt(master_key_, strlen(master_key_), raw_key_, + strlen(raw_key_), OB_MAX_ENCRYPTION_KEY_NAME_LENGTH, nullptr, 0, nullptr, 0, + 0, mode, encrypt_key_, encrypt_key_len_, nullptr)); } void TestMicroBlockEncryption::TearDown() diff --git a/unittest/storage/blocksstable/test_sstable_meta.cpp b/unittest/storage/blocksstable/test_sstable_meta.cpp index f395242878..5f623020be 100644 --- a/unittest/storage/blocksstable/test_sstable_meta.cpp +++ b/unittest/storage/blocksstable/test_sstable_meta.cpp @@ -76,7 +76,7 @@ void TestRootBlockInfo::SetUp() oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); TestDataFilePrepare::SetUp(); prepare_tablet_read_info(); - des_meta_.encrypt_id_ = ObAesOpMode::ob_invalid_mode; + des_meta_.encrypt_id_ = ObCipherOpMode::ob_invalid_mode; des_meta_.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; ASSERT_TRUE(!root_info_.is_valid()); root_info_.reset();