diff --git a/be/src/exprs/encryption_functions.cpp b/be/src/exprs/encryption_functions.cpp index 8ba3709b0a..5d919a3b48 100644 --- a/be/src/exprs/encryption_functions.cpp +++ b/be/src/exprs/encryption_functions.cpp @@ -21,53 +21,177 @@ #include "exprs/expr.h" #include "runtime/string_value.h" #include "runtime/tuple_row.h" -#include "util/aes_util.h" #include "util/debug_util.h" +#include "util/encryption_util.h" #include "util/md5.h" +#include "util/sm3.h" +#include "util/string_util.h" #include "util/url_coding.h" namespace doris { void EncryptionFunctions::init() {} -StringVal EncryptionFunctions::aes_encrypt(FunctionContext* ctx, const StringVal& src, - const StringVal& key) { +StringCaseUnorderedMap aes_mode_map { + {"AES_128_ECB", AES_128_ECB}, {"AES_192_ECB", AES_192_ECB}, + {"AES_256_ECB", AES_256_ECB}, {"AES_128_CBC", AES_128_CBC}, + {"AES_192_CBC", AES_192_CBC}, {"AES_256_CBC", AES_256_CBC}, + {"AES_128_CFB", AES_128_CFB}, {"AES_192_CFB", AES_192_CFB}, + {"AES_256_CFB", AES_256_CFB}, {"AES_128_CFB1", AES_128_CFB1}, + {"AES_192_CFB1", AES_192_CFB1}, {"AES_256_CFB1", AES_256_CFB1}, + {"AES_128_CFB8", AES_128_CFB8}, {"AES_192_CFB8", AES_192_CFB8}, + {"AES_256_CFB8", AES_256_CFB8}, {"AES_128_CFB128", AES_128_CFB128}, + {"AES_192_CFB128", AES_192_CFB128}, {"AES_256_CFB128", AES_256_CFB128}, + {"AES_128_CTR", AES_128_CTR}, {"AES_192_CTR", AES_192_CTR}, + {"AES_256_CTR", AES_256_CTR}, {"AES_128_OFB", AES_128_OFB}, + {"AES_192_OFB", AES_192_OFB}, {"AES_256_OFB", AES_256_OFB}}; +StringCaseUnorderedMap sm4_mode_map {{"SM4_128_ECB", SM4_128_ECB}, + {"SM4_128_CBC", SM4_128_CBC}, + {"SM4_128_CFB128", SM4_128_CFB128}, + {"SM4_128_OFB", SM4_128_OFB}, + {"SM4_128_CTR", SM4_128_CTR}}; + +StringVal encrypt(FunctionContext* ctx, const StringVal& src, const StringVal& key, + const StringVal& iv, EncryptionMode mode) { if (src.len == 0 || src.is_null) { return StringVal::null(); } - - // cipher_len = (clearLen/16 + 1) * 16; int cipher_len = src.len + 16; std::unique_ptr p; p.reset(new char[cipher_len]); - - int ret_code = - AesUtil::encrypt(AES_128_ECB, (unsigned char*)src.ptr, src.len, (unsigned char*)key.ptr, - key.len, nullptr, true, (unsigned char*)p.get()); + int ret_code = 0; + if (mode != AES_128_ECB && mode != AES_192_ECB && mode != AES_256_ECB && mode != AES_256_ECB && + mode != SM4_128_ECB) { + if (iv.len == 0 || iv.is_null) { + return StringVal::null(); + } + int iv_len = 32; // max key length 256 / 8 + std::unique_ptr init_vec; + init_vec.reset(new char[iv_len]); + std::memset(init_vec.get(), 0, iv.len + 1); + memcpy(init_vec.get(), iv.ptr, iv.len); + ret_code = EncryptionUtil::encrypt( + mode, (unsigned char*)src.ptr, src.len, (unsigned char*)key.ptr, key.len, + (unsigned char*)init_vec.get(), true, (unsigned char*)p.get()); + } else { + ret_code = EncryptionUtil::encrypt(mode, (unsigned char*)src.ptr, src.len, + (unsigned char*)key.ptr, key.len, nullptr, true, + (unsigned char*)p.get()); + } if (ret_code < 0) { return StringVal::null(); } return AnyValUtil::from_buffer_temp(ctx, p.get(), ret_code); } -StringVal EncryptionFunctions::aes_decrypt(FunctionContext* ctx, const StringVal& src, - const StringVal& key) { +StringVal decrypt(FunctionContext* ctx, const StringVal& src, const StringVal& key, + const StringVal& iv, EncryptionMode mode) { if (src.len == 0 || src.is_null) { return StringVal::null(); } - int cipher_len = src.len; std::unique_ptr p; p.reset(new char[cipher_len]); - - int ret_code = - AesUtil::decrypt(AES_128_ECB, (unsigned char*)src.ptr, src.len, (unsigned char*)key.ptr, - key.len, nullptr, true, (unsigned char*)p.get()); + int ret_code = 0; + if (mode != AES_128_ECB && mode != AES_192_ECB && mode != AES_256_ECB && mode != AES_256_ECB && + mode != SM4_128_ECB) { + if (iv.len == 0 || iv.is_null) { + return StringVal::null(); + } + int iv_len = 32; // max key length 256 / 8 + std::unique_ptr init_vec; + init_vec.reset(new char[iv_len]); + std::memset(init_vec.get(), 0, iv.len + 1); + memcpy(init_vec.get(), iv.ptr, iv.len); + ret_code = EncryptionUtil::decrypt( + mode, (unsigned char*)src.ptr, src.len, (unsigned char*)key.ptr, key.len, + (unsigned char*)init_vec.get(), true, (unsigned char*)p.get()); + } else { + ret_code = EncryptionUtil::decrypt(mode, (unsigned char*)src.ptr, src.len, + (unsigned char*)key.ptr, key.len, nullptr, true, + (unsigned char*)p.get()); + } if (ret_code < 0) { return StringVal::null(); } return AnyValUtil::from_buffer_temp(ctx, p.get(), ret_code); } +StringVal EncryptionFunctions::aes_encrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key) { + return aes_encrypt(ctx, src, key, StringVal::null(), StringVal("AES_128_ECB")); +} + +StringVal EncryptionFunctions::aes_decrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key) { + return aes_decrypt(ctx, src, key, StringVal::null(), StringVal("AES_128_ECB")); +} + +StringVal EncryptionFunctions::aes_encrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key, const StringVal& iv, + const StringVal& mode) { + EncryptionMode encryption_mode = AES_128_ECB; + if (mode.len != 0 && !mode.is_null) { + std::string mode_str(reinterpret_cast(mode.ptr), mode.len); + if (aes_mode_map.count(mode_str) == 0) { + return StringVal::null(); + } + encryption_mode = aes_mode_map.at(mode_str); + } + return encrypt(ctx, src, key, iv, encryption_mode); +} + +StringVal EncryptionFunctions::aes_decrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key, const StringVal& iv, + const StringVal& mode) { + EncryptionMode encryption_mode = AES_128_ECB; + if (mode.len != 0 && !mode.is_null) { + std::string mode_str(reinterpret_cast(mode.ptr), mode.len); + if (aes_mode_map.count(mode_str) == 0) { + return StringVal::null(); + } + encryption_mode = aes_mode_map.at(mode_str); + } + return decrypt(ctx, src, key, iv, encryption_mode); +} + +StringVal EncryptionFunctions::sm4_encrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key) { + return sm4_encrypt(ctx, src, key, StringVal::null(), StringVal("SM4_128_ECB")); +} + +StringVal EncryptionFunctions::sm4_decrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key) { + return sm4_decrypt(ctx, src, key, StringVal::null(), StringVal("SM4_128_ECB")); +} + +StringVal EncryptionFunctions::sm4_encrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key, const StringVal& iv, + const StringVal& mode) { + EncryptionMode encryption_mode = SM4_128_ECB; + if (mode.len != 0 && !mode.is_null) { + std::string mode_str(reinterpret_cast(mode.ptr), mode.len); + if (sm4_mode_map.count(mode_str) == 0) { + return StringVal::null(); + } + encryption_mode = sm4_mode_map.at(mode_str); + } + return encrypt(ctx, src, key, iv, encryption_mode); +} + +StringVal EncryptionFunctions::sm4_decrypt(FunctionContext* ctx, const StringVal& src, + const StringVal& key, const StringVal& iv, + const StringVal& mode) { + EncryptionMode encryption_mode = SM4_128_ECB; + if (mode.len != 0 && !mode.is_null) { + std::string mode_str(reinterpret_cast(mode.ptr), mode.len); + if (sm4_mode_map.count(mode_str) == 0) { + return StringVal::null(); + } + encryption_mode = sm4_mode_map.at(mode_str); + } + return decrypt(ctx, src, key, iv, encryption_mode); +} + StringVal EncryptionFunctions::from_base64(FunctionContext* ctx, const StringVal& src) { if (src.len == 0 || src.is_null) { return StringVal::null(); @@ -123,4 +247,27 @@ StringVal EncryptionFunctions::md5(FunctionContext* ctx, const StringVal& src) { return AnyValUtil::from_buffer_temp(ctx, digest.hex().c_str(), digest.hex().size()); } +StringVal EncryptionFunctions::sm3sum(FunctionContext* ctx, int num_args, const StringVal* args) { + SM3Digest digest; + for (int i = 0; i < num_args; ++i) { + const StringVal& arg = args[i]; + if (arg.is_null) { + continue; + } + digest.update(arg.ptr, arg.len); + } + digest.digest(); + return AnyValUtil::from_buffer_temp(ctx, digest.hex().c_str(), digest.hex().size()); +} + +StringVal EncryptionFunctions::sm3(FunctionContext* ctx, const StringVal& src) { + if (src.is_null) { + return StringVal::null(); + } + SM3Digest digest; + digest.update(src.ptr, src.len); + digest.digest(); + return AnyValUtil::from_buffer_temp(ctx, digest.hex().c_str(), digest.hex().size()); +} + } // namespace doris diff --git a/be/src/exprs/encryption_functions.h b/be/src/exprs/encryption_functions.h index 853ec97f56..8a7e45d421 100644 --- a/be/src/exprs/encryption_functions.h +++ b/be/src/exprs/encryption_functions.h @@ -32,12 +32,38 @@ class TupleRow; class EncryptionFunctions { public: static void init(); - static doris_udf::StringVal aes_encrypt(doris_udf::FunctionContext* context, - const doris_udf::StringVal& val1, - const doris_udf::StringVal& val2); - static doris_udf::StringVal aes_decrypt(doris_udf::FunctionContext* context, - const doris_udf::StringVal& val1, - const doris_udf::StringVal& val2); + static doris_udf::StringVal aes_encrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key); + static doris_udf::StringVal aes_decrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key); + static doris_udf::StringVal aes_encrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key, + const doris_udf::StringVal& iv, + const doris_udf::StringVal& mode); + static doris_udf::StringVal aes_decrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key, + const doris_udf::StringVal& iv, + const doris_udf::StringVal& mode); + static doris_udf::StringVal sm4_encrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key); + static doris_udf::StringVal sm4_decrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key); + static doris_udf::StringVal sm4_encrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key, + const doris_udf::StringVal& iv, + const doris_udf::StringVal& mode); + static doris_udf::StringVal sm4_decrypt(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src, + const doris_udf::StringVal& key, + const doris_udf::StringVal& iv, + const doris_udf::StringVal& mode); static doris_udf::StringVal from_base64(doris_udf::FunctionContext* context, const doris_udf::StringVal& val1); static doris_udf::StringVal to_base64(doris_udf::FunctionContext* context, @@ -46,6 +72,10 @@ public: const doris_udf::StringVal* args); static doris_udf::StringVal md5(doris_udf::FunctionContext* ctx, const doris_udf::StringVal& src); + static doris_udf::StringVal sm3sum(doris_udf::FunctionContext* ctx, int num_args, + const doris_udf::StringVal* args); + static doris_udf::StringVal sm3(doris_udf::FunctionContext* ctx, + const doris_udf::StringVal& src); }; } // namespace doris diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index da43f5b369..22a6c03da5 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -76,8 +76,8 @@ DataDir::DataDir(const std::string& path, int64_t capacity_bytes, _cluster_id(-1), _cluster_id_incomplete(false), _to_be_deleted(false), - _current_shard(0), _env(Env::get_env(storage_medium)), + _current_shard(0), _meta(nullptr) { _path_desc.storage_medium = storage_medium; _path_desc.remote_path = remote_path; @@ -99,9 +99,9 @@ DataDir::~DataDir() { Status DataDir::init() { if (!Env::Default()->path_exists(_path_desc.filepath).ok()) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::IOError(strings::Substitute("opendir failed, path=$0", _path_desc.filepath)), - "check file exist failed"); + RETURN_NOT_OK_STATUS_WITH_WARN(Status::IOError(strings::Substitute( + "opendir failed, path=$0", _path_desc.filepath)), + "check file exist failed"); } RETURN_NOT_OK_STATUS_WITH_WARN(update_capacity(), "update_capacity failed"); @@ -129,14 +129,15 @@ Status DataDir::_init_cluster_id() { } if (is_remote()) { int32_t remote_cluster_id; - RETURN_IF_ERROR(read_cluster_id(_env, cluster_id_path_desc.remote_path, &remote_cluster_id)); + RETURN_IF_ERROR( + read_cluster_id(_env, cluster_id_path_desc.remote_path, &remote_cluster_id)); if (remote_cluster_id == -1) { _cluster_id_incomplete = true; } if (remote_cluster_id != -1 && _cluster_id != -1 && _cluster_id != remote_cluster_id) { - return Status::InternalError(strings::Substitute( - "cluster id $0 is not equal with remote cluster id $1", - _cluster_id, remote_cluster_id)); + return Status::InternalError( + strings::Substitute("cluster id $0 is not equal with remote cluster id $1", + _cluster_id, remote_cluster_id)); } if (_cluster_id == -1) { _cluster_id = remote_cluster_id; @@ -151,8 +152,9 @@ Status DataDir::read_cluster_id(Env* env, const std::string& cluster_id_path, in Status exist_status = env->path_exists(cluster_id_path); if (exist_status.ok()) { Status status = env->new_random_access_file(cluster_id_path, &input_file); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute("open file failed: $0, err=$1", - cluster_id_path, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN( + status, strings::Substitute("open file failed: $0, err=$1", cluster_id_path, + status.to_string())); std::string content; RETURN_IF_ERROR(input_file->read_all(&content)); if (content.size() > 0) { @@ -163,8 +165,9 @@ Status DataDir::read_cluster_id(Env* env, const std::string& cluster_id_path, in } else if (exist_status.is_not_found()) { *cluster_id = -1; } else { - RETURN_NOT_OK_STATUS_WITH_WARN(exist_status, strings::Substitute("check exist failed: $0, err=$1", - cluster_id_path, exist_status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN( + exist_status, strings::Substitute("check exist failed: $0, err=$1", cluster_id_path, + exist_status.to_string())); } return Status::OK(); } @@ -172,23 +175,26 @@ Status DataDir::read_cluster_id(Env* env, const std::string& cluster_id_path, in Status DataDir::_init_capacity() { int64_t disk_capacity = -1; int64_t available = -1; - RETURN_NOT_OK_STATUS_WITH_WARN(Env::Default()->get_space_info(_path_desc.filepath, &disk_capacity, &available), - strings::Substitute("get_space_info failed: $0", _path_desc.filepath)); + RETURN_NOT_OK_STATUS_WITH_WARN( + Env::Default()->get_space_info(_path_desc.filepath, &disk_capacity, &available), + strings::Substitute("get_space_info failed: $0", _path_desc.filepath)); if (_capacity_bytes == -1) { _capacity_bytes = disk_capacity; } else if (_capacity_bytes > disk_capacity) { RETURN_NOT_OK_STATUS_WITH_WARN( Status::InvalidArgument(strings::Substitute( - "root path $0's capacity $1 should not larger than disk capacity $2", _path_desc.filepath, - _capacity_bytes, disk_capacity)), + "root path $0's capacity $1 should not larger than disk capacity $2", + _path_desc.filepath, _capacity_bytes, disk_capacity)), "init capacity failed"); } std::string data_path = _path_desc.filepath + DATA_PREFIX; Status exist_status = Env::Default()->path_exists(data_path); - if (!exist_status.ok() && (!exist_status.is_not_found() || !Env::Default()->create_dirs(data_path).ok())) { + if (!exist_status.ok() && + (!exist_status.is_not_found() || !Env::Default()->create_dirs(data_path).ok())) { RETURN_NOT_OK_STATUS_WITH_WARN(Status::IOError(strings::Substitute( - "failed to create data root path $0", data_path)), "create_dirs failed"); + "failed to create data root path $0", data_path)), + "create_dirs failed"); } return Status::OK(); @@ -209,7 +215,8 @@ Status DataDir::_init_meta() { OLAPStatus res = _meta->init(); if (res != OLAP_SUCCESS) { RETURN_NOT_OK_STATUS_WITH_WARN( - Status::IOError(strings::Substitute("open rocksdb failed, path=$0", _path_desc.filepath)), + Status::IOError( + strings::Substitute("open rocksdb failed, path=$0", _path_desc.filepath)), "init OlapMeta failed"); } return Status::OK(); @@ -234,12 +241,14 @@ Status DataDir::_write_cluster_id_to_path(const FilePathDesc& path_desc, int32_t cluster_id_ss << cluster_id; std::unique_ptr wfile; if (!Env::Default()->path_exists(path_desc.filepath).ok()) { - RETURN_IF_ERROR(env_util::write_string_to_file_sync(Env::Default(), Slice(cluster_id_ss.str()), path_desc.filepath)); + RETURN_IF_ERROR(env_util::write_string_to_file_sync( + Env::Default(), Slice(cluster_id_ss.str()), path_desc.filepath)); } if (_env->is_remote_env()) { Status exist_status = _env->path_exists(path_desc.remote_path); if (exist_status.is_not_found()) { - RETURN_IF_ERROR(env_util::write_string_to_file_sync(_env, Slice(cluster_id_ss.str()), path_desc.remote_path)); + RETURN_IF_ERROR(env_util::write_string_to_file_sync(_env, Slice(cluster_id_ss.str()), + path_desc.remote_path)); } else { RETURN_IF_ERROR(exist_status); } @@ -252,7 +261,8 @@ void DataDir::health_check() { if (_is_used) { OLAPStatus res = OLAP_SUCCESS; if ((res = _read_and_write_test_file()) != OLAP_SUCCESS) { - LOG(WARNING) << "store read/write test file occur IO Error. path=" << _path_desc.filepath; + LOG(WARNING) << "store read/write test file occur IO Error. path=" + << _path_desc.filepath; if (is_io_error(res)) { _is_used = false; } @@ -355,7 +365,8 @@ OLAPStatus DataDir::_clean_unfinished_converting_data() { _meta, clean_unifinished_tablet_meta_func, HEADER_PREFIX); if (clean_unfinished_meta_status != OLAP_SUCCESS) { // If failed to clean meta just skip the error, there will be useless metas in rocksdb column family - LOG(WARNING) << "there is failure when clean temp tablet meta from data dir=" << _path_desc.filepath; + LOG(WARNING) << "there is failure when clean temp tablet meta from data dir=" + << _path_desc.filepath; } else { LOG(INFO) << "successfully clean temp tablet meta from data dir=" << _path_desc.filepath; } @@ -384,7 +395,8 @@ bool DataDir::convert_old_data_success() { OLAPStatus DataDir::set_convert_finished() { OLAPStatus res = _meta->set_tablet_convert_finished(); if (res != OLAP_SUCCESS) { - LOG(FATAL) << "save convert flag failed after convert old tablet. dir=" << _path_desc.filepath; + LOG(FATAL) << "save convert flag failed after convert old tablet. dir=" + << _path_desc.filepath; return res; } return OLAP_SUCCESS; @@ -449,7 +461,8 @@ OLAPStatus DataDir::load() { RowsetMetaManager::traverse_rowset_metas(_meta, load_rowset_func); if (load_rowset_status != OLAP_SUCCESS) { - LOG(WARNING) << "errors when load rowset meta from meta env, skip this data dir:" << _path_desc.filepath; + LOG(WARNING) << "errors when load rowset meta from meta env, skip this data dir:" + << _path_desc.filepath; } else { LOG(INFO) << "load rowset from meta finished, data dir: " << _path_desc.filepath; } @@ -492,19 +505,23 @@ OLAPStatus DataDir::load() { if (failed_tablet_ids.size() != 0) { LOG(WARNING) << "load tablets from header failed" << ", loaded tablet: " << tablet_ids.size() - << ", error tablet: " << failed_tablet_ids.size() << ", path: " << _path_desc.filepath; + << ", error tablet: " << failed_tablet_ids.size() + << ", path: " << _path_desc.filepath; if (!config::ignore_load_tablet_failure) { - LOG(FATAL) << "load tablets encounter failure. stop BE process. path: " << _path_desc.filepath; + LOG(FATAL) << "load tablets encounter failure. stop BE process. path: " + << _path_desc.filepath; } } if (load_tablet_status != OLAP_SUCCESS) { LOG(WARNING) << "there is failure when loading tablet headers" << ", loaded tablet: " << tablet_ids.size() - << ", error tablet: " << failed_tablet_ids.size() << ", path: " << _path_desc.filepath; + << ", error tablet: " << failed_tablet_ids.size() + << ", path: " << _path_desc.filepath; } else { LOG(INFO) << "load tablet from meta finished" << ", loaded tablet: " << tablet_ids.size() - << ", error tablet: " << failed_tablet_ids.size() << ", path: " << _path_desc.filepath; + << ", error tablet: " << failed_tablet_ids.size() + << ", path: " << _path_desc.filepath; } // traverse rowset @@ -731,8 +748,8 @@ void DataDir::perform_path_scan() { } } } - LOG(INFO) << "scan data dir path: " << _path_desc.filepath - << " finished. path size: " << _all_check_paths.size() + _all_tablet_schemahash_paths.size(); + LOG(INFO) << "scan data dir path: " << _path_desc.filepath << " finished. path size: " + << _all_check_paths.size() + _all_tablet_schemahash_paths.size(); _check_path_cv.notify_one(); } @@ -749,8 +766,9 @@ bool DataDir::_check_pending_ids(const std::string& id) { } Status DataDir::update_capacity() { - RETURN_NOT_OK_STATUS_WITH_WARN(_env->get_space_info(_path_desc.filepath, &_disk_capacity_bytes, &_available_bytes), - strings::Substitute("get_space_info failed: $0", _path_desc.filepath)); + RETURN_NOT_OK_STATUS_WITH_WARN( + _env->get_space_info(_path_desc.filepath, &_disk_capacity_bytes, &_available_bytes), + strings::Substitute("get_space_info failed: $0", _path_desc.filepath)); if (_disk_capacity_bytes < 0) { _disk_capacity_bytes = _capacity_bytes; } diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 985b78f239..a5f244864a 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -77,9 +77,10 @@ set(UTIL_FILES cidr.cpp core_local.cpp uid_util.cpp - aes_util.cpp + encryption_util.cpp string_util.cpp md5.cpp + sm3.cpp thrift_rpc_helper.cpp faststring.cc slice.cpp diff --git a/be/src/util/aes_util.cpp b/be/src/util/aes_util.cpp deleted file mode 100644 index 76f17cccb5..0000000000 --- a/be/src/util/aes_util.cpp +++ /dev/null @@ -1,176 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "aes_util.h" - -#include -#include -#include - -#include -#include -#include -#include - -namespace doris { - -static const int AES_MAX_KEY_LENGTH = 256; - -const EVP_CIPHER* get_evp_type(const AesMode mode) { - switch (mode) { - case AES_128_ECB: - return EVP_aes_128_ecb(); - case AES_128_CBC: - return EVP_aes_128_cbc(); - case AES_192_ECB: - return EVP_aes_192_ecb(); - case AES_192_CBC: - return EVP_aes_192_cbc(); - case AES_256_ECB: - return EVP_aes_256_ecb(); - case AES_256_CBC: - return EVP_aes_256_cbc(); - default: - return nullptr; - } -} - -static uint aes_mode_key_sizes[] = { - 128 /* AES_128_ECB */, 192 /* AES_192_ECB */, 256 /* AES_256_ECB */, - 128 /* AES_128_CBC */, 192 /* AES_192_CBC */, 256 /* AES_256_CBC */ -}; - -static void aes_create_key(const unsigned char* origin_key, uint32_t key_length, - uint8_t* encrypt_key, AesMode mode) { - const uint key_size = aes_mode_key_sizes[mode] / 8; - uint8_t* origin_key_end = ((uint8_t*)origin_key) + key_length; /* origin key boundary*/ - - uint8_t* encrypt_key_end; /* encrypt key boundary */ - encrypt_key_end = encrypt_key + key_size; - - std::memset(encrypt_key, 0, key_size); /* initialize key */ - - uint8_t* ptr; /* Start of the encrypt key*/ - uint8_t* origin_ptr; /* Start of the origin key */ - for (ptr = encrypt_key, origin_ptr = (uint8_t*)origin_key; origin_ptr < origin_key_end; - ptr++, origin_ptr++) { - if (ptr == encrypt_key_end) { - /* loop over origin key until we used all key */ - ptr = encrypt_key; - } - *ptr ^= *origin_ptr; - } -} - -static int do_encrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher, - const unsigned char* source, uint32_t source_length, - const unsigned char* encrypt_key, const unsigned char* iv, bool padding, - unsigned char* encrypt, int* length_ptr) { - int ret = EVP_EncryptInit(aes_ctx, cipher, encrypt_key, iv); - if (ret == 0) { - return ret; - } - ret = EVP_CIPHER_CTX_set_padding(aes_ctx, padding); - if (ret == 0) { - return ret; - } - int u_len = 0; - ret = EVP_EncryptUpdate(aes_ctx, encrypt, &u_len, source, source_length); - if (ret == 0) { - return ret; - } - int f_len = 0; - ret = EVP_EncryptFinal(aes_ctx, encrypt + u_len, &f_len); - *length_ptr = u_len + f_len; - return ret; -} - -int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_length, - const unsigned char* key, uint32_t key_length, const unsigned char* iv, - bool padding, unsigned char* encrypt) { - const EVP_CIPHER* cipher = get_evp_type(mode); - /* The encrypt key to be used for encryption */ - unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8]; - aes_create_key(key, key_length, encrypt_key, mode); - - if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) { - return AES_BAD_DATA; - } - EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX_reset(aes_ctx); - int length = 0; - int ret = do_encrypt(aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt, - &length); - EVP_CIPHER_CTX_free(aes_ctx); - if (ret == 0) { - ERR_clear_error(); - return AES_BAD_DATA; - } else { - return length; - } -} - -static int do_decrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher, - const unsigned char* encrypt, uint32_t encrypt_length, - const unsigned char* encrypt_key, const unsigned char* iv, bool padding, - unsigned char* decrypt_content, int* length_ptr) { - int ret = EVP_DecryptInit(aes_ctx, cipher, encrypt_key, iv); - if (ret == 0) { - return ret; - } - ret = EVP_CIPHER_CTX_set_padding(aes_ctx, padding); - if (ret == 0) { - return ret; - } - int u_len = 0; - ret = EVP_DecryptUpdate(aes_ctx, decrypt_content, &u_len, encrypt, encrypt_length); - if (ret == 0) { - return ret; - } - int f_len = 0; - ret = EVP_DecryptFinal_ex(aes_ctx, decrypt_content + u_len, &f_len); - *length_ptr = u_len + f_len; - return ret; -} - -int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length, - const unsigned char* key, uint32_t key_length, const unsigned char* iv, - bool padding, unsigned char* decrypt_content) { - const EVP_CIPHER* cipher = get_evp_type(mode); - - /* The encrypt key to be used for decryption */ - unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8]; - aes_create_key(key, key_length, encrypt_key, mode); - - if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) { - return AES_BAD_DATA; - } - EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX_reset(aes_ctx); - int length = 0; - int ret = do_decrypt(aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding, - decrypt_content, &length); - EVP_CIPHER_CTX_free(aes_ctx); - if (ret > 0) { - return length; - } else { - ERR_clear_error(); - return AES_BAD_DATA; - } -} - -} // namespace doris diff --git a/be/src/util/encryption_util.cpp b/be/src/util/encryption_util.cpp new file mode 100644 index 0000000000..5854a7ff25 --- /dev/null +++ b/be/src/util/encryption_util.cpp @@ -0,0 +1,249 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/encryption_util.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace doris { + +static const int ENCRYPTION_MAX_KEY_LENGTH = 256; + +const EVP_CIPHER* get_evp_type(const EncryptionMode mode) { + switch (mode) { + case AES_128_ECB: + return EVP_aes_128_ecb(); + case AES_128_CBC: + return EVP_aes_128_cbc(); + case AES_128_CFB: + return EVP_aes_128_cfb(); + case AES_128_CFB1: + return EVP_aes_128_cfb1(); + case AES_128_CFB8: + return EVP_aes_128_cfb8(); + case AES_128_CFB128: + return EVP_aes_128_cfb128(); + case AES_128_CTR: + return EVP_aes_128_ctr(); + case AES_128_OFB: + return EVP_aes_128_ofb(); + case AES_192_ECB: + return EVP_aes_192_ecb(); + case AES_192_CBC: + return EVP_aes_192_cbc(); + case AES_192_CFB: + return EVP_aes_192_cfb(); + case AES_192_CFB1: + return EVP_aes_192_cfb1(); + case AES_192_CFB8: + return EVP_aes_192_cfb8(); + case AES_192_CFB128: + return EVP_aes_192_cfb128(); + case AES_192_CTR: + return EVP_aes_192_ctr(); + case AES_192_OFB: + return EVP_aes_192_ofb(); + case AES_256_ECB: + return EVP_aes_256_ecb(); + case AES_256_CBC: + return EVP_aes_256_cbc(); + case AES_256_CFB: + return EVP_aes_256_cfb(); + case AES_256_CFB1: + return EVP_aes_256_cfb1(); + case AES_256_CFB8: + return EVP_aes_256_cfb8(); + case AES_256_CFB128: + return EVP_aes_256_cfb128(); + case AES_256_CTR: + return EVP_aes_256_ctr(); + case AES_256_OFB: + return EVP_aes_256_ofb(); + case SM4_128_CBC: + return EVP_sm4_cbc(); + case SM4_128_ECB: + return EVP_sm4_ecb(); + case SM4_128_CFB128: + return EVP_sm4_cfb128(); + case SM4_128_OFB: + return EVP_sm4_ofb(); + case SM4_128_CTR: + return EVP_sm4_ctr(); + default: + return nullptr; + } +} + +static uint mode_key_sizes[] = { + 128 /* AES_128_ECB */, + 192 /* AES_192_ECB */, + 256 /* AES_256_ECB */, + 128 /* AES_128_CBC */, + 192 /* AES_192_CBC */, + 256 /* AES_256_CBC */, + 128 /* AES_128_CFB */, + 192 /* AES_192_CFB */, + 256 /* AES_256_CFB */, + 128 /* AES_128_CFB1 */, + 192 /* AES_192_CFB1 */, + 256 /* AES_256_CFB1 */, + 128 /* AES_128_CFB8 */, + 192 /* AES_192_CFB8 */, + 256 /* AES_256_CFB8 */, + 128 /* AES_128_CFB128 */, + 192 /* AES_192_CFB128 */, + 256 /* AES_256_CFB128 */, + 128 /* AES_128_CTR */, + 192 /* AES_192_CTR */, + 256 /* AES_256_CTR */, + 128 /* AES_128_OFB */, + 192 /* AES_192_OFB */, + 256 /* AES_256_OFB */, + 128 /* SM4_128_ECB */, + 128 /* SM4_128_CBC */, + 128 /* SM4_128_CFB128 */, + 128 /* SM4_128_OFB */, + 128 /* SM4_128_CTR */ +}; + +static void create_key(const unsigned char* origin_key, uint32_t key_length, uint8_t* encrypt_key, + EncryptionMode mode) { + const uint key_size = mode_key_sizes[mode] / 8; + uint8_t* origin_key_end = ((uint8_t*)origin_key) + key_length; /* origin key boundary*/ + + uint8_t* encrypt_key_end; /* encrypt key boundary */ + encrypt_key_end = encrypt_key + key_size; + + std::memset(encrypt_key, 0, key_size); /* initialize key */ + + uint8_t* ptr; /* Start of the encrypt key*/ + uint8_t* origin_ptr; /* Start of the origin key */ + for (ptr = encrypt_key, origin_ptr = (uint8_t*)origin_key; origin_ptr < origin_key_end; + ptr++, origin_ptr++) { + if (ptr == encrypt_key_end) { + /* loop over origin key until we used all key */ + ptr = encrypt_key; + } + *ptr ^= *origin_ptr; + } +} + +static int do_encrypt(EVP_CIPHER_CTX* cipher_ctx, const EVP_CIPHER* cipher, + const unsigned char* source, uint32_t source_length, + const unsigned char* encrypt_key, const unsigned char* iv, bool padding, + unsigned char* encrypt, int* length_ptr) { + int ret = EVP_EncryptInit(cipher_ctx, cipher, encrypt_key, iv); + if (ret == 0) { + return ret; + } + ret = EVP_CIPHER_CTX_set_padding(cipher_ctx, padding); + if (ret == 0) { + return ret; + } + int u_len = 0; + ret = EVP_EncryptUpdate(cipher_ctx, encrypt, &u_len, source, source_length); + if (ret == 0) { + return ret; + } + int f_len = 0; + ret = EVP_EncryptFinal(cipher_ctx, encrypt + u_len, &f_len); + *length_ptr = u_len + f_len; + return ret; +} + +int EncryptionUtil::encrypt(EncryptionMode mode, const unsigned char* source, + uint32_t source_length, const unsigned char* key, uint32_t key_length, + const unsigned char* iv, bool padding, unsigned char* encrypt) { + const EVP_CIPHER* cipher = get_evp_type(mode); + /* The encrypt key to be used for encryption */ + unsigned char encrypt_key[ENCRYPTION_MAX_KEY_LENGTH / 8]; + create_key(key, key_length, encrypt_key, mode); + + if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) { + return AES_BAD_DATA; + } + EVP_CIPHER_CTX* cipher_ctx = EVP_CIPHER_CTX_new(); + EVP_CIPHER_CTX_reset(cipher_ctx); + int length = 0; + int ret = do_encrypt(cipher_ctx, cipher, source, source_length, encrypt_key, iv, padding, + encrypt, &length); + EVP_CIPHER_CTX_free(cipher_ctx); + if (ret == 0) { + ERR_clear_error(); + return AES_BAD_DATA; + } else { + return length; + } +} + +static int do_decrypt(EVP_CIPHER_CTX* cipher_ctx, const EVP_CIPHER* cipher, + const unsigned char* encrypt, uint32_t encrypt_length, + const unsigned char* encrypt_key, const unsigned char* iv, bool padding, + unsigned char* decrypt_content, int* length_ptr) { + int ret = EVP_DecryptInit(cipher_ctx, cipher, encrypt_key, iv); + if (ret == 0) { + return ret; + } + ret = EVP_CIPHER_CTX_set_padding(cipher_ctx, padding); + if (ret == 0) { + return ret; + } + int u_len = 0; + ret = EVP_DecryptUpdate(cipher_ctx, decrypt_content, &u_len, encrypt, encrypt_length); + if (ret == 0) { + return ret; + } + int f_len = 0; + ret = EVP_DecryptFinal_ex(cipher_ctx, decrypt_content + u_len, &f_len); + *length_ptr = u_len + f_len; + return ret; +} + +int EncryptionUtil::decrypt(EncryptionMode mode, const unsigned char* encrypt, + uint32_t encrypt_length, const unsigned char* key, uint32_t key_length, + const unsigned char* iv, bool padding, unsigned char* decrypt_content) { + const EVP_CIPHER* cipher = get_evp_type(mode); + + /* The encrypt key to be used for decryption */ + unsigned char encrypt_key[ENCRYPTION_MAX_KEY_LENGTH / 8]; + create_key(key, key_length, encrypt_key, mode); + + if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) { + return AES_BAD_DATA; + } + EVP_CIPHER_CTX* cipher_ctx = EVP_CIPHER_CTX_new(); + EVP_CIPHER_CTX_reset(cipher_ctx); + int length = 0; + int ret = do_decrypt(cipher_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding, + decrypt_content, &length); + EVP_CIPHER_CTX_free(cipher_ctx); + if (ret > 0) { + return length; + } else { + ERR_clear_error(); + return AES_BAD_DATA; + } +} + +} // namespace doris diff --git a/be/src/util/aes_util.h b/be/src/util/encryption_util.h similarity index 59% rename from be/src/util/aes_util.h rename to be/src/util/encryption_util.h index d3627419a5..e051d28013 100644 --- a/be/src/util/aes_util.h +++ b/be/src/util/encryption_util.h @@ -15,21 +15,53 @@ // specific language governing permissions and limitations // under the License. +#pragma once + #include namespace doris { -enum AesMode { AES_128_ECB, AES_192_ECB, AES_256_ECB, AES_128_CBC, AES_192_CBC, AES_256_CBC }; +enum EncryptionMode { + AES_128_ECB, + AES_192_ECB, + AES_256_ECB, + AES_128_CBC, + AES_192_CBC, + AES_256_CBC, + AES_128_CFB, + AES_192_CFB, + AES_256_CFB, + AES_128_CFB1, + AES_192_CFB1, + AES_256_CFB1, + AES_128_CFB8, + AES_192_CFB8, + AES_256_CFB8, + AES_128_CFB128, + AES_192_CFB128, + AES_256_CFB128, + AES_128_CTR, + AES_192_CTR, + AES_256_CTR, + AES_128_OFB, + AES_192_OFB, + AES_256_OFB, + SM4_128_ECB, + SM4_128_CBC, + SM4_128_CFB128, + SM4_128_OFB, + SM4_128_CTR +}; -enum AesState { AES_SUCCESS = 0, AES_BAD_DATA = -1 }; +enum EncryptionState { AES_SUCCESS = 0, AES_BAD_DATA = -1 }; -class AesUtil { +class EncryptionUtil { public: - static int encrypt(AesMode mode, const unsigned char* source, uint32_t source_length, + static int encrypt(EncryptionMode mode, const unsigned char* source, uint32_t source_length, const unsigned char* key, uint32_t key_length, const unsigned char* iv, bool padding, unsigned char* encrypt); - static int decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length, + static int decrypt(EncryptionMode mode, const unsigned char* encrypt, uint32_t encrypt_length, const unsigned char* key, uint32_t key_length, const unsigned char* iv, bool padding, unsigned char* decrypt_content); }; diff --git a/be/src/util/sm3.cpp b/be/src/util/sm3.cpp new file mode 100644 index 0000000000..c2a6f8e9fc --- /dev/null +++ b/be/src/util/sm3.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/sm3.h" + +#include "util/logging.h" + +namespace doris { + +SM3Digest::SM3Digest() { + _md = EVP_sm3(); + _ctx = EVP_MD_CTX_new(); + EVP_DigestInit_ex(_ctx, _md, NULL); +} + +void SM3Digest::update(const void* data, size_t length) { + EVP_DigestUpdate(_ctx, data, length); +} + +void SM3Digest::digest() { + unsigned char buf[SM3_DIGEST_LENGTH]; + uint32_t length; + EVP_DigestFinal_ex(_ctx, buf, &length); + DCHECK_EQ(SM3_DIGEST_LENGTH, length); + + char hex_buf[2 * SM3_DIGEST_LENGTH]; + + static char dig_vec_lower[] = "0123456789abcdef"; + char* to = hex_buf; + for (int i = 0; i < SM3_DIGEST_LENGTH; ++i) { + *to++ = dig_vec_lower[buf[i] >> 4]; + *to++ = dig_vec_lower[buf[i] & 0x0F]; + } + _hex.assign(hex_buf, 2 * SM3_DIGEST_LENGTH); +} + +} // namespace doris diff --git a/be/src/util/sm3.h b/be/src/util/sm3.h new file mode 100644 index 0000000000..910841072e --- /dev/null +++ b/be/src/util/sm3.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#define SM3_DIGEST_LENGTH 32 + +#include + +#include + +namespace doris { + +class SM3Digest { +public: + SM3Digest(); + + void update(const void* data, size_t length); + void digest(); + + const std::string& hex() const { return _hex; } + +private: + EVP_MD_CTX* _ctx; + const EVP_MD* _md; + + std::string _hex; +}; + +} // namespace doris diff --git a/be/test/exprs/encryption_functions_test.cpp b/be/test/exprs/encryption_functions_test.cpp index 2c13e05c88..ec01fd94df 100644 --- a/be/test/exprs/encryption_functions_test.cpp +++ b/be/test/exprs/encryption_functions_test.cpp @@ -79,23 +79,190 @@ TEST_F(EncryptionFunctionsTest, to_base64) { TEST_F(EncryptionFunctionsTest, aes_decrypt) { std::unique_ptr context(new doris_udf::FunctionContext()); { - StringVal encryptWord = EncryptionFunctions::aes_encrypt(context.get(), doris_udf::StringVal("hello"), - doris_udf::StringVal("key")); + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key")); StringVal result = EncryptionFunctions::aes_decrypt(context.get(), encryptWord, - doris_udf::StringVal("key")); + doris_udf::StringVal("key")); StringVal expected = doris_udf::StringVal("hello"); ASSERT_EQ(expected, result); } { - StringVal encryptWord = EncryptionFunctions::aes_encrypt(context.get(), doris_udf::StringVal("hello"), - doris_udf::StringVal("key")); + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key")); encryptWord.is_null = true; StringVal result = EncryptionFunctions::aes_decrypt(context.get(), encryptWord, - doris_udf::StringVal("key")); + doris_udf::StringVal("key")); StringVal expected = doris_udf::StringVal::null(); ASSERT_EQ(expected, result); } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal::null(), doris_udf::StringVal("AES_128_ECB")); + encryptWord.is_null = true; + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal::null(), doris_udf::StringVal("AES_128_ECB")); + StringVal expected = doris_udf::StringVal::null(); + ASSERT_EQ(expected, result); + } + + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal::null(), doris_udf::StringVal("AES_128_ECB")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal::null(), doris_udf::StringVal("AES_128_ECB")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_256_CBC")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_256_CBC")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB1")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB1")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB8")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB8")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB128")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CFB128")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CTR")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_CTR")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::aes_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_OFB")); + StringVal result = EncryptionFunctions::aes_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("AES_192_OFB")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } +} + +TEST_F(EncryptionFunctionsTest, sm4_decrypt) { + std::unique_ptr context(new doris_udf::FunctionContext()); + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key")); + StringVal result = EncryptionFunctions::sm4_decrypt(context.get(), encryptWord, + doris_udf::StringVal("key")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key")); + encryptWord.is_null = true; + StringVal result = EncryptionFunctions::sm4_decrypt(context.get(), encryptWord, + doris_udf::StringVal("key")); + StringVal expected = doris_udf::StringVal::null(); + ASSERT_EQ(expected, result); + } + + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal::null(), doris_udf::StringVal("SM4_128_ECB")); + encryptWord.is_null = true; + StringVal result = EncryptionFunctions::sm4_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal::null(), doris_udf::StringVal("SM4_128_ECB")); + StringVal expected = doris_udf::StringVal::null(); + ASSERT_EQ(expected, result); + } + + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_CBC")); + StringVal result = EncryptionFunctions::sm4_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_CBC")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_CFB128")); + StringVal result = EncryptionFunctions::sm4_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_CFB128")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_CTR")); + StringVal result = EncryptionFunctions::sm4_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_CTR")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } + { + StringVal encryptWord = EncryptionFunctions::sm4_encrypt( + context.get(), doris_udf::StringVal("hello"), doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_OFB")); + StringVal result = EncryptionFunctions::sm4_decrypt( + context.get(), encryptWord, doris_udf::StringVal("key"), + doris_udf::StringVal("01234567890"), doris_udf::StringVal("SM4_128_OFB")); + StringVal expected = doris_udf::StringVal("hello"); + ASSERT_EQ(expected, result); + } } } // namespace doris diff --git a/be/test/util/CMakeLists.txt b/be/test/util/CMakeLists.txt index ec88f8c12a..e9f75a1462 100644 --- a/be/test/util/CMakeLists.txt +++ b/be/test/util/CMakeLists.txt @@ -37,8 +37,9 @@ ADD_BE_TEST(types_test) ADD_BE_TEST(json_util_test) ADD_BE_TEST(byte_buffer_test2) ADD_BE_TEST(uid_util_test) -ADD_BE_TEST(aes_util_test) +ADD_BE_TEST(encryption_util_test) ADD_BE_TEST(md5_test) +ADD_BE_TEST(sm3_test) ADD_BE_TEST(bitmap_test) ADD_BE_TEST(bitmap_value_test) ADD_BE_TEST(faststring_test) diff --git a/be/test/util/aes_util_test.cpp b/be/test/util/aes_util_test.cpp deleted file mode 100644 index 93517e2e58..0000000000 --- a/be/test/util/aes_util_test.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/aes_util.h" - -#include - -#include -#include - -#include "util/url_coding.h" - -namespace doris { - -class AesUtilTest : public testing::Test { -public: - AesUtilTest() { _aes_key = "doris_aes_key"; } - -private: - std::string _aes_key; -}; - -void do_aes_test(const std::string& source, const std::string& key) { - int cipher_len = source.length() + 16; - std::unique_ptr dest(new unsigned char[cipher_len]); - int ret_code = - AesUtil::encrypt(AES_128_ECB, (unsigned char*)source.c_str(), source.length(), - (unsigned char*)key.c_str(), key.length(), nullptr, true, dest.get()); - ASSERT_TRUE(ret_code > 0); - int encrypted_length = ret_code; - std::unique_ptr decrypted(new char[cipher_len]); - ret_code = - AesUtil::decrypt(AES_128_ECB, dest.get(), encrypted_length, (unsigned char*)key.c_str(), - key.length(), nullptr, true, (unsigned char*)decrypted.get()); - ASSERT_TRUE(ret_code > 0); - std::string decrypted_content(decrypted.get(), ret_code); - ASSERT_EQ(source, decrypted_content); -} - -TEST_F(AesUtilTest, aes_test_basic) { - std::string source_1 = "hello, doris"; - do_aes_test(source_1, _aes_key); - std::string source_2 = "doris test"; - do_aes_test(source_2, _aes_key); -} - -TEST_F(AesUtilTest, aes_test_by_case) { - std::string case_1 = "9qYx8l1601oWHEVCREAqZg=="; // base64 for encrypted "hello, doris" - std::string source_1 = "hello, doris"; - std::string case_2 = "nP/db4j4yqMjXv/pItaOVA=="; // base64 for encrypted "doris test" - std::string source_2 = "doris test"; - - std::unique_ptr encrypt_1(new char[case_1.length()]); - int length_1 = base64_decode(case_1.c_str(), case_1.length(), encrypt_1.get()); - std::unique_ptr decrypted_1(new char[case_1.length()]); - int ret_code = AesUtil::decrypt(AES_128_ECB, (unsigned char*)encrypt_1.get(), length_1, - (unsigned char*)_aes_key.c_str(), _aes_key.length(), nullptr, - true, (unsigned char*)decrypted_1.get()); - ASSERT_TRUE(ret_code > 0); - std::string decrypted_content_1(decrypted_1.get(), ret_code); - ASSERT_EQ(source_1, decrypted_content_1); - - std::unique_ptr encrypt_2(new char[case_2.length()]); - int length_2 = base64_decode(case_2.c_str(), case_2.length(), encrypt_2.get()); - std::unique_ptr decrypted_2(new char[case_2.length()]); - ret_code = AesUtil::decrypt(AES_128_ECB, (unsigned char*)encrypt_2.get(), length_2, - (unsigned char*)_aes_key.c_str(), _aes_key.length(), nullptr, true, - (unsigned char*)decrypted_2.get()); - ASSERT_TRUE(ret_code > 0); - std::string decrypted_content_2(decrypted_2.get(), ret_code); - ASSERT_EQ(source_2, decrypted_content_2); -} - -} // namespace doris - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/be/test/util/encryption_util_test.cpp b/be/test/util/encryption_util_test.cpp new file mode 100644 index 0000000000..30c9752a89 --- /dev/null +++ b/be/test/util/encryption_util_test.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/encryption_util.h" + +#include + +#include +#include + +#include "util/url_coding.h" + +namespace doris { + +class EncryptionUtilTest : public testing::Test { +public: + EncryptionUtilTest() { _aes_key = "doris_aes_key"; } + +private: + std::string _aes_key; +}; + +void do_aes_test(const std::string& source, const std::string& key) { + int cipher_len = source.length() + 16; + std::unique_ptr dest(new unsigned char[cipher_len]); + int ret_code = EncryptionUtil::encrypt(AES_128_ECB, (unsigned char*)source.c_str(), + source.length(), (unsigned char*)key.c_str(), + key.length(), nullptr, true, dest.get()); + ASSERT_TRUE(ret_code > 0); + int encrypted_length = ret_code; + std::unique_ptr decrypted(new char[cipher_len]); + ret_code = EncryptionUtil::decrypt(AES_128_ECB, dest.get(), encrypted_length, + (unsigned char*)key.c_str(), key.length(), nullptr, true, + (unsigned char*)decrypted.get()); + ASSERT_TRUE(ret_code > 0); + std::string decrypted_content(decrypted.get(), ret_code); + ASSERT_EQ(source, decrypted_content); +} + +void do_sm4_test(const std::string& source, const std::string& key) { + int cipher_len = source.length() + 16; + std::unique_ptr dest(new unsigned char[cipher_len]); + int ret_code = EncryptionUtil::encrypt(SM4_128_ECB, (unsigned char*)source.c_str(), + source.length(), (unsigned char*)key.c_str(), + key.length(), nullptr, true, dest.get()); + ASSERT_TRUE(ret_code > 0); + int encrypted_length = ret_code; + std::unique_ptr decrypted(new char[cipher_len]); + ret_code = EncryptionUtil::decrypt(SM4_128_ECB, dest.get(), encrypted_length, + (unsigned char*)key.c_str(), key.length(), nullptr, true, + (unsigned char*)decrypted.get()); + ASSERT_TRUE(ret_code > 0); + std::string decrypted_content(decrypted.get(), ret_code); + ASSERT_EQ(source, decrypted_content); +} + +TEST_F(EncryptionUtilTest, aes_test_basic) { + std::string source_1 = "hello, doris"; + do_aes_test(source_1, _aes_key); + std::string source_2 = "doris test"; + do_aes_test(source_2, _aes_key); +} + +TEST_F(EncryptionUtilTest, sm4_test_basic) { + std::string source_1 = "hello, doris"; + do_sm4_test(source_1, _aes_key); + std::string source_2 = "doris test"; + do_sm4_test(source_2, _aes_key); +} + +TEST_F(EncryptionUtilTest, aes_test_by_case) { + std::string case_1 = "9qYx8l1601oWHEVCREAqZg=="; // base64 for encrypted "hello, doris" + std::string source_1 = "hello, doris"; + std::string case_2 = "nP/db4j4yqMjXv/pItaOVA=="; // base64 for encrypted "doris test" + std::string source_2 = "doris test"; + + std::unique_ptr encrypt_1(new char[case_1.length()]); + int length_1 = base64_decode(case_1.c_str(), case_1.length(), encrypt_1.get()); + std::unique_ptr decrypted_1(new char[case_1.length()]); + int ret_code = EncryptionUtil::decrypt(AES_128_ECB, (unsigned char*)encrypt_1.get(), length_1, + (unsigned char*)_aes_key.c_str(), _aes_key.length(), + nullptr, true, (unsigned char*)decrypted_1.get()); + ASSERT_TRUE(ret_code > 0); + std::string decrypted_content_1(decrypted_1.get(), ret_code); + ASSERT_EQ(source_1, decrypted_content_1); + + std::unique_ptr encrypt_2(new char[case_2.length()]); + int length_2 = base64_decode(case_2.c_str(), case_2.length(), encrypt_2.get()); + std::unique_ptr decrypted_2(new char[case_2.length()]); + ret_code = EncryptionUtil::decrypt(AES_128_ECB, (unsigned char*)encrypt_2.get(), length_2, + (unsigned char*)_aes_key.c_str(), _aes_key.length(), nullptr, + true, (unsigned char*)decrypted_2.get()); + ASSERT_TRUE(ret_code > 0); + std::string decrypted_content_2(decrypted_2.get(), ret_code); + ASSERT_EQ(source_2, decrypted_content_2); +} + +TEST_F(EncryptionUtilTest, sm4_test_by_case) { + std::string case_1 = "P/Ub8/arZ22TW+rAT5sgYg=="; // base64 for encrypted "hello, doris" + std::string source_1 = "hello, doris"; + std::string case_2 = "2I+UW9axOP2Tv35BGYgy+g=="; // base64 for encrypted "doris test" + std::string source_2 = "doris test"; + + std::unique_ptr encrypt_1(new char[case_1.length()]); + int length_1 = base64_decode(case_1.c_str(), case_1.length(), encrypt_1.get()); + std::cout << encrypt_1.get(); + std::unique_ptr decrypted_1(new char[case_1.length()]); + int ret_code = EncryptionUtil::decrypt(SM4_128_ECB, (unsigned char*)encrypt_1.get(), length_1, + (unsigned char*)_aes_key.c_str(), _aes_key.length(), + nullptr, true, (unsigned char*)decrypted_1.get()); + ASSERT_TRUE(ret_code > 0); + std::string decrypted_content_1(decrypted_1.get(), ret_code); + ASSERT_EQ(source_1, decrypted_content_1); + + std::unique_ptr encrypt_2(new char[case_2.length()]); + int length_2 = base64_decode(case_2.c_str(), case_2.length(), encrypt_2.get()); + std::unique_ptr decrypted_2(new char[case_2.length()]); + ret_code = EncryptionUtil::decrypt(SM4_128_ECB, (unsigned char*)encrypt_2.get(), length_2, + (unsigned char*)_aes_key.c_str(), _aes_key.length(), nullptr, + true, (unsigned char*)decrypted_2.get()); + ASSERT_TRUE(ret_code > 0); + std::string decrypted_content_2(decrypted_2.get(), ret_code); + ASSERT_EQ(source_2, decrypted_content_2); +} + +} // namespace doris + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/be/test/util/sm3_test.cpp b/be/test/util/sm3_test.cpp new file mode 100644 index 0000000000..ff039ff196 --- /dev/null +++ b/be/test/util/sm3_test.cpp @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/sm3.h" + +#include + +namespace doris { + +class SM3Test : public testing::Test { +public: + SM3Test() {} + virtual ~SM3Test() {} +}; + +TEST_F(SM3Test, empty) { + SM3Digest digest; + digest.digest(); + ASSERT_STREQ("1ab21d8355cfa17f8e61194831e81a8f22bec8c728fefb747ed035eb5082aa2b", + digest.hex().c_str()); +} + +TEST_F(SM3Test, normal) { + { + SM3Digest digest; + digest.update("abc", 3); + digest.digest(); + ASSERT_STREQ("66c7f0f462eeedd9d1f2d46bdc10e4e24167c4875cf2f7a2297da02b8f4ba8e0", + digest.hex().c_str()); + } + { + SM3Digest digest; + digest.update("0123456789", 10); + digest.digest(); + ASSERT_STREQ("09093b72553f5d9d622d6c62f5ffd916ee959679b1bd4d169c3e12aa8328e743", + digest.hex().c_str()); + } + + { + SM3Digest digest; + digest.update("01234", 5); + digest.update("56789", 5); + digest.digest(); + ASSERT_STREQ("09093b72553f5d9d622d6c62f5ffd916ee959679b1bd4d169c3e12aa8328e743", + digest.hex().c_str()); + } +} + +} // namespace doris + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index a6a49871ba..87b929b772 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -432,6 +432,18 @@ module.exports = [ "orthogonal_bitmap_union_count", ], }, + { + title: "Encryption and Digest Functions", + directoryPath: "encrypt-digest-functions/", + children: [ + "aes", + "md5", + "md5sum", + "sm4", + "sm3", + "sm3sum" + ], + }, { title: "Hash Functions", directoryPath: "hash-functions/", diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index 8a113dd408..ac4150ab06 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -441,6 +441,18 @@ module.exports = [ directoryPath: "hash-functions/", children: ["murmur_hash3_32"], }, + { + title: "加密和信息摘要函数", + directoryPath: "encrypt-digest-functions/", + children: [ + "aes", + "md5", + "md5sum", + "sm4", + "sm3", + "sm3sum" + ], + }, { title: "table functions", directoryPath: "table-functions/", diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md index a77b658059..c2794249de 100644 --- a/docs/en/administrator-guide/variables.md +++ b/docs/en/administrator-guide/variables.md @@ -445,4 +445,38 @@ Translated with www.DeepL.com/Translator (free version) Used to turn off all automatic join reorder algorithms in the system. There are two values: true and false.It is closed by default, that is, the automatic join reorder algorithm of the system is adopted. After set to true, the system will close all automatic sorting algorithms, adopt the original SQL table order, and execute join * `return_object_data_as_binary` - Used to identify whether to return the bitmap/hll result in the select result. In the select into outfile statement, if the export file format is csv, the bimap/hll data will be base64-encoded, if it is the parquet file format, the data will be stored as a byte array \ No newline at end of file + Used to identify whether to return the bitmap/hll result in the select result. In the select into outfile statement, if the export file format is csv, the bimap/hll data will be base64-encoded, if it is the parquet file format, the data will be stored as a byte array +* `block_encryption_mode` + The block_encryption_mode variable controls the block encryption mode. The default setting is empty, when use AES equal to `AES_128_ECB`, when use SM4 equal to `SM3_128_ECB` + available values: +``` + AES_128_ECB, + AES_192_ECB, + AES_256_ECB, + AES_128_CBC, + AES_192_CBC, + AES_256_CBC, + AES_128_CFB, + AES_192_CFB, + AES_256_CFB, + AES_128_CFB1, + AES_192_CFB1, + AES_256_CFB1, + AES_128_CFB8, + AES_192_CFB8, + AES_256_CFB8, + AES_128_CFB128, + AES_192_CFB128, + AES_256_CFB128, + AES_128_CTR, + AES_192_CTR, + AES_256_CTR, + AES_128_OFB, + AES_192_OFB, + AES_256_OFB, + SM4_128_ECB, + SM4_128_CBC, + SM4_128_CFB128, + SM4_128_OFB, + SM4_128_CTR, +``` \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/encrypt-digest-functions/aes.md b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/aes.md new file mode 100644 index 0000000000..7b3083d9a2 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/aes.md @@ -0,0 +1,88 @@ +--- +{ +"title": "AES", +"language": "en" +} +--- + + + +# AES_ENCRYPT + +## description +encryption of data using the official AES +### Syntax + +`VARCHAR AES_ENCRYPT(str,key_str[,init_vector])` + +## example + +``` +MySQL > select to_base64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3')); ++--------------------------------+ +| to_base64(aes_encrypt('text')) | ++--------------------------------+ +| wr2JEDVXzL9+2XtRhgIloA== | ++--------------------------------+ +1 row in set (0.010 sec) + +MySQL> set block_encryption_mode="AES_256_CBC"; +Query OK, 0 rows affected (0.006 sec) + +MySQL > select to_base64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789')); ++----------------------------------------------------------------------------------+ +| to_base64(aes_encrypt('text', 'F3229A0B371ED2D9441B830D21A390C3', '0123456789')) | ++----------------------------------------------------------------------------------+ +| mvZT1KJw7N0RJf27aipUpg== | ++----------------------------------------------------------------------------------+ +1 row in set (0.011 sec) +``` + +# AES_DECRYPT + +## description +decryption of data using the official AES (### Syntax + +`VARCHAR AES_DECRYPT(str,key_str[,init_vector])` + +## example + +``` +MySQL > select AES_DECRYPT(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3'); ++------------------------------------------------------+ +| aes_decrypt(from_base64('wr2JEDVXzL9+2XtRhgIloA==')) | ++------------------------------------------------------+ +| text | ++------------------------------------------------------+ +1 row in set (0.012 sec) + +MySQL> set block_encryption_mode="AES_256_CBC"; +Query OK, 0 rows affected (0.006 sec) + +MySQL > select AES_DECRYPT(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789'); ++--------------------------------------------------------------------------------------------------------+ +| aes_decrypt(from_base64('mvZT1KJw7N0RJf27aipUpg=='), 'F3229A0B371ED2D9441B830D21A390C3', '0123456789') | ++--------------------------------------------------------------------------------------------------------+ +| text | ++--------------------------------------------------------------------------------------------------------+ +1 row in set (0.012 sec) +``` + +## keyword + + AES_ENCRYPT, AES_DECRYPT \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5.md b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5.md new file mode 100644 index 0000000000..de2cca23ec --- /dev/null +++ b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5.md @@ -0,0 +1,47 @@ +--- +{ +"title": "MD5", +"language": "en" +} +--- + + + +# MD5 + +## description +Calculates an MD5 128-bit checksum for the string +### Syntax + +`MD5(str)` + +## example + +``` +MySQL [(none)]> select md5("abc"); ++----------------------------------+ +| md5('abc') | ++----------------------------------+ +| 900150983cd24fb0d6963f7d28e17f72 | ++----------------------------------+ +1 row in set (0.013 sec) +``` + +## keyword + + MD5 \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md new file mode 100644 index 0000000000..7ff1d18e75 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md @@ -0,0 +1,56 @@ +--- +{ +"title": "MD5SUM", +"language": "en" +} +--- + + + +# MD5SUM + +## description +Calculates an MD5 128-bit checksum for the strings +### Syntax + +`MD5SUM(str[,str])` + +## example + +``` +MySQL > select md5("abcd"); ++----------------------------------+ +| md5('abcd') | ++----------------------------------+ +| e2fc714c4727ee9395f324cd2e7f331f | ++----------------------------------+ +1 row in set (0.011 sec) + +MySQL > select md5sum("ab","cd"); ++----------------------------------+ +| md5sum('ab', 'cd') | ++----------------------------------+ +| e2fc714c4727ee9395f324cd2e7f331f | ++----------------------------------+ +1 row in set (0.008 sec) + +``` + +## keyword + + MD5SUM \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3.md b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3.md new file mode 100644 index 0000000000..85d09e87e7 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3.md @@ -0,0 +1,47 @@ +--- +{ +"title": "SM3", +"language": "en" +} +--- + + + +# SM3 + +## description +Calculates an SM3 256-bit checksum for the string +### Syntax + +`SM3(str)` + +## example + +``` +MySQL > select sm3("abcd"); ++------------------------------------------------------------------+ +| sm3('abcd') | ++------------------------------------------------------------------+ +| 82ec580fe6d36ae4f81cae3c73f4a5b3b5a09c943172dc9053c69fd8e18dca1e | ++------------------------------------------------------------------+ +1 row in set (0.009 sec) +``` + +## keyword + + SM3 \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md new file mode 100644 index 0000000000..33e8b131c5 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md @@ -0,0 +1,56 @@ +--- +{ +"title": "SM3SUM", +"language": "en" +} +--- + + + +# SM3SUM + +## description +Calculates an SM3 128-bit checksum for the strings +### Syntax + +`SM3SUM(str[,str])` + +## example + +``` +MySQL > select sm3("abcd"); ++------------------------------------------------------------------+ +| sm3('abcd') | ++------------------------------------------------------------------+ +| 82ec580fe6d36ae4f81cae3c73f4a5b3b5a09c943172dc9053c69fd8e18dca1e | ++------------------------------------------------------------------+ +1 row in set (0.009 sec) + +MySQL > select sm3sum("ab","cd"); ++------------------------------------------------------------------+ +| sm3sum('ab', 'cd') | ++------------------------------------------------------------------+ +| 82ec580fe6d36ae4f81cae3c73f4a5b3b5a09c943172dc9053c69fd8e18dca1e | ++------------------------------------------------------------------+ +1 row in set (0.009 sec) + +``` + +## keyword + + SM3SUM \ No newline at end of file diff --git a/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm4.md b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm4.md new file mode 100644 index 0000000000..29f860a796 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/encrypt-digest-functions/sm4.md @@ -0,0 +1,93 @@ +--- +{ +"title": "SM4", +"language": "en" +} +--- + + + +# SM4_ENCRYPT + +## description +SM4 加密函数 +### Syntax + +`VARCHAR SM4_ENCRYPT(str,key_str[,init_vector])` + +返回加密后的结果 + +## example + +``` +MySQL > select TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3')); ++--------------------------------+ +| to_base64(sm4_encrypt('text')) | ++--------------------------------+ +| aDjwRflBrDjhBZIOFNw3Tg== | ++--------------------------------+ +1 row in set (0.010 sec) + +MySQL > set block_encryption_mode="SM4_128_CBC"; +Query OK, 0 rows affected (0.001 sec) + +MySQL > select to_base64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789')); ++----------------------------------------------------------------------------------+ +| to_base64(sm4_encrypt('text', 'F3229A0B371ED2D9441B830D21A390C3', '0123456789')) | ++----------------------------------------------------------------------------------+ +| G7yqOKfEyxdagboz6Qf01A== | ++----------------------------------------------------------------------------------+ +1 row in set (0.014 sec) +``` + +# SM4_DECRYPT + +## description +Aes 解密函数 +### Syntax + +`VARCHAR AES_DECRYPT(str,key_str[,init_vector])` + +返回解密后的结果 + +## example + +``` +MySQL [(none)]> select SM4_DECRYPT(FROM_BASE64('aDjwRflBrDjhBZIOFNw3Tg=='),'F3229A0B371ED2D9441B830D21A390C3'); ++------------------------------------------------------+ +| sm4_decrypt(from_base64('aDjwRflBrDjhBZIOFNw3Tg==')) | ++------------------------------------------------------+ +| text | ++------------------------------------------------------+ +1 row in set (0.009 sec) + +MySQL> set block_encryption_mode="SM4_128_CBC"; +Query OK, 0 rows affected (0.006 sec) + +MySQL > select SM4_DECRYPT(FROM_BASE64('G7yqOKfEyxdagboz6Qf01A=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789'); ++--------------------------------------------------------------------------------------------------------+ +| sm4_decrypt(from_base64('G7yqOKfEyxdagboz6Qf01A=='), 'F3229A0B371ED2D9441B830D21A390C3', '0123456789') | ++--------------------------------------------------------------------------------------------------------+ +| text | ++--------------------------------------------------------------------------------------------------------+ +1 row in set (0.012 sec) +``` + +## keyword + + SM4_ENCRYPT, SM4_DECRYPT \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md index cc63cdffe2..1470352afd 100644 --- a/docs/zh-CN/administrator-guide/variables.md +++ b/docs/zh-CN/administrator-guide/variables.md @@ -439,3 +439,37 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3); * `return_object_data_as_binary` 用于标识是否在select 结果中返回bitmap/hll 结果。在 select into outfile 语句中，如果导出文件格式为csv 则会将 bimap/hll 数据进行base64编码，如果是parquet 文件格式 将会把数据作为byte array 存储 +* `block_encryption_mode` + 可以通过block_encryption_mode参数，控制块加密模式，默认值为：空。当使用AES算法加密时相当于`AES_128_ECB`, 当时用SM3算法加密时相当于`SM3_128_ECB` + 可选值： +``` + AES_128_ECB, + AES_192_ECB, + AES_256_ECB, + AES_128_CBC, + AES_192_CBC, + AES_256_CBC, + AES_128_CFB, + AES_192_CFB, + AES_256_CFB, + AES_128_CFB1, + AES_192_CFB1, + AES_256_CFB1, + AES_128_CFB8, + AES_192_CFB8, + AES_256_CFB8, + AES_128_CFB128, + AES_192_CFB128, + AES_256_CFB128, + AES_128_CTR, + AES_192_CTR, + AES_256_CTR, + AES_128_OFB, + AES_192_OFB, + AES_256_OFB, + SM4_128_ECB, + SM4_128_CBC, + SM4_128_CFB128, + SM4_128_OFB, + SM4_128_CTR, +``` \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/aes.md b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/aes.md new file mode 100644 index 0000000000..8f1770772b --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/aes.md @@ -0,0 +1,93 @@ +--- +{ +"title": "AES", +"language": "zh-CN" +} +--- + + + +# AES_ENCRYPT + +## description +Aes 加密函数 +### Syntax + +`VARCHAR AES_ENCRYPT(str,key_str[,init_vector])` + +返回加密后的结果 + +## example + +``` +MySQL > select to_base64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3')); ++--------------------------------+ +| to_base64(aes_encrypt('text')) | ++--------------------------------+ +| wr2JEDVXzL9+2XtRhgIloA== | ++--------------------------------+ +1 row in set (0.010 sec) + +MySQL> set block_encryption_mode="AES_256_CBC"; +Query OK, 0 rows affected (0.006 sec) + +MySQL > select to_base64(AES_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789')); ++----------------------------------------------------------------------------------+ +| to_base64(aes_encrypt('text', 'F3229A0B371ED2D9441B830D21A390C3', '0123456789')) | ++----------------------------------------------------------------------------------+ +| mvZT1KJw7N0RJf27aipUpg== | ++----------------------------------------------------------------------------------+ +1 row in set (0.011 sec) +``` + +# AES_DECRYPT + +## description +Aes 解密函数 +### Syntax + +`VARCHAR AES_DECRYPT(str,key_str[,init_vector])` + +返回解密后的结果 + +## example + +``` +MySQL > select AES_DECRYPT(FROM_BASE64('wr2JEDVXzL9+2XtRhgIloA=='),'F3229A0B371ED2D9441B830D21A390C3'); ++------------------------------------------------------+ +| aes_decrypt(from_base64('wr2JEDVXzL9+2XtRhgIloA==')) | ++------------------------------------------------------+ +| text | ++------------------------------------------------------+ +1 row in set (0.012 sec) + +MySQL> set block_encryption_mode="AES_256_CBC"; +Query OK, 0 rows affected (0.006 sec) + +MySQL > select AES_DECRYPT(FROM_BASE64('mvZT1KJw7N0RJf27aipUpg=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789'); ++--------------------------------------------------------------------------------------------------------+ +| aes_decrypt(from_base64('mvZT1KJw7N0RJf27aipUpg=='), 'F3229A0B371ED2D9441B830D21A390C3', '0123456789') | ++--------------------------------------------------------------------------------------------------------+ +| text | ++--------------------------------------------------------------------------------------------------------+ +1 row in set (0.012 sec) +``` + +## keyword + + AES_ENCRYPT, AES_DECRYPT \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5.md b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5.md new file mode 100644 index 0000000000..38da45be42 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5.md @@ -0,0 +1,47 @@ +--- +{ +"title": "MD5", +"language": "zh-CN" +} +--- + + + +# MD5 + +## description +计算 MD5 128-bit +### Syntax + +`MD5(str)` + +## example + +``` +MySQL [(none)]> select md5("abc"); ++----------------------------------+ +| md5('abc') | ++----------------------------------+ +| 900150983cd24fb0d6963f7d28e17f72 | ++----------------------------------+ +1 row in set (0.013 sec) +``` + +## keyword + + MD5 \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md new file mode 100644 index 0000000000..9374fc46a0 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/md5sum.md @@ -0,0 +1,56 @@ +--- +{ +"title": "MD5SUM", +"language": "zh-CN" +} +--- + + + +# MD5SUM + +## description +计算 多个字符串 MD5 128-bit +### Syntax + +`MD5SUM(str[,str])` + +## example + +``` +MySQL > select md5("abcd"); ++----------------------------------+ +| md5('abcd') | ++----------------------------------+ +| e2fc714c4727ee9395f324cd2e7f331f | ++----------------------------------+ +1 row in set (0.011 sec) + +MySQL > select md5sum("ab","cd"); ++----------------------------------+ +| md5sum('ab', 'cd') | ++----------------------------------+ +| e2fc714c4727ee9395f324cd2e7f331f | ++----------------------------------+ +1 row in set (0.008 sec) + +``` + +## keyword + + MD5SUM \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3.md b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3.md new file mode 100644 index 0000000000..f0efd68f38 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3.md @@ -0,0 +1,47 @@ +--- +{ +"title": "SM3", +"language": "zh-CN" +} +--- + + + +# SM3 + +## description +计算 SM3 256-bit +### Syntax + +`SM3(str)` + +## example + +``` +MySQL > select sm3("abcd"); ++------------------------------------------------------------------+ +| sm3('abcd') | ++------------------------------------------------------------------+ +| 82ec580fe6d36ae4f81cae3c73f4a5b3b5a09c943172dc9053c69fd8e18dca1e | ++------------------------------------------------------------------+ +1 row in set (0.009 sec) +``` + +## keyword + + SM3 \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md new file mode 100644 index 0000000000..5f0f9712da --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm3sum.md @@ -0,0 +1,56 @@ +--- +{ +"title": "SM3SUM", +"language": "zh-CN" +} +--- + + + +# SM3SUM + +## description +计算 多个字符串 SM3 256-bit +### Syntax + +`SM3SUM(str[,str])` + +## example + +``` +MySQL > select sm3("abcd"); ++------------------------------------------------------------------+ +| sm3('abcd') | ++------------------------------------------------------------------+ +| 82ec580fe6d36ae4f81cae3c73f4a5b3b5a09c943172dc9053c69fd8e18dca1e | ++------------------------------------------------------------------+ +1 row in set (0.009 sec) + +MySQL > select sm3sum("ab","cd"); ++------------------------------------------------------------------+ +| sm3sum('ab', 'cd') | ++------------------------------------------------------------------+ +| 82ec580fe6d36ae4f81cae3c73f4a5b3b5a09c943172dc9053c69fd8e18dca1e | ++------------------------------------------------------------------+ +1 row in set (0.009 sec) + +``` + +## keyword + + SM3SUM \ No newline at end of file diff --git a/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm4.md b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm4.md new file mode 100644 index 0000000000..35b6575fe4 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/encrypt-digest-functions/sm4.md @@ -0,0 +1,93 @@ +--- +{ +"title": "SM4", +"language": "zh-CN" +} +--- + + + +# SM4_ENCRYPT + +## description +SM4 加密函数 +### Syntax + +`VARCHAR SM4_ENCRYPT(str,key_str[,init_vector])` + +返回加密后的结果 + +## example + +``` +MySQL > select TO_BASE64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3')); ++--------------------------------+ +| to_base64(sm4_encrypt('text')) | ++--------------------------------+ +| aDjwRflBrDjhBZIOFNw3Tg== | ++--------------------------------+ +1 row in set (0.010 sec) + +MySQL > set block_encryption_mode="SM4_128_CBC"; +Query OK, 0 rows affected (0.001 sec) + +MySQL > select to_base64(SM4_ENCRYPT('text','F3229A0B371ED2D9441B830D21A390C3', '0123456789')); ++----------------------------------------------------------------------------------+ +| to_base64(sm4_encrypt('text', 'F3229A0B371ED2D9441B830D21A390C3', '0123456789')) | ++----------------------------------------------------------------------------------+ +| G7yqOKfEyxdagboz6Qf01A== | ++----------------------------------------------------------------------------------+ +1 row in set (0.014 sec) +``` + +# SM4_DECRYPT + +## description +Aes 解密函数 +### Syntax + +`VARCHAR AES_DECRYPT(str,key_str[,init_vector])` + +返回解密后的结果 + +## example + +``` +MySQL [(none)]> select SM4_DECRYPT(FROM_BASE64('aDjwRflBrDjhBZIOFNw3Tg=='),'F3229A0B371ED2D9441B830D21A390C3'); ++------------------------------------------------------+ +| sm4_decrypt(from_base64('aDjwRflBrDjhBZIOFNw3Tg==')) | ++------------------------------------------------------+ +| text | ++------------------------------------------------------+ +1 row in set (0.009 sec) + +MySQL> set block_encryption_mode="SM4_128_CBC"; +Query OK, 0 rows affected (0.006 sec) + +MySQL > select SM4_DECRYPT(FROM_BASE64('G7yqOKfEyxdagboz6Qf01A=='),'F3229A0B371ED2D9441B830D21A390C3', '0123456789'); ++--------------------------------------------------------------------------------------------------------+ +| sm4_decrypt(from_base64('G7yqOKfEyxdagboz6Qf01A=='), 'F3229A0B371ED2D9441B830D21A390C3', '0123456789') | ++--------------------------------------------------------------------------------------------------------+ +| text | ++--------------------------------------------------------------------------------------------------------+ +1 row in set (0.012 sec) +``` + +## keyword + + SM4_ENCRYPT, SM4_DECRYPT \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 87f891913e..f6a350adac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -46,6 +46,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,7 +55,9 @@ import java.io.DataOutput; import java.io.IOException; import java.text.StringCharacterIterator; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; // TODO: for aggregations, we need to unify the code paths for builtins and UDAs. public class FunctionCallExpr extends Expr { @@ -253,13 +256,26 @@ public class FunctionCallExpr extends Expr { boolean isJsonFunction = false; int len = children.size(); List result = Lists.newArrayList(); - if ((fnName.getFunction().equalsIgnoreCase("json_array")) || - (fnName.getFunction().equalsIgnoreCase("json_object"))) { + if (fnName.getFunction().equalsIgnoreCase("json_array") || + fnName.getFunction().equalsIgnoreCase("json_object")) { len = len - 1; isJsonFunction = true; } + if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") || + fnName.getFunction().equalsIgnoreCase("aes_encrypt") || + fnName.getFunction().equalsIgnoreCase("sm4_decrypt") || + fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) { + len = len - 1; + } for (int i = 0; i < len; ++i) { - result.add(children.get(i).toSql()); + if (i == 1 && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") || + fnName.getFunction().equalsIgnoreCase("aes_encrypt") || + fnName.getFunction().equalsIgnoreCase("sm4_decrypt") || + fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) { + result.add("\'***\'"); + } else { + result.add(children.get(i).toSql()); + } } sb.append(Joiner.on(", ").join(result)).append(")"); if (fnName.getFunction().equalsIgnoreCase("json_quote") || isJsonFunction) { @@ -584,6 +600,70 @@ public class FunctionCallExpr extends Expr { } } } + if ((fnName.getFunction().equalsIgnoreCase("aes_decrypt") + || fnName.getFunction().equalsIgnoreCase("aes_encrypt") + || fnName.getFunction().equalsIgnoreCase("sm4_decrypt") + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) + && children.size() == 3) { + String blockEncryptionMode = ""; + Set aesModes = new HashSet<>(Arrays.asList( + "AES_128_ECB", + "AES_192_ECB", + "AES_256_ECB", + "AES_128_CBC", + "AES_192_CBC", + "AES_256_CBC", + "AES_128_CFB", + "AES_192_CFB", + "AES_256_CFB", + "AES_128_CFB1", + "AES_192_CFB1", + "AES_256_CFB1", + "AES_128_CFB8", + "AES_192_CFB8", + "AES_256_CFB8", + "AES_128_CFB128", + "AES_192_CFB128", + "AES_256_CFB128", + "AES_128_CTR", + "AES_192_CTR", + "AES_256_CTR", + "AES_128_OFB", + "AES_192_OFB", + "AES_256_OFB" + )); + Set sm4Modes = new HashSet<>(Arrays.asList( + "SM4_128_ECB", + "SM4_128_CBC", + "SM4_128_CFB128", + "SM4_128_OFB", + "SM4_128_CTR")); + + if (ConnectContext.get() != null) { + blockEncryptionMode = ConnectContext.get().getSessionVariable().getBlockEncryptionMode(); + if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") + || fnName.getFunction().equalsIgnoreCase("aes_encrypt")) { + if (StringUtils.isAllBlank(blockEncryptionMode)) { + blockEncryptionMode = "AES_128_ECB"; + } + if (!aesModes.contains(blockEncryptionMode.toUpperCase())) { + throw new AnalysisException("session variable block_encryption_mode is invalid with aes"); + + } + } + if (fnName.getFunction().equalsIgnoreCase("sm4_decrypt") + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) { + if (StringUtils.isAllBlank(blockEncryptionMode)) { + blockEncryptionMode = "SM4_128_ECB"; + } + if (!sm4Modes.contains(blockEncryptionMode.toUpperCase())) { + throw new AnalysisException("session variable block_encryption_mode is invalid with sm4"); + + } + } + } + children.add(new StringLiteral(blockEncryptionMode)); + } } @@ -849,6 +929,7 @@ public class FunctionCallExpr extends Expr { /** * rewrite alias function to real function * reset function name, function params and it's children to real function's + * * @return * @throws AnalysisException */ @@ -888,6 +969,7 @@ public class FunctionCallExpr extends Expr { /** * replace origin function expr and it's children with input params exprs depending on parameter name + * * @param parameters * @param inputParamsExprs * @param oriExpr diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 2f3a5fcb27..7ccaffafd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -169,6 +169,8 @@ public class SessionVariable implements Serializable, Writable { public static final String RETURN_OBJECT_DATA_AS_BINARY = "return_object_data_as_binary"; + public static final String BLOCK_ENCRYPTION_MODE = "block_encryption_mode"; + // session origin value public Map sessionOriginValue = new HashMap(); // check stmt is or not [select /*+ SET_VAR(...)*/ ...] @@ -400,6 +402,16 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = RETURN_OBJECT_DATA_AS_BINARY) private boolean returnObjectDataAsBinary = false; + @VariableMgr.VarAttr(name = BLOCK_ENCRYPTION_MODE) + private String blockEncryptionMode = ""; + + public String getBlockEncryptionMode() { + return blockEncryptionMode; + } + + public void setBlockEncryptionMode(String blockEncryptionMode) { + this.blockEncryptionMode = blockEncryptionMode; + } public long getMaxExecMemByte() { return maxExecMemByte; } diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 40d08c9b8e..8202793c0a 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1266,18 +1266,54 @@ visible_functions = [ [['aes_decrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '_ZN5doris19EncryptionFunctions11aes_decryptEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], + [['aes_encrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris19EncryptionFunctions11aes_encryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], + [['aes_decrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris19EncryptionFunctions11aes_decryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], + [['sm4_encrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], + '_ZN5doris19EncryptionFunctions11sm4_encryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], + [['sm4_decrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], + '_ZN5doris19EncryptionFunctions11sm4_decryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], + [['sm4_encrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris19EncryptionFunctions11sm4_encryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], + [['sm4_decrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR', 'VARCHAR'], + '_ZN5doris19EncryptionFunctions11sm4_decryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], [['from_base64'], 'VARCHAR', ['VARCHAR'], '_ZN5doris19EncryptionFunctions11from_base64EPN9doris_udf' '15FunctionContextERKNS1_9StringValE', '', '', '', ''], [['to_base64'], 'VARCHAR', ['VARCHAR'], '_ZN5doris19EncryptionFunctions9to_base64EPN9doris_udf' '15FunctionContextERKNS1_9StringValE', '', '', '', ''], - [['aes_encrypt'], 'STRING', ['STRING', 'STRING'], + [['aes_encrypt'], 'STRING', ['STRING', 'STRING'], '_ZN5doris19EncryptionFunctions11aes_encryptEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], [['aes_decrypt'], 'STRING', ['STRING', 'STRING'], '_ZN5doris19EncryptionFunctions11aes_decryptEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], + [['aes_encrypt'], 'STRING', ['STRING', 'STRING', 'STRING', 'STRING'], + '_ZN5doris19EncryptionFunctions11aes_encryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], + [['aes_decrypt'], 'STRING', ['STRING', 'STRING', 'STRING', 'STRING'], + '_ZN5doris19EncryptionFunctions11aes_decryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], + [['sm4_encrypt'], 'STRING', ['STRING', 'STRING'], + '_ZN5doris19EncryptionFunctions11sm4_encryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], + [['sm4_decrypt'], 'STRING', ['STRING', 'STRING'], + '_ZN5doris19EncryptionFunctions11sm4_decryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], + [['sm4_encrypt'], 'STRING', ['STRING', 'STRING', 'STRING', 'STRING'], + '_ZN5doris19EncryptionFunctions11sm4_encryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], + [['sm4_decrypt'], 'STRING', ['STRING', 'STRING', 'STRING', 'STRING'], + '_ZN5doris19EncryptionFunctions11sm4_decryptEPN9doris_udf' + '15FunctionContextERKNS1_9StringValES6_S6_S6_', '', '', '', ''], [['from_base64'], 'STRING', ['STRING'], '_ZN5doris19EncryptionFunctions11from_base64EPN9doris_udf' '15FunctionContextERKNS1_9StringValE', '', '', '', ''], @@ -1294,6 +1330,15 @@ visible_functions = [ [['md5sum'], 'VARCHAR', ['STRING', '...'], '_ZN5doris19EncryptionFunctions6md5sumEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', '', '', '', ''], + [['sm3'], 'VARCHAR', ['VARCHAR'], + '_ZN5doris19EncryptionFunctions3sm3EPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', '', ''], + [['sm3sum'], 'VARCHAR', ['VARCHAR', '...'], + '_ZN5doris19EncryptionFunctions6sm3sumEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', '', '', '', ''], + [['sm3'], 'VARCHAR', ['STRING'], + '_ZN5doris19EncryptionFunctions3sm3EPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', '', ''], + [['sm3sum'], 'VARCHAR', ['STRING', '...'], + '_ZN5doris19EncryptionFunctions6sm3sumEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', '', '', '', ''], + # geo functions [['ST_Point'], 'VARCHAR', ['DOUBLE', 'DOUBLE'], '_ZN5doris12GeoFunctions8st_pointEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_', '', '', '', ''],