hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

init push

Browse Source
This commit is contained in:
oceanbase-admin
2021-05-31 22:56:52 +08:00
commit cea7de1475
7020 changed files with 5689869 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
src/sql/engine/user_defined_function/ob_udf_ctx_mgr.cpp Normal file
View File

@ -0,0 +1,114 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "ob_udf_ctx_mgr.h"
#include "sql/engine/expr/ob_expr_dll_udf.h"
#include "sql/engine/user_defined_function/ob_udf_util.h"
namespace oceanbase {
namespace sql {
using namespace common;
using ObUdfCtx = ObUdfFunction::ObUdfCtx;
ObUdfCtxMgr::~ObUdfCtxMgr()
{
if (ctxs_.created()) {
common::hash::ObHashMap<uint64_t, ObNormalUdfExeUnit*, common::hash::NoPthreadDefendMode>::iterator iter =
ctxs_.begin();
for (; iter != ctxs_.end(); iter++) {
ObNormalUdfExeUnit* normal_unit = iter->second;
if (OB_NOT_NULL(normal_unit->normal_func_) && OB_NOT_NULL(normal_unit->udf_ctx_)) {
IGNORE_RETURN normal_unit->normal_func_->process_deinit_func(*normal_unit->udf_ctx_);
}
}
}
allocator_.reset();
ctxs_.destroy();
}
int ObUdfCtxMgr::register_udf_expr(
const ObExprDllUdf* expr, const ObNormalUdfFunction* func, ObNormalUdfExeUnit*& udf_exec_unit)
{
int ret = OB_SUCCESS;
uint64_t expr_id = common::OB_INVALID_ID;
ObUdfCtx* new_udf_ctx = nullptr;
ObNormalUdfExeUnit* tmp_udf_exec_unit = nullptr;
if (OB_FAIL(try_init_map())) {
LOG_WARN("failed to init udf ctx map", K(ret));
} else if (OB_ISNULL(expr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the expr is null", K(ret));
} else if (FALSE_IT(expr_id = expr->get_id())) {
} else if (OB_ISNULL(new_udf_ctx = (ObUdfCtx*)allocator_.alloc(sizeof(ObUdfCtx)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret));
} else if (OB_ISNULL(tmp_udf_exec_unit = (ObNormalUdfExeUnit*)allocator_.alloc(sizeof(ObNormalUdfExeUnit)))) {
} else {
new_udf_ctx->state_ = ObUdfFunction::UDF_UNINITIALIZED;
IGNORE_RETURN ObUdfUtil::construct_udf_args(new_udf_ctx->udf_args_);
IGNORE_RETURN ObUdfUtil::construct_udf_init(new_udf_ctx->udf_init_);
tmp_udf_exec_unit->udf_ctx_ = new_udf_ctx;
tmp_udf_exec_unit->normal_func_ = func;
if (OB_FAIL(ctxs_.set_refactored(expr_id, tmp_udf_exec_unit))) {
LOG_WARN("failed to set new udf ctx", K(ret));
} else {
udf_exec_unit = tmp_udf_exec_unit;
}
}
return ret;
}
int ObUdfCtxMgr::get_udf_ctx(uint64_t expr_id, ObNormalUdfExeUnit*& udf_exec_unit)
{
int ret = OB_SUCCESS;
if (OB_FAIL(try_init_map())) {
LOG_WARN("failed to init udf ctx map", K(ret));
} else if (OB_FAIL(ctxs_.get_refactored(expr_id, udf_exec_unit))) {
LOG_WARN("failed to get udf_ctx", K(ret));
}
return ret;
}
int ObUdfCtxMgr::try_init_map()
{
int ret = OB_SUCCESS;
if (!ctxs_.created()) {
if (OB_FAIL(ctxs_.create(
common::hash::cal_next_prime(BUKET_NUM), common::ObModIds::OB_SQL_UDF, common::ObModIds::OB_SQL_UDF))) {
LOG_WARN("create hash failed", K(ret));
}
}
return ret;
}
int ObUdfCtxMgr::reset()
{
int ret = OB_SUCCESS;
if (ctxs_.created()) {
common::hash::ObHashMap<uint64_t, ObNormalUdfExeUnit*, common::hash::NoPthreadDefendMode>::iterator iter =
ctxs_.begin();
for (; iter != ctxs_.end(); iter++) {
ObNormalUdfExeUnit* normal_unit = iter->second;
if (OB_NOT_NULL(normal_unit->normal_func_) && OB_NOT_NULL(normal_unit->udf_ctx_)) {
IGNORE_RETURN normal_unit->normal_func_->process_deinit_func(*normal_unit->udf_ctx_);
}
}
ctxs_.clear();
}
allocator_.reset();
return ret;
}
} // namespace sql
} // namespace oceanbase

52
src/sql/engine/user_defined_function/ob_udf_ctx_mgr.h Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_UDF_CTX_MGR_H_
#define OB_UDF_CTX_MGR_H_
#include "sql/engine/user_defined_function/ob_user_defined_function.h"
namespace oceanbase {
namespace sql {
class ObExprDllUdf;
class ObUdfCtxMgr {
private:
static const int64_t BUKET_NUM = 100;
public:
ObUdfCtxMgr() : allocator_(common::ObModIds::OB_SQL_UDF), ctxs_()
{}
~ObUdfCtxMgr();
int register_udf_expr(const ObExprDllUdf* expr, const ObNormalUdfFunction* func, ObNormalUdfExeUnit*& udf_exec_unit);
int get_udf_ctx(uint64_t expr_id, ObNormalUdfExeUnit*& udf_exec_unit);
int try_init_map();
common::ObIAllocator& get_allocator()
{
return allocator_;
}
int reset();
private:
common::ObArenaAllocator allocator_;
common::hash::ObHashMap<uint64_t, ObNormalUdfExeUnit*, common::hash::NoPthreadDefendMode> ctxs_;
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObUdfCtxMgr);
};
} // namespace sql
} // namespace oceanbase
#endif

96
src/sql/engine/user_defined_function/ob_udf_registration_types.h Normal file
View File

@ -0,0 +1,96 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef UDF_REGISTRATION_TYPES_H_
#define UDF_REGISTRATION_TYPES_H_
namespace oceanbase {
namespace sql {
/*
* All these types are also defined in mysql.
* Change all names.
* */
/*
Type of the user defined function return slot and arguments
*/
enum UdfItemResult {
INVALID_RESULT = -1, /* not valid for UDFs */
STRING_RESULT = 0, /* char * */
REAL_RESULT, /* double */
INT_RESULT, /* long long */
ROW_RESULT, /* not valid for UDFs */
DECIMAL_RESULT /* char *, to be converted to/from a decimal */
};
enum ItemUdfType { UDFTYPE_FUNCTION = 1, UDFTYPE_AGGREGATE };
/*
The user defined function args.
We fill all the data in stage of resolver except args and lengths.
Only got the row we are working on can we fill args and lengths.
*/
typedef struct UDF_ARGS {
unsigned int arg_count; /* Number of arguments */
enum UdfItemResult* arg_type; /* Pointer to item_results */
char** args; /* Pointer to argument */
unsigned long* lengths; /* Length of string arguments */
char* maybe_null; /* Set to 1 (not '1') for all maybe_null args */
char** attributes; /* Pointer to attribute name */
unsigned long* attribute_lengths; /* Length of attribute arguments */
void* extension;
} ObUdfArgs;
/*
Information about the result of a user defined function
@todo add a notion for determinism of the UDF.
@sa Item_udf_func::update_used_tables()
*/
typedef struct UDF_INIT {
bool maybe_null; /* 1 if function can return NULL */
unsigned int decimals; /* for real functions */
unsigned long max_length; /* For string functions */
char* ptr; /* free pointer for function data */
bool const_item; /* 1 if function always returns the same value */
void* extension;
} ObUdfInit;
/* udf helper function */
typedef void (*ObUdfFuncClear)(ObUdfInit* udf_init, unsigned char* is_null, unsigned char* error);
typedef void (*ObUdfFuncAdd)(ObUdfInit* udf_init, ObUdfArgs* udf_args, unsigned char* is_null, unsigned char* error);
typedef void (*ObUdfFuncDeinit)(ObUdfInit* udf_init);
typedef bool (*ObUdfFuncInit)(ObUdfInit* udf_init, ObUdfArgs* udf_args, char* message);
/* udf process row function */
typedef void (*ObUdfFuncAny)(void);
typedef double (*ObUdfFuncDouble)(
ObUdfInit* udf_init, ObUdfArgs* udf_args, unsigned char* is_null, unsigned char* error);
typedef long long (*ObUdfFuncLonglong)(
ObUdfInit* udf_init, ObUdfArgs* udf_args, unsigned char* is_null, unsigned char* error);
typedef char* (*ObUdfFuncString)(ObUdfInit* initid, ObUdfArgs* args, char* result, unsigned long* length,
unsigned char* is_null, unsigned char* error);
typedef void* ObUdfSoHandler;
} // namespace sql
} // namespace oceanbase
#endif

760
src/sql/engine/user_defined_function/ob_udf_util.cpp Normal file
View File

@ -0,0 +1,760 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "ob_udf_util.h"
#include "observer/mysql/obsm_utils.h"
#include "lib/file/file_directory_utils.h"
#include "sql/engine/expr/ob_expr_operator.h"
#include "sql/engine/user_defined_function/ob_user_defined_function.h"
#include "sql/session/ob_sql_session_info.h"
namespace oceanbase {
namespace sql {
using namespace common;
using namespace obmysql;
const char* ObUdfUtil::load_function_postfix[5] = {
"",
"_init",
"_deinit",
"_clear",
"_add",
};
int ObUdfUtil::calc_udf_result_type(common::ObIAllocator& allocator, const ObUdfFunction* udf_func,
const share::schema::ObUDFMeta& udf_meta, const common::ObIArray<common::ObString>& udf_attributes,
const common::ObIArray<ObExprResType>& udf_attributes_types, ObExprResType& type)
{
int ret = OB_SUCCESS;
bool init_succ = false;
// Based on the following facts:
// 1. OB calculates the result type of expr when it is called ExprDeduceType,
// and it is called in the resolver phase/rewrite phase/CG phase. In the CG phase,
// the ObAccuracy of ObExprResType will be set to ObPostExprItem as the basis for
// data regularization during calculation.
// 2. The init function of the user UDF can assign values to the UDF_INIT structure,
// set max_length and decimal, and specify the precision of the type. It is also
// required to execute the init function in the earliest plan generation stage to
// obtain max_length and decimal.
// 3. During the execution of a physical Expr, the UDF_INIT and UDF_ARG structures
// used by the udf function interface are used as member variables. Expr needs to
// pass these two structures to the user-written function in a non-const manner.
// For example, users can perform operations on the memory pointed to by ptr
// in the UDF_INIT structure in their own udf function.
// From the perspective of 2, UDF_INIT and UDF_ARG are static properties; from the perspective of 3,
// they are more like executing ctx. Under this premise, Expr must use UDF_INIT/UDF_ARG as
// the ctx component of the execution period, and must execute the init functions
// to obtain accuracy, making it a necessary parameter in the logic stage. Then a better solution
// is to first Call the init and deinit functions of udf once, specifically to obtain accuracy.
// But there is another problem here, that is, if the init function is relatively obsolete or
// blocked, the generation phase of this plan will be very slow.
/* do the user defined init func */
ObUdfFunction::ObUdfCtx udf_ctx;
if (OB_FAIL(ObUdfUtil::init_udf_args(allocator, udf_attributes, udf_attributes_types, udf_ctx.udf_args_))) {
LOG_WARN("failed to set udf args", K(ret));
} else if (OB_FAIL(udf_func->process_init_func(udf_ctx))) {
LOG_WARN("do agg init func failed", K(ret));
} else {
/* further infomation about scale, precision and length, just see the ob_resolver_utils.cpp */
init_succ = true;
type.set_default_collation_type();
switch (udf_meta.ret_) {
case share::schema::ObUDF::STRING: {
/* for string and decimal, mysql use char* as result */
type.set_varchar();
const ObAccuracy& default_accuracy = ObAccuracy::DDL_DEFAULT_ACCURACY[ObVarcharType];
/* OB_MAX_VARCHAR_LENGTH(1024*1024) is bigger than ObUdfUtil::MAX_FIELD_WIDTH(255*3 + 1) */
if (udf_ctx.udf_init_.max_length > ObUdfUtil::UDF_MAX_FIELD_WIDTH) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("the length is invalid", K(ret), K(udf_ctx.udf_init_.max_length));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (udf_ctx.udf_init_.max_length != 0) {
type.set_length((ObLength)udf_ctx.udf_init_.max_length);
} else {
type.set_length(default_accuracy.get_length());
// TODO how about collation?
/*
* type.set_charset_type(charset_type);
* type.set_collation_type(collation_type);
* type.set_binary_collation(is_binary);
* */
}
break;
}
case share::schema::ObUDF::UDFRetType::DECIMAL: {
/* for string and decimal, mysql use char* as result */
type.set_number();
const ObAccuracy& default_accuracy = ObAccuracy::DDL_DEFAULT_ACCURACY[ObNumberType];
/*
* set scale
* OB_MAX_DECIMAL_SCALE is 30, OB_MAX_DOUBLE_FLOAT_PRECISION is 53,
* ObUdfUtil::DECIMAL_MAX_STR_LENGTH is 9*9 + 2, the same.
* */
if (udf_ctx.udf_init_.decimals > OB_MAX_DECIMAL_SCALE) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("the decimal is invalid", K(ret), K(udf_ctx.udf_init_.decimals));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (udf_ctx.udf_init_.decimals != 0) {
type.set_scale((ObScale)udf_ctx.udf_init_.decimals);
} else {
type.set_scale(default_accuracy.get_scale());
}
/* set precision */
if (OB_SUCC(ret)) {
if (udf_ctx.udf_init_.max_length > OB_MAX_DECIMAL_PRECISION) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("the max length is invalid", K(ret), K(udf_ctx.udf_init_.max_length));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (udf_ctx.udf_init_.max_length != 0) {
type.set_precision((ObPrecision)udf_ctx.udf_init_.max_length);
} else {
type.set_precision(default_accuracy.get_precision());
}
}
break;
}
case share::schema::ObUDF::UDFRetType::REAL: {
// for real, mysql use double as result
type.set_double();
const ObAccuracy& default_accuracy = ObAccuracy::DDL_DEFAULT_ACCURACY[ObDoubleType];
/* set scale */
if (udf_ctx.udf_init_.decimals > OB_MAX_DOUBLE_FLOAT_SCALE) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("the decimal is invalid", K(ret), K(udf_ctx.udf_init_.decimals));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (udf_ctx.udf_init_.decimals != 0) {
type.set_scale((ObScale)udf_ctx.udf_init_.decimals);
} else {
type.set_scale(default_accuracy.get_scale());
}
/* set precision */
if (OB_SUCC(ret)) {
if (udf_ctx.udf_init_.max_length > OB_MAX_DOUBLE_FLOAT_PRECISION) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("the max length is invalid", K(ret), K(udf_ctx.udf_init_.max_length));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (udf_ctx.udf_init_.max_length != 0) {
type.set_precision((ObPrecision)udf_ctx.udf_init_.max_length);
} else {
type.set_precision(default_accuracy.get_precision());
}
}
break;
}
case share::schema::ObUDF::UDFRetType::INTEGER: {
// for integer, mysql use long long as result
type.set_int();
const ObAccuracy& default_accuracy = ObAccuracy::DDL_DEFAULT_ACCURACY[ObIntType];
if (udf_ctx.udf_init_.max_length > OB_MAX_INTEGER_DISPLAY_WIDTH) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("the max length is invalid", K(ret), K(udf_ctx.udf_init_.max_length), K(udf_ctx.udf_init_.decimals));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (udf_ctx.udf_init_.max_length != 0) {
type.set_precision((ObPrecision)udf_ctx.udf_init_.max_length);
type.set_scale(0);
} else {
type.set_precision(default_accuracy.get_precision());
}
break;
}
default:
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unhandled udf result type", K(ret));
}
/* do udf deinit function */
if (init_succ) {
IGNORE_RETURN udf_func->process_deinit_func(udf_ctx);
}
}
LOG_DEBUG("udf get result type", K(type));
return ret;
}
int ObUdfUtil::load_so(const common::ObString dl, ObUdfSoHandler& handler)
{
int ret = OB_SUCCESS;
ObUdfSoHandler handler_tmp = nullptr;
// Since ObString does not store the last \0, this method is still used, otherwise dlopen will fail.
int so_name_max_len = OB_MAX_UDF_NAME_LENGTH + ObUdfUtil::UDF_MAX_EXPAND_LENGTH;
char so_name[OB_MAX_UDF_NAME_LENGTH + ObUdfUtil::UDF_MAX_EXPAND_LENGTH];
MEMSET(so_name, 0, so_name_max_len);
MEMCPY(so_name, dl.ptr(), dl.length());
if (OB_ISNULL(handler_tmp = dlopen(so_name, RTLD_NOW))) {
/* some error happened */
ret = OB_CANT_OPEN_LIBRARY;
char* error_msg = dlerror();
LOG_WARN("failed to open dl", K(ret), K(error_msg));
} else {
/* we got the so handler success */
handler = handler_tmp;
LOG_DEBUG("udf get dll handler", K(handler));
}
return ret;
}
void ObUdfUtil::unload_so(ObUdfSoHandler& handler)
{
if (nullptr != handler) {
dlclose(handler);
}
}
int ObUdfUtil::deep_copy_udf_args_cell(
common::ObIAllocator& allocator, int64_t param_num, const common::ObObj* src_objs, common::ObObj*& dst_objs)
{
int ret = OB_SUCCESS;
ObObj* objs_tmp = nullptr;
if (OB_ISNULL(src_objs) || param_num < 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid input", K(ret), K(src_objs), K(param_num));
} else if (param_num == 0) {
} else if (OB_ISNULL(objs_tmp = (ObObj*)allocator.alloc(param_num * sizeof(ObObj)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate failed", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < param_num; ++i) {
if (OB_FAIL(ob_write_obj(allocator, src_objs[i], objs_tmp[i]))) {
LOG_WARN("deep copy obj failed", K(ret));
}
}
if (OB_SUCC(ret)) {
dst_objs = objs_tmp;
}
return ret;
}
int ObUdfUtil::init_udf_args(ObIAllocator& allocator, const common::ObIArray<common::ObString>& udf_attributes,
const common::ObIArray<ObExprResType>& udf_attributes_types, ObUdfArgs& udf_args)
{
int ret = OB_SUCCESS;
if (udf_attributes.count() > INT64_MAX) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("too much args", K(ret));
} else if (udf_attributes.count() != udf_attributes_types.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected input", K(ret), K(udf_attributes.count()), K(udf_attributes_types.count()));
} else {
udf_args.arg_count = (unsigned int)udf_attributes.count();
udf_args.args = (char**)allocator.alloc(sizeof(char*) * udf_args.arg_count);
udf_args.lengths = (unsigned long*)allocator.alloc(sizeof(unsigned long) * udf_args.arg_count);
udf_args.arg_type = (enum UdfItemResult*)allocator.alloc(sizeof(enum UdfItemResult) * udf_args.arg_count);
udf_args.maybe_null = (char*)allocator.alloc(sizeof(char) * udf_args.arg_count);
udf_args.attributes = (char**)allocator.alloc(sizeof(char*) * udf_args.arg_count);
udf_args.attribute_lengths = (unsigned long*)allocator.alloc(sizeof(unsigned long) * udf_args.arg_count);
}
if (OB_SUCC(ret)) {
if (udf_args.arg_count != 0 && (OB_ISNULL(udf_args.args) || OB_ISNULL(udf_args.lengths) ||
OB_ISNULL(udf_args.arg_type) || OB_ISNULL(udf_args.maybe_null) ||
OB_ISNULL(udf_args.attributes) || OB_ISNULL(udf_args.attribute_lengths))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), K(udf_args.arg_count));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < udf_args.arg_count; ++i) {
udf_args.lengths[i] = 0;
udf_args.args[i] = nullptr;
udf_args.attribute_lengths[i] = udf_attributes.at(i).length();
if (OB_FAIL(convert_ob_type_to_udf_type(udf_attributes_types.at(i).get_type(), udf_args.arg_type[i]))) {
LOG_WARN("failt to convert ob type to udf type",
K(udf_attributes_types.at(i).get_type()),
K(udf_attributes_types.at(i).get_calc_type()));
} else if (udf_args.attribute_lengths[i] > 0) {
udf_args.attributes[i] = (char*)allocator.alloc(udf_args.attribute_lengths[i]);
IGNORE_RETURN MEMCPY(udf_args.attributes[i], udf_attributes.at(i).ptr(), udf_attributes.at(i).length());
} else {
udf_args.attributes[i] = nullptr;
}
if (OB_SUCC(ret)) {
udf_args.maybe_null[i] = udf_attributes_types.at(i).has_result_flag(OB_MYSQL_NOT_NULL_FLAG);
}
}
IGNORE_RETURN print_udf_args_to_log(udf_args);
return ret;
}
int ObUdfUtil::init_const_args(ObIAllocator& allocator, const common::ObIArray<ObUdfConstArgs>& const_results,
ObUdfArgs& udf_args, ObExprCtx& expr_ctx)
{
int ret = OB_SUCCESS;
UNUSED(allocator);
ObNewRow empty_row;
const auto& Objs = expr_ctx.phy_plan_ctx_->get_param_store();
UNUSED(Objs);
for (int64_t i = 0; i < const_results.count() && OB_SUCC(ret); ++i) {
const ObUdfConstArgs& const_args = const_results.at(i);
ObObj value;
if (const_args.idx_in_udf_arg_ >= udf_args.arg_count || OB_ISNULL(const_args.sql_calc_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid idx", K(ret), K(const_args.idx_in_udf_arg_), K(udf_args.arg_count));
} else if (OB_FAIL(const_args.sql_calc_->calc(expr_ctx, empty_row, value))) {
LOG_WARN("failed to calc sql expression", K(ret));
} else {
udf_args.args[const_args.idx_in_udf_arg_] = (char*)value.get_data_ptr();
udf_args.lengths[const_args.idx_in_udf_arg_] = value.get_string_len();
}
LOG_DEBUG("calculable value", K(value));
}
return ret;
}
int ObUdfUtil::set_udf_args(
common::ObExprCtx& expr_ctx, int64_t param_num, common::ObObj* src_objs, ObUdfArgs& udf_args)
{
int ret = OB_SUCCESS;
// MySQL needs various buffers to copy data to temporary memory for UDF to use.
// Since we have already deep copyed all the passed ObObj in the previous section,
// the memory is directly used by the UDF and there is no need to copy it again.
// The difference is that the int/double area of mysql is continuous. If UDF is used
// this hidden condition, it may cause core
if (param_num != udf_args.arg_count) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid input", K(ret), K(param_num), K(udf_args.arg_count));
}
for (uint64_t i = 0; i < udf_args.arg_count && OB_SUCC(ret); i++) {
udf_args.args[i] = 0;
switch (udf_args.arg_type[i]) {
case STRING_RESULT:
case DECIMAL_RESULT: {
if (src_objs[i].is_null()) {
udf_args.lengths[i] = 0;
} else {
if (!src_objs[i].is_string_type()) {
EXPR_DEFINE_CAST_CTX(expr_ctx, CM_WARN_ON_FAIL);
if (OB_FAIL(ObObjCaster::to_type(ObVarcharType, cast_ctx, src_objs[i], src_objs[i]))) {
LOG_WARN("failed to cast", K(ret));
}
}
if (OB_SUCC(ret)) {
udf_args.args[i] = (char*)(src_objs[i].get_data_ptr());
udf_args.lengths[i] = src_objs[i].get_string_len();
}
}
break;
}
case INT_RESULT:
/* int, udf should not use lengths */
if (OB_UNLIKELY(ObIntTC != src_objs[i].get_type_class())) {
EXPR_DEFINE_CAST_CTX(expr_ctx, CM_WARN_ON_FAIL);
if (OB_FAIL(ObObjCaster::to_type(ObIntType, cast_ctx, src_objs[i], src_objs[i]))) {
LOG_WARN("failed to cast", K(ret));
}
}
if (OB_SUCC(ret) && !src_objs[i].is_null()) {
int64_t num = src_objs[i].get_int();
src_objs[i].set_int(num);
udf_args.args[i] = (char*)src_objs[i].get_data_ptr();
}
break;
case REAL_RESULT:
/* double, udf should not use lengths */
if (!src_objs[i].is_double()) {
EXPR_DEFINE_CAST_CTX(expr_ctx, CM_WARN_ON_FAIL);
if (OB_FAIL(ObObjCaster::to_type(ObDoubleType, cast_ctx, src_objs[i], src_objs[i]))) {
LOG_WARN("failed to cast", K(ret));
}
}
if (OB_SUCC(ret) && !src_objs[i].is_null()) {
udf_args.args[i] = (char*)src_objs[i].get_data_ptr();
}
break;
case ROW_RESULT:
default:
// This case should never be chosen
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected type", K(ret), K(udf_args.arg_type[i]));
break;
}
}
IGNORE_RETURN ObUdfUtil::print_udf_args_to_log(udf_args);
return ret;
}
int ObUdfUtil::process_udf_func(share::schema::ObUDF::UDFRetType ret_type, common::ObIAllocator& allocator,
ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAny func_origin_, common::ObObj& result)
{
int ret = OB_SUCCESS;
switch (ret_type) {
case share::schema::ObUDF::STRING: {
if (OB_FAIL(ObUdfUtil::process_str(allocator, udf_init, udf_args, func_origin_, result))) {
LOG_WARN("failed to process str", K(ret));
}
break;
}
case share::schema::ObUDF::DECIMAL: {
if (OB_FAIL(ObUdfUtil::process_dec(allocator, udf_init, udf_args, func_origin_, result))) {
LOG_WARN("failed to process dec", K(ret));
}
break;
}
case share::schema::ObUDF::INTEGER: {
if (OB_FAIL(ObUdfUtil::process_int(allocator, udf_init, udf_args, func_origin_, result))) {
LOG_WARN("failed to process int", K(ret));
}
break;
}
case share::schema::ObUDF::REAL: {
if (OB_FAIL(ObUdfUtil::process_real(allocator, udf_init, udf_args, func_origin_, result))) {
LOG_WARN("failed to process real", K(ret));
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected udf ret type", K(ret), K(ret_type));
}
}
return ret;
}
int ObUdfUtil::process_str(
common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAny func, ObObj& result)
{
int ret = OB_SUCCESS;
unsigned long res_length = UDF_MAX_FIELD_WIDTH;
unsigned char is_null_tmp = 0;
unsigned char error = 0;
char* str_for_user = (char*)allocator.alloc(UDF_MAX_FIELD_WIDTH);
IGNORE_RETURN MEMSET(str_for_user, 0, UDF_MAX_FIELD_WIDTH);
ObUdfFuncString func_str = (ObUdfFuncString)func;
char* res = func_str(&udf_init, &udf_args, str_for_user, &res_length, &is_null_tmp, &error);
if (error) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error happened while do the user's code", K(ret));
} else if (nullptr == res || is_null_tmp) {
/* set null to result */
result.set_null();
} else if (res_length >= UDF_MAX_FIELD_WIDTH) {
/*
* if res_length is bigger than MAX_FIELD_WIDTH, it must be some error happened.
* */
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the string is too large, maybe some error happen in user defined function", K(ret), K(res_length));
} else {
// result.set_char_value(res, (ObString::obstr_size_t)res_length);
result.set_string(ObVarcharType, res, (ObString::obstr_size_t)res_length);
result.set_default_collation_type();
}
return ret;
}
int ObUdfUtil::process_dec(
common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAny func, ObObj& result)
{
int ret = OB_SUCCESS;
unsigned long res_length = UDF_DECIMAL_MAX_STR_LENGTH;
unsigned char is_null_tmp = 0;
unsigned char error = 0;
char* buf_for_user = (char*)allocator.alloc(UDF_DECIMAL_MAX_STR_LENGTH);
IGNORE_RETURN MEMSET(buf_for_user, 0, UDF_DECIMAL_MAX_STR_LENGTH);
ObUdfFuncString func_str = (ObUdfFuncString)func;
common::number::ObNumber num;
ObPrecision res_precision = -1;
ObScale res_scale = -1;
char* res = func_str(&udf_init, &udf_args, buf_for_user, &res_length, &is_null_tmp, &error);
if (error) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error happened while do the user's code", K(ret));
} else if (nullptr == res || is_null_tmp) {
/* set null to result */
result.set_null();
} else if (res_length >= UDF_DECIMAL_MAX_STR_LENGTH) {
/*
* if res_length is bigger than DECIMAL_MAX_STR_LENGTH, it must be some error happened.
* */
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the string is too large, maybe some error happen in user defined function", K(ret), K(res_length));
} else if (OB_FAIL(num.from(res, (long int)res_length, allocator, &res_precision, &res_scale))) {
LOG_WARN("fail to convert char* to decimal/obnumber", K(ret));
} else {
result.set_number(num);
}
return ret;
}
int ObUdfUtil::process_int(
common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAny func, ObObj& result)
{
int ret = OB_SUCCESS;
UNUSED(allocator);
unsigned char is_null_tmp = 0;
unsigned char error = 0;
ObUdfFuncLonglong func_int = (ObUdfFuncLonglong)func;
long long int tmp = func_int(&udf_init, &udf_args, &is_null_tmp, &error);
if (error) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error happened while do the user's code", K(ret));
} else if (is_null_tmp) {
/* set null to result */
result.set_null();
} else {
result.set_int(tmp);
}
return ret;
}
int ObUdfUtil::process_real(
common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAny func, ObObj& result)
{
int ret = OB_SUCCESS;
UNUSED(allocator);
unsigned char is_null_tmp = 0;
unsigned char error = 0;
ObUdfFuncDouble func_double = (ObUdfFuncDouble)func;
double tmp = func_double(&udf_init, &udf_args, &is_null_tmp, &error);
if (error) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error happened while do the user's code", K(ret));
} else if (is_null_tmp) {
/* set null to result */
result.set_null();
} else {
result.set_double(tmp);
}
return ret;
}
int ObUdfUtil::process_add_func(ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAdd func)
{
int ret = OB_SUCCESS;
unsigned char is_null_tmp = 0;
unsigned char error = 0;
IGNORE_RETURN func(&udf_init, &udf_args, &is_null_tmp, &error);
if (error) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error happened while do the user's code", K(ret));
}
UNUSED(is_null_tmp);
return ret;
}
int ObUdfUtil::process_clear_func(ObUdfInit& udf_init, const ObUdfFuncClear func)
{
int ret = OB_SUCCESS;
unsigned char error = 0;
unsigned char is_null_tmp = 0;
IGNORE_RETURN func(&udf_init, &is_null_tmp, &error);
if (error) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error happened while do the user's code", K(ret), K(error), K(is_null_tmp));
}
return ret;
}
void ObUdfUtil::construct_udf_args(ObUdfArgs& args)
{
args.arg_count = 0;
args.arg_type = nullptr;
args.args = nullptr;
args.attribute_lengths = nullptr;
args.attributes = nullptr;
args.extension = nullptr;
args.maybe_null = nullptr;
}
void ObUdfUtil::construct_udf_init(ObUdfInit& init)
{
init.const_item = 0;
init.decimals = 0;
init.extension = nullptr;
init.max_length = 0;
init.maybe_null = 0;
init.ptr = nullptr;
}
const char* ObUdfUtil::result_type_name(enum UdfItemResult result)
{
int64_t offset = 0;
if (INVALID_RESULT == result) {
offset = 0;
} else {
offset = (int64_t)result;
}
static const char* result_string[7] = {
"INVALID_RESULT",
"STRING_RESULT",
"REAL_RESULT",
"INT_RESULT",
"ROW_RESULT",
"DECIMAL_RESULT",
};
return result_string[offset + 1];
}
void ObUdfUtil::print_udf_args_to_log(const ObUdfArgs& args)
{
int ret = OB_SUCCESS;
ObSEArray<ObString, 16> args_strings;
ObSEArray<ObString, 16> atts_strings;
ObSEArray<int64_t, 16> maybe_nulls;
ObSEArray<ObString, 16> arg_type_strings;
for (int64_t i = 0; i < args.arg_count && OB_SUCC(ret); ++i) {
if (args.lengths[i] > 0 && OB_FAIL(args_strings.push_back(ObString(args.lengths[i], args.args[i])))) {
LOG_WARN("push back failed", K(ret));
}
if (OB_SUCC(ret)) {
if (args.attribute_lengths[i] > 0 &&
OB_FAIL(atts_strings.push_back(ObString(args.attribute_lengths[i], args.attributes[i])))) {
LOG_WARN("push back failed", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(maybe_nulls.push_back(args.maybe_null[i] == 1 ? 1 : 0))) {
LOG_WARN("push back failed", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(arg_type_strings.push_back(
ObString(STRLEN(result_type_name(args.arg_type[i])), result_type_name(args.arg_type[i]))))) {
LOG_WARN("push back failed", K(ret));
}
}
}
LOG_DEBUG("UDF ARGS",
K(ret),
K(args.arg_count),
K(args_strings),
K(atts_strings),
K(maybe_nulls),
K(arg_type_strings),
K(lbt()));
}
void ObUdfUtil::print_udf_init_to_log(const ObUdfInit& init)
{
LOG_DEBUG("UDF INIT", K(init.maybe_null), K(init.decimals), K(init.max_length), K(init.ptr), K(init.const_item));
}
void ObUdfUtil::assign_udf_args(const ObUdfArgs& src_args, ObUdfArgs& dst_args)
{
dst_args.arg_count = src_args.arg_count;
dst_args.extension = src_args.extension;
}
void ObUdfUtil::assign_udf_init(const ObUdfInit& src_init, ObUdfInit& dst_init)
{
dst_init.const_item = src_init.const_item;
dst_init.decimals = src_init.decimals;
dst_init.extension = src_init.extension;
dst_init.max_length = src_init.max_length;
dst_init.maybe_null = src_init.maybe_null;
dst_init.ptr = src_init.ptr;
}
int ObUdfUtil::is_udf_ctx_valid(const ObUdfArgs& udf_args, const ObUdfInit& udf_init)
{
int ret = OB_SUCCESS;
UNUSED(udf_init);
if (udf_args.arg_type == nullptr || udf_args.args == nullptr || udf_args.lengths == nullptr ||
udf_args.maybe_null == nullptr || udf_args.attributes == nullptr || udf_args.attribute_lengths == nullptr) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("some field is null",
K(udf_args.arg_type),
K(udf_args.args),
K(udf_args.maybe_null),
K(udf_args.attributes),
K(udf_args.attribute_lengths),
K(udf_args.arg_count));
} else {
for (int64_t i = 0; i < udf_args.arg_count && OB_SUCC(ret); ++i) {
if (udf_args.args[i] == nullptr || udf_args.attributes[i] == nullptr) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("some field is null", K(ret), K(udf_args.args[i]), K(udf_args.attributes[i]));
}
}
}
return ret;
}
int ObUdfUtil::convert_ob_type_to_udf_type(common::ObObjType ob_type, UdfItemResult& udf_type)
{
int ret = OB_SUCCESS;
// mysql 5.6 type.
obmysql::EMySQLFieldType mysql_type;
uint16_t flags;
ObScale num_decimals;
if (OB_FAIL(ObSMUtils::get_mysql_type(ob_type, mysql_type, flags, num_decimals))) {
LOG_WARN("get mysql type failed", K(ret));
} else if (OB_FAIL(convert_mysql_type_to_udf_type(mysql_type, udf_type))) {
LOG_WARN("get udf type failed", K(ret));
}
LOG_DEBUG("udf type change", K(ob_type), K(mysql_type), K(udf_type));
return ret;
}
int ObUdfUtil::convert_mysql_type_to_udf_type(const obmysql::EMySQLFieldType& mysql_type, UdfItemResult& udf_type)
{
int ret = OB_SUCCESS;
// In the expression item of mysql, it seems that each category returns the hard-coded result type and field type.
switch (mysql_type) {
case EMySQLFieldType::MYSQL_TYPE_TINY:
case EMySQLFieldType::MYSQL_TYPE_SHORT:
case EMySQLFieldType::MYSQL_TYPE_INT24:
case EMySQLFieldType::MYSQL_TYPE_LONG:
case EMySQLFieldType::MYSQL_TYPE_LONGLONG:
udf_type = INT_RESULT;
break;
case EMySQLFieldType::MYSQL_TYPE_DECIMAL:
case EMySQLFieldType::MYSQL_TYPE_NEWDECIMAL:
case EMySQLFieldType::MYSQL_TYPE_OB_NUMBER_FLOAT:
udf_type = DECIMAL_RESULT;
break;
case EMySQLFieldType::MYSQL_TYPE_FLOAT:
case EMySQLFieldType::MYSQL_TYPE_DOUBLE:
udf_type = REAL_RESULT;
break;
case EMySQLFieldType::MYSQL_TYPE_VARCHAR:
case EMySQLFieldType::MYSQL_TYPE_VAR_STRING:
case EMySQLFieldType::MYSQL_TYPE_STRING:
case EMySQLFieldType::MYSQL_TYPE_OB_NCHAR:
case EMySQLFieldType::MYSQL_TYPE_OB_NVARCHAR2:
case EMySQLFieldType::MYSQL_TYPE_OB_UROWID:
udf_type = STRING_RESULT;
break;
case EMySQLFieldType::MYSQL_TYPE_YEAR:
udf_type = INT_RESULT;
break;
case EMySQLFieldType::MYSQL_TYPE_TIMESTAMP:
case EMySQLFieldType::MYSQL_TYPE_DATE:
case EMySQLFieldType::MYSQL_TYPE_TIME:
case EMySQLFieldType::MYSQL_TYPE_DATETIME:
case EMySQLFieldType::MYSQL_TYPE_NEWDATE:
case EMySQLFieldType::MYSQL_TYPE_BIT:
// case MYSQL_TYPE_TIMESTAMP2:
// case MYSQL_TYPE_DATETIME2:
// case MYSQL_TYPE_TIME2:
// case MYSQL_TYPE_JSON:
case EMySQLFieldType::MYSQL_TYPE_ENUM:
case EMySQLFieldType::MYSQL_TYPE_SET:
case EMySQLFieldType::MYSQL_TYPE_GEOMETRY:
case EMySQLFieldType::MYSQL_TYPE_NULL:
case EMySQLFieldType::MYSQL_TYPE_TINY_BLOB:
case EMySQLFieldType::MYSQL_TYPE_BLOB:
case EMySQLFieldType::MYSQL_TYPE_MEDIUM_BLOB:
case EMySQLFieldType::MYSQL_TYPE_LONG_BLOB:
udf_type = STRING_RESULT;
break;
default:
udf_type = INVALID_RESULT;
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unhandled mysql type", K(ret));
}
return ret;
}
} // namespace sql
} // namespace oceanbase

151
src/sql/engine/user_defined_function/ob_udf_util.h Normal file
View File

@ -0,0 +1,151 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_UDF_UTIL_H_
#define OB_UDF_UTIL_H_
#include <dlfcn.h>
#include "share/schema/ob_udf.h"
#include "sql/engine/user_defined_function/ob_user_defined_function.h"
#include "sql/engine/user_defined_function/ob_udf_registration_types.h"
#include "rpc/obmysql/ob_mysql_global.h"
#include "sql/engine/expr/ob_expr_res_type.h"
//#include "common/expression/ob_i_sql_expression.h"
namespace oceanbase {
namespace common {
class ObExprCtx;
}
namespace sql {
#define GET_AGG_UDF_ID(GROUP_ID, COL_COUNT, AGG_COUNT, AGG_IDX) ((GROUP_ID) * ((COL_COUNT) + (AGG_COUNT)) + AGG_IDX)
class ObPhysicalPlanCtx;
class obUdfConstArgs;
class ObUdfFunction;
class ObUdfUtil {
public:
/*
* for example, if the defined func named 'my_udf_add', we wanna load the auxiliary function.
* then we use 'my_udf_add_init' 'my_udf_add_deinit' 'my_udf_add_clear' or 'my_udf_add_add' to load
* these functions (actually the max_expand_length is '_deinit\0').
* */
static const int UDF_MAX_EXPAND_LENGTH = 10;
/* from mysql */
static const int UDF_MAX_MBWIDTH = 3; /* Max multibyte sequence */
static const int UDF_MAX_FIELD_CHARLENGTH = 255;
static const int UDF_MAX_FIELD_WIDTH = (UDF_MAX_FIELD_CHARLENGTH * UDF_MAX_MBWIDTH + 1); /* Max column width +1 */
static const int UDF_DECIMAL_BUFF_LENGTH = 9; /** maximum length of buffer in our big digits (uint32). */
static const int UDF_DECIMAL_MAX_POSSIBLE_PRECISION =
(UDF_DECIMAL_BUFF_LENGTH * 9); /* the number of digits that my_decimal can possibly contain */
static const int UDF_DECIMAL_MAX_STR_LENGTH = (UDF_DECIMAL_MAX_POSSIBLE_PRECISION + 2);
public:
enum load_function_type {
UDF_ORIGIN,
UDF_INIT,
UDF_DEINIT,
UDF_CLEAR,
UDF_ADD,
};
static const char* load_function_postfix[5];
const char* get_postfix_name(load_function_type type)
{
return load_function_postfix[type];
}
public:
/*
* calc udf's result type
* */
static int calc_udf_result_type(common::ObIAllocator& allocator, const ObUdfFunction* udf_func,
const share::schema::ObUDFMeta& udf_meta, const common::ObIArray<common::ObString>& udf_attributes,
const common::ObIArray<ObExprResType>& udf_attributes_types, ObExprResType& type);
/*
* load all function to our server
* */
static void unload_so(ObUdfSoHandler& handler);
static int load_so(const common::ObString dl, ObUdfSoHandler& handler);
template <typename T>
static int load_function(const common::ObString& name, const ObUdfSoHandler handler, const common::ObString& postfix,
const bool ignore_error, T& func);
/*
* helper function
* */
static int deep_copy_udf_args_cell(
common::ObIAllocator& allocator, int64_t param_num, const common::ObObj* src_objs, common::ObObj*& dst_objs);
static int set_udf_args(common::ObExprCtx& expr_ctx, int64_t param_num, common::ObObj* src_objs, ObUdfArgs& udf_args);
static int init_udf_args(common::ObIAllocator& allocator, const common::ObIArray<common::ObString>& udf_attributes,
const common::ObIArray<ObExprResType>& udf_attributes_types, ObUdfArgs& udf_args);
static int init_const_args(common::ObIAllocator& allocator, const common::ObIArray<ObUdfConstArgs>& const_results,
ObUdfArgs& udf_args, common::ObExprCtx& expr_ctx);
static void construct_udf_args(ObUdfArgs& args);
static void construct_udf_init(ObUdfInit& init);
static const char* result_type_name(enum UdfItemResult result);
static void print_udf_args_to_log(const ObUdfArgs& args);
static void print_udf_init_to_log(const ObUdfInit& init);
static void assign_udf_args(const ObUdfArgs& src_args, ObUdfArgs& dst_args);
static void assign_udf_init(const ObUdfInit& src_init, ObUdfInit& dst_init);
static int is_udf_ctx_valid(const ObUdfArgs& src_args, const ObUdfInit& dst_init);
/*
* interface to invoke udf
* */
static int process_udf_func(share::schema::ObUDF::UDFRetType ret_type, common::ObIAllocator& allocator,
ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAny func, common::ObObj& result);
static int process_str(common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args,
const ObUdfFuncAny func, common::ObObj& result);
static int process_dec(common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args,
const ObUdfFuncAny func, common::ObObj& result);
static int process_int(common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args,
const ObUdfFuncAny func, common::ObObj& result);
static int process_real(common::ObIAllocator& allocator, ObUdfInit& udf_init, ObUdfArgs& udf_args,
const ObUdfFuncAny func, common::ObObj& result);
static int process_add_func(ObUdfInit& udf_init, ObUdfArgs& udf_args, const ObUdfFuncAdd func);
static int process_clear_func(ObUdfInit& udf_init, const ObUdfFuncClear func);
/*
* convert function, from ob type to mysql field type
* */
static int convert_ob_type_to_udf_type(common::ObObjType ob_type, UdfItemResult& udf_type);
static int convert_mysql_type_to_udf_type(const obmysql::EMySQLFieldType& mysql_type, UdfItemResult& udf_type);
};
template <typename T>
int ObUdfUtil::load_function(const common::ObString& name, const ObUdfSoHandler handler,
const common::ObString& postfix, const bool ignore_error, T& func)
{
int ret = common::OB_SUCCESS;
T func_tmp = nullptr;
int func_len = common::OB_MAX_UDF_NAME_LENGTH + ObUdfUtil::UDF_MAX_EXPAND_LENGTH;
char func_name[common::OB_MAX_UDF_NAME_LENGTH + ObUdfUtil::UDF_MAX_EXPAND_LENGTH];
MEMSET(func_name, 0, func_len);
MEMCPY(func_name, name.ptr(), name.length());
char* postfix_start_pos = func_name + name.length();
MEMCPY(postfix_start_pos, postfix.ptr(), postfix.length());
if (OB_ISNULL(((func_tmp = (T)dlsym(handler, func_name))))) {
ret = ignore_error ? common::OB_SUCCESS : common::OB_ERR_UNEXPECTED;
SQL_LOG(WARN, "Can't find symbol", K(ret), K(name), K(func_name));
} else {
func = func_tmp;
SQL_LOG(DEBUG, "get func_init function", K(handler), K(func));
}
return ret;
}
} // namespace sql
} // namespace oceanbase
#endif

226
src/sql/engine/user_defined_function/ob_user_defined_function.cpp Normal file
View File

@ -0,0 +1,226 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "ob_user_defined_function.h"
#include "sql/engine/user_defined_function/ob_udf_util.h"
#include "share/ob_i_sql_expression.h"
namespace oceanbase {
using namespace common;
namespace sql {
ObUdfFunction::~ObUdfFunction()
{
IGNORE_RETURN ObUdfUtil::unload_so(dlhandle_);
}
/*
* try to load the so file to ob.
* try to load origin function and all helper function to ob.
*
* */
int ObUdfFunction::init(const share::schema::ObUDFMeta& udf_meta)
{
int ret = OB_SUCCESS;
if (udf_meta.dl_.empty() || udf_meta.name_.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the udf meta is invalid", K(ret));
} else if (OB_FAIL(ObUdfUtil::load_so(udf_meta.dl_, dlhandle_))) {
LOG_WARN("load so error", K(ret));
} else if (OB_FAIL(ObUdfUtil::load_function(udf_meta.name_,
dlhandle_,
ObString::make_string(""),
false, /* can't ignore error */
func_origin_))) {
// change the error code
ret = OB_CANT_FIND_DL_ENTRY;
LOG_WARN("load origin function failed", K(ret));
LOG_USER_ERROR(OB_CANT_FIND_DL_ENTRY, udf_meta.name_.length(), udf_meta.name_.ptr());
} else if (OB_FAIL(ObUdfUtil::load_function(udf_meta.name_,
dlhandle_,
ObString::make_string("_init"),
false, /* can't ignore error */
func_init_))) {
LOG_WARN("load init function failed", K(ret));
} else if (OB_FAIL(ObUdfUtil::load_function(udf_meta.name_,
dlhandle_,
ObString::make_string("_deinit"),
true, /* ignore error */
func_deinit_))) {
LOG_WARN("load deinit function failed", K(ret));
} else if (udf_meta.type_ == share::schema::ObUDF::UDFType::FUNCTION) {
// do nothing
} else if (OB_FAIL(ObUdfUtil::load_function(udf_meta.name_,
dlhandle_,
ObString::make_string("_clear"),
false, /* ignore error */
func_clear_))) {
LOG_WARN("load clear function error", K(ret));
} else if (OB_FAIL(ObUdfUtil::load_function(udf_meta.name_,
dlhandle_,
ObString::make_string("_add"),
false, /* ignore error */
func_add_))) {
LOG_WARN("load add function error", K(ret));
}
if (OB_SUCC(ret)) {
IGNORE_RETURN udf_meta_.assign(udf_meta);
}
return ret;
}
int ObUdfFunction::process_init_func(ObUdfFunction::ObUdfCtx& udf_ctx) const
{
int ret = OB_SUCCESS;
if (OB_ISNULL(func_init_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the init function is null", K(ret));
} else {
char init_msg_buff[OB_MYSQL_ERRMSG_SIZE];
MEMSET(init_msg_buff, 0, OB_MYSQL_ERRMSG_SIZE);
bool error = func_init_(&udf_ctx.udf_init_, &udf_ctx.udf_args_, init_msg_buff);
if (error) {
ret = OB_CANT_INITIALIZE_UDF;
LOG_WARN("do init func failed", K(init_msg_buff), K(ret), K(error));
LOG_USER_ERROR(OB_CANT_INITIALIZE_UDF, udf_meta_.name_.length(), udf_meta_.name_.ptr());
}
}
IGNORE_RETURN ObUdfUtil::print_udf_args_to_log(udf_ctx.udf_args_);
return ret;
}
void ObUdfFunction::process_deinit_func(ObUdfFunction::ObUdfCtx& udf_ctx) const
{
if (OB_ISNULL(func_deinit_)) {
LOG_DEBUG("the deinit function is null");
} else {
IGNORE_RETURN func_deinit_(&udf_ctx.udf_init_);
}
}
int ObNormalUdfFunction::process_origin_func(
ObObj& result, const ObObj* objs_stack, int64_t param_num, ObExprCtx& expr_ctx, ObUdfCtx& udf_ctx) const
{
int ret = OB_SUCCESS;
ObObj* objs = nullptr;
IGNORE_RETURN ObUdfUtil::print_udf_args_to_log(udf_ctx.udf_args_);
ObUdfCtx* tmp_ctx = (ObUdfCtx*)&udf_ctx;
// COPY_UDF_CTX_TO_STACK(udf_ctx, tmp_ctx);
/*
* step 1. Deep copy the objs. It's not a good idea to assume the udf do not change
* the input row cell.
* step 2. Set cell to args.
* step 3. Invoke the interface defined by user.
* */
if (OB_FAIL(ret)) {
/* do nothing */
} else if (OB_ISNULL(expr_ctx.calc_buf_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the calc_buf_ is null", K(ret));
} else if (OB_FAIL(ObUdfUtil::deep_copy_udf_args_cell(*expr_ctx.calc_buf_, param_num, objs_stack, objs))) {
LOG_WARN("failed to deep copy udf", K(ret));
} else if (OB_FAIL(ObUdfUtil::set_udf_args(expr_ctx, param_num, objs, tmp_ctx->udf_args_))) {
LOG_WARN("failed to set udf args", K(ret));
} else if (OB_FAIL(ObUdfUtil::process_udf_func(
udf_meta_.ret_, *expr_ctx.calc_buf_, udf_ctx.udf_init_, tmp_ctx->udf_args_, func_origin_, result))) {
LOG_WARN("failed to process udf function", K(ret));
}
return ret;
}
int ObAggUdfFunction::process_origin_func(ObIAllocator& allocator, ObObj& agg_result, ObUdfCtx& udf_ctx) const
{
int ret = OB_SUCCESS;
ObUdfCtx* tmp_ctx = &udf_ctx;
IGNORE_RETURN ObUdfUtil::print_udf_args_to_log(udf_ctx.udf_args_);
// COPY_UDF_CTX_TO_STACK(udf_ctx, tmp_ctx);
if (OB_FAIL(ObUdfUtil::process_udf_func(
udf_meta_.ret_, allocator, udf_ctx.udf_init_, tmp_ctx->udf_args_, func_origin_, agg_result))) {
LOG_WARN("failed to process udf function", K(ret));
}
return ret;
}
int ObAggUdfFunction::process_add_func(
ObExprCtx& expr_ctx, ObArenaAllocator& allocator, const ObNewRow& row, ObUdfCtx& udf_ctx) const
{
int ret = OB_SUCCESS;
ObObj* objs = nullptr;
int64_t cells_count = row.get_count();
/* do some prepare works: copy udf ctx to this function stack; deep copy this cell; */
ObUdfCtx* tmp_ctx = &udf_ctx;
// COPY_UDF_CTX_TO_STACK(udf_ctx, tmp_ctx);
if (OB_SUCC(ret)) {
if (OB_ISNULL(func_add_)) {
ret = OB_NOT_INIT;
LOG_WARN("the add func is null", K(ret));
} else if (OB_ISNULL(objs = (ObObj*)allocator.alloc(sizeof(ObObj) * cells_count))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), K(objs));
} else {
/* be careful with the cells' order */
for (int64_t i = 0; i < cells_count && OB_SUCC(ret); ++i) {
if (OB_FAIL(ob_write_obj(allocator, row.get_cell(i), objs[i]))) {
LOG_WARN("failed to copy obj", K(ret));
}
}
}
}
/* do add row */
if (OB_SUCC(ret)) {
if (OB_FAIL(ObUdfUtil::set_udf_args(expr_ctx, cells_count, objs, tmp_ctx->udf_args_))) {
LOG_WARN("failed to set udf args", K(ret));
} else if (OB_FAIL(ObUdfUtil::process_add_func(udf_ctx.udf_init_, tmp_ctx->udf_args_, func_add_))) {
LOG_WARN("failed to process add row", K(ret));
}
}
/* no matter success or failed, reuse this memory */
allocator.free(objs);
return ret;
}
int ObAggUdfFunction::process_clear_func(ObUdfCtx& udf_ctx) const
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObUdfUtil::process_clear_func(udf_ctx.udf_init_, func_clear_))) {
LOG_WARN("failed to process add row", K(ret));
}
return ret;
}
ObAggUdfMeta& ObAggUdfMeta::operator=(const ObAggUdfMeta& other)
{
if (this != &other) {
udf_meta_ = other.udf_meta_;
int ret = OB_SUCCESS;
if (OB_FAIL(udf_attributes_.assign(other.udf_attributes_))) {
LOG_ERROR("assign searray error", K(ret));
} else if (OB_FAIL(udf_attributes_types_.assign(other.udf_attributes_types_))) {
LOG_ERROR("assign searray error", K(ret));
} else if (OB_FAIL(calculable_results_.assign(other.calculable_results_))) {
LOG_ERROR("assign searray error", K(ret));
}
}
return *this;
}
OB_SERIALIZE_MEMBER(ObAggUdfMeta, udf_meta_, udf_attributes_, udf_attributes_types_, calculable_results_);
// we will des/ser the sql_calc_ at the operator group
OB_SERIALIZE_MEMBER(ObUdfConstArgs, idx_in_udf_arg_);
} // namespace sql
} // namespace oceanbase

208
src/sql/engine/user_defined_function/ob_user_defined_function.h Normal file
View File

@ -0,0 +1,208 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_USER_DEFINED_FUNCTION_H_
#define OB_USER_DEFINED_FUNCTION_H_
#include "ob_udf_registration_types.h"
#include "share/schema/ob_udf.h"
#include "sql/engine/expr/ob_expr_res_type.h"
namespace oceanbase {
namespace common {
class ObExprCtx;
}
namespace sql {
#define COPY_UDF_CTX_TO_STACK(SRC_UDF_CTX, TMP_UDF_CTX) \
tmp_ctx = (ObUdfCtx*)alloca(sizeof(ObUdfCtx)); \
if (OB_ISNULL(tmp_ctx)) { \
ret = OB_ALLOCATE_MEMORY_FAILED; \
LOG_WARN("failed to allocate memory from stack", K(ret)); \
} else { \
IGNORE_RETURN ObUdfUtil::assign_udf_args((SRC_UDF_CTX).udf_args_, (TMP_UDF_CTX)->udf_args_); \
if (OB_FAIL(ObUdfUtil::is_udf_ctx_valid((SRC_UDF_CTX).udf_args_, (SRC_UDF_CTX).udf_init_))) { \
LOG_WARN("the udf ctx is invalid", K(ret)); \
} else { \
(TMP_UDF_CTX)->udf_args_.args = (char**)alloca(sizeof(char*) * (SRC_UDF_CTX).udf_args_.arg_count); \
(TMP_UDF_CTX)->udf_args_.lengths = \
(unsigned long*)alloca(sizeof(unsigned long) * (SRC_UDF_CTX).udf_args_.arg_count); \
(TMP_UDF_CTX)->udf_args_.arg_type = \
(enum UdfItemResult*)alloca(sizeof(enum UdfItemResult) * (SRC_UDF_CTX).udf_args_.arg_count); \
(TMP_UDF_CTX)->udf_args_.maybe_null = (char*)alloca(sizeof(char) * (SRC_UDF_CTX).udf_args_.arg_count); \
(TMP_UDF_CTX)->udf_args_.attributes = (char**)alloca(sizeof(char*) * (SRC_UDF_CTX).udf_args_.arg_count); \
(TMP_UDF_CTX)->udf_args_.attribute_lengths = \
(unsigned long*)alloca(sizeof(unsigned long) * (SRC_UDF_CTX).udf_args_.arg_count); \
for (int64_t i = 0; OB_SUCC(ret) && i < (SRC_UDF_CTX).udf_args_.arg_count; ++i) { \
(TMP_UDF_CTX)->udf_args_.lengths[i] = (SRC_UDF_CTX).udf_args_.lengths[i]; \
(TMP_UDF_CTX)->udf_args_.attribute_lengths[i] = (SRC_UDF_CTX).udf_args_.attribute_lengths[i]; \
(TMP_UDF_CTX)->udf_args_.arg_type[i] = (SRC_UDF_CTX).udf_args_.arg_type[i]; \
if ((SRC_UDF_CTX).udf_args_.lengths[i] > 0) { \
(TMP_UDF_CTX)->udf_args_.args[i] = (char*)alloca((SRC_UDF_CTX).udf_args_.lengths[i]); \
IGNORE_RETURN MEMCPY( \
(TMP_UDF_CTX)->udf_args_.args[i], (SRC_UDF_CTX).udf_args_.args[i], (SRC_UDF_CTX).udf_args_.lengths[i]); \
} else { \
(TMP_UDF_CTX)->udf_args_.args[i] = nullptr; \
} \
if ((SRC_UDF_CTX).udf_args_.attribute_lengths[i] > 0) { \
(TMP_UDF_CTX)->udf_args_.attributes[i] = (char*)alloca((SRC_UDF_CTX).udf_args_.attribute_lengths[i]); \
IGNORE_RETURN MEMCPY((TMP_UDF_CTX)->udf_args_.attributes[i], \
(SRC_UDF_CTX).udf_args_.attributes[i], \
(SRC_UDF_CTX).udf_args_.attribute_lengths[i]); \
} else { \
(TMP_UDF_CTX)->udf_args_.attributes[i] = nullptr; \
} \
} \
} \
}
class ObAggUdfMeta;
class ObSqlExpression;
struct ObUdfConstArgs {
OB_UNIS_VERSION_V(1);
public:
ObUdfConstArgs() : sql_calc_(nullptr), idx_in_udf_arg_(common::OB_INVALID_INDEX){};
virtual ~ObUdfConstArgs() = default;
ObUdfConstArgs& operator=(const ObUdfConstArgs& other)
{
sql_calc_ = other.sql_calc_;
idx_in_udf_arg_ = other.idx_in_udf_arg_;
return *this;
}
TO_STRING_KV(K_(idx_in_udf_arg));
ObSqlExpression* sql_calc_;
int64_t idx_in_udf_arg_;
};
class ObUdfFunction {
private:
static const int OB_MYSQL_ERRMSG_SIZE = 512;
public:
friend class ObGetUdfFunctor;
friend class ObResetUdfFunctor;
friend class ObForceDelUdfFunctor;
enum ObUdfState {
UDF_UNINITIALIZED,
UDF_INIT,
UDF_EXE,
UDF_DEINIT,
};
class ObUdfCtx {
public:
ObUdfCtx() : state_(UDF_UNINITIALIZED), udf_init_(), udf_args_()
{}
virtual ~ObUdfCtx() = default;
ObUdfState state_;
ObUdfInit udf_init_;
ObUdfArgs udf_args_;
};
public:
ObUdfFunction()
: udf_meta_(),
dlhandle_(nullptr),
func_origin_(nullptr),
func_init_(nullptr),
func_deinit_(nullptr),
func_clear_(nullptr),
func_add_(nullptr)
{}
virtual ~ObUdfFunction();
// try to load the .so to ob
virtual int init(const share::schema::ObUDFMeta& udf_meta);
virtual int process_init_func(ObUdfFunction::ObUdfCtx& udf_ctx) const;
virtual void process_deinit_func(ObUdfFunction::ObUdfCtx& udf_ctx) const;
protected:
share::schema::ObUDFMeta udf_meta_;
ObUdfSoHandler dlhandle_;
ObUdfFuncAny func_origin_;
ObUdfFuncInit func_init_;
ObUdfFuncDeinit func_deinit_;
// helper function for aggregation udf
ObUdfFuncClear func_clear_;
ObUdfFuncAdd func_add_;
};
class ObNormalUdfFunction : public ObUdfFunction {
private:
public:
ObNormalUdfFunction() = default;
virtual ~ObNormalUdfFunction() = default;
int process_origin_func(common::ObObj& result, const common::ObObj* objs_stack, int64_t param_num,
common::ObExprCtx& expr_ctx, ObUdfCtx& udf_ctx) const;
};
class ObAggUdfFunction : public ObUdfFunction {
private:
public:
ObAggUdfFunction() = default;
virtual ~ObAggUdfFunction() = default;
int process_origin_func(common::ObIAllocator& allocator, common::ObObj& result, ObUdfCtx& udf_ctx) const;
int process_add_func(common::ObExprCtx& expr_ctx, common::ObArenaAllocator& allocator, const common::ObNewRow& row,
ObUdfCtx& udf_ctx) const;
int process_clear_func(ObUdfCtx& udf_ctx) const;
};
class ObAggUdfMeta {
OB_UNIS_VERSION_V(1);
public:
ObAggUdfMeta() : udf_meta_(), udf_attributes_(), udf_attributes_types_(), calculable_results_()
{}
explicit ObAggUdfMeta(const share::schema::ObUDFMeta& meta)
: udf_meta_(meta), udf_attributes_(), udf_attributes_types_(), calculable_results_()
{}
virtual ~ObAggUdfMeta() = default;
ObAggUdfMeta& operator=(const ObAggUdfMeta& other);
TO_STRING_KV(K_(udf_meta), K_(udf_attributes), K_(udf_attributes_types), K_(calculable_results));
share::schema::ObUDFMeta udf_meta_; /* all the info we need about udf*/
common::ObSEArray<common::ObString, 16> udf_attributes_; /* udf's input, args' name */
common::ObSEArray<ObExprResType, 16> udf_attributes_types_; /* udf's input, aatribute type */
common::ObSEArray<ObUdfConstArgs, 16> calculable_results_; /* const input expr' param idx */
};
class ObAggUdfExeUnit {
public:
ObAggUdfExeUnit(ObAggUdfFunction* agg_func, ObUdfFunction::ObUdfCtx* udf_ctx) : agg_func_(agg_func), udf_ctx_(udf_ctx)
{}
ObAggUdfExeUnit() : agg_func_(nullptr), udf_ctx_(nullptr)
{}
virtual ~ObAggUdfExeUnit() = default;
ObAggUdfFunction* agg_func_;
ObUdfFunction::ObUdfCtx* udf_ctx_;
};
class ObNormalUdfExeUnit {
public:
ObNormalUdfExeUnit(ObNormalUdfFunction* normal_func, ObUdfFunction::ObUdfCtx* udf_ctx)
: normal_func_(normal_func), udf_ctx_(udf_ctx)
{}
ObNormalUdfExeUnit() : normal_func_(nullptr), udf_ctx_(nullptr)
{}
virtual ~ObNormalUdfExeUnit() = default;
const ObNormalUdfFunction* normal_func_;
ObUdfFunction::ObUdfCtx* udf_ctx_;
};
} // namespace sql
} // namespace oceanbase
#endif