From 044fadf593e3063413577d81059c7a8be0a5c938 Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 30 Aug 2023 13:10:42 +0000 Subject: [PATCH] [FEAT MERGE]:Merge foreign key feature to master Co-authored-by: YangEfei --- deps/oblib/src/common/ob_common_types.h | 8 +- src/share/schema/ob_schema_struct.cpp | 48 ++ src/share/schema/ob_schema_struct.h | 3 + src/share/schema/ob_table_schema.cpp | 82 +++ src/share/schema/ob_table_schema.h | 4 + src/sql/CMakeLists.txt | 1 + src/sql/code_generator/ob_dml_cg_service.cpp | 301 +++++++-- src/sql/code_generator/ob_dml_cg_service.h | 45 +- .../code_generator/ob_static_engine_cg.cpp | 89 +++ src/sql/das/ob_das_define.h | 6 +- src/sql/das/ob_das_scan_op.cpp | 5 +- src/sql/das/ob_das_utils.cpp | 3 +- src/sql/engine/dml/ob_dml_ctx_define.cpp | 85 ++- src/sql/engine/dml/ob_dml_ctx_define.h | 60 +- src/sql/engine/dml/ob_dml_service.cpp | 224 +++++-- src/sql/engine/dml/ob_dml_service.h | 19 +- src/sql/engine/dml/ob_fk_checker.cpp | 617 ++++++++++++++++++ src/sql/engine/dml/ob_fk_checker.h | 119 ++++ src/sql/engine/dml/ob_table_delete_op.cpp | 1 + src/sql/engine/dml/ob_table_insert_op.cpp | 6 +- src/sql/engine/dml/ob_table_insert_up_op.cpp | 18 +- src/sql/engine/dml/ob_table_merge_op.cpp | 14 +- src/sql/engine/dml/ob_table_modify_op.cpp | 191 +++++- src/sql/engine/dml/ob_table_modify_op.h | 34 +- src/sql/engine/dml/ob_table_replace_op.cpp | 8 +- src/sql/engine/dml/ob_table_update_op.cpp | 9 +- .../static/ob_px_multi_part_insert_op.cpp | 3 +- .../static/ob_px_multi_part_update_op.cpp | 3 +- src/sql/engine/table/ob_table_scan_op.cpp | 1 + src/sql/ob_sql_trans_control.cpp | 40 ++ src/sql/ob_sql_trans_control.h | 1 + src/sql/optimizer/ob_log_del_upd.cpp | 157 +++++ src/sql/optimizer/ob_log_del_upd.h | 13 +- src/sql/optimizer/ob_log_delete.cpp | 9 + src/sql/optimizer/ob_log_delete.h | 1 + src/sql/optimizer/ob_log_insert.cpp | 52 ++ src/sql/optimizer/ob_log_insert.h | 1 + src/sql/optimizer/ob_log_merge.cpp | 39 ++ src/sql/optimizer/ob_log_merge.h | 1 + src/sql/optimizer/ob_log_update.cpp | 19 + src/sql/optimizer/ob_log_update.h | 1 + src/sql/resolver/ddl/ob_ddl_resolver.cpp | 59 +- src/sql/resolver/dml/ob_dml_resolver.cpp | 199 +++++- src/sql/resolver/dml/ob_dml_resolver.h | 19 +- src/sql/resolver/dml/ob_insert_resolver.cpp | 3 + src/sql/resolver/dml/ob_merge_resolver.cpp | 2 + .../dml/ob_multi_table_insert_resolver.cpp | 2 + src/sql/resolver/dml/ob_update_resolver.cpp | 2 + .../ob_micro_block_row_scanner.cpp | 8 +- src/storage/memtable/mvcc/ob_mvcc_engine.cpp | 8 +- src/storage/memtable/mvcc/ob_mvcc_engine.h | 3 +- .../memtable/mvcc/ob_mvcc_iterator.cpp | 43 +- src/storage/memtable/mvcc/ob_mvcc_iterator.h | 3 +- src/storage/memtable/mvcc/ob_mvcc_row.cpp | 3 + src/storage/memtable/ob_memtable.cpp | 55 +- src/storage/memtable/ob_memtable_iterator.cpp | 40 +- .../memtable/ob_row_conflict_handler.cpp | 10 +- .../memtable/ob_row_conflict_handler.h | 6 +- src/storage/tx_storage/ob_access_service.cpp | 4 - 59 files changed, 2587 insertions(+), 223 deletions(-) create mode 100644 src/sql/engine/dml/ob_fk_checker.cpp create mode 100644 src/sql/engine/dml/ob_fk_checker.h diff --git a/deps/oblib/src/common/ob_common_types.h b/deps/oblib/src/common/ob_common_types.h index a638e759cf..1680cf79c9 100644 --- a/deps/oblib/src/common/ob_common_types.h +++ b/deps/oblib/src/common/ob_common_types.h @@ -50,6 +50,7 @@ struct ObQueryFlag #define OBSF_BIT_IS_SHOW_SEED 1 #define OBSF_BIT_SKIP_READ_LOB 1 #define OBSF_BIT_IS_LOOKUP_FOR_4377 1 +#define OBSF_BIT_FOR_FOREING_KEY_CHECK 1 #define OBSF_BIT_RESERVED 31 static const uint64_t OBSF_MASK_SCAN_ORDER = (0x1UL << OBSF_BIT_SCAN_ORDER) - 1; @@ -74,6 +75,7 @@ struct ObQueryFlag static const uint64_t OBSF_MASK_IS_LARGE_QUERY = (0x1UL << OBSF_BIT_IS_LARGE_QUERY) - 1; static const uint64_t OBSF_MASK_IS_SSTABLE_CUT = (0x1UL << OBSF_BIT_IS_SSTABLE_CUT) - 1; static const uint64_t OBSF_MASK_SKIP_READ_LOB = (0x1UL << OBSF_BIT_SKIP_READ_LOB) - 1; + static const uint64_t OBSF_MASK_FOR_FOREING_KEY_CHECK = (0x1UL << OBSF_BIT_FOR_FOREING_KEY_CHECK) - 1; enum ScanOrder { @@ -134,6 +136,7 @@ struct ObQueryFlag uint64_t is_show_seed_ : OBSF_BIT_IS_SHOW_SEED; uint64_t skip_read_lob_ : OBSF_BIT_SKIP_READ_LOB; uint64_t is_lookup_for_4377_ : OBSF_BIT_IS_LOOKUP_FOR_4377; + uint64_t for_foreign_key_check_ : OBSF_BIT_FOR_FOREING_KEY_CHECK; uint64_t reserved_ : OBSF_BIT_RESERVED; }; }; @@ -212,9 +215,11 @@ struct ObQueryFlag inline void set_use_fast_agg() { use_fast_agg_ = UseFastAgg; } inline void set_iter_uncommitted_row() { iter_uncommitted_row_ = true; } inline void set_not_iter_uncommitted_row() { iter_uncommitted_row_ = false; } - inline bool iter_uncommitted_row() const { return iter_uncommitted_row_; } + inline void set_for_foreign_key_check() { for_foreign_key_check_ = true; } inline void set_ignore_trans_stat() { ignore_trans_stat_ = true; } inline void set_not_ignore_trans_stat() { ignore_trans_stat_ = false; } + inline bool iter_uncommitted_row() const { return iter_uncommitted_row_; } + inline bool is_for_foreign_key_check() const { return for_foreign_key_check_; } inline bool is_ignore_trans_stat() const { return ignore_trans_stat_; } inline bool is_sstable_cut() const { return is_sstable_cut_; } inline bool is_skip_read_lob() const { return skip_read_lob_; } @@ -250,6 +255,7 @@ struct ObQueryFlag "is_sstable_cut", is_sstable_cut_, "skip_read_lob", skip_read_lob_, "is_lookup_for_4377", is_lookup_for_4377_, + "is_for_foreign_key_check", for_foreign_key_check_, "reserved", reserved_); OB_UNIS_VERSION(1); }; diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 20e1a4452e..eb1061df36 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -13434,6 +13434,54 @@ int ObTableLatestSchemaVersion::assign(const ObTableLatestSchemaVersion &other) return ret; } +int ObForeignKeyInfo::get_child_column_id(const uint64_t parent_column_id, uint64_t &child_column_id) const +{ + int ret = OB_SUCCESS; + child_column_id = OB_INVALID_ID; + if (parent_column_ids_.count() == child_column_ids_.count()) { + for (int64_t i = 0; i < parent_column_ids_.count(); ++i) { + if (parent_column_ids_.at(i) == parent_column_id) { + child_column_id = child_column_ids_.at(i); + break; + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("The number of parent key columns and foreign key columns is different", + K(ret), K(parent_column_ids_.count()), K(child_column_ids_.count())); + } + + if (OB_SUCC(ret) && OB_INVALID_ID == child_column_id) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Not find corresponding child column id", K(ret), K(parent_column_id)); + } + return ret; +} + +int ObForeignKeyInfo::get_parent_column_id(const uint64_t child_column_id, uint64_t &parent_column_id) const +{ + int ret = OB_SUCCESS; + parent_column_id = OB_INVALID_ID; + if (parent_column_ids_.count() == child_column_ids_.count()) { + for (int64_t i = 0; i < child_column_ids_.count(); ++i) { + if (child_column_ids_.at(i) == child_column_id) { + parent_column_id = parent_column_ids_.at(i); + break; + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("The number of parent key columns and foreign key columns is different", + K(ret), K(parent_column_ids_.count()), K(child_column_ids_.count())); + } + + if (OB_SUCC(ret) && OB_INVALID_ID == parent_column_id) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Not find corresponding parent column id", K(ret), K(child_column_id)); + } + return ret; +} + // // } //namespace schema diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index c9f33a6717..e729f12338 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -6161,6 +6161,9 @@ public: } return ret; } + + int get_child_column_id(const uint64_t parent_column_id, uint64_t &child_column_id) const; + int get_parent_column_id(const uint64_t child_column_id, uint64_t &parent_column_id) const; inline void reset() { table_id_ = common::OB_INVALID_ID; diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 6973d26008..6925b903f4 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -7177,6 +7177,68 @@ int ObTableSchema::add_foreign_key_info(const ObForeignKeyInfo &foreign_key_info return ret; } +int ObTableSchema::get_fk_check_index_tid(ObSchemaGetterGuard &schema_guard, const common::ObIArray &parent_column_ids, uint64_t &scan_index_tid) const +{ + int ret = OB_SUCCESS; + scan_index_tid = OB_INVALID_ID; + bool is_rowkey_column = false; + if (0 >= parent_column_ids.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column number in parent key is zero", K(ret), K(parent_column_ids.count())); + } else if (OB_FAIL(check_rowkey_column(parent_column_ids, is_rowkey_column))) { + LOG_WARN("failed to check if parent key is rowkey", K(ret)); + } else if (is_rowkey_column) { + scan_index_tid = table_id_; + } else { + bool find = false; + for (int i = 0; OB_SUCC(ret) && i < simple_index_infos_.count() && !find; ++i) { + const ObAuxTableMetaInfo &index_info = simple_index_infos_.at(i); + const uint64_t index_tid = index_info.table_id_; + const ObTableSchema *index_schema = NULL; + if (!is_unique_index(index_info.index_type_)) { + // do nothing + } else if (OB_FAIL(schema_guard.get_table_schema(get_tenant_id(), + index_tid, + index_schema))) { + LOG_WARN("fail to get table schema", K(ret)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("index schema from schema guard is NULL", K(ret), K(index_schema)); + } else if (parent_column_ids.count() == index_schema->get_index_column_num()) { + const ObIndexInfo &index_info = index_schema->get_index_info(); + bool is_rowkey = true; + int j = 0; + for (; OB_SUCC(ret) && j < parent_column_ids.count() && is_rowkey; ++j) { + if (OB_FAIL(index_info.is_rowkey_column(parent_column_ids.at(j), is_rowkey))) { + LOG_WARN("failed to check parent column is unique index column", K(ret)); + } + } + if (OB_SUCC(ret) && is_rowkey) { + find = true; + scan_index_tid = index_tid; + } + } + } + } + return ret; +} + +int ObTableSchema::check_rowkey_column(const common::ObIArray &parent_column_ids, bool &is_rowkey) const +{ + int ret = OB_SUCCESS; + is_rowkey = true; + if (parent_column_ids.count() != rowkey_column_num_) { + is_rowkey = false; + } else { + for (int i = 0; OB_SUCC(ret) && i < parent_column_ids.count() && is_rowkey; ++i) { + const uint64_t parent_column_id = parent_column_ids.at(i); + ret = rowkey_info_.is_rowkey_column(parent_column_id, is_rowkey); + } + } + return ret; +} + + int ObTableSchema::add_simple_index_info(const ObAuxTableMetaInfo &simple_index_info) { int ret = OB_SUCCESS; @@ -7678,6 +7740,26 @@ int ObTableSchema::convert_column_udt_set_ids(const ObHashMapis_stored_generated_column() + && column_schema->has_cascaded_column_id(column_id)) { + is_stored_base_col = true; + } + } + return ret; +} + int64_t ObPrintableTableSchema::to_string(char *buf, const int64_t buf_len) const { int64_t pos = 0; diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 0a24fb668f..8c1a84737b 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1177,6 +1177,7 @@ public: inline bool has_generated_column() const { return generated_columns_.num_members() > 0; } // The table has a generated column that is a partition key. bool has_generated_and_partkey_column() const; + int check_is_stored_generated_column_base_column(uint64_t column_id, bool &is_stored_base_col) const; // Check whether the data table column has prefix index column deps. int check_prefix_index_columns_depend(const ObColumnSchemaV2 &data_column_schema, ObSchemaGetterGuard &schema_guard, bool &has_prefix_idx_col_deps) const; int check_functional_index_columns_depend(const ObColumnSchemaV2 &data_column_schema, ObSchemaGetterGuard &schema_guard, bool &has_prefix_idx_col_deps) const; @@ -1327,6 +1328,9 @@ public: inline void reset_foreign_key_infos() { foreign_key_infos_.reset(); } int add_simple_index_info(const ObAuxTableMetaInfo &simple_index_info); + int get_fk_check_index_tid(ObSchemaGetterGuard &schema_guard, const common::ObIArray &parent_column_ids, uint64_t &scan_index_tid) const; + int check_rowkey_column(const common::ObIArray &parent_column_ids, bool &is_rowkey) const; + // trigger inline const common::ObIArray &get_trigger_list() const { diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 34627903eb..95f2a0819f 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -233,6 +233,7 @@ ob_set_subtarget(ob_sql engine_dml engine/dml/ob_link_dml_op.cpp engine/dml/ob_link_op.cpp engine/dml/ob_trigger_handler.cpp + engine/dml/ob_fk_checker.cpp ) ob_set_subtarget(ob_sql engine_expr diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index d59569dba9..41bdf7122d 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -1006,7 +1006,7 @@ int ObDmlCgService::generate_scan_ctdef(ObLogInsert &op, return ret; } -int ObDmlCgService::generate_dml_column_ids(ObLogicalOperator &op, +int ObDmlCgService::generate_dml_column_ids(const ObLogicalOperator &op, const ObIArray &columns_exprs, ObIArray &column_ids) { @@ -1033,7 +1033,7 @@ int ObDmlCgService::generate_dml_column_ids(ObLogicalOperator &op, return ret; } -int ObDmlCgService::generate_updated_column_ids(ObLogDelUpd &log_op, +int ObDmlCgService::generate_updated_column_ids(const ObLogDelUpd &log_op, const ObAssignments &assigns, const ObIArray &column_ids, ObIArray &updated_column_ids) @@ -2605,11 +2605,11 @@ int ObDmlCgService::need_foreign_key_handle(const ObForeignKeyArg &fk_arg, } int ObDmlCgService::generate_fk_arg(ObForeignKeyArg &fk_arg, - uint64_t name_table_id, - const common::ObIArray &column_ids, - const common::ObIArray &updated_column_ids, - const ObIArray &name_column_ids, - const ObIArray &value_column_ids, + bool check_parent_table, + const IndexDMLInfo &index_dml_info, + const ObForeignKeyInfo &fk_info, + const ObLogDelUpd &op, + ObRawExpr* fk_part_id_expr, ObSchemaGetterGuard &schema_guard, ObDMLBaseCtDef &dml_ctdef) { @@ -2620,7 +2620,18 @@ int ObDmlCgService::generate_fk_arg(ObForeignKeyArg &fk_arg, const ObColumnSchemaV2 *column_schema = NULL; ObIAllocator &allocator = cg_.phy_plan_->get_allocator(); const ObDASDMLBaseCtDef &das_ctdef = dml_ctdef.das_base_ctdef_; - if (OB_FAIL(need_foreign_key_handle(fk_arg, updated_column_ids, + ObArray column_ids; + ObArray updated_column_ids; + fk_arg.use_das_scan_ = check_parent_table; + const ObIArray &value_column_ids = check_parent_table ? fk_info.child_column_ids_ : fk_info.parent_column_ids_; + const ObIArray &name_column_ids = check_parent_table ? fk_info.parent_column_ids_ : fk_info.child_column_ids_; + uint64_t name_table_id = check_parent_table ? fk_info.parent_table_id_ : fk_info.child_table_id_; + + if (OB_FAIL(generate_dml_column_ids(op, index_dml_info.column_exprs_, column_ids))) { + LOG_WARN("add column ids failed", K(ret)); + } else if (OB_FAIL(generate_updated_column_ids(op, index_dml_info.assignments_, column_ids, updated_column_ids))) { + LOG_WARN("add updated column ids failed", K(ret), K(index_dml_info.assignments_)); + } else if (OB_FAIL(need_foreign_key_handle(fk_arg, updated_column_ids, value_column_ids, das_ctdef.op_type_, need_handle))) { LOG_WARN("failed to check if need handle foreign key", K(ret)); @@ -2681,15 +2692,237 @@ int ObDmlCgService::generate_fk_arg(ObForeignKeyArg &fk_arg, ret = OB_ERR_UNEXPECTED; LOG_WARN("foreign key column id is not in colunm ids", K(fk_arg), K(value_column_ids.at(i)), K(ret)); - } else if (OB_FAIL(fk_arg.columns_.push_back(fk_column))) { - LOG_WARN("failed to push foreign key column", K(fk_arg), K(fk_column), K(ret)); + } else { + fk_column.obj_meta_ = column_schema->get_meta_type(); + if (OB_FAIL(fk_arg.columns_.push_back(fk_column))) { + LOG_WARN("failed to push foreign key column", K(fk_arg), K(fk_column), K(ret)); + } } } - if (OB_SUCC(ret) && need_handle) { - if (OB_FAIL(dml_ctdef.fk_args_.push_back(fk_arg))) { - LOG_WARN("failed to add foreign key arg", K(fk_arg), K(ret)); + + // if need use das scan to perform foreign key check, create fk_check_ctdef for fk_arg + if (OB_FAIL(ret)) { + // do nothing + } else if (need_handle) { + if (check_parent_table) { + ObDMLCtDefAllocator fk_allocator(cg_.phy_plan_->get_allocator()); + if (OB_ISNULL(fk_arg.fk_ctdef_ = fk_allocator.alloc())) { + LOG_WARN("failed to alocate foreign key ctdef", K(ret)); + } else if (OB_FAIL(generate_fk_check_ctdef(op, name_table_id, + fk_part_id_expr, + name_column_ids, + schema_guard, + *fk_arg.fk_ctdef_))) { + LOG_WARN("failed to check foreign key check ctdef", K(ret)); + } else if (OB_FAIL(dml_ctdef.fk_args_.push_back(fk_arg))) { + LOG_WARN("failed to add foreign key arg", K(fk_arg), K(ret)); + } } else { - cg_.phy_plan_->set_has_nested_sql(true); + if (OB_FAIL(dml_ctdef.fk_args_.push_back(fk_arg))) { + LOG_WARN("failed to add foreign key arg", K(fk_arg), K(ret)); + } else { + cg_.phy_plan_->set_has_nested_sql(true); + } + } + } else if (!need_handle) { + fk_arg.use_das_scan_ = false; + } + return ret; +} + +int ObDmlCgService::generate_fk_check_ctdef(const ObLogDelUpd &op, + uint64_t name_table_id, + ObRawExpr* fk_part_id_expr, + const common::ObIArray &name_column_ids, + share::schema::ObSchemaGetterGuard &schema_guard, + ObForeignKeyCheckerCtdef &fk_ctdef) +{ + int ret = OB_SUCCESS; + uint64_t index_tid = OB_INVALID_ID; + // check if need create check ctdef + if (get_fk_check_scan_table_id(name_table_id, name_column_ids, schema_guard, index_tid)) { + LOG_WARN("failed to get foreign key check scan table id", K(name_table_id), K(ret)); + } else if (OB_INVALID_ID == index_tid) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid index table id to build das scan task for foreign key check", K(ret)); + } else if (OB_FAIL(generate_fk_scan_ctdef(schema_guard, index_tid, fk_ctdef.das_scan_ctdef_))) { + LOG_WARN("failed to generate das scan ctdef for foreign key check", K(ret)); + } else if (OB_FAIL(generate_fk_table_loc_info(index_tid, fk_ctdef.loc_meta_, fk_ctdef.tablet_id_, fk_ctdef.is_part_table_))) { + LOG_WARN("failed to generate table location meta for foreign key check", K(ret)); + } else { + const uint64_t tenant_id = MTL_ID(); + const ObTableSchema *table_schema = nullptr; + fk_ctdef.rowkey_ids_.set_capacity(name_column_ids.count()); + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, index_tid, table_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_FAIL(generate_rowkey_idx_for_foreign_key(name_column_ids, table_schema, fk_ctdef.rowkey_ids_))) { + LOG_WARN("failed to generate rowkey ids for foreign key", K(ret)); + } else { + fk_ctdef.rowkey_count_ = table_schema->get_rowkey_column_num(); + } + } + // generate the part expr used for building das task to perform foreign key check if parent table is partitioned + if (OB_SUCC(ret)) { + ObRawExpr *part_id_expr_for_lookup = NULL; + ObExpr *rt_part_id_expr = NULL; + ObSEArray constraint_dep_exprs; + ObSEArray constraint_raw_exprs; + if (OB_ISNULL(part_id_expr_for_lookup = fk_part_id_expr)) { + // check if table to perform das task is partiton table + } else if (OB_FAIL(cg_.generate_calc_part_id_expr(*part_id_expr_for_lookup, nullptr, rt_part_id_expr))) { + LOG_WARN("generate rt part_id_expr failed", K(ret), KPC(part_id_expr_for_lookup)); + } else if (OB_ISNULL(rt_part_id_expr)) { + LOG_WARN("rt part_id_expr for lookup is null", K(ret)); + } else if (OB_FAIL(constraint_raw_exprs.push_back(part_id_expr_for_lookup))) { + LOG_WARN("fail to push part_id_expr to constraint_raw_exprs", K(ret)); + } else if (OB_FAIL(cg_.generate_calc_exprs(constraint_dep_exprs, + constraint_raw_exprs, + fk_ctdef.part_id_dep_exprs_, + op.get_type(), + false))) { + LOG_WARN("fail to generate part_id_expr depend calc_expr", K(constraint_dep_exprs), + K(constraint_raw_exprs), K(fk_ctdef.part_id_dep_exprs_)); + } else { + fk_ctdef.calc_part_id_expr_ = rt_part_id_expr; + } + } + return ret; +} + +int ObDmlCgService::generate_rowkey_idx_for_foreign_key(const ObIArray &name_column_ids, + const ObTableSchema *parent_table, + ObIArray &rowkey_ids) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(parent_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema used for foreign key check is null", K(ret)); + } else if (OB_FAIL(rowkey_ids.reserve(name_column_ids.count()))) { + LOG_WARN("failed to reverse rowkey ids", K(ret)); + } else { + const ObRowkeyInfo &rowkey_info = parent_table->get_rowkey_info(); + for (int64_t i = 0; OB_SUCC(ret) && i < name_column_ids.count(); ++i) { + const uint64_t column_id = name_column_ids.at(i); + int64_t index = -1; + if (OB_FAIL(rowkey_info.get_index(column_id, index))) { + LOG_WARN("failed to get the index of parent column in primary key", K(ret)); + } else if (OB_FAIL(rowkey_ids.push_back(index))) { + LOG_WARN("failed to push back rowkey index", K(ret)); + } + } + } + return ret; +} + +int ObDmlCgService::generate_fk_table_loc_info(uint64_t index_table_id, + ObDASTableLocMeta &loc_meta, + ObTabletID &tablet_id, + bool &is_part_table) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = nullptr; + ObSchemaGetterGuard *schema_guard = nullptr; + if (OB_ISNULL(cg_.opt_ctx_) + || OB_ISNULL(schema_guard = cg_.opt_ctx_->get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid argument", K(ret), K(cg_.opt_ctx_), K(schema_guard)); + } else if (OB_FAIL(schema_guard->get_table_schema(MTL_ID(), index_table_id, table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(index_table_id)); + } else { + loc_meta.table_loc_id_ = index_table_id; + loc_meta.ref_table_id_ = index_table_id; + loc_meta.select_leader_ = 1; + loc_meta.is_dup_table_ = (ObDuplicateScope::DUPLICATE_SCOPE_NONE != table_schema->get_duplicate_scope()); + if (PARTITION_LEVEL_ZERO == table_schema->get_part_level()) { + tablet_id = table_schema->get_tablet_id(); + } else { + is_part_table = true; + } + } + + return ret; +} + +int ObDmlCgService::get_fk_check_scan_table_id(const uint64_t parent_table_id, + const common::ObIArray &name_column_ids, + share::schema::ObSchemaGetterGuard &schema_guard, + uint64_t &index_table_id) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, parent_table_id, table_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_FAIL(table_schema->get_fk_check_index_tid(schema_guard, name_column_ids, index_table_id))) { + LOG_WARN("failed to get scan table id for foreign key check", K(ret)); + } + return ret; +} + +int ObDmlCgService::generate_fk_scan_ctdef(share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t index_tid, + ObDASScanCtDef &scan_ctdef) +{ + int ret = OB_SUCCESS; + scan_ctdef.ref_table_id_ = index_tid; + const uint64_t tenant_id = MTL_ID(); + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, index_tid, table_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_FAIL(schema_guard.get_schema_version( + TABLE_SCHEMA, tenant_id, index_tid, scan_ctdef.schema_version_))) { + LOG_WARN("fail to get schema version", K(ret), K(tenant_id), K(index_tid)); + } else { + scan_ctdef.table_param_.get_enable_lob_locator_v2() + = (cg_.get_cur_cluster_version() >= CLUSTER_VERSION_4_1_0_0); + if (OB_FAIL(scan_ctdef.table_param_.convert(*table_schema, scan_ctdef.access_column_ids_))) { + LOG_WARN("convert table param failed", K(ret)); + } + } + return ret; +} + +int ObDmlCgService::generate_fk_scan_part_id_expr(ObLogDelUpd &op, + uint64_t parent_table_id, + uint64_t index_tid, + ObForeignKeyCheckerCtdef &fk_ctdef) +{ + int ret = OB_SUCCESS; + + // check if the table to perform das scan task is partition table + bool is_part_table = false; + if (OB_SUCC(ret) && is_part_table) { + ObRawExpr *part_id_expr_for_lookup = NULL; + ObExpr *rt_part_id_expr = NULL; + ObSEArray constraint_dep_exprs; + ObSEArray constraint_raw_exprs; + ObLogPlan *log_plan = nullptr; + if (OB_ISNULL(log_plan = op.get_plan())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("plan is null", K(ret)); + } else if (OB_FAIL(log_plan->gen_calc_part_id_expr(parent_table_id, + index_tid, + CALC_PARTITION_TABLET_ID, + part_id_expr_for_lookup))) { + LOG_WARN("failed to gen calc part id expr", K(ret)); + } else if (OB_ISNULL(part_id_expr_for_lookup)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("part_id_expr for lookup is null", K(ret)); + } else if (OB_FAIL(cg_.generate_calc_part_id_expr(*part_id_expr_for_lookup, nullptr, rt_part_id_expr))) { + LOG_WARN("generate rt part_id_expr failed", K(ret), KPC(part_id_expr_for_lookup)); + } else if (OB_ISNULL(rt_part_id_expr)) { + LOG_WARN("rt part_id_expr for lookup is null", K(ret)); + } else if (OB_FAIL(constraint_raw_exprs.push_back(part_id_expr_for_lookup))) { + LOG_WARN("fail to push part_id_expr to constraint_raw_exprs", K(ret)); + } else if (OB_FAIL(cg_.generate_calc_exprs(constraint_dep_exprs, + constraint_raw_exprs, + fk_ctdef.part_id_dep_exprs_, + op.get_type(), + false))) { + LOG_WARN("fail to generate part_id_expr depend calc_expr", K(constraint_dep_exprs), + K(constraint_raw_exprs), K(fk_ctdef.part_id_dep_exprs_)); + } else { + fk_ctdef.calc_part_id_expr_ = rt_part_id_expr; } } return ret; @@ -2703,12 +2936,8 @@ int ObDmlCgService::convert_foreign_keys(ObLogDelUpd &op, ObLogPlan *log_plan = NULL; ObSqlSchemaGuard *schema_guard = NULL; const ObIArray *fk_infos = NULL; - const ObIArray *value_column_ids = NULL; - const ObIArray *name_column_ids = NULL; - ObArray column_ids; - ObArray updated_column_ids; - uint64_t name_table_id = OB_INVALID_ID; const ObTableSchema *table_schema = NULL; + bool check_parent_table = false; if (OB_ISNULL(log_plan = op.get_plan()) || OB_ISNULL(cg_.phy_plan_) || OB_ISNULL(schema_guard = log_plan->get_optimizer_context().get_sql_schema_guard()) || OB_ISNULL(schema_guard->get_schema_guard())) { @@ -2729,10 +2958,6 @@ int ObDmlCgService::convert_foreign_keys(ObLogDelUpd &op, LOG_WARN("foreign key infos is null", K(ret)); } else if (OB_FAIL(dml_ctdef.fk_args_.init(table_schema->get_foreign_key_real_count()))) { LOG_WARN("failed to init foreign key stmts", K(ret)); - } else if (OB_FAIL(generate_dml_column_ids(op, index_dml_info.column_exprs_, column_ids))) { - LOG_WARN("add column ids failed", K(ret)); - } else if (OB_FAIL(generate_updated_column_ids(op, index_dml_info.assignments_, column_ids, updated_column_ids))) { - LOG_WARN("add updated column ids failed", K(ret), K(index_dml_info.assignments_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < fk_infos->count(); i++) { const ObForeignKeyInfo &fk_info = fk_infos->at(i); @@ -2768,9 +2993,6 @@ int ObDmlCgService::convert_foreign_keys(ObLogDelUpd &op, fk_arg.is_self_ref_ = true; } if (OB_SUCC(ret) && fk_info.table_id_ == fk_info.child_table_id_) { - name_table_id = fk_info.parent_table_id_; - name_column_ids = &fk_info.parent_column_ids_; - value_column_ids = &fk_info.child_column_ids_; if (DAS_OP_TABLE_INSERT == dml_ctdef.dml_type_ || DAS_OP_TABLE_UPDATE == dml_ctdef.dml_type_) { if (lib::is_mysql_mode() && fk_info.is_parent_table_mock_) { @@ -2791,22 +3013,20 @@ int ObDmlCgService::convert_foreign_keys(ObLogDelUpd &op, } else { fk_arg.ref_action_ = ACTION_CHECK_EXIST; } + check_parent_table = true; } else { fk_arg.ref_action_ = ACTION_INVALID; } if (OB_FAIL(ret)) { - } else if (OB_FAIL(generate_fk_arg(fk_arg, name_table_id, - column_ids, updated_column_ids, - *name_column_ids, *value_column_ids, - *schema_guard->get_schema_guard(), - dml_ctdef))) { + } else if (fk_arg.ref_action_ != ACTION_INVALID && + OB_FAIL(generate_fk_arg(fk_arg, check_parent_table, index_dml_info, fk_info, op, + index_dml_info.fk_lookup_part_id_expr_.at(i), + *schema_guard->get_schema_guard(), + dml_ctdef))) { LOG_WARN("failed to add fk arg to dml ctdef", K(ret)); } } if (OB_SUCC(ret) && fk_info.table_id_ == fk_info.parent_table_id_) { - name_table_id = fk_info.child_table_id_; - name_column_ids = &fk_info.child_column_ids_; - value_column_ids = &fk_info.parent_column_ids_; if (DAS_OP_TABLE_UPDATE == dml_ctdef.dml_type_) { fk_arg.ref_action_ = fk_info.update_action_; } else if (DAS_OP_TABLE_DELETE == dml_ctdef.dml_type_) { @@ -2814,11 +3034,14 @@ int ObDmlCgService::convert_foreign_keys(ObLogDelUpd &op, } else { fk_arg.ref_action_ = ACTION_INVALID; } - if (OB_FAIL(generate_fk_arg(fk_arg, name_table_id, - column_ids, updated_column_ids, - *name_column_ids, *value_column_ids, - *schema_guard->get_schema_guard(), - dml_ctdef))) { + check_parent_table = false; + if (fk_arg.ref_action_ != ACTION_INVALID && OB_FAIL(generate_fk_arg(fk_arg, + check_parent_table, + index_dml_info, + fk_info, op, + nullptr, // fk_lookup_part_id_expr_, for parent table, don't use scan task to perform foreign key check + *schema_guard->get_schema_guard(), + dml_ctdef))) { LOG_WARN("failed to add fk arg to dml ctdef", K(ret)); } } diff --git a/src/sql/code_generator/ob_dml_cg_service.h b/src/sql/code_generator/ob_dml_cg_service.h index 5016f0a859..fab5d1a3b0 100644 --- a/src/sql/code_generator/ob_dml_cg_service.h +++ b/src/sql/code_generator/ob_dml_cg_service.h @@ -109,10 +109,10 @@ public: bool &is_heap_table); private: - int generate_dml_column_ids(ObLogicalOperator &op, + int generate_dml_column_ids(const ObLogicalOperator &op, const common::ObIArray &columns_exprs, common::ObIArray &column_ids); - int generate_updated_column_ids(ObLogDelUpd &log_op, + int generate_updated_column_ids(const ObLogDelUpd &log_op, const ObAssignments &assigns, const common::ObIArray &column_ids, common::ObIArray &updated_column_ids); @@ -232,13 +232,44 @@ private: const ObDASOpType &op_type, bool &need_handle); int generate_fk_arg(ObForeignKeyArg &fk_arg, - uint64_t name_table_id, - const common::ObIArray &column_ids, - const common::ObIArray &updated_column_ids, - const common::ObIArray &name_column_ids, - const common::ObIArray &value_column_ids, + bool check_parent_table, + const IndexDMLInfo &index_dml_info, + const ObForeignKeyInfo &fk_info, + const ObLogDelUpd &op, + ObRawExpr* fk_part_id_expr, share::schema::ObSchemaGetterGuard &schema_guard, ObDMLBaseCtDef &dml_ctdef); + + int get_fk_check_scan_table_id(const uint64_t parent_table_id, + const common::ObIArray &name_column_ids, + share::schema::ObSchemaGetterGuard &schema_guard, + uint64_t &index_table_id); + + int generate_fk_check_ctdef(const ObLogDelUpd &op, + uint64_t name_table_id, + ObRawExpr* fk_part_id_expr, + const common::ObIArray &name_column_ids, + share::schema::ObSchemaGetterGuard &schema_guard, + ObForeignKeyCheckerCtdef &fk_chk_ctdef); + + int generate_fk_scan_ctdef(share::schema::ObSchemaGetterGuard &schema_guard, + const uint64_t index_tid, + ObDASScanCtDef &scan_ctdef); + + int generate_fk_scan_part_id_expr(ObLogDelUpd &op, + uint64_t parent_table_id, + uint64_t index_tid, + ObForeignKeyCheckerCtdef &fk_ctdef); + + int generate_fk_table_loc_info(uint64_t index_table_id, + ObDASTableLocMeta &loc_meta, + ObTabletID &tablet_id, + bool &is_part_table_); + + int generate_rowkey_idx_for_foreign_key(const common::ObIArray &name_column_ids, + const ObTableSchema *parent_table, + ObIArray &rowkey_ids_); + int convert_foreign_keys(ObLogDelUpd &op, const IndexDMLInfo &index_dml_info, ObDMLBaseCtDef &dml_ctdef); diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index e29773ead0..471df021a3 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -2025,6 +2025,37 @@ int ObStaticEngineCG::generate_merge_with_das(ObLogMerge &op, OZ(generate_rt_exprs(op.get_delete_condition(), spec.delete_conds_)); } + if (OB_SUCC(ret)) { + ObMergeCtDef *merge_ctdef = spec.merge_ctdefs_.at(0); + bool find = false; + + if (OB_NOT_NULL(merge_ctdef->upd_ctdef_)) { + const ObUpdCtDef &upd_ctdef = *merge_ctdef->upd_ctdef_; + for (int64_t i = 0; OB_SUCC(ret) && !find && i < upd_ctdef.fk_args_.count(); ++i) { + const ObForeignKeyArg &fk_arg = upd_ctdef.fk_args_.at(i); + if (!fk_arg.use_das_scan_) { + find = true; + } + } + } + + if (OB_SUCC(ret) && !find) { + if (OB_NOT_NULL(merge_ctdef->del_ctdef_)) { + const ObDelCtDef &del_ctdef = *merge_ctdef->del_ctdef_; + if (del_ctdef.fk_args_.count() > 0) { + find = true; + } + } + } + + if (OB_SUCC(ret)) { + if (find) { + spec.check_fk_batch_ = false; + } else { + spec.check_fk_batch_ = true; + } + } + } return ret; } @@ -2053,6 +2084,7 @@ int ObStaticEngineCG::generate_spec(ObLogicalOperator &op, int ObStaticEngineCG::generate_insert_with_das(ObLogInsert &op, ObTableInsertSpec &spec) { int ret = OB_SUCCESS; + spec.check_fk_batch_ = true; const ObIArray &index_dml_infos = op.get_index_dml_infos(); if (OB_ISNULL(phy_plan_)) { ret = OB_INVALID_ARGUMENT; @@ -2148,6 +2180,7 @@ int ObStaticEngineCG::generate_delete_with_das(ObLogDelete &op, ObTableDeleteSpe { int ret = OB_SUCCESS; const ObIArray &delete_table_list = op.get_table_list(); + spec.check_fk_batch_ = false; if (OB_ISNULL(phy_plan_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KP(phy_plan_)); @@ -2283,6 +2316,27 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableReplaceSpec &spec, c LOG_DEBUG("print replace_ctdef", K(ret), KPC(replace_ctdef)); } // for index_dml_infos end } + + if (OB_SUCC(ret)) { + ObReplaceCtDef *replace_ctdef = spec.replace_ctdefs_.at(0); + if (OB_ISNULL(replace_ctdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replace ctdef is null", K(ret)); + } else { + const ObInsCtDef *ins_ctdef = replace_ctdef->ins_ctdef_; + const ObDelCtDef *del_ctdef = replace_ctdef->del_ctdef_; + if (OB_NOT_NULL(ins_ctdef) && OB_NOT_NULL(del_ctdef)) { + if (del_ctdef->fk_args_.count() > 0) { + spec.check_fk_batch_ = false; + } else { + spec.check_fk_batch_ = true; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert or delete ctdef is null", K(ret), K(ins_ctdef), K(del_ctdef)); + } + } + } return ret; } @@ -2319,6 +2373,7 @@ int ObStaticEngineCG::generate_update_with_das(ObLogUpdate &op, ObTableUpdateSpe LOG_WARN("generate ab stmt id expr failed", K(ret)); } } + bool find = false; for (int64_t i = 0; OB_SUCC(ret) && i < table_list.count(); ++i) { const uint64_t loc_table_id = table_list.at(i); ObSEArray index_dml_infos; @@ -2342,9 +2397,18 @@ int ObStaticEngineCG::generate_update_with_das(ObLogUpdate &op, ObTableUpdateSpe } else { upd_ctdef->has_instead_of_trigger_ = op.has_instead_of_trigger(); ctdefs.at(j) = upd_ctdef; + for (int64_t j = 0; j < upd_ctdef->fk_args_.count() && !find; ++j) { + const ObForeignKeyArg &fk_arg = upd_ctdef->fk_args_.at(j); + if (!fk_arg.use_das_scan_) { + find = true; + } + } } } // for index_dml_infos end } //for table_columns end + if (OB_SUCC(ret)) { + spec.check_fk_batch_ = !find; + } return ret; } @@ -2485,6 +2549,30 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObTableInsertUpSpec &spec, } } } + + if (OB_SUCC(ret)) { + const ObInsertUpCtDef *insert_up_ctdef = spec.insert_up_ctdefs_.at(0); + if (OB_ISNULL(insert_up_ctdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert update ctdef is nullptr", K(ret)); + } else { + const ObInsCtDef *ins_ctdef = insert_up_ctdef->ins_ctdef_; + const ObUpdCtDef *upd_ctdef = insert_up_ctdef->upd_ctdef_; + if (OB_ISNULL(ins_ctdef) || OB_ISNULL(upd_ctdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert or update ctdef is null", K(ret)); + } else { + spec.check_fk_batch_ = true; + for (int64_t i = 0; i < upd_ctdef->fk_args_.count() && spec.check_fk_batch_; ++i) { + const ObForeignKeyArg &fk_arg = upd_ctdef->fk_args_.at(i); + if (!fk_arg.use_das_scan_) { + spec.check_fk_batch_ = false; + break; + } + } + } + } + } return ret; } @@ -6528,6 +6616,7 @@ int ObStaticEngineCG::generate_spec(ObLogInsertAll &op, int ObStaticEngineCG::generate_insert_all_with_das(ObLogInsertAll &op, ObTableInsertAllSpec &spec) { int ret = OB_SUCCESS; + spec.check_fk_batch_ = true; if (OB_ISNULL(op.get_insert_all_table_info()) || OB_ISNULL(phy_plan_) || OB_UNLIKELY(op.get_table_list().count() != op.get_insert_all_table_info()->count())) { diff --git a/src/sql/das/ob_das_define.h b/src/sql/das/ob_das_define.h index 94146166e0..cf79c8acb9 100644 --- a/src/sql/das/ob_das_define.h +++ b/src/sql/das/ob_das_define.h @@ -269,7 +269,8 @@ public: K_(tablet_locs), K_(is_writing), K_(is_reading), - K_(need_refresh)); + K_(need_refresh), + K_(is_fk_check)); /** * BE CAREFUL!!! can't declare implicit allocator or @@ -291,7 +292,8 @@ public: uint64_t is_reading_ : 1; //mark this table is reading uint64_t rebuild_reference_ : 1; //mark whether rebuild the related reference uint64_t need_refresh_ : 1; - uint64_t reserved_ : 60; + uint64_t is_fk_check_ : 1; //mark this table is used for foreign key checking + uint64_t reserved_ : 59; }; }; diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index a76bc00fab..4f4cb23281 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -1250,7 +1250,10 @@ OB_INLINE int ObLocalIndexLookupOp::init_scan_param() } // lookup to main table should invoke multi get scan_param_.is_get_ = true; - + scan_param_.is_for_foreign_check_ = lookup_rtdef_->is_for_foreign_check_; + if (lookup_rtdef_->is_for_foreign_check_) { + scan_param_.trans_desc_ = tx_desc_; + } if (OB_NOT_NULL(snapshot_)) { scan_param_.snapshot_ = *snapshot_; } else { diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index 332dd9a481..8cb0c01347 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -88,7 +88,8 @@ int ObDASUtils::check_nested_sql_mutating(ObTableID ref_table_id, ObExecContext FOREACH_X(node, parent_das_ctx.get_table_loc_list(), OB_SUCC(ret)) { ObDASTableLoc *table_loc = *node; if (table_loc->loc_meta_->ref_table_id_ == ref_table_id - && (table_loc->is_writing_ || (!is_reading && lib::is_mysql_mode()))) { + && (table_loc->is_writing_ || (!is_reading && lib::is_mysql_mode())) + && !table_loc->is_fk_check_ ) { ObSchemaGetterGuard schema_guard; const ObTableSchema *table_schema = NULL; uint64_t tenant_id = exec_ctx.get_my_session()->get_effective_tenant_id(); diff --git a/src/sql/engine/dml/ob_dml_ctx_define.cpp b/src/sql/engine/dml/ob_dml_ctx_define.cpp index 2029b1057a..0b39d17eea 100644 --- a/src/sql/engine/dml/ob_dml_ctx_define.cpp +++ b/src/sql/engine/dml/ob_dml_ctx_define.cpp @@ -12,6 +12,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/dml/ob_dml_ctx_define.h" +#include "sql/engine/dml/ob_fk_checker.h" #include "sql/das/ob_das_utils.h" namespace oceanbase { @@ -112,11 +113,79 @@ int ObTriggerColumnsInfo::set_trigger_rowid() return set_trigger_column(true, false, false, false, true); } +OB_SERIALIZE_MEMBER(ObForeignKeyCheckerCtdef, + calc_part_id_expr_, + part_id_dep_exprs_, + das_scan_ctdef_, + loc_meta_, + is_part_table_, + tablet_id_, + rowkey_count_, + rowkey_ids_); + OB_SERIALIZE_MEMBER(ObTriggerArg, trigger_id_, trigger_events_.bit_value_, timing_points_.bit_value_); -OB_SERIALIZE_MEMBER(ObForeignKeyColumn, name_, idx_, name_idx_); +OB_SERIALIZE_MEMBER(ObForeignKeyColumn, name_, idx_, name_idx_, obj_meta_); -OB_SERIALIZE_MEMBER(ObForeignKeyArg, ref_action_, table_name_, columns_, database_name_, is_self_ref_, table_id_); +// OB_SERIALIZE_MEMBER(ObForeignKeyArg, +// ref_action_, +// table_name_, +// columns_, +// database_name_, +// is_self_ref_, +// table_id_, +// fk_ctdef_); + +OB_DEF_SERIALIZE(ObForeignKeyArg) +{ + int ret = OB_SUCCESS; + LST_DO_CODE(OB_UNIS_ENCODE, + ref_action_, + table_name_, + columns_, + database_name_, + is_self_ref_, + table_id_ + ); + if (OB_NOT_NULL(fk_ctdef_)) { + OB_UNIS_ENCODE(*fk_ctdef_); + } + return ret; +} + +OB_DEF_DESERIALIZE(ObForeignKeyArg) +{ + int ret = OB_SUCCESS; + LST_DO_CODE(OB_UNIS_DECODE, + ref_action_, + table_name_, + columns_, + database_name_, + is_self_ref_, + table_id_ + ); + if (OB_NOT_NULL(fk_ctdef_)) { + OB_UNIS_DECODE(*fk_ctdef_); + } + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObForeignKeyArg) +{ + int64_t len = 0; + LST_DO_CODE(OB_UNIS_ADD_LEN, + ref_action_, + table_name_, + columns_, + database_name_, + is_self_ref_, + table_id_ + ); + if (OB_NOT_NULL(fk_ctdef_)) { + OB_UNIS_ADD_LEN(*fk_ctdef_); + } + return len; +} OB_SERIALIZE_MEMBER(ColumnContent, projector_index_, @@ -690,5 +759,17 @@ OB_DEF_SERIALIZE_SIZE(ObInsertUpCtDef) OB_UNIS_ADD_LEN(*upd_ctdef_); return len; } + +ObDMLBaseRtDef::~ObDMLBaseRtDef() +{ + for (int64_t i = 0; i < fk_checker_array_.count(); ++i) { + ObForeignKeyChecker *fk_checker = fk_checker_array_.at(i); + if (OB_NOT_NULL(fk_checker)) { + fk_checker->reset(); + fk_checker = nullptr; + } + } + fk_checker_array_.release_array(); +} } // namespace sql } // namespace oceanbase diff --git a/src/sql/engine/dml/ob_dml_ctx_define.h b/src/sql/engine/dml/ob_dml_ctx_define.h index 755704e306..cf1b7bd8b4 100644 --- a/src/sql/engine/dml/ob_dml_ctx_define.h +++ b/src/sql/engine/dml/ob_dml_ctx_define.h @@ -14,6 +14,7 @@ #define DEV_SRC_SQL_ENGINE_DML_OB_DML_CTX_DEFINE_H_ #include "sql/das/ob_das_dml_ctx_define.h" #include "sql/das/ob_das_ref.h" +#include "sql/das/ob_das_scan_op.h" namespace oceanbase { namespace sql @@ -21,6 +22,8 @@ namespace sql typedef ObDASOpType ObDMLOpType; class ObTableModifyOp; +class ObForeignKeyChecker; +typedef common::ObArrayWrap FkCheckerArray; struct ObErrLogCtDef { @@ -296,17 +299,56 @@ public: ObForeignKeyColumn() : name_(), idx_(-1), - name_idx_(-1) + name_idx_(-1), + obj_meta_() {} inline void reset() { name_.reset(); idx_ = -1; name_idx_ = -1; } TO_STRING_KV(N_COLUMN_NAME, name_, N_INDEX, idx_, - N_INDEX, name_idx_); + N_INDEX, name_idx_, + N_META, obj_meta_); common::ObString name_; int32_t idx_; // index of the column id in column_ids_ of ObTableModify. value column idx int32_t name_idx_; + ObObjMeta obj_meta_; // This is used to cast fk type to parent key type }; +struct ObForeignKeyCheckerCtdef +{ + OB_UNIS_VERSION(1); +public: + ObForeignKeyCheckerCtdef(ObIAllocator &alloc) + : calc_part_id_expr_(nullptr), + part_id_dep_exprs_(alloc), + das_scan_ctdef_(alloc), + loc_meta_(alloc), + is_part_table_(false), + tablet_id_(), + rowkey_count_(0), + rowkey_ids_(alloc) + {} + // 父表的主表/unique索引表对应的分区键 + + TO_STRING_KV(KPC_(calc_part_id_expr), + K_(part_id_dep_exprs), + K_(das_scan_ctdef), + K_(loc_meta), + K_(is_part_table), + K_(tablet_id), + K_(rowkey_count)); + ObExpr *calc_part_id_expr_; + // calc_part_id_expr_计算所依赖的表达式，用于clear_eval_flag + ExprFixedArray part_id_dep_exprs_; + // 回表查询，为了结构统一，主表也需要一次回表，第一次的insert都返回主表的主键 + ObDASScanCtDef das_scan_ctdef_; + ObDASTableLocMeta loc_meta_; + bool is_part_table_; + ObTabletID tablet_id_; + int64_t rowkey_count_; + IntFixedArray rowkey_ids_; //save the index of parent key column in rowkey +}; + + class ObForeignKeyArg { OB_UNIS_VERSION(1); @@ -317,7 +359,9 @@ public: table_name_(), columns_(), is_self_ref_(false), - table_id_(OB_INVALID_ID) + table_id_(0), + fk_ctdef_(nullptr), + use_das_scan_(false) {} ObForeignKeyArg(common::ObIAllocator &alloc) @@ -326,7 +370,9 @@ public: table_name_(), columns_(alloc), is_self_ref_(false), - table_id_(OB_INVALID_ID) + table_id_(0), + fk_ctdef_(nullptr), + use_das_scan_(false) {} inline void reset() { @@ -343,7 +389,10 @@ public: common::ObString table_name_; common::ObFixedArray columns_; bool is_self_ref_; + // the index table id of unique index for parent key, used to build das task to scan index table uint64_t table_id_; + ObForeignKeyCheckerCtdef *fk_ctdef_; + bool use_das_scan_; }; typedef common::ObFixedArray ObForeignKeyArgArray; @@ -471,7 +520,7 @@ protected: //dml base runtime context definition struct ObDMLBaseRtDef { - virtual ~ObDMLBaseRtDef() = default; + virtual ~ObDMLBaseRtDef(); VIRTUAL_TO_STRING_KV(K_(trig_rtdef), K_(cur_row_num)); ObTrigDMLRtDef trig_rtdef_; @@ -481,6 +530,7 @@ struct ObDMLBaseRtDef //reference the das base ctdef to facilitate //some modules to access the das rtdef ObDASDMLBaseRtDef &das_base_rtdef_; + FkCheckerArray fk_checker_array_; protected: ObDMLBaseRtDef(ObDASDMLBaseRtDef &das_base_rtdef) : trig_rtdef_(), diff --git a/src/sql/engine/dml/ob_dml_service.cpp b/src/sql/engine/dml/ob_dml_service.cpp index 298625048b..b3906b417e 100644 --- a/src/sql/engine/dml/ob_dml_service.cpp +++ b/src/sql/engine/dml/ob_dml_service.cpp @@ -29,7 +29,7 @@ #include "sql/engine/expr/ob_expr_lob_utils.h" #include "lib/geo/ob_geo_utils.h" #include "sql/ob_sql_utils.h" - +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { using namespace common; @@ -1207,11 +1207,61 @@ int ObDMLService::init_trigger_for_insert( return ret; } +int ObDMLService::init_fk_checker_array(ObDMLRtCtx &dml_rtctx, + const ObDMLBaseCtDef &dml_ctdef, + FkCheckerArray &fk_checker_array) +{ + int ret = OB_SUCCESS; + ObIAllocator &allocator = dml_rtctx.get_exec_ctx().get_allocator(); + const ObForeignKeyArgArray &fk_args = dml_ctdef.fk_args_; + if (!fk_args.empty()) { + if (OB_FAIL(fk_checker_array.allocate_array(allocator, fk_args.count()))) { + LOG_WARN("failed to create foreign key checker array", K(ret)); + } else { + for (int i = 0; OB_SUCC(ret) && i < fk_args.count(); ++i) { + fk_checker_array.at(i) = nullptr; + } + } + } + for (int i = 0; OB_SUCC(ret) && i < fk_args.count(); ++i) { + const ObForeignKeyArg &fk_arg = fk_args.at(i); + ObForeignKeyChecker *fk_checker = nullptr; + if (fk_arg.use_das_scan_) { + if (OB_ISNULL(fk_arg.fk_ctdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("need to perform foreign key check by das scan task, but scan ctdef is null", K(ret)); + } else { + // create fk_checker here + void *checker_buf = allocator.alloc(sizeof(ObForeignKeyChecker)); + if (OB_ISNULL(checker_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("cllocate foreign key checker buffer failed", K(ret)); + } else { + fk_checker = new(checker_buf) ObForeignKeyChecker(dml_rtctx.get_eval_ctx(), *fk_arg.fk_ctdef_); + fk_checker_array.at(i) = fk_checker; + const ObExprFrameInfo *expr_frame_info = NULL; + expr_frame_info = nullptr != dml_rtctx.op_.get_spec().expr_frame_info_ + ? dml_rtctx.op_.get_spec().expr_frame_info_ + : &(dml_rtctx.op_.get_spec().plan_->get_expr_frame_info()); + int64_t estimate_row = dml_rtctx.op_.get_spec().rows_; + ObIAllocator *allocator = &dml_rtctx.op_.get_exec_ctx().get_allocator(); + if (OB_FAIL(fk_checker->init_foreign_key_checker(estimate_row, expr_frame_info, *fk_arg.fk_ctdef_, + dml_ctdef.new_row_, allocator))) { + LOG_WARN("failed to init foreign key checker", K(ret)); + } + } + } + } + } + return ret; +} + int ObDMLService::init_ins_rtdef( ObDMLRtCtx &dml_rtctx, ObInsRtDef &ins_rtdef, const ObInsCtDef &ins_ctdef, - ObIArray &clear_exprs) + ObIArray &clear_exprs, + ObIArray &fk_checkers) { int ret = OB_SUCCESS; dml_rtctx.get_exec_ctx().set_dml_event(ObDmlEventType::DE_INSERTING); @@ -1230,9 +1280,16 @@ int ObDMLService::init_ins_rtdef( LOG_WARN("init related das ctdef failed", K(ret)); } else if (OB_FAIL(init_trigger_for_insert(dml_rtctx, ins_ctdef, ins_rtdef, clear_exprs))) { LOG_WARN("failed to init trigger for insert", K(ret)); + } else if (OB_FAIL(init_fk_checker_array(dml_rtctx, ins_ctdef, ins_rtdef.fk_checker_array_))) { + LOG_WARN("failed to init foreign key checker array", K(ret)); } else { ins_rtdef.das_rtdef_.related_ctdefs_ = &ins_ctdef.related_ctdefs_; ins_rtdef.das_rtdef_.related_rtdefs_ = &ins_rtdef.related_rtdefs_; + for (int64_t i = 0; OB_SUCC(ret) && i < ins_rtdef.fk_checker_array_.count(); ++i) { + if (OB_NOT_NULL(ins_rtdef.fk_checker_array_.at(i))) { + fk_checkers.push_back(ins_rtdef.fk_checker_array_.at(i)); + } + } } return ret; } @@ -1271,6 +1328,8 @@ int ObDMLService::init_del_rtdef(ObDMLRtCtx &dml_rtctx, LOG_WARN("init related das ctdef failed", K(ret)); } else if (OB_FAIL(init_trigger_for_delete(dml_rtctx, del_ctdef, del_rtdef))) { LOG_WARN("failed to init trigger for delete", K(ret)); + } else if (OB_FAIL(init_fk_checker_array(dml_rtctx, del_ctdef, del_rtdef.fk_checker_array_))) { + LOG_WARN("failed to init foreign key checker array", K(ret)); } else { del_rtdef.das_rtdef_.related_ctdefs_ = &del_ctdef.related_ctdefs_; del_rtdef.das_rtdef_.related_rtdefs_ = &del_rtdef.related_rtdefs_; @@ -1397,7 +1456,8 @@ int ObDMLService::init_upd_rtdef( ObDMLRtCtx &dml_rtctx, ObUpdRtDef &upd_rtdef, const ObUpdCtDef &upd_ctdef, - ObIArray &clear_exprs) + ObIArray &clear_exprs, + ObIArray &fk_checkers) { int ret = OB_SUCCESS; const ObDASTableLocMeta *loc_meta = get_table_loc_meta(upd_ctdef.multi_ctdef_); @@ -1416,10 +1476,17 @@ int ObDMLService::init_upd_rtdef( LOG_WARN("init related das ctdef failed", K(ret)); } else if (OB_FAIL(init_trigger_for_update(dml_rtctx, upd_ctdef, upd_rtdef, dml_rtctx.op_, clear_exprs))) { LOG_WARN("failed to init trigger for update", K(ret)); + } else if (OB_FAIL(init_fk_checker_array(dml_rtctx, upd_ctdef, upd_rtdef.fk_checker_array_))) { + LOG_WARN("failed to init foreign key checker array", K(ret)); } else { upd_rtdef.dupd_rtdef_.related_ctdefs_ = &upd_ctdef.related_upd_ctdefs_; upd_rtdef.dupd_rtdef_.related_rtdefs_ = &upd_rtdef.related_upd_rtdefs_; dml_rtctx.get_exec_ctx().set_update_columns(&upd_ctdef.assign_columns_); + for (int64_t i = 0; OB_SUCC(ret) && i < upd_rtdef.fk_checker_array_.count(); ++i) { + if (OB_NOT_NULL(upd_rtdef.fk_checker_array_.at(i))) { + fk_checkers.push_back(upd_rtdef.fk_checker_array_.at(i)); + } + } } if (OB_SUCC(ret) && T_DISTINCT_NONE != upd_ctdef.distinct_algo_) { @@ -2079,68 +2146,82 @@ int ObDMLService::get_nested_dup_table_ctx(const uint64_t table_id, DASDelCtxLi return ret; } -int ObDMLService::handle_after_row_processing_batch(ObDMLModifyRowsList *dml_modify_rows) +int ObDMLService::handle_after_processing_multi_row(ObDMLModifyRowsList *dml_modify_rows, ObTableModifyOp *dml_op) { int ret = OB_SUCCESS; const ObDmlEventType t_insert = ObDmlEventType::DE_INSERTING; const ObDmlEventType t_update = ObDmlEventType::DE_UPDATING; const ObDmlEventType t_delete = ObDmlEventType::DE_DELETING; - ObDMLModifyRowsList::iterator row_iter = dml_modify_rows->begin(); - for (; OB_SUCC(ret) && row_iter != dml_modify_rows->end(); row_iter++) { - ObDMLModifyRowNode &modify_row = *row_iter; - if (OB_ISNULL(modify_row.full_row_) && OB_ISNULL(modify_row.new_row_) && OB_ISNULL(modify_row.old_row_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid parameter for batch post row processing", K(ret)); - } else if (OB_ISNULL(modify_row.dml_op_) || OB_ISNULL(modify_row.dml_ctdef_) || OB_ISNULL(modify_row.dml_rtdef_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid parameter for batch post row processing", K(ret)); - } else { - ObTableModifyOp &op = *modify_row.dml_op_; - const ObDMLBaseCtDef &dml_ctdef = *modify_row.dml_ctdef_; - ObDMLBaseRtDef &dml_rtdef = *modify_row.dml_rtdef_; - const ObDmlEventType dml_event = modify_row.dml_event_; - // process foreign key - if (OB_NOT_NULL(modify_row.full_row_) && OB_FAIL(modify_row.full_row_->to_expr(modify_row.dml_ctdef_->full_row_, op.get_eval_ctx()))) { - LOG_WARN("failed to covert stored full row to expr", K(ret)); - } else if (OB_NOT_NULL(modify_row.old_row_) && OB_FAIL(modify_row.old_row_->to_expr(dml_ctdef.old_row_, op.get_eval_ctx()))) { - LOG_WARN("failed to covert stored old row to expr", K(ret)); - } else if (OB_NOT_NULL(modify_row.new_row_) && OB_FAIL(modify_row.new_row_->to_expr(dml_ctdef.new_row_, op.get_eval_ctx()))) { - LOG_WARN("failed to covert stored new row to expr", K(ret)); + if (OB_ISNULL(dml_modify_rows) || OB_ISNULL(dml_op)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dml operator or modify rows list is null", K(dml_modify_rows), K(dml_op)); + } else { + ObDMLModifyRowsList::iterator row_iter = dml_modify_rows->begin(); + for (; OB_SUCC(ret) && row_iter != dml_modify_rows->end(); row_iter++) { + ObDMLModifyRowNode &modify_row = *row_iter; + if (OB_ISNULL(modify_row.full_row_) && OB_ISNULL(modify_row.new_row_) && OB_ISNULL(modify_row.old_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parameter for batch post row processing", K(ret)); + } else if (OB_ISNULL(modify_row.dml_op_) || OB_ISNULL(modify_row.dml_ctdef_) || OB_ISNULL(modify_row.dml_rtdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parameter for batch post row processing", K(ret)); } else { - if (t_update == dml_event) { - // for update op, Foreign key checks need to be performed only if the value has changed - if (reinterpret_cast(dml_rtdef).is_row_changed_ && OB_FAIL(ForeignKeyHandle::do_handle(op, dml_ctdef, dml_rtdef))) { - LOG_WARN("failed to handle foreign key constraints", K(ret)); + ObTableModifyOp &op = *modify_row.dml_op_; + const ObDMLBaseCtDef &dml_ctdef = *modify_row.dml_ctdef_; + ObDMLBaseRtDef &dml_rtdef = *modify_row.dml_rtdef_; + const ObDmlEventType dml_event = modify_row.dml_event_; + // process foreign key + if (OB_NOT_NULL(modify_row.full_row_) && OB_FAIL(modify_row.full_row_->to_expr(modify_row.dml_ctdef_->full_row_, op.get_eval_ctx()))) { + LOG_WARN("failed to covert stored full row to expr", K(ret)); + } else if (OB_NOT_NULL(modify_row.old_row_) && OB_FAIL(modify_row.old_row_->to_expr(dml_ctdef.old_row_, op.get_eval_ctx()))) { + LOG_WARN("failed to covert stored old row to expr", K(ret)); + } else if (OB_NOT_NULL(modify_row.new_row_) && OB_FAIL(modify_row.new_row_->to_expr(dml_ctdef.new_row_, op.get_eval_ctx()))) { + LOG_WARN("failed to covert stored new row to expr", K(ret)); + } else if (op.need_foreign_key_checks()) { + if (t_update == dml_event && !reinterpret_cast(dml_rtdef).is_row_changed_) { + LOG_DEBUG("update operation don't change any value, no need to perform foreign key check"); + } else { + // if check foreign key in batch, build fk check tasks here + if (dml_op->get_spec().check_fk_batch_) { + if (OB_FAIL(build_batch_fk_check_tasks(dml_ctdef, dml_rtdef))) { + LOG_WARN("failed to build batch check foreign key checks", K(ret)); + } + } else if (OB_FAIL(ForeignKeyHandle::do_handle(op, dml_ctdef, dml_rtdef))) { + LOG_WARN("failed to handle foreign key constraints", K(ret)); + } + } + } + // process after row trigger + if (OB_SUCC(ret) && dml_ctdef.trig_ctdef_.all_tm_points_.has_after_row()) { + ObEvalCtx &eval_ctx = op.get_eval_ctx(); + if (dml_event != t_insert && dml_event != t_update && dml_event != t_delete) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid trigger event", K(ret)); + } else if (t_insert == dml_event && OB_FAIL(TriggerHandle::init_param_new_row( + eval_ctx, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_))) { + LOG_WARN("failed to init trigger parameter for new row", K(ret)); + } else if (t_delete == dml_event && OB_FAIL(TriggerHandle::init_param_old_row( + eval_ctx, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_))) { + LOG_WARN("failed to init trigger parameter for old row", K(ret)); + } else if (t_update == dml_event && OB_FAIL(TriggerHandle::init_param_rows( + eval_ctx, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_))) { + LOG_WARN("failed to init trigger parameter for old row and new row", K(ret)); + } else if (OB_FAIL(TriggerHandle::do_handle_after_row(op, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_, dml_event))) { + LOG_WARN("failed to handle after trigger", K(ret)); } - } else if (OB_FAIL(ForeignKeyHandle::do_handle(op, dml_ctdef, dml_rtdef))) { - LOG_WARN("failed to handle foreign key constraints", K(ret)); - } - } - // process after row trigger - if (OB_SUCC(ret) && dml_ctdef.trig_ctdef_.all_tm_points_.has_after_row()) { - ObEvalCtx &eval_ctx = op.get_eval_ctx(); - if (dml_event != t_insert && dml_event != t_update && dml_event != t_delete) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid trigger event", K(ret)); - } else if (t_insert == dml_event && OB_FAIL(TriggerHandle::init_param_new_row( - eval_ctx, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_))) { - LOG_WARN("failed to init trigger parameter for new row", K(ret)); - } else if (t_delete == dml_event && OB_FAIL(TriggerHandle::init_param_old_row( - eval_ctx, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_))) { - LOG_WARN("failed to init trigger parameter for old row", K(ret)); - } else if (t_update == dml_event && OB_FAIL(TriggerHandle::init_param_rows( - eval_ctx, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_))) { - LOG_WARN("failed to init trigger parameter for old row and new row", K(ret)); - } else if (OB_FAIL(TriggerHandle::do_handle_after_row(op, dml_ctdef.trig_ctdef_, dml_rtdef.trig_rtdef_, dml_event))) { - LOG_WARN("failed to handle after trigger", K(ret)); } } } + + // check the result of batch foreign key check results + if (OB_SUCC(ret) && dml_op->get_spec().check_fk_batch_ && OB_FAIL(dml_op->perform_batch_fk_check())) { + LOG_WARN("failed to perform batch foreign key check", K(ret)); + } } return ret; } -int ObDMLService::handle_after_row_processing_single(ObDMLModifyRowsList *dml_modify_rows) +int ObDMLService::handle_after_processing_single_row(ObDMLModifyRowsList *dml_modify_rows) { int ret = OB_SUCCESS; // for single-row processing, the expr defined in ctdef and trig parameters haven't been refreshed @@ -2185,18 +2266,49 @@ int ObDMLService::handle_after_row_processing_single(ObDMLModifyRowsList *dml_mo return ret; } -int ObDMLService::handle_after_row_processing(bool execute_single_row, ObDMLModifyRowsList *dml_modify_rows) +int ObDMLService::handle_after_row_processing(ObTableModifyOp *op, + ObDMLModifyRowsList *dml_modify_rows) { int ret = OB_SUCCESS; - if (1 > dml_modify_rows->size()) { + if (OB_ISNULL(op) || OB_ISNULL(dml_modify_rows)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table modify operator or list of the modify rows is null", K(ret), K(op), K(dml_modify_rows)); + } else if (1 > dml_modify_rows->size()) { // after row processing list is empty, nothing to do #ifndef NDEBUG LOG_INFO("No row need to perform foreign key check or after row trigger"); #endif - } else if (execute_single_row) { - ret = handle_after_row_processing_single(dml_modify_rows); + } else if (op->execute_single_row_) { + ret = handle_after_processing_single_row(dml_modify_rows); } else { - ret = handle_after_row_processing_batch(dml_modify_rows); + ret = handle_after_processing_multi_row(dml_modify_rows, op); + } + return ret; +} + +int ObDMLService::build_batch_fk_check_tasks(const ObDMLBaseCtDef &dml_ctdef, + ObDMLBaseRtDef &dml_rtdef) +{ + int ret = OB_SUCCESS; + if (dml_rtdef.fk_checker_array_.count() != dml_ctdef.fk_args_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key aruments count is not equal with foreign key checker count", + K(ret),K(dml_rtdef.fk_checker_array_.count()), K(dml_ctdef.fk_args_.count())); + } else { + for (int i = 0; OB_SUCC(ret) && i < dml_rtdef.fk_checker_array_.count(); ++i) { + ObForeignKeyChecker *fk_checker = dml_rtdef.fk_checker_array_.at(i); + const ObForeignKeyArg &fk_arg = dml_ctdef.fk_args_.at(i); + bool need_check = true; + if (OB_ISNULL(fk_checker)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key checker is nullptr", K(ret), K(i)); + } else if (!fk_arg.use_das_scan_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key check can not use das scan", K(ret), K(i)); + } else if (OB_FAIL(fk_checker->build_fk_check_das_task(fk_arg.columns_, dml_ctdef.new_row_, need_check))) { + LOG_WARN("failed to build batch foreign key check scan task", K(ret)); + } + } } return ret; } diff --git a/src/sql/engine/dml/ob_dml_service.h b/src/sql/engine/dml/ob_dml_service.h index 15e5fe3dad..d048776ee9 100644 --- a/src/sql/engine/dml/ob_dml_service.h +++ b/src/sql/engine/dml/ob_dml_service.h @@ -152,7 +152,8 @@ public: static int init_ins_rtdef(ObDMLRtCtx &dml_rtctx, ObInsRtDef &ins_rtdef, const ObInsCtDef &ins_ctdef, - ObIArray &clear_exprs); + ObIArray &clear_exprs, + ObIArray &fk_checkers); static int init_trigger_for_insert(ObDMLRtCtx &dml_rtctx, const ObInsCtDef &ins_ctdef, ObInsRtDef &ins_rtdef, @@ -162,7 +163,8 @@ public: static int init_upd_rtdef(ObDMLRtCtx &dml_rtctx, ObUpdRtDef &upd_rtdef, const ObUpdCtDef &upd_ctdef, - ObIArray &clear_exprs); + ObIArray &clear_exprs, + ObIArray &fk_checkers); static int init_das_ins_rtdef_for_update(ObDMLRtCtx &dml_rtctx, const ObUpdCtDef &upd_ctdef, ObUpdRtDef &upd_rtdef); @@ -228,13 +230,18 @@ public: static int get_nested_dup_table_ctx(const uint64_t table_id, DASDelCtxList& del_ctx_list, SeRowkeyDistCtx *&rowkey_dist_ctx); - static int handle_after_row_processing_single(ObDMLModifyRowsList *dml_modify_rows); - static int handle_after_row_processing_batch(ObDMLModifyRowsList *dml_modify_rows); - static int handle_after_row_processing(bool execute_single_row, ObDMLModifyRowsList *dml_modify_rows); + static int handle_after_processing_single_row(ObDMLModifyRowsList *dml_modify_rows); + static int handle_after_processing_multi_row(ObDMLModifyRowsList *dml_modify_rows, ObTableModifyOp *op); + // static int handle_after_processing_batch(const ObTableModifyOp *op, + // ObDMLModifyRowsList *dml_modify_rows); + static int handle_after_row_processing(ObTableModifyOp *op, ObDMLModifyRowsList *dml_modify_rows); static int init_ob_rowkey( ObIAllocator &allocator, const int64_t rowkey_cnt, ObRowkey &table_rowkey); static int add_trans_info_datum(ObExpr *trans_info_expr, ObEvalCtx &eval_ctx, ObChunkDatumStore::StoredRow *stored_row); + static int init_fk_checker_array(ObDMLRtCtx &dml_rtctx, + const ObDMLBaseCtDef &dml_ctdef, + FkCheckerArray &fk_checker_array); static int log_user_error_inner(int ret, int64_t row_num, common::ObString &column_name, @@ -260,6 +267,8 @@ private: ObEvalCtx &eval_ctx, ObDMLBaseRtDef &dml_rtdef, common::ObIAllocator &allocator); + static int build_batch_fk_check_tasks(const ObDMLBaseCtDef &dml_ctdef, + ObDMLBaseRtDef &dml_rtdef); }; template diff --git a/src/sql/engine/dml/ob_fk_checker.cpp b/src/sql/engine/dml/ob_fk_checker.cpp new file mode 100644 index 0000000000..5be2067c31 --- /dev/null +++ b/src/sql/engine/dml/ob_fk_checker.cpp @@ -0,0 +1,617 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "sql/engine/dml/ob_fk_checker.h" +#include "sql/engine/ob_exec_context.h" +#include "sql/engine/expr/ob_expr.h" +#include "sql/engine/dml/ob_dml_service.h" +namespace oceanbase +{ +namespace sql +{ + + +int ObForeignKeyChecker::reset() { + int ret = OB_SUCCESS; + if (se_rowkey_dist_ctx_ != nullptr) { + se_rowkey_dist_ctx_->destroy(); + se_rowkey_dist_ctx_ = nullptr; + } + clear_exprs_.reset(); + table_rowkey_.reset(); + if (das_ref_.has_task()) { + if (OB_FAIL(das_ref_.close_all_task())) { + LOG_WARN("close all das task failed", K(ret)); + } + } + das_ref_.reset(); + return ret; +} + +int ObForeignKeyChecker::reuse() { + int ret = OB_SUCCESS; + batch_distinct_fk_cnt_ = 0; + if (das_ref_.has_task()) { + if (OB_FAIL(das_ref_.close_all_task())) { + LOG_WARN("close all das task failed", K(ret)); + } + } + das_ref_.reuse(); + return ret; +} + +int ObForeignKeyChecker::do_fk_check_batch(bool &all_has_result) +{ + int ret = OB_SUCCESS; + + int64_t get_row_count = 0; + if (0 == batch_distinct_fk_cnt_) { + LOG_INFO("distinct foreign key count is 0 in a batch"); + all_has_result = true; + } else if (OB_FAIL(das_ref_.execute_all_task())) { + LOG_WARN("execute all scan das task failed", K(ret)); + } else if (OB_FAIL(get_scan_result_count(get_row_count))) { + LOG_WARN("failed to check the result count od foreign key scan task", K(ret)); + } else if (get_row_count > batch_distinct_fk_cnt_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result row count exceeds the number of unique key", K(ret), K(get_row_count), K(batch_distinct_fk_cnt_)); + } else if (get_row_count == batch_distinct_fk_cnt_) { + all_has_result = true; + } else { + all_has_result = false; + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(das_ref_.close_all_task())) { + LOG_WARN("close all das task failed", K(ret)); + } else { + reuse(); + } + } + return ret; +} + +int ObForeignKeyChecker::get_scan_result_count(int64_t &get_row_count) +{ + int ret = OB_SUCCESS; + get_row_count = 0; + DASOpResultIter result_iter = das_ref_.begin_result_iter(); + while (OB_SUCC(ret)) { + if (OB_FAIL(result_iter.get_next_row())) { + if (OB_ITER_END == ret) { + if (OB_FAIL(result_iter.next_result())) { + if (OB_ITER_END != ret) { + LOG_WARN("fetch next task failed", K(ret)); + } + } + } else { + LOG_WARN("get next row from das result failed", K(ret)); + } + } else { + get_row_count++; + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + return ret; +} + +int ObForeignKeyChecker::do_fk_check_single_row(const ObIArray &columns, const ObExprPtrIArray &row, bool &has_result) +{ + int ret = OB_SUCCESS; + bool need_check = true; + bool is_all_null = false; + bool has_null = false; + + if (OB_FAIL(build_fk_check_das_task(columns, row, need_check))) { + LOG_WARN("failed to build table look up das task", K(ret)); + } else if (!need_check) { + has_result = true; + LOG_TRACE("This value has been checked successfully in previous row", K(row)); + } else if (OB_FAIL(das_ref_.execute_all_task())) { + LOG_WARN("execute all scan das task failed", K(ret)); + } else { + // check if result is empty + DASOpResultIter result_iter = das_ref_.begin_result_iter(); + int64_t result_cnt = 0; + bool got_row = false; + if (1 != das_ref_.get_das_task_cnt()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("has more than one das task", K(das_ref_.get_das_task_cnt()), K(ret)); + } else if (OB_FAIL(result_iter.get_next_row())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + has_result = false; + } else { + has_result = true; + } + } + + if (OB_SUCC(ret) && OB_FAIL(das_ref_.close_all_task())) { + LOG_WARN("close all das task failed", K(ret)); + } else { + reuse(); + } + return ret; +} + +int ObForeignKeyChecker::build_fk_check_das_task(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &need_check) +{ + int ret = OB_SUCCESS; + ObDASScanOp *das_scan_op = nullptr; + ObDASTabletLoc *tablet_loc = nullptr; + ObNewRange lookup_range; + need_check = true; + bool is_all_null = false; + bool has_null = false; + if (OB_FAIL(check_fk_columns_has_null(columns, row, is_all_null, has_null))) { + LOG_WARN("failed to check foreign key columns are all null", K(ret)); + } else if (has_null) { + // Match simple is the ony one match method of OB, if foreign key columns has null, it will pass foreign key check; + // Note: we need to support match partial and match full method for a more strict foreign key check in MySQL mode + need_check = false; + LOG_INFO("foreign key columns has null, pass foreign key check"); + } else if (OB_FAIL(build_table_range(columns, row, lookup_range, need_check))) { + LOG_WARN("build data table range failed", K(ret), KPC(tablet_loc)); + } else if (!need_check) { + LOG_TRACE("The current foreign key has been successfully checked before", K(lookup_range)); + } else if (OB_FAIL(calc_lookup_tablet_loc(tablet_loc))) { + LOG_WARN("calc lookup pkey fail", K(ret)); + } else if (OB_FAIL(get_das_scan_op(tablet_loc, das_scan_op))) { + LOG_WARN("get_das_scan_op failed", K(ret), K(tablet_loc)); + } else if (OB_ISNULL(das_scan_op)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("das_scan_op should be not null", K(ret)); + } else { + storage::ObTableScanParam &scan_param = das_scan_op->get_scan_param(); + if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) { + LOG_WARN("store lookup key range failed", K(ret), K(lookup_range), K(scan_param)); + } else { + batch_distinct_fk_cnt_ += 1; + LOG_TRACE("after build conflict rowkey", K(scan_param.tablet_id_), + K(scan_param.key_ranges_.count()), K(lookup_range), K(batch_distinct_fk_cnt_)); + } + } + return ret; +} + +int ObForeignKeyChecker::calc_lookup_tablet_loc(ObDASTabletLoc *&tablet_loc) +{ + int ret = OB_SUCCESS; + ObExpr *part_id_expr = NULL; + ObTabletID tablet_id; + ObObjectID partition_id = OB_INVALID_ID; + tablet_loc = nullptr; + if (OB_ISNULL(part_id_expr = checker_ctdef_.calc_part_id_expr_)) { + tablet_loc = local_tablet_loc_; + if (OB_ISNULL(tablet_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet loc is null", K(ret)); + } + } + else if (OB_FAIL(ObSQLUtils::clear_evaluated_flag(clear_exprs_, eval_ctx_))) { + LOG_WARN("fail to clear rowkey flag", K(ret), K(checker_ctdef_.part_id_dep_exprs_)); + } else if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(part_id_expr, eval_ctx_, partition_id, tablet_id))) { + LOG_WARN("fail to calc part id", K(ret), KPC(part_id_expr)); + } else if (OB_FAIL(DAS_CTX(das_ref_.get_exec_ctx()).extended_tablet_loc(*table_loc_, tablet_id, tablet_loc))) { + LOG_WARN("extended tablet loc failed", K(ret)); + } + LOG_INFO("tablet_id and partition id is", K(tablet_id), K(partition_id)); + return ret; +} + +int ObForeignKeyChecker::get_das_scan_op(ObDASTabletLoc *tablet_loc, ObDASScanOp *&das_scan_op) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!das_ref_.has_das_op(tablet_loc, das_scan_op))) { + if (OB_FAIL(das_ref_.prepare_das_task(tablet_loc, das_scan_op))) { + LOG_WARN("prepare das task failed", K(ret)); + } else { + das_scan_op->set_scan_ctdef(&checker_ctdef_.das_scan_ctdef_); + das_scan_op->set_scan_rtdef(&das_scan_rtdef_); + } + } + return ret; +} + +int ObForeignKeyChecker::init_foreign_key_checker(int64_t estimate_row, + const ObExprFrameInfo *expr_frame_info, + ObForeignKeyCheckerCtdef &fk_ctdef, + const ObExprPtrIArray &row, + common::ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = eval_ctx_.exec_ctx_.get_my_session(); + const ObDASTableLocMeta &loc_meta = fk_ctdef.loc_meta_; + ObMemAttr mem_attr; + mem_attr.tenant_id_ = session->get_effective_tenant_id(); + mem_attr.label_ = "SqlFKeyCkr"; + das_ref_.set_expr_frame_info(expr_frame_info); + das_ref_.set_mem_attr(mem_attr); + das_ref_.set_execute_directly(false); // 确认这个参数设置的合理性 + if (OB_FAIL(DAS_CTX(das_ref_.get_exec_ctx()).extended_table_loc(loc_meta, table_loc_))) { + LOG_WARN("failed to extend table_loc", K(ret), K(loc_meta)); + } else if (OB_ISNULL(table_loc_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table location is null", K(ret)); + } else if (OB_FAIL(init_das_scan_rtdef())) { + LOG_WARN("failed to init das scan rtdef for foreign key check", K(ret)); + } else if (OB_FAIL(ObDMLService::create_rowkey_check_hashset(estimate_row, &das_ref_.get_exec_ctx(), se_rowkey_dist_ctx_))) { + LOG_WARN("failed to create hash set used for foreign key check", K(ret)); + } else if (OB_FAIL(ObSqlTransControl::set_fk_check_snapshot(das_ref_.get_exec_ctx()))) { + LOG_WARN("failed to set snapshot for foreign key check", K(ret)); + } else if (OB_FAIL(init_clear_exprs(fk_ctdef, row))) { + LOG_WARN("failed to init clear exprs", K(ret)); + } else if (OB_ISNULL(allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("allocator used to init foreign key checker is null", K(ret)); + } else { + allocator_ = allocator; + table_loc_->is_fk_check_ = true; //mark the table location with fk checking action + } + + if (OB_SUCC(ret) && !fk_ctdef.is_part_table_) { + if (OB_FAIL(DAS_CTX(das_ref_.get_exec_ctx()).extended_tablet_loc(*table_loc_, fk_ctdef.tablet_id_, local_tablet_loc_))) { + LOG_WARN("failed to extended tablet loc for foreign key check", K(ret), K(fk_ctdef.tablet_id_)); + } + } + return ret; +} + +int ObForeignKeyChecker::init_clear_exprs(ObForeignKeyCheckerCtdef &fk_ctdef, const ObExprPtrIArray &row) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < fk_ctdef.part_id_dep_exprs_.count(); ++i) { + ObExpr* expr = fk_ctdef.part_id_dep_exprs_.at(i); + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("partition id calc expr is null", K(ret), K(i)); + } else if (!has_exist_in_array(row, expr)) { + clear_exprs_.push_back(expr); + } + } + return ret; +} + +int ObForeignKeyChecker::init_das_scan_rtdef() +{ + int ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = eval_ctx_.exec_ctx_.get_physical_plan_ctx(); + ObSQLSessionInfo *my_session = eval_ctx_.exec_ctx_.get_my_session(); + ObTaskExecutorCtx &task_exec_ctx = eval_ctx_.exec_ctx_.get_task_exec_ctx(); + das_scan_rtdef_.timeout_ts_ = plan_ctx->get_ps_timeout_timestamp(); + das_scan_rtdef_.sql_mode_ = my_session->get_sql_mode(); + das_scan_rtdef_.stmt_allocator_.set_alloc(&das_ref_.get_das_alloc()); + das_scan_rtdef_.scan_allocator_.set_alloc(&das_ref_.get_das_alloc()); + ObQueryFlag query_flag(ObQueryFlag::Forward, // scan_order + false, // daily_merge + false, // optimize + false, // sys scan + false, // full_row + false, // index_back + false, // query_stat + lib::is_mysql_mode() ? ObQueryFlag::MysqlMode : ObQueryFlag::AnsiMode, // sql_mode + true // read_latest + ); + das_scan_rtdef_.scan_flag_.flag_ = query_flag.flag_; + int64_t schema_version = task_exec_ctx.get_query_tenant_begin_schema_version(); + das_scan_rtdef_.tenant_schema_version_ = schema_version; + das_scan_rtdef_.eval_ctx_ = &eval_ctx_; + das_scan_rtdef_.is_for_foreign_check_ = true; + das_scan_rtdef_.scan_flag_.set_for_foreign_key_check(); + // No Need to init pushdown op? + if (OB_FAIL(das_scan_rtdef_.init_pd_op(eval_ctx_.exec_ctx_, checker_ctdef_.das_scan_ctdef_))) { + LOG_WARN("init pushdown storage filter failed", K(ret)); + } + return ret; +} + +int ObForeignKeyChecker::build_table_range(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check) +{ + int ret = OB_SUCCESS; + int64_t rowkey_cnt = checker_ctdef_.rowkey_count_; + int64_t fk_cnt = columns.count(); + bool need_shadow_columns = false; + if (0 == rowkey_cnt || 0 == fk_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid rowkey count of foreign key count", K(ret), K(rowkey_cnt), K(fk_cnt)); + } else if (rowkey_cnt == fk_cnt) { + ret = build_primary_table_range(columns, row, lookup_range, need_check); + } else if (OB_FAIL(check_need_shadow_columns(columns, row, need_shadow_columns))) { + LOG_WARN("failed to check need shadow columns", K(ret)); + } else { + if (need_shadow_columns) { + ret = build_index_table_range_need_shadow_column(columns, row, lookup_range, need_check); + } else { + ret = build_index_table_range(columns, row, lookup_range, need_check); + } + } + return ret; +} + +int ObForeignKeyChecker::check_fk_column_type(const ObObjMeta &col_obj_meta, const ObObjMeta &dst_obj_meta) +{ + int ret = OB_SUCCESS; + if (col_obj_meta.get_type() != dst_obj_meta.get_type()) { + if (lib::is_oracle_mode() && ob_is_number_tc(col_obj_meta.get_type()) && ob_is_number_tc(dst_obj_meta.get_type())) { + // oracle mode, numberfloat and number type are same + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid type to perform foreign key check", K(col_obj_meta), K(dst_obj_meta)); + } + } + return ret; +} + +int ObForeignKeyChecker::build_primary_table_range(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check) +{ + int ret = OB_SUCCESS; + ObObj *obj_ptr = nullptr; + void *buf = nullptr; + int64_t rowkey_cnt = checker_ctdef_.rowkey_count_; + int64_t fk_cnt = columns.count(); + // check parent table + if (fk_cnt != checker_ctdef_.rowkey_ids_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid foreign key column count", K(ret), K(fk_cnt), K(checker_ctdef_.rowkey_ids_.count())); + } else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObObj) * rowkey_cnt))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate buffer failed", K(ret), K(rowkey_cnt)); + } else { + obj_ptr = new(buf) ObObj[rowkey_cnt]; + } + + for (int64_t i = 0; OB_SUCC(ret) && i < fk_cnt; ++i) { + ObObj tmp_obj; + ObDatum *col_datum = nullptr; + const ObObjMeta &col_obj_meta = row.at(columns.at(i).idx_)->obj_meta_; + const ObObjMeta &dst_obj_meta = columns.at(i).obj_meta_; + const ObObjDatumMapType &obj_datum_map = row.at(columns.at(i).idx_)->obj_datum_map_; + int64_t rowkey_index = checker_ctdef_.rowkey_ids_.at(i); + if (rowkey_index < 0 || rowkey_index >= rowkey_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Invalid woekey index to build scan range", K(ret), K(rowkey_index)); + } else if (OB_FAIL(check_fk_column_type(col_obj_meta, dst_obj_meta))) { + LOG_WARN("failed to perform foreign key column type check", K(ret), K(i)); + } else if (OB_FAIL(row.at(columns.at(i).idx_)->eval(eval_ctx_, col_datum))) { + LOG_WARN("evaluate expr failed", K(ret), K(i)); + } else if (OB_FAIL(col_datum->to_obj(tmp_obj, dst_obj_meta, obj_datum_map))) { + LOG_WARN("convert datum to obj failed", K(ret), K(i)); + } + // 这里需要做深拷贝 + else if (OB_FAIL(ob_write_obj(*allocator_, tmp_obj, obj_ptr[rowkey_index]))) { + LOG_WARN("deep copy rowkey value failed", K(ret), K(tmp_obj)); + } + } + + if (OB_SUCC(ret)) { + ObRowkey table_rowkey(obj_ptr, rowkey_cnt); + // check if the foreign key has been checked before + ret = se_rowkey_dist_ctx_->exist_refactored(table_rowkey); + if (OB_HASH_EXIST == ret) { + ret = OB_SUCCESS; + need_check = false; + LOG_TRACE("This foreign key has been checked before", K(table_rowkey)); + } else if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + need_check = true; + uint64_t ref_table_id = checker_ctdef_.das_scan_ctdef_.ref_table_id_; + if (OB_FAIL(lookup_range.build_range(ref_table_id, table_rowkey))) { + LOG_WARN("build lookup range failed", K(ret), K(ref_table_id), K(table_rowkey)); + } else if (OB_FAIL(se_rowkey_dist_ctx_->set_refactored(table_rowkey))) { + LOG_WARN("failed to add foreign key to cached hash_set", K(ret)); + } + } else { + LOG_WARN("check if foreign key item exists failed", K(ret), K(table_rowkey)); + } + } + return ret; +} + +int ObForeignKeyChecker::build_index_table_range(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check) +{ + int ret = OB_SUCCESS; + ObObj *obj_ptr = nullptr; + void *buf = nullptr; + int64_t rowkey_cnt = checker_ctdef_.rowkey_count_; + int64_t fk_cnt = columns.count(); + if (fk_cnt != checker_ctdef_.rowkey_ids_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid foreign key column count", K(ret), K(fk_cnt), K(checker_ctdef_.rowkey_ids_.count())); + } else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObObj) * rowkey_cnt))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate buffer failed", K(ret), K(rowkey_cnt)); + } else { + obj_ptr = new(buf) ObObj[rowkey_cnt]; + } + + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) { + if (i < fk_cnt) { + ObObj tmp_obj; + ObDatum *col_datum = nullptr; + const ObObjMeta &col_obj_meta = row.at(columns.at(i).idx_)->obj_meta_; + const ObObjMeta &dst_obj_meta = columns.at(i).obj_meta_; + const ObObjDatumMapType &obj_datum_map = row.at(columns.at(i).idx_)->obj_datum_map_; + int64_t rowkey_index = checker_ctdef_.rowkey_ids_.at(i); + if (rowkey_index < 0 || rowkey_index >= fk_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Invalid woekey index to build scan range", K(ret), K(rowkey_index)); + } else if (OB_FAIL(check_fk_column_type(col_obj_meta, dst_obj_meta))) { + LOG_WARN("failed to perform foreign key column type check", K(ret), K(i)); + } else if (OB_FAIL(row.at(columns.at(i).idx_)->eval(eval_ctx_, col_datum))) { + LOG_WARN("evaluate expr failed", K(ret), K(i)); + } else if (OB_FAIL(col_datum->to_obj(tmp_obj, dst_obj_meta, obj_datum_map))) { + LOG_WARN("convert datum to obj failed", K(ret), K(i)); + } + // 这里需要做深拷贝 + else if (OB_FAIL(ob_write_obj(*allocator_, tmp_obj, obj_ptr[rowkey_index]))) { + LOG_WARN("deep copy rowkey value failed", K(ret), K(tmp_obj)); + } + } else { + // is parent key is unique key, the store key of the unique index is unique key + optional primary key + // if unique key is not null, the column of optional primary key is null + obj_ptr[i].set_null(); + } + } + + if (OB_SUCC(ret)) { + ObRowkey table_rowkey(obj_ptr, rowkey_cnt); + // check if the foreign key has been checked before + ret = se_rowkey_dist_ctx_->exist_refactored(table_rowkey); // check if the foreign key has been check before + if (OB_HASH_EXIST == ret) { + ret = OB_SUCCESS; + need_check = false; + LOG_TRACE("This foreign key has been checked before", K(table_rowkey)); + } else if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + need_check = true; + uint64_t ref_table_id = checker_ctdef_.das_scan_ctdef_.ref_table_id_; + if (OB_FAIL(lookup_range.build_range(ref_table_id, table_rowkey))) { + LOG_WARN("build lookup range failed", K(ret), K(ref_table_id), K(table_rowkey)); + } else if (OB_FAIL(se_rowkey_dist_ctx_->set_refactored(table_rowkey))) { // add the foreign key that has not been checked before to the cached hash-set + LOG_WARN("failed to add foreign key to cached hash_set", K(ret)); + } + } else { + LOG_WARN("check if foreign key item exists failed", K(ret), K(table_rowkey)); + } + } + return ret; +} + +int ObForeignKeyChecker::build_index_table_range_need_shadow_column(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check) +{ + int ret = OB_SUCCESS; + ObObj *obj_ptr_start = nullptr; + ObObj *obj_ptr_end = nullptr; + void *buf_start = nullptr; + void *buf_end = nullptr; + int64_t rowkey_cnt = checker_ctdef_.rowkey_count_; + int64_t fk_cnt = columns.count(); + if (fk_cnt != checker_ctdef_.rowkey_ids_.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid foreign key column count", K(ret), K(fk_cnt), K(checker_ctdef_.rowkey_ids_.count())); + } else if (OB_ISNULL(buf_start = allocator_->alloc(sizeof(ObObj) * rowkey_cnt))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate buffer failed", K(ret), K(rowkey_cnt)); + } else if (OB_ISNULL(buf_end = allocator_->alloc(sizeof(ObObj) * rowkey_cnt))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate buffer failed", K(ret), K(rowkey_cnt)); + } else { + obj_ptr_start = new(buf_start) ObObj[rowkey_cnt]; + obj_ptr_end = new(buf_end) ObObj[rowkey_cnt]; + } + + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) { + if (i < fk_cnt) { + ObObj tmp_obj; + ObDatum *col_datum = nullptr; + const ObObjMeta &col_obj_meta = row.at(columns.at(i).idx_)->obj_meta_; + const ObObjMeta &dst_obj_meta = columns.at(i).obj_meta_; + const ObObjDatumMapType &obj_datum_map = row.at(columns.at(i).idx_)->obj_datum_map_; + int64_t rowkey_index = checker_ctdef_.rowkey_ids_.at(i); + if (rowkey_index < 0 || rowkey_index >= fk_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Invalid woekey index to build scan range", K(ret), K(rowkey_index)); + } else if (OB_FAIL(check_fk_column_type(col_obj_meta, dst_obj_meta))) { + LOG_WARN("failed to perform foreign key column type check", K(ret), K(i)); + } else if (OB_FAIL(row.at(columns.at(i).idx_)->eval(eval_ctx_, col_datum))) { + LOG_WARN("evaluate expr failed", K(ret), K(i)); + } else if (OB_FAIL(col_datum->to_obj(tmp_obj, dst_obj_meta, obj_datum_map))) { + LOG_WARN("convert datum to obj failed", K(ret), K(i)); + } else if (OB_FAIL(ob_write_obj(*allocator_, tmp_obj, obj_ptr_start[rowkey_index]))) { + LOG_WARN("deep copy rowkey value failed", K(ret), K(tmp_obj)); + } else if (OB_FAIL(ob_write_obj(*allocator_, tmp_obj, obj_ptr_end[rowkey_index]))) { + LOG_WARN("deep copy rowkey value failed", K(ret), K(tmp_obj)); + } + } else { + obj_ptr_start[i].set_min_value(); + obj_ptr_end[i].set_max_value(); + } + } + + if (OB_SUCC(ret)) { + ObRowkey start_key(obj_ptr_start, rowkey_cnt); + ObRowkey end_key(obj_ptr_end, rowkey_cnt); + lookup_range.start_key_ = start_key; + lookup_range.end_key_ = end_key; + } + return ret; +} + +int ObForeignKeyChecker::check_need_shadow_columns(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &need_shadow_columns) +{ + int ret = OB_SUCCESS; + need_shadow_columns = false; + if (lib::is_mysql_mode()) { + bool rowkey_has_null = false; + for (int64_t i = 0; OB_SUCC(ret) && !rowkey_has_null && i < columns.count(); ++i) { + ObDatum *col_datum = nullptr; + OZ(row.at(columns.at(i).idx_)->eval(eval_ctx_, col_datum)); + if (col_datum->is_null()) { + rowkey_has_null = true; + } + } + need_shadow_columns = rowkey_has_null; + } else { + bool is_rowkey_all_null = true; + for (int64_t i = 0; OB_SUCC(ret) && is_rowkey_all_null && i < columns.count(); ++i) { + ObDatum *col_datum = nullptr; + OZ(row.at(columns.at(i).idx_)->eval(eval_ctx_, col_datum)); + if (!col_datum->is_null()) { + is_rowkey_all_null = false; + } + } + need_shadow_columns = is_rowkey_all_null; + } + LOG_INFO("need shadow columns", K(need_shadow_columns)); + return ret; +} + +int ObForeignKeyChecker::check_fk_columns_has_null(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &is_all_null, + bool &has_null) +{ + int ret = OB_SUCCESS; + is_all_null = true; + has_null = false; + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); ++i) { + ObDatum *col_datum = nullptr; + OZ(row.at(columns.at(i).idx_)->eval(eval_ctx_, col_datum)); + if (!col_datum->is_null()) { + is_all_null = false; + } else { + has_null = true; + } + } + return ret; +} + +} // end namespace sql +} // end namespace oceanbase \ No newline at end of file diff --git a/src/sql/engine/dml/ob_fk_checker.h b/src/sql/engine/dml/ob_fk_checker.h new file mode 100644 index 0000000000..4d1a3c4d63 --- /dev/null +++ b/src/sql/engine/dml/ob_fk_checker.h @@ -0,0 +1,119 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_ENGINE_DML_OB_FOREIGN_KEY_CHECKER_H_ +#define OBDEV_SRC_SQL_ENGINE_DML_OB_FOREIGN_KEY_CHECKER_H_ +#include "sql/engine/basic/ob_chunk_datum_store.h" +#include "sql/das/ob_das_scan_op.h" +#include "ob_dml_ctx_define.h" + +namespace oceanbase +{ +namespace sql +{ + +class ObForeignKeyChecker +{ +public: + ObForeignKeyChecker(ObEvalCtx &eval_ctx, const ObForeignKeyCheckerCtdef &checker_ctdef) + : eval_ctx_(eval_ctx), + das_scan_rtdef_(), + checker_ctdef_(checker_ctdef), + das_ref_(eval_ctx, eval_ctx.exec_ctx_), + table_loc_(nullptr), + local_tablet_loc_(nullptr), + se_rowkey_dist_ctx_(nullptr), + table_rowkey_(), + batch_distinct_fk_cnt_(0), + allocator_(nullptr) + { + } + + ~ObForeignKeyChecker() { + } + + TO_STRING_KV(K_(das_scan_rtdef), + K_(checker_ctdef), + KPC_(table_loc), + KPC_(local_tablet_loc), + KPC_(se_rowkey_dist_ctx), + K_(table_rowkey), + K_(batch_distinct_fk_cnt), + K_(clear_exprs)); + + int reset(); + int reuse(); + int do_fk_check_single_row(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &has_result); + + int do_fk_check_batch(bool &all_has_result); + + int init_foreign_key_checker(int64_t estimate_row, + const ObExprFrameInfo *expr_frame_info, + ObForeignKeyCheckerCtdef &fk_ctdef, + const ObExprPtrIArray &row, + ObIAllocator *allocator); + + int build_fk_check_das_task(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &need_check); +private: + int init_das_scan_rtdef(); + int calc_lookup_tablet_loc(ObDASTabletLoc *&tablet_loc); + int get_das_scan_op(ObDASTabletLoc *tablet_loc, ObDASScanOp *&das_scan_op); + // map fk column to the rowkey column of parent/parent index table + int build_table_range(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check); + int build_index_table_range(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check); + int build_index_table_range_need_shadow_column(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check); + int build_primary_table_range(const ObIArray &columns, + const ObExprPtrIArray &row, + ObNewRange &lookup_range, + bool &need_check); + int check_need_shadow_columns(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &need_shadow_columns); + int check_fk_columns_has_null(const ObIArray &columns, + const ObExprPtrIArray &row, + bool &is_all_null, + bool &has_null); + int get_scan_result_count(int64_t &get_row_count); + int init_clear_exprs(ObForeignKeyCheckerCtdef &fk_ctdef, const ObExprPtrIArray &row); + int check_fk_column_type(const ObObjMeta &col_obj_meta, const ObObjMeta &dst_obj_meta); +public: + ObEvalCtx &eval_ctx_; // 用于表达式的计算 + ObDASScanRtDef das_scan_rtdef_; + // 存储计算table loc以及构造das-task需要的分区键，主键expr等 + const ObForeignKeyCheckerCtdef &checker_ctdef_; + // 外键检查回表构造das-task用 + ObDASRef das_ref_; + ObDASTableLoc *table_loc_; + ObDASTabletLoc *local_tablet_loc_; //用于外键检查的表是非分区表的时候用 + SeRowkeyDistCtx *se_rowkey_dist_ctx_; + ObRowkey table_rowkey_; + int64_t batch_distinct_fk_cnt_; + ObSEArray clear_exprs_; + ObIAllocator *allocator_; +}; + +} // namespace sql +} // namespace oceanbase +#endif /* OBDEV_SRC_SQL_ENGINE_DML_OB_CONFLICT_ROW_CHECKER_H_ */ \ No newline at end of file diff --git a/src/sql/engine/dml/ob_table_delete_op.cpp b/src/sql/engine/dml/ob_table_delete_op.cpp index 168e0d0450..5abcc56ca7 100644 --- a/src/sql/engine/dml/ob_table_delete_op.cpp +++ b/src/sql/engine/dml/ob_table_delete_op.cpp @@ -165,6 +165,7 @@ OB_INLINE int ObTableDeleteOp::open_table_for_each() if (OB_FAIL(del_rtdefs_.allocate_array(ctx_.get_allocator(), MY_SPEC.del_ctdefs_.count()))) { LOG_WARN("allocate delete rtdef failed", K(ret), K(MY_SPEC.del_ctdefs_.count())); } + fk_checkers_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < del_rtdefs_.count(); ++i) { DelRtDefArray &rtdefs = del_rtdefs_.at(i); const ObTableDeleteSpec::DelCtDefArray &ctdefs = MY_SPEC.del_ctdefs_.at(i); diff --git a/src/sql/engine/dml/ob_table_insert_op.cpp b/src/sql/engine/dml/ob_table_insert_op.cpp index 3dd3392a14..8a72125c37 100644 --- a/src/sql/engine/dml/ob_table_insert_op.cpp +++ b/src/sql/engine/dml/ob_table_insert_op.cpp @@ -26,6 +26,8 @@ #include "lib/profile/ob_perf_event.h" #include "share/schema/ob_table_dml_param.h" #include "share/ob_tablet_autoincrement_service.h" +#include "sql/engine/cmd/ob_table_direct_insert_service.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase @@ -142,6 +144,7 @@ OB_INLINE int ObTableInsertOp::open_table_for_each() LOG_WARN("allocate insert rtdef failed", K(ret), K(MY_SPEC.ins_ctdefs_.count())); } trigger_clear_exprs_.reset(); + fk_checkers_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < ins_rtdefs_.count(); ++i) { InsRtDefArray &rtdefs = ins_rtdefs_.at(i); const ObTableInsertSpec::InsCtDefArray &ctdefs = MY_SPEC.ins_ctdefs_.at(i); @@ -151,7 +154,7 @@ OB_INLINE int ObTableInsertOp::open_table_for_each() for (int64_t j = 0; OB_SUCC(ret) && j < rtdefs.count(); ++j) { ObInsRtDef &ins_rtdef = rtdefs.at(j); const ObInsCtDef &ins_ctdef = *ctdefs.at(j); - if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef, ins_ctdef, trigger_clear_exprs_))) { + if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef, ins_ctdef, trigger_clear_exprs_, fk_checkers_))) { LOG_WARN("init insert rtdef failed", K(ret)); } } @@ -455,5 +458,6 @@ OB_INLINE int ObTableInsertOp::close_table_for_each() return ret; } + } // namespace sql } // namespace oceanbase diff --git a/src/sql/engine/dml/ob_table_insert_up_op.cpp b/src/sql/engine/dml/ob_table_insert_up_op.cpp index ad2e9d5b54..129bc4d220 100644 --- a/src/sql/engine/dml/ob_table_insert_up_op.cpp +++ b/src/sql/engine/dml/ob_table_insert_up_op.cpp @@ -20,6 +20,7 @@ #include "sql/engine/expr/ob_expr_calc_partition_id.h" #include "sql/engine/dml/ob_trigger_handler.h" #include "lib/utility/ob_tracepoint.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { @@ -183,18 +184,27 @@ OB_INLINE int ObTableInsertUpOp::init_insert_up_rtdef() LOG_WARN("allocate insert up rtdef failed", K(ret), K(MY_SPEC.insert_up_ctdefs_.count())); } trigger_clear_exprs_.reset(); + fk_checkers_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < insert_up_rtdefs_.count(); ++i) { ObInsertUpCtDef *insert_up_ctdef = MY_SPEC.insert_up_ctdefs_.at(i); const ObInsCtDef *ins_ctdef = insert_up_ctdef->ins_ctdef_; const ObUpdCtDef *upd_ctdef = insert_up_ctdef->upd_ctdef_; ObInsRtDef &ins_rtdef = insert_up_rtdefs_.at(i).ins_rtdef_; ObUpdRtDef &upd_rtdef = insert_up_rtdefs_.at(i).upd_rtdef_; - if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef, *ins_ctdef, trigger_clear_exprs_))) { + if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, + ins_rtdef, + *ins_ctdef, + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init insert rt_def failed", K(ret), KPC(ins_ctdef)); } else if (OB_ISNULL(upd_ctdef)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("upd_ctdef is null", K(ret)); - } else if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, upd_rtdef, *upd_ctdef, trigger_clear_exprs_))) { + } else if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, + upd_rtdef, + *upd_ctdef, + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init upd_rt_def failed", K(ret), KPC(upd_ctdef)); } else { ins_rtdef.das_rtdef_.table_loc_->is_writing_ = true; @@ -848,7 +858,7 @@ int ObTableInsertUpOp::do_insert_up() LOG_WARN("fail to load all row", K(ret)); } else if (OB_FAIL(post_all_dml_das_task(dml_rtctx_, false))) { LOG_WARN("fail to post all das task", K(ret)); - } else if (!check_is_duplicated() && OB_FAIL(ObDMLService::handle_after_row_processing(execute_single_row_, &dml_modify_rows_))) { + } else if (!check_is_duplicated() && OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) { LOG_WARN("try insert is not duplicated, failed to process foreign key handle", K(ret)); } else if (!check_is_duplicated()) { insert_rows_ += insert_rows; @@ -872,7 +882,7 @@ int ObTableInsertUpOp::do_insert_up() LOG_WARN("do insert rows post process failed", K(ret)); } else if (OB_FAIL(post_all_dml_das_task(dml_rtctx_, false))) { LOG_WARN("do insert rows post process failed", K(ret)); - } else if (OB_FAIL(ObDMLService::handle_after_row_processing(execute_single_row_, &dml_modify_rows_))) { + } else if (OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) { LOG_WARN("try insert is duplicated, failed to process foreign key handle", K(ret)); } diff --git a/src/sql/engine/dml/ob_table_merge_op.cpp b/src/sql/engine/dml/ob_table_merge_op.cpp index 456fabf6fb..2f8d2bfebb 100644 --- a/src/sql/engine/dml/ob_table_merge_op.cpp +++ b/src/sql/engine/dml/ob_table_merge_op.cpp @@ -25,6 +25,7 @@ #include "sql/engine/dml/ob_dml_service.h" #include "sql/engine/dml/ob_trigger_handler.h" #include "sql/engine/expr/ob_expr_calc_partition_id.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { @@ -214,13 +215,18 @@ int ObTableMergeOp::open_table_for_each() LOG_WARN("fail to init ObRowkey used for distinct check", K(ret)); } trigger_clear_exprs_.reset(); + fk_checkers_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.merge_ctdefs_.count(); ++i) { ObMergeCtDef *merge_ctdef = MY_SPEC.merge_ctdefs_.at(i); if (OB_NOT_NULL(merge_ctdef->ins_ctdef_)) { // init insert rtdef const ObInsCtDef &ins_ctdef = *merge_ctdef->ins_ctdef_; ObInsRtDef &ins_rtdef = merge_rtdefs_.at(i).ins_rtdef_; - if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef, ins_ctdef, trigger_clear_exprs_))) { + if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, + ins_rtdef, + ins_ctdef, + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init ins rtdef failed", K(ret)); } else if (ins_ctdef.is_primary_index_) { if (OB_FAIL(ObDMLService::process_before_stmt_trigger(ins_ctdef, ins_rtdef, dml_rtctx_, @@ -237,7 +243,11 @@ int ObTableMergeOp::open_table_for_each() const ObUpdCtDef &upd_ctdef = *merge_ctdef->upd_ctdef_; ObUpdRtDef &upd_rtdef = merge_rtdefs_.at(i).upd_rtdef_; upd_rtdef.primary_rtdef_ = &merge_rtdefs_.at(0).upd_rtdef_; - if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, upd_rtdef, upd_ctdef, trigger_clear_exprs_))) { + if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, + upd_rtdef, + upd_ctdef, + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init upd rtdef failed", K(ret)); } else if (upd_ctdef.is_primary_index_) { if (OB_FAIL(ObDMLService::process_before_stmt_trigger(upd_ctdef, upd_rtdef, dml_rtctx_, diff --git a/src/sql/engine/dml/ob_table_modify_op.cpp b/src/sql/engine/dml/ob_table_modify_op.cpp index 354fb603d7..5398e073ae 100644 --- a/src/sql/engine/dml/ob_table_modify_op.cpp +++ b/src/sql/engine/dml/ob_table_modify_op.cpp @@ -26,6 +26,7 @@ #include "observer/ob_inner_sql_connection_pool.h" #include "lib/worker.h" #include "share/ob_debug_sync.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { @@ -52,6 +53,7 @@ int ForeignKeyHandle::do_handle(ObTableModifyOp &op, } for (int i = 0; OB_SUCC(ret) && i < dml_ctdef.fk_args_.count(); i++) { const ObForeignKeyArg &fk_arg = dml_ctdef.fk_args_.at(i); + ObForeignKeyChecker *fk_checker = dml_rtdef.fk_checker_array_.at(i); if (OB_SUCC(ret) && !new_row.empty()) { if (ACTION_CHECK_EXIST == fk_arg.ref_action_) { // insert or update. @@ -61,7 +63,7 @@ int ForeignKeyHandle::do_handle(ObTableModifyOp &op, } else if (is_foreign_key_cascade) { // nested update can not check parent row. LOG_DEBUG("skip foreign_key_check_exist in nested session"); - } else if (OB_FAIL(check_exist(op, fk_arg, new_row, false))) { + } else if (OB_FAIL(check_exist(op, fk_arg, new_row, fk_checker, false))) { LOG_WARN("failed to check exist", K(ret), K(fk_arg), K(new_row)); } } @@ -75,7 +77,7 @@ int ForeignKeyHandle::do_handle(ObTableModifyOp &op, K(ret), K(fk_arg), K(old_row), K(new_row)); } else if (!has_changed) { // nothing. - } else if (OB_FAIL(check_exist(op, fk_arg, old_row, true))) { + } else if (OB_FAIL(check_exist(op, fk_arg, old_row, fk_checker, true))) { LOG_WARN("failed to check exist", K(ret), K(fk_arg), K(old_row)); } } else if (ACTION_CASCADE == fk_arg.ref_action_) { @@ -111,6 +113,10 @@ int ForeignKeyHandle::do_handle(ObTableModifyOp &op, } } } + } else if (ACTION_SET_NULL == fk_arg.ref_action_) { + if (OB_FAIL(set_null(op, fk_arg, old_row))) { + LOG_WARN("failed to perform set null for foreign key", K(ret)); + } } } // if (old_row.is_valid()) } // for @@ -149,7 +155,40 @@ int ForeignKeyHandle::value_changed(ObTableModifyOp &op, return ret; } -int ForeignKeyHandle::check_exist(ObTableModifyOp &op, +int ForeignKeyHandle::check_exist(ObTableModifyOp &modify_op, const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &row, ObForeignKeyChecker *fk_checker, bool expect_zero) +{ + int ret = OB_SUCCESS; + if (!expect_zero) { + ret = check_exist_scan_task(modify_op, fk_arg, row, fk_checker, expect_zero); + } else { + ret = check_exist_inner_sql(modify_op, fk_arg, row, expect_zero); + } + return ret; +} + +int ForeignKeyHandle::check_exist_scan_task(ObTableModifyOp &modify_op, const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &row, ObForeignKeyChecker *fk_checker, bool expect_zero) +{ + int ret = OB_SUCCESS; + bool has_result = false; + DEBUG_SYNC(BEFORE_FOREIGN_KEY_CONSTRAINT_CHECK); + if (OB_ISNULL(fk_checker)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key checker is nullptr", K(ret)); + } else if (OB_FAIL(fk_checker->do_fk_check_single_row(fk_arg.columns_, row, has_result))) { + LOG_WARN("failed to perform foreign key check by das scan tasks", K(ret)); + } else { + if (!has_result) { + ret = OB_ERR_NO_REFERENCED_ROW; + } else { + ret = OB_SUCCESS; + } + } + return ret; +} + +int ForeignKeyHandle::check_exist_inner_sql(ObTableModifyOp &op, const ObForeignKeyArg &fk_arg, const ObExprPtrIArray &row, bool expect_zero) @@ -382,6 +421,85 @@ int ForeignKeyHandle::cascade(ObTableModifyOp &op, return ret; } +int ForeignKeyHandle::set_null(ObTableModifyOp &op, + const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &old_row) +{ + int ret = OB_SUCCESS; + static const char *UPDATE_FMT_MYSQL = "update `%.*s`.`%.*s` set %.*s where %.*s"; + static const char *UPDATE_FMT_ORACLE = "update \"%.*s\".\"%.*s\" set %.*s where %.*s"; + + ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, + OB_MALLOC_NORMAL_BLOCK_SIZE, + MTL_ID()); + char *stmt_buf = nullptr; + int64_t stmt_len = 0; + char *where_buf = nullptr; + int64_t where_len = 0; + int64_t stmt_pos = 0; + int64_t where_pos = 0; + const ObString &database_name = fk_arg.database_name_; + const ObString &table_name = fk_arg.table_name_; + if (old_row.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("old row is invalid", K(ret)); + } else if (OB_FAIL(gen_where(op.get_eval_ctx(), where_buf, where_len, where_pos, + fk_arg.columns_, old_row, alloc, op.get_obj_print_params()))) { + if (OB_LIKELY(OB_ERR_NULL_VALUE == ret)) { + // skip cascade if any column in foreign key is NULL. + ret = OB_SUCCESS; + stmt_pos = 0; + } else { + LOG_WARN("failed to gen foreign key where", K(ret), K(old_row), K(fk_arg.columns_)); + } + } else { + const char *update_fmt = lib::is_mysql_mode() ? UPDATE_FMT_MYSQL : UPDATE_FMT_ORACLE; + char *set_buf = nullptr; + int64_t set_len = 0; + int64_t set_pos = 0; + if (OB_FAIL(gen_column_null_value(op.get_eval_ctx(), set_buf, set_len, set_pos, fk_arg.columns_, alloc, op.get_obj_print_params()))) { + LOG_WARN("failed to gen foreign key set null", K(ret), K(fk_arg.columns_)); + } else if (OB_FAIL(databuff_printf(stmt_buf, stmt_len, stmt_pos, alloc, update_fmt, + database_name.length(), database_name.ptr(), + table_name.length(), table_name.ptr(), + static_cast(set_pos), set_buf, + static_cast(where_pos), where_buf))) { + LOG_WARN("failed to print stmt", K(ret), K(table_name), K(set_buf), K(where_buf)); + } else { + stmt_buf[stmt_pos++] = 0; + } + } + + if (OB_SUCC(ret) && stmt_pos > 0) { + LOG_DEBUG("foreign key cascade set null", "stmt", stmt_buf, K(old_row), K(fk_arg)); + if (OB_FAIL(op.begin_nested_session(fk_arg.is_self_ref_))) { + LOG_WARN("failed to begin nested session", K(ret)); + } else { + if (OB_FAIL(op.set_foreign_key_cascade(true))) { + LOG_WARN("failed to set foreign key cascade", K(ret)); + } else if (OB_FAIL(op.execute_write(stmt_buf))) { + LOG_WARN("failed to execute stmt", K(ret), K(stmt_buf)); + } + int reset_ret = op.set_foreign_key_cascade(false); + if (OB_SUCCESS != reset_ret) { + LOG_WARN("failed to reset foreign key cascade", K(reset_ret)); + if (OB_SUCCESS == ret) { + ret = reset_ret; + } + } + int end_ret = op.end_nested_session(); + if (OB_SUCCESS != end_ret) { + LOG_WARN("failed to end nested session", K(end_ret)); + if (OB_SUCCESS == ret) { + ret = end_ret; + } + } + } + } + + return ret; +} + int ForeignKeyHandle::gen_set(ObEvalCtx &eval_ctx, char *&buf, int64_t &len, int64_t &pos, const ObIArray &columns, const ObExprPtrIArray &row, @@ -448,6 +566,45 @@ int ForeignKeyHandle::gen_column_value(ObEvalCtx &eval_ctx, char *&buf, int64_t return ret; } +int ForeignKeyHandle::gen_column_null_value(ObEvalCtx &ctx, char *&buf, int64_t &len, int64_t &pos, + const common::ObIArray &columns, + common::ObIAllocator &alloc, + const common::ObObjPrintParams &print_params) +{ + int ret = OB_SUCCESS; + static const char *COLUMN_FMT_MYSQL = "`%.*s` = "; + static const char *COLUMN_FMT_ORACLE = "\"%.*s\" = "; + const char *column_fmt = lib::is_mysql_mode() ? COLUMN_FMT_MYSQL : COLUMN_FMT_ORACLE; + const char *delimiter = ", "; + if (OB_ISNULL(delimiter) || OB_ISNULL(print_params.tz_info_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("argument is invalid", K(ret), KP(buf), KP(delimiter), KP(print_params.tz_info_)); + } else if (columns.count() <= 0) { + ret= OB_ERR_UNEXPECTED; + LOG_ERROR("columns count of fk is zero or less than zero", K(ret), K(columns.count()), K(columns)); + } + ObObj col_obj; + col_obj.set_null(); + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { + if (OB_SUCC(ret)) { + const ObString &col_name = columns.at(i).name_; + if (OB_FAIL(databuff_printf(buf, len, pos, alloc, column_fmt, + col_name.length(), col_name.ptr()))) { + LOG_WARN("failed to print column name", K(ret), K(col_name)); + } else if (OB_FAIL(col_obj.print_sql_literal(buf, len, pos, alloc, print_params))) { + LOG_WARN("failed to print column value", K(ret), K(col_obj)); + } else if (OB_FAIL(databuff_printf(buf, len, pos, alloc, "%s", delimiter))) { + LOG_WARN("failed to print delimiter", K(ret), K(delimiter)); + } + } + } + if (OB_SUCC(ret)) { + pos -= STRLEN(delimiter); + buf[pos++] = 0; + } + return ret; +} + int ForeignKeyHandle::is_self_ref_row(ObEvalCtx &eval_ctx, const ObExprPtrIArray &row, const ObForeignKeyArg &fk_arg, @@ -925,8 +1082,12 @@ int ObTableModifyOp::set_foreign_key_cascade(bool is_cascade) int ObTableModifyOp::get_foreign_key_cascade(bool &is_cascade) const { int ret = OB_SUCCESS; - OV (OB_NOT_NULL(inner_conn_)); - OZ (inner_conn_->get_foreign_key_cascade(is_cascade)); + if (OB_NOT_NULL(inner_conn_)) { + OZ (inner_conn_->get_foreign_key_cascade(is_cascade)); + } else { + //if inner connection is null, it means not need use inner sql, so set is_cascade false + is_cascade = false; + } return ret; } @@ -1028,7 +1189,7 @@ int ObTableModifyOp::submit_all_dml_task() LOG_WARN("execute all dml das task failed", K(ret)); } else if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { LOG_WARN("close all das task failed", K(ret)); - } else if (OB_FAIL(ObDMLService::handle_after_row_processing(execute_single_row_, &get_dml_modify_row_list()))) { + } else if (OB_FAIL(ObDMLService::handle_after_row_processing(this, &get_dml_modify_row_list()))) { LOG_WARN("perform batch foreign key constraints and after row trigger failed", K(ret)); } else { dml_modify_rows_.clear(); @@ -1129,5 +1290,23 @@ int ObTableModifyOp::inner_get_next_row() } return ret; } + +int ObTableModifyOp::perform_batch_fk_check() +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < fk_checkers_.count(); ++i) { + bool all_has_result = false; + ObForeignKeyChecker *fk_checker = fk_checkers_.at(i); + if (OB_ISNULL(fk_checker)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key checker is null", K(ret), K(i)); + } else if (OB_FAIL(fk_checker->do_fk_check_batch(all_has_result))) { + LOG_WARN("failed to perform batch foreign key check", K(ret)); + } else if (!all_has_result) { + ret = OB_ERR_NO_REFERENCED_ROW; + } + } + return ret; +} } // namespace sql } // namespace oceanbase diff --git a/src/sql/engine/dml/ob_table_modify_op.h b/src/sql/engine/dml/ob_table_modify_op.h index fb6d972c0f..a59a623eb9 100644 --- a/src/sql/engine/dml/ob_table_modify_op.h +++ b/src/sql/engine/dml/ob_table_modify_op.h @@ -17,6 +17,7 @@ #include "sql/engine/dml/ob_dml_ctx_define.h" #include "observer/ob_inner_sql_connection.h" #include "sql/engine/ob_exec_context.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { @@ -42,10 +43,26 @@ private: const ObExprPtrIArray &old_row, const ObExprPtrIArray &new_row, bool &has_changed); - static int check_exist(ObTableModifyOp &modify_op, const ObForeignKeyArg &fk_arg, - const ObExprPtrIArray &row, bool expect_zero); + static int check_exist(ObTableModifyOp &modify_op, + const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &row, + ObForeignKeyChecker *fk_checker, + bool expect_zero); + static int check_exist_inner_sql(ObTableModifyOp &modify_op, + const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &row, + bool expect_zero); + static int check_exist_scan_task(ObTableModifyOp &modify_op, + const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &row, + ObForeignKeyChecker *fk_checker, + bool expect_zero); static int cascade(ObTableModifyOp &modify_op, const ObForeignKeyArg &fk_arg, const ObExprPtrIArray &old_row, const ObExprPtrIArray &new_row); + + static int set_null(ObTableModifyOp &modify_op, const ObForeignKeyArg &fk_arg, + const ObExprPtrIArray &old_row); + static int gen_set(ObEvalCtx &eval_ctx, char *&buf, int64_t &len, int64_t &pos, const common::ObIArray &columns, const ObExprPtrIArray &row, common::ObIAllocator &alloc, @@ -59,6 +76,12 @@ private: const ObExprPtrIArray &row, const char *delimiter, common::ObIAllocator &alloc, const common::ObObjPrintParams &print_params, bool forbid_null); + + static int gen_column_null_value(ObEvalCtx &ctx, char *&buf, int64_t &len, int64_t &pos, + const common::ObIArray &columns, + common::ObIAllocator &alloc, + const common::ObObjPrintParams &print_params); + static int is_self_ref_row(ObEvalCtx &ctx, const ObExprPtrIArray &row, const ObForeignKeyArg &fk_arg, bool &is_self_ref); }; @@ -110,7 +133,8 @@ public: uint64_t use_dist_das_ : 1; uint64_t has_instead_of_trigger_ : 1; // abandoned, don't use again uint64_t is_pdml_update_split_ : 1; // 标记delete, insert op是否由update拆分而来 - uint64_t reserved_ : 56; + uint64_t check_fk_batch_ : 1; // mark if the foreign key constraint can be checked in batch + uint64_t reserved_ : 55; }; }; private: @@ -181,6 +205,7 @@ public: { dml_rtctx_.cleanup(); trigger_clear_exprs_.reset(); + fk_checkers_.reset(); ObOperator::destroy(); } @@ -216,6 +241,8 @@ public: ObDMLModifyRowsList& get_dml_modify_row_list() { return dml_modify_rows_;} int submit_all_dml_task(); + + int perform_batch_fk_check(); protected: OperatorOpenOrder get_operator_open_order() const; virtual int inner_open(); @@ -264,6 +291,7 @@ public: ObErrLogRtDef err_log_rt_def_; ObSEArray trigger_clear_exprs_; ObDMLModifyRowsList dml_modify_rows_; + ObSEArray fk_checkers_; private: ObSQLSessionInfo::StmtSavedValue *saved_session_; char saved_session_buf_[sizeof(ObSQLSessionInfo::StmtSavedValue)] __attribute__((aligned (16)));; diff --git a/src/sql/engine/dml/ob_table_replace_op.cpp b/src/sql/engine/dml/ob_table_replace_op.cpp index c518a6e3d4..94e3797275 100644 --- a/src/sql/engine/dml/ob_table_replace_op.cpp +++ b/src/sql/engine/dml/ob_table_replace_op.cpp @@ -23,6 +23,7 @@ #include "sql/das/ob_das_insert_op.h" #include "sql/das/ob_data_access_service.h" #include "sql/engine/dml/ob_trigger_handler.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { @@ -179,13 +180,14 @@ OB_INLINE int ObTableReplaceOp::init_replace_rtdef() LOG_WARN("allocate insert rtdef failed", K(ret), K(MY_SPEC.replace_ctdefs_.count())); } trigger_clear_exprs_.reset(); + fk_checkers_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < replace_rtdefs_.count(); ++i) { ObReplaceCtDef *replace_ctdef = MY_SPEC.replace_ctdefs_.at(i); const ObInsCtDef *ins_ctdef = replace_ctdef->ins_ctdef_; const ObDelCtDef *del_ctdef = replace_ctdef->del_ctdef_; ObInsRtDef &ins_rtdef = replace_rtdefs_.at(i).ins_rtdef_; ObDelRtDef &del_rtdef = replace_rtdefs_.at(i).del_rtdef_; - OZ(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef, *ins_ctdef, trigger_clear_exprs_)); + OZ(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef, *ins_ctdef, trigger_clear_exprs_, fk_checkers_)); OZ(ObDMLService::init_del_rtdef(dml_rtctx_, del_rtdef, *del_ctdef)); if (OB_SUCC(ret)) { ins_rtdef.das_rtdef_.table_loc_->is_writing_ = true; @@ -472,7 +474,7 @@ int ObTableReplaceOp::do_replace_into() LOG_WARN("fail to load all row", K(ret)); } else if (OB_FAIL(post_all_dml_das_task())) { LOG_WARN("fail to post all das task", K(ret)); - } else if (!check_is_duplicated() && OB_FAIL(ObDMLService::handle_after_row_processing(execute_single_row_, &dml_modify_rows_))) { + } else if (!check_is_duplicated() && OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) { LOG_WARN("try insert is not duplicated, failed to process foreign key handle", K(ret)); } else if (!check_is_duplicated()) { LOG_DEBUG("try insert is not duplicated", K(ret)); @@ -493,7 +495,7 @@ int ObTableReplaceOp::do_replace_into() LOG_WARN("fail to prepare final das task", K(ret)); } else if (OB_FAIL(post_all_dml_das_task())) { LOG_WARN("do insert rows post process failed", K(ret)); - } else if (OB_FAIL(ObDMLService::handle_after_row_processing(execute_single_row_, &dml_modify_rows_))) { + } else if (OB_FAIL(ObDMLService::handle_after_row_processing(this, &dml_modify_rows_))) { LOG_WARN("try insert is duplicated, failed to process foreign key handle", K(ret)); } diff --git a/src/sql/engine/dml/ob_table_update_op.cpp b/src/sql/engine/dml/ob_table_update_op.cpp index 223dab7372..f76477365f 100644 --- a/src/sql/engine/dml/ob_table_update_op.cpp +++ b/src/sql/engine/dml/ob_table_update_op.cpp @@ -17,6 +17,7 @@ #include "sql/engine/dml/ob_dml_service.h" #include "sql/engine/dml/ob_trigger_handler.h" #include "sql/engine/expr/ob_expr_calc_partition_id.h" +#include "sql/engine/dml/ob_fk_checker.h" namespace oceanbase { @@ -214,6 +215,7 @@ OB_INLINE int ObTableUpdateOp::open_table_for_each() LOG_WARN("allocate update rtdef failed", K(ret), K(MY_SPEC.upd_ctdefs_.count())); } trigger_clear_exprs_.reset(); + fk_checkers_.reset(); for (int64_t i = 0; OB_SUCC(ret) && i < upd_rtdefs_.count(); ++i) { UpdRtDefArray &rtdefs = upd_rtdefs_.at(i); const ObTableUpdateSpec::UpdCtDefArray &ctdefs = MY_SPEC.upd_ctdefs_.at(i); @@ -224,7 +226,11 @@ OB_INLINE int ObTableUpdateOp::open_table_for_each() const ObUpdCtDef &upd_ctdef = *ctdefs.at(j); ObUpdRtDef &upd_rtdef = rtdefs.at(j); upd_rtdef.primary_rtdef_ = &rtdefs.at(0); - if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, upd_rtdef, upd_ctdef, trigger_clear_exprs_))) { + if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, + upd_rtdef, + upd_ctdef, + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init upd rtdef failed", K(ret)); } } @@ -481,5 +487,6 @@ int ObTableUpdateOp::write_rows_post_proc(int last_errno) return ret; } + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp index a03a034208..ef097295c4 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp @@ -39,7 +39,8 @@ int ObPxMultiPartInsertOp::inner_open() } else if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, ins_rtdef_, MY_SPEC.ins_ctdef_, - trigger_clear_exprs_))) { + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init insert rtdef failed", K(ret)); } else if (!(MY_SPEC.row_desc_.is_valid())) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp index 8e2ca85daa..47d180905e 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp @@ -42,7 +42,8 @@ int ObPxMultiPartUpdateOp::inner_open() } else if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, upd_rtdef_, MY_SPEC.upd_ctdef_, - trigger_clear_exprs_))) { + trigger_clear_exprs_, + fk_checkers_))) { LOG_WARN("init update rtdef failed", K(ret)); } LOG_TRACE("pdml static update op", K(ret), K_(MY_SPEC.row_desc)); diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index d985ace92f..208f609a21 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -889,6 +889,7 @@ OB_INLINE int ObTableScanOp::init_das_scan_rtdef(const ObDASScanCtDef &das_ctdef das_rtdef.scan_flag_.is_show_seed_ = plan_ctx->get_show_seed(); if(is_foreign_check_nested_session()) { das_rtdef.is_for_foreign_check_ = true; + das_rtdef.scan_flag_.set_for_foreign_key_check(); } if (MY_SPEC.batch_scan_flag_ || is_lookup) { das_rtdef.scan_flag_.scan_order_ = ObQueryFlag::KeepOrder; diff --git a/src/sql/ob_sql_trans_control.cpp b/src/sql/ob_sql_trans_control.cpp index 52fb7a1bb7..fe72c7a64a 100644 --- a/src/sql/ob_sql_trans_control.cpp +++ b/src/sql/ob_sql_trans_control.cpp @@ -737,6 +737,46 @@ int ObSqlTransControl::stmt_refresh_snapshot(ObExecContext &exec_ctx) { return ret; } +int ObSqlTransControl::set_fk_check_snapshot(ObExecContext &exec_ctx) +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = GET_MY_SESSION(exec_ctx); + ObDASCtx &das_ctx = DAS_CTX(exec_ctx); + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(exec_ctx); + const ObPhysicalPlan *plan = plan_ctx->get_phy_plan(); + // insert stmt does not set snapshot by default, set snapshopt for foreign key check induced by insert heres + if (plan->is_plain_insert()) { + transaction::ObTransService *txs = NULL; + auto &snapshot = das_ctx.get_snapshot(); + auto &tx_desc = *session->get_tx_desc(); + int64_t stmt_expire_ts = get_stmt_expire_ts(plan_ctx, *session); + share::ObLSID local_ls_id; + bool local_single_ls_plan = plan->is_local_plan() + && OB_PHY_PLAN_LOCAL == plan->get_location_type() + && has_same_lsid(das_ctx, snapshot, local_ls_id); + if (OB_FAIL(get_tx_service(session, txs))) { + LOG_WARN("failed to get transaction service", K(ret)); + } else { + if (local_single_ls_plan) { + ret = txs->get_ls_read_snapshot(tx_desc, + session->get_tx_isolation(), + local_ls_id, + stmt_expire_ts, + snapshot); + } else { + ret = txs->get_read_snapshot(tx_desc, + session->get_tx_isolation(), + stmt_expire_ts, + snapshot); + } + if (OB_FAIL(ret)) { + LOG_WARN("fail to get snapshot", K(ret), K(local_ls_id), KPC(session)); + } + } + } + return ret; +} + int ObSqlTransControl::stmt_setup_savepoint_(ObSQLSessionInfo *session, ObDASCtx &das_ctx, ObPhysicalPlanCtx *plan_ctx, diff --git a/src/sql/ob_sql_trans_control.h b/src/sql/ob_sql_trans_control.h index a663d19d95..68c41b6d3c 100644 --- a/src/sql/ob_sql_trans_control.h +++ b/src/sql/ob_sql_trans_control.h @@ -197,6 +197,7 @@ public: const ObPhysicalPlanCtx *plan_ctx, transaction::ObTransService *txs); static int stmt_refresh_snapshot(ObExecContext &ctx); + static int set_fk_check_snapshot(ObExecContext &exec_ctx); static int stmt_setup_savepoint_(ObSQLSessionInfo *session, ObDASCtx &das_ctx, ObPhysicalPlanCtx *plan_ctx, diff --git a/src/sql/optimizer/ob_log_del_upd.cpp b/src/sql/optimizer/ob_log_del_upd.cpp index e855f76cbc..ad1a46f1f6 100644 --- a/src/sql/optimizer/ob_log_del_upd.cpp +++ b/src/sql/optimizer/ob_log_del_upd.cpp @@ -660,6 +660,8 @@ int ObLogDelUpd::inner_get_op_exprs(ObIArray &all_exprs, bool need_c LOG_WARN("failed to find trasn info producer", K(ret)); } else if (OB_FAIL(generate_rowid_expr_for_trigger())) { LOG_WARN("failed to try add rowid col expr for trigger", K(ret)); + } else if (OB_FAIL(generate_part_id_expr_for_foreign_key(all_exprs))) { + LOG_WARN("failed to generate part expr for foreign key", K(ret)); } else if (NULL != lock_row_flag_expr_ && OB_FAIL(all_exprs.push_back(lock_row_flag_expr_))) { LOG_WARN("failed to push back expr", K(ret)); } else if (OB_FAIL(append(all_exprs, view_check_exprs_))) { @@ -1307,6 +1309,134 @@ int ObLogDelUpd::generate_lookup_part_id_expr(IndexDMLInfo &index_info) return ret; } +int ObLogDelUpd::generate_fk_lookup_part_id_expr(IndexDMLInfo &index_dml_info) +{ + int ret = OB_SUCCESS; + const uint64_t table_id = index_dml_info.ref_table_id_; + ObLogPlan *log_plan = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + ObSQLSessionInfo *session_info = NULL; + const ObIArray *fk_infos = NULL; + const ObTableSchema *table_schema = NULL; + if (OB_ISNULL(log_plan = get_plan())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema or plan is NULL", K(ret), KP(schema_guard), KP(get_plan())); + } else if (OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_schema_guard()) || + OB_ISNULL(session_info = get_plan()->get_optimizer_context().get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get schama guart", K(ret), K(schema_guard), K(session_info)); + } else if (OB_FAIL(schema_guard->get_table_schema(session_info->get_effective_tenant_id(), + index_dml_info.ref_table_id_, + table_schema))) { + LOG_WARN("failed to get table schema", K(index_dml_info.ref_table_id_), K(ret)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(index_dml_info.ref_table_id_), K(ret)); + } else if (!table_schema->is_user_table()) { + // do nothing, especially for global index. + LOG_DEBUG("skip generate partition key for foreign key", + "table_name", table_schema->get_table_name_str(), + "table_type", table_schema->get_table_type()); + } else if (OB_ISNULL(fk_infos = &table_schema->get_foreign_key_infos())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key infos is null", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < fk_infos->count(); i++) { + const ObForeignKeyInfo &fk_info = fk_infos->at(i); + ObRawExpr* fk_scan_part_expr = nullptr; + if (fk_info.table_id_ != fk_info.child_table_id_) { + // update parent table, check child table, don't use das task to perform foreign key check + ret = index_dml_info.fk_lookup_part_id_expr_.push_back(fk_scan_part_expr); + } else if (!fk_info.is_parent_table_mock_) { + const uint64_t parent_table_id = fk_info.parent_table_id_; + const ObTableSchema *parent_table_schema = NULL; + uint64_t scan_index_tid = OB_INVALID_ID; + if (OB_FAIL(schema_guard->get_table_schema(session_info->get_effective_tenant_id(), + parent_table_id, + parent_table_schema))) { + LOG_WARN("failed to get table schema of parent table", K(ret), K(parent_table_id)); + } else if (OB_ISNULL(parent_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("parent table not exist", K(ret), K(parent_table_id)); + } else if (OB_FAIL(parent_table_schema->get_fk_check_index_tid(*schema_guard, fk_info.parent_column_ids_, scan_index_tid))) { + LOG_WARN("failed to get index tid used to build scan das task for foreign key checks", K(ret)); + } else if (OB_INVALID_ID == scan_index_tid) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid table id to build das scan task for foreign key checks", K(ret)); + } else { + ObRawExpr* fk_look_up_part_id_expr = nullptr; + const ObTableSchema* scan_table_schema = nullptr; + if (schema_guard->get_table_schema(session_info->get_effective_tenant_id(), + scan_index_tid, + scan_table_schema)) { + LOG_WARN("failed to get scan table schema to perform foreign key check", K(ret), K(scan_index_tid)); + } else if (OB_ISNULL(scan_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table schema scanned for foreign key check not exist", K(ret)); + } else if (scan_table_schema->get_part_level() != PARTITION_LEVEL_ZERO && + OB_FAIL(log_plan->gen_calc_part_id_expr(fk_info.foreign_key_id_, // init table_id by foreign key id + scan_index_tid, + CALC_PARTITION_TABLET_ID, + fk_look_up_part_id_expr))) { + LOG_WARN("failed to gen calc part id expr", K(ret)); + } else if (OB_FAIL(index_dml_info.fk_lookup_part_id_expr_.push_back(fk_look_up_part_id_expr))) { + LOG_WARN("failed to push part id expr to array", K(ret), K(index_dml_info.fk_lookup_part_id_expr_)); + } + } + } + } + } + return ret; +} + +int ObLogDelUpd::convert_insert_new_fk_lookup_part_id_expr(ObIArray &all_exprs, IndexDMLInfo &index_dml_info) +{ + int ret = OB_SUCCESS; + ObSEArray dml_columns; + ObSEArray dml_values; + if (OB_FAIL(get_insert_exprs(index_dml_info, dml_columns, dml_values))) { + LOG_WARN("failed to get insert exprs", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < index_dml_info.fk_lookup_part_id_expr_.count(); ++i) { + ObRawExpr *fk_look_up_part_id_expr = index_dml_info.fk_lookup_part_id_expr_.at(i); + if (OB_ISNULL(fk_look_up_part_id_expr)) { + //nothing to do + } else if (OB_FAIL(replace_expr_for_fk_part_expr(dml_columns, + dml_values, + fk_look_up_part_id_expr))) { + LOG_WARN("failed to replace column ref expr for partition expr used for foreign key check", K(ret)); + } else if (OB_FAIL(all_exprs.push_back(fk_look_up_part_id_expr))) { + LOG_WARN("failed to push foreign key check partition expr to all exprs", K(ret)); + } + } + } + return ret; +} + +int ObLogDelUpd::convert_update_new_fk_lookup_part_id_expr(ObIArray &all_exprs, IndexDMLInfo &index_dml_info) +{ + int ret = OB_SUCCESS; + ObSEArray dml_columns; + ObSEArray dml_values; + if (OB_FAIL(get_update_exprs(index_dml_info, dml_columns, dml_values))) { + LOG_WARN("failed to get insert exprs", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < index_dml_info.fk_lookup_part_id_expr_.count(); ++i) { + ObRawExpr *fk_look_up_part_id_expr = index_dml_info.fk_lookup_part_id_expr_.at(i); + if (OB_ISNULL(fk_look_up_part_id_expr)) { + + } else if (OB_FAIL(replace_expr_for_fk_part_expr(dml_columns, + dml_values, + fk_look_up_part_id_expr))) { + LOG_WARN("failed to replace column ref expr for partition expr used for foreign key check", K(ret)); + } else if (OB_FAIL(all_exprs.push_back(fk_look_up_part_id_expr))) { + LOG_WARN("failed to push foreign key check partition expr to all exprs", K(ret)); + } + } + } + return ret; +} + int ObLogDelUpd::generate_insert_new_calc_partid_expr(IndexDMLInfo &index_dml_info) { int ret = OB_SUCCESS; @@ -1364,6 +1494,33 @@ int ObLogDelUpd::convert_expr_by_dml_operation(const ObIArray &dml_ return ret; } +int ObLogDelUpd::replace_expr_for_fk_part_expr(const ObIArray &dml_columns, + const ObIArray &dml_new_values, + ObRawExpr *fk_part_id_expr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(get_plan())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("current value expr is null", K(ret), K(get_plan())); + } else { + ObRawExprCopier copier(get_plan()->get_optimizer_context().get_expr_factory()); + if (OB_FAIL(copier.add_replaced_expr(dml_columns, dml_new_values))) { + LOG_WARN("failed to add replace pair", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < fk_part_id_expr->get_param_count(); ++i) { + ObRawExpr *param = fk_part_id_expr->get_param_expr(i); + ObRawExpr *new_param = NULL; + if (OB_FAIL(copier.copy_on_replace(param, new_param))) { + LOG_WARN("failed to static replace expr", K(ret)); + } else { + fk_part_id_expr->get_param_expr(i) = new_param; + } + } + } + } + return ret; +} + int ObLogDelUpd::get_update_exprs(const IndexDMLInfo &dml_info, ObIArray &dml_columns, ObIArray &dml_values) diff --git a/src/sql/optimizer/ob_log_del_upd.h b/src/sql/optimizer/ob_log_del_upd.h index 2f5da1774e..c03713376c 100644 --- a/src/sql/optimizer/ob_log_del_upd.h +++ b/src/sql/optimizer/ob_log_del_upd.h @@ -43,7 +43,8 @@ public: old_rowid_expr_(NULL), new_rowid_expr_(NULL), trans_info_expr_(NULL), - related_index_ids_() + related_index_ids_(), + fk_lookup_part_id_expr_() { } inline void reset() @@ -72,6 +73,7 @@ public: new_rowid_expr_ = NULL, trans_info_expr_ = NULL, related_index_ids_.reset(); + fk_lookup_part_id_expr_.reset(); } int64_t to_explain_string(char *buf, int64_t buf_len, ExplainType type) const; int init_assignment_info(const ObAssignments &assignments, @@ -169,6 +171,8 @@ public: // local index id related to current dml TableIDArray related_index_ids_; + common::ObSEArray fk_lookup_part_id_expr_; + TO_STRING_KV(K_(table_id), K_(ref_table_id), K_(loc_table_id), @@ -330,12 +334,19 @@ public: virtual int is_my_fixed_expr(const ObRawExpr *expr, bool &is_fixed) override; protected: virtual int generate_rowid_expr_for_trigger() = 0; + virtual int generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) = 0; virtual int generate_multi_part_partition_id_expr() = 0; int generate_old_rowid_expr(IndexDMLInfo &table_dml_info); int generate_update_new_rowid_expr(IndexDMLInfo &table_dml_info); int generate_insert_new_rowid_expr(IndexDMLInfo &table_dml_info); int generate_old_calc_partid_expr(IndexDMLInfo &index_info); int generate_lookup_part_id_expr(IndexDMLInfo &index_info); + int generate_fk_lookup_part_id_expr(IndexDMLInfo &index_info); + int convert_insert_new_fk_lookup_part_id_expr(ObIArray &all_exprs,IndexDMLInfo &index_dml_info); + int convert_update_new_fk_lookup_part_id_expr(ObIArray &all_exprs, IndexDMLInfo &index_dml_info); + int replace_expr_for_fk_part_expr(const ObIArray &dml_columns, + const ObIArray &dml_new_values, + ObRawExpr *fk_part_id_expr); int generate_insert_new_calc_partid_expr(IndexDMLInfo &index_dml_info); int generate_update_new_calc_partid_expr(IndexDMLInfo &index_dml_info); diff --git a/src/sql/optimizer/ob_log_delete.cpp b/src/sql/optimizer/ob_log_delete.cpp index b38c7e68b8..f91b91a0a9 100644 --- a/src/sql/optimizer/ob_log_delete.cpp +++ b/src/sql/optimizer/ob_log_delete.cpp @@ -149,6 +149,15 @@ int ObLogDelete::generate_rowid_expr_for_trigger() return ret; } +int ObLogDelete::generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) +{ + // NOTE: for delete parent table, don't support foregin key checks use das task now, + // no need to generate part id expr, do nothing here + int ret = OB_SUCCESS; + + return ret; +} + int ObLogDelete::get_plan_item_info(PlanText &plan_text, ObSqlPlanItem &plan_item) { diff --git a/src/sql/optimizer/ob_log_delete.h b/src/sql/optimizer/ob_log_delete.h index 957f93bc24..5d84e4a778 100644 --- a/src/sql/optimizer/ob_log_delete.h +++ b/src/sql/optimizer/ob_log_delete.h @@ -36,6 +36,7 @@ public: ObSqlPlanItem &plan_item) override; protected: virtual int generate_rowid_expr_for_trigger() override; + virtual int generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) override; virtual int generate_multi_part_partition_id_expr() override; private: DISALLOW_COPY_AND_ASSIGN(ObLogDelete); diff --git a/src/sql/optimizer/ob_log_insert.cpp b/src/sql/optimizer/ob_log_insert.cpp index 9d72c54407..3c1f5c2ffd 100644 --- a/src/sql/optimizer/ob_log_insert.cpp +++ b/src/sql/optimizer/ob_log_insert.cpp @@ -365,6 +365,58 @@ int ObLogInsert::generate_rowid_expr_for_trigger() return ret; } +int ObLogInsert::generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < get_index_dml_infos().count(); ++i) { + IndexDMLInfo *dml_info = get_index_dml_infos().at(i); + if (OB_ISNULL(dml_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dml info is null", K(ret), K(dml_info)); + } else if (!dml_info->is_primary_index_) { + // do nothing + } else if (OB_FAIL(generate_fk_lookup_part_id_expr(*dml_info))) { + LOG_WARN("failed to generate lookup part expr for foreign key", K(ret)); + } else if (OB_FAIL(convert_insert_new_fk_lookup_part_id_expr(all_exprs, *dml_info))) { + LOG_WARN("failed to convert lookup part expr for foreign key", K(ret)); + } + } + + if (OB_FAIL(ret)) { + /*do nothing*/ + } else if (is_replace()) { + for (int64_t i = 0; OB_SUCC(ret) && i < get_replace_index_dml_infos().count(); ++i) { + IndexDMLInfo *dml_info = get_replace_index_dml_infos().at(i); + if (OB_ISNULL(dml_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dml info is null", K(ret), K(dml_info)); + } else if (!dml_info->is_primary_index_) { + // do nothing + } else if (OB_FAIL(generate_fk_lookup_part_id_expr(*dml_info))) { + LOG_WARN("failed to generate lookup part expr for foreign key", K(ret)); + } else if (OB_FAIL(convert_update_new_fk_lookup_part_id_expr(all_exprs, *dml_info))) { + LOG_WARN("failed to convert lookup part expr for foreign key", K(ret)); + } + } + } else if (get_insert_up()) { + for (int64_t i = 0; OB_SUCC(ret) && i < get_insert_up_index_dml_infos().count(); ++i) { + IndexDMLInfo *dml_info = get_insert_up_index_dml_infos().at(i); + if (OB_ISNULL(dml_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dml info is null", K(ret), K(dml_info)); + } else if (!dml_info->is_primary_index_) { + // do nothing + } else if (OB_FAIL(generate_fk_lookup_part_id_expr(*dml_info))) { + LOG_WARN("failed to generate lookup part expr for foreign key", K(ret)); + } else if (OB_FAIL(convert_update_new_fk_lookup_part_id_expr(all_exprs, *dml_info))) { + LOG_WARN("failed to convert lookup part expr for foreign key", K(ret)); + } + } + } + + return ret; +} + int ObLogInsert::generate_multi_part_partition_id_expr() { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_insert.h b/src/sql/optimizer/ob_log_insert.h index 80c1f82d5c..2afdf9a323 100644 --- a/src/sql/optimizer/ob_log_insert.h +++ b/src/sql/optimizer/ob_log_insert.h @@ -113,6 +113,7 @@ public: protected: int get_constraint_info_exprs(ObIArray &all_exprs); virtual int generate_rowid_expr_for_trigger() override; + virtual int generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) override; virtual int generate_multi_part_partition_id_expr() override; protected: bool is_replace_; diff --git a/src/sql/optimizer/ob_log_merge.cpp b/src/sql/optimizer/ob_log_merge.cpp index 3a0e85f05b..c0c3fb55c2 100644 --- a/src/sql/optimizer/ob_log_merge.cpp +++ b/src/sql/optimizer/ob_log_merge.cpp @@ -300,6 +300,45 @@ int ObLogMerge::generate_rowid_expr_for_trigger() return ret; } +int ObLogMerge::generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) +{ + int ret = OB_SUCCESS; + const ObMergeStmt *merge_stmt = static_cast(get_stmt()); + if (OB_ISNULL(merge_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("merge stmt is null", K(ret), K(merge_stmt)); + } else if (merge_stmt->has_insert_clause()) { + for (int64_t i = 0; OB_SUCC(ret) && i < get_index_dml_infos().count(); ++i) { + IndexDMLInfo *ins_info = get_index_dml_infos().at(i); + if (OB_ISNULL(ins_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dml info is null", K(ret), K(ins_info)); + } else if (!ins_info->is_primary_index_) { + // do nothing + } else if (OB_FAIL(generate_fk_lookup_part_id_expr(*ins_info))) { + LOG_WARN("failed to generate lookup part expr for foreign key", K(ret)); + } else if (OB_FAIL(convert_insert_new_fk_lookup_part_id_expr(all_exprs,*ins_info))) { + LOG_WARN("failed to convert lookup part expr for foreign key", K(ret)); + } + } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < get_update_infos().count(); ++i) { + IndexDMLInfo *upd_info = get_update_infos().at(i); + if (OB_ISNULL(upd_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update info is null", K(ret), K(upd_info)); + } else if (!upd_info->is_primary_index_) { + // do nothing + } else if (OB_FAIL(generate_fk_lookup_part_id_expr(*upd_info))) { + LOG_WARN("failed to generate lookup part expr for foreign key", K(ret)); + } else if (OB_FAIL(convert_update_new_fk_lookup_part_id_expr(all_exprs, *upd_info))) { + LOG_WARN("failed to convert lookup part expr for foreign key", K(ret)); + } + } + return ret; +} + int ObLogMerge::generate_multi_part_partition_id_expr() { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_merge.h b/src/sql/optimizer/ob_log_merge.h index 23732a690c..986c62ba44 100644 --- a/src/sql/optimizer/ob_log_merge.h +++ b/src/sql/optimizer/ob_log_merge.h @@ -57,6 +57,7 @@ public: ObSqlPlanItem &plan_item) override; protected: int generate_rowid_expr_for_trigger() override; + virtual int generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) override; int generate_multi_part_partition_id_expr() override; virtual int gen_location_constraint(void *ctx) override; DISALLOW_COPY_AND_ASSIGN(ObLogMerge); diff --git a/src/sql/optimizer/ob_log_update.cpp b/src/sql/optimizer/ob_log_update.cpp index ca13da79dc..474a931242 100644 --- a/src/sql/optimizer/ob_log_update.cpp +++ b/src/sql/optimizer/ob_log_update.cpp @@ -200,6 +200,25 @@ int ObLogUpdate::generate_rowid_expr_for_trigger() return ret; } +int ObLogUpdate::generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < get_index_dml_infos().count(); ++i) { + IndexDMLInfo *dml_info = get_index_dml_infos().at(i); + if (OB_ISNULL(dml_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dml info is null", K(ret), K(dml_info)); + } else if (!dml_info->is_primary_index_) { + // do nothing + } else if (OB_FAIL(generate_fk_lookup_part_id_expr(*dml_info))) { + LOG_WARN("failed to generate lookup part expr for foreign key", K(ret)); + } else if (OB_FAIL(convert_update_new_fk_lookup_part_id_expr(all_exprs, *dml_info))) { + LOG_WARN("failed to convert lookup part expr for foreign key", K(ret)); + } + } + return ret; +} + int ObLogUpdate::generate_multi_part_partition_id_expr() { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_update.h b/src/sql/optimizer/ob_log_update.h index 477177d26b..e522aad3f6 100644 --- a/src/sql/optimizer/ob_log_update.h +++ b/src/sql/optimizer/ob_log_update.h @@ -35,6 +35,7 @@ public: ObSqlPlanItem &plan_item) override; private: virtual int generate_rowid_expr_for_trigger() override; + virtual int generate_part_id_expr_for_foreign_key(ObIArray &all_exprs) override; virtual int generate_multi_part_partition_id_expr() override; private: DISALLOW_COPY_AND_ASSIGN(ObLogUpdate); diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 7801044ad2..e4d5baa02f 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -7901,9 +7901,18 @@ int ObDDLResolver::resolve_foreign_key_options(const ParseNode *node, action = ACTION_CASCADE; break; case T_SET_NULL: - // action = ACTION_SET_NULL; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "create foreign key with set null option"); + { + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), data_version))) { + LOG_WARN("failed to get data version", K(ret), K(session_info_->get_effective_tenant_id())); + } else if (data_version >= DATA_VERSION_4_2_1_0) { + action = ACTION_SET_NULL; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("current tenant data version is less than 4.2.1.0, foreign key set null not supported", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "create foreign key with set null option"); + } + } break; case T_NO_ACTION: action = ACTION_NO_ACTION; @@ -7975,9 +7984,18 @@ int ObDDLResolver::resolve_foreign_key_option(const ParseNode *option_node, action = ACTION_CASCADE; break; case T_SET_NULL: - // action = ACTION_SET_NULL; - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "create foreign key with set null option"); + { + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), data_version))) { + LOG_WARN("failed to get data version", K(ret), K(session_info_->get_effective_tenant_id())); + } else if (data_version >= DATA_VERSION_4_2_1_0) { + action = ACTION_SET_NULL; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("current tenant data version is less than 4.2.1.0, foreign key set null not supported", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "create foreign key with set null option"); + } + } break; default: ret = OB_INVALID_ARGUMENT; @@ -8119,6 +8137,7 @@ int ObDDLResolver::resolve_foreign_key_state( // description: 检查创建外键时父表和子表的外键列数量和类型是否匹配 // 检查创建外键时父表的外键列是否是主键或者唯一索引 // 检查当外键属于自引用时，外键列是否不为同一列 +// 检查外键的casacde行为是否和schema里的定义相冲突 例如 set_null 和 not_null // // @param [in] arg 已经存有外键信息的 ObCreateForeignKeyArg // @return oceanbase error code defined in lib/ob_errno.def @@ -8303,6 +8322,34 @@ int ObDDLResolver::check_foreign_key_reference( } } } + if (OB_SUCC(ret) && (arg.delete_action_ == ACTION_SET_NULL || arg.update_action_ == ACTION_SET_NULL)) { + // To compatible with oracle and mysql, check if set null ref action is valid + // More detail can be found in: + for (int64_t i = 0; OB_SUCC(ret) && i < arg.child_columns_.count(); ++i) { + const ObString &fk_col_name = arg.child_columns_.at(i); + const ObColumnSchemaV2 *fk_col_schema = child_table_schema->get_column_schema(fk_col_name); + if (OB_ISNULL(fk_col_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key column schema is null", K(ret), K(i)); + } else if (fk_col_schema->is_generated_column()) { + ret = OB_ERR_UNSUPPORTED_FK_SET_NULL_ON_GENERATED_COLUMN; + LOG_WARN("foreign key column is generated column", K(ret), K(i), K(fk_col_name)); + } else if (!fk_col_schema->is_nullable() && lib::is_mysql_mode()) { + ret = OB_ERR_FK_COLUMN_NOT_NULL; + LOG_USER_ERROR(OB_ERR_FK_COLUMN_NOT_NULL, to_cstring(fk_col_name), to_cstring(arg.foreign_key_name_)); + } else if (lib::is_mysql_mode() ) { + // check if fk column is base column of virtual generated column in MySQL mode + const uint64_t fk_col_id = fk_col_schema->get_column_id(); + bool is_stored_base_col = false; + if (OB_FAIL(child_table_schema->check_is_stored_generated_column_base_column(fk_col_id, is_stored_base_col))) { + LOG_WARN("failed to check foreign key column is virtual generated column base column", K(ret), K(i)); + } else if (is_stored_base_col) { + ret = OB_ERR_CANNOT_ADD_FOREIGN; + LOG_WARN("foreign key column is the base column of stored generated column in mysql mode", K(ret), K(i), K(fk_col_name)); + } + } + } + } } return ret; diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index eefcbf52f0..fda25bb91d 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -5326,6 +5326,191 @@ int ObDMLResolver::resolve_table_partition_expr(const TableItem &table_item, con return ret; } +int ObDMLResolver::resolve_foreign_key_constraint(const TableItem *table_item) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = nullptr; + if (OB_ISNULL(table_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret)); + } else if (!table_item->is_basic_table()) { + //nothing to do, only resolve foreign key constraint for basic table + } else if (OB_FAIL(schema_checker_->get_table_schema(MTL_ID(), table_item->ref_id_, table_schema))) { + LOG_WARN("get table schema failed", K_(table_item->table_name), K(table_item->ref_id_), K(ret)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table schema not exist", K(ret), K(table_item->ref_id_)); + } else if (OB_FAIL(resolve_fk_table_partition_expr(*table_item, *table_schema))) { + LOG_WARN("failed to resolve partition expr used for foreign key check", K(ret)); + } + return ret; +} + +int ObDMLResolver::resolve_fk_table_partition_expr(const TableItem &table_item, const ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + ObDMLStmt *dml_stmt = get_stmt(); + if (OB_ISNULL(dml_stmt)) { + ret = OB_NOT_INIT; + LOG_WARN("dml_stmt is null", K(ret)); + } else if (table_schema.get_foreign_key_infos().count() > 0) { + const common::ObIArray & foreign_key_infos = table_schema.get_foreign_key_infos(); + for (int64_t i = 0; i < foreign_key_infos.count(); i++) { + const ObForeignKeyInfo &foreign_key_info = foreign_key_infos.at(i); + const uint64_t parent_table_id = foreign_key_info.parent_table_id_; + const uint64_t child_table_id = foreign_key_info.child_table_id_; + if (child_table_id == table_schema.get_table_id() && !foreign_key_info.is_parent_table_mock_) { + const ObTableSchema *parent_table_schema = nullptr; + bool parent_key_is_pkey = false; + const common::ObSEArray &parent_column_ids = foreign_key_info.parent_column_ids_; + const ObTableSchema *resolve_table_schema = nullptr; + uint64_t fk_scan_tid = OB_INVALID_ID; + if (OB_FAIL(schema_checker_->get_table_schema(session_info_->get_effective_tenant_id(), parent_table_id, parent_table_schema))) { //NOTE: Can we use this function to get schema here + if (OB_TABLE_NOT_EXIST == ret) { + // Note: Parent table not exist, return OB_ERR_NO_REFERENCED_ROW instead of table not exist to ensure compatibility + ret = OB_ERR_NO_REFERENCED_ROW; + } + LOG_WARN("get parent table schema from schema checker failed", K(ret), K(parent_table_id)); + } else if (OB_ISNULL(parent_table_schema)) { + // Note: Parent table not exist, return OB_ERR_NO_REFERENCED_ROW instead of table not exist to ensure compatibility + ret = OB_ERR_NO_REFERENCED_ROW; + LOG_WARN("parent table not exists", K(parent_table_id)); + } else if (OB_FAIL(parent_table_schema->get_fk_check_index_tid(*schema_checker_->get_schema_guard(), parent_column_ids, fk_scan_tid))) { + LOG_WARN("failed to get table id to perform scan task for foreign key check", K(ret)); + } else if (OB_INVALID_ID == fk_scan_tid) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid table id to perform scan task for foregin key check", K(ret)); + } else if (OB_FAIL(schema_checker_->get_table_schema(session_info_->get_effective_tenant_id(), fk_scan_tid, resolve_table_schema))) { + LOG_WARN("failed to get table schema to perform foreign key check", K(ret), K(fk_scan_tid)); + } else if (OB_ISNULL(resolve_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table schema used to perform foreign key check not exist", K(ret), K(fk_scan_tid)); + } else { + ObRawExpr *parent_part_expr = nullptr; + ObRawExpr *parent_subpart_expr = nullptr; + const ObString &part_str = resolve_table_schema->get_part_option().get_part_func_expr_str(); + ObPartitionFuncType part_type = resolve_table_schema->get_part_option().get_part_func_type(); + // NOTE: for parent index table, can we still use table_item of child table here + if (resolve_table_schema->get_part_level() != PARTITION_LEVEL_ZERO) { + if (OB_FAIL(resolve_partition_expr(table_item, *resolve_table_schema, part_type, part_str, parent_part_expr, true, &foreign_key_info))) { + LOG_WARN("Failed to resolve partition expr", K(ret), K(part_str), K(part_type)); + } else if (PARTITION_LEVEL_TWO == resolve_table_schema->get_part_level()) { + const ObString &parent_subpart_str = resolve_table_schema->get_sub_part_option().get_part_func_expr_str(); + ObPartitionFuncType parent_subpart_type = resolve_table_schema->get_sub_part_option().get_part_func_type(); + if (OB_FAIL(resolve_partition_expr(table_item, *parent_table_schema, parent_subpart_type, parent_subpart_str, parent_subpart_expr, true, &foreign_key_info))) { + LOG_WARN("Failed to resolve partition expr", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(dml_stmt->set_part_expr(foreign_key_info.foreign_key_id_, fk_scan_tid, + parent_part_expr, parent_subpart_expr))) { + LOG_WARN("set part expr to dml stmt failed", K(ret)); + } else { + LOG_TRACE("resolve partition expr", K(table_item), KPC(parent_part_expr), K(part_str)); + } + } + } + } + } + } + return ret; +} + +int ObDMLResolver::map_to_fk_column_name(const ObTableSchema &child_table_schema, + const ObTableSchema &parent_table_schema, + const ObForeignKeyInfo &fk_info, + const ObString &pk_col_name, + ObString &fk_col_name) +{ + int ret = OB_SUCCESS; + bool is_column_exist = false; + if (OB_ISNULL(parent_table_schema.get_column_schema(pk_col_name))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parent column schema is null", K(ret),K(pk_col_name)); + } else { + const uint64_t pk_col_id = parent_table_schema.get_column_schema(pk_col_name)->get_column_id(); + uint64_t fk_col_id = OB_INVALID_ID; + if OB_FAIL(fk_info.get_child_column_id(pk_col_id, fk_col_id)) { + LOG_WARN("failed to get child column id according parent column id", K(ret)); + } else { + child_table_schema.get_column_name_by_column_id(fk_col_id, fk_col_name, is_column_exist); + if (!is_column_exist) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child column is not exist", K(ret), K(fk_col_id)); + } + } + } + return ret; +} + +int ObDMLResolver::resolve_columns_for_fk_partition_expr(ObRawExpr *&expr, + ObIArray &columns, + const TableItem &table_item, // table_item of dml table(child_table) + const ObTableSchema &parent_table_schema, + const ObForeignKeyInfo *fk_info) +{ + int ret = OB_SUCCESS; + const uint64_t child_table_id = fk_info->child_table_id_; + const ObTableSchema *child_table_schema = nullptr; + if (OB_ISNULL(fk_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("foreign key info is null", K(ret)); + } else if (OB_FAIL(schema_checker_->get_table_schema(MTL_ID(),child_table_id, child_table_schema))) { + LOG_WARN("failed to get child table schema", K(ret)); + } else { + ObArray real_exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { + ObQualifiedName &q_name = columns.at(i); + ObRawExpr *real_ref_expr = NULL; + ObString child_col_name; + if (q_name.is_sys_func()) { + if (OB_FAIL(resolve_qualified_identifier(q_name, columns, real_exprs, real_ref_expr))) { + LOG_WARN("resolve sysfunc expr failed", K(q_name), K(ret)); + } else if (OB_ISNULL(real_ref_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is NULL", K(ret)); + } else if (!real_ref_expr->is_sys_func_expr()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid exor", K(*real_ref_expr), K(ret)); + } else { + ObSysFunRawExpr *sys_func_expr = static_cast(real_ref_expr); + if (OB_FAIL(sys_func_expr->check_param_num())) { + LOG_WARN("sys func check param failed", K(ret)); + } + } + } else if (OB_FAIL(map_to_fk_column_name(*child_table_schema, parent_table_schema, *fk_info, q_name.col_name_, child_col_name))) { + LOG_WARN("failed to map parent column name to child column name", K(ret)); + } else { + ColumnItem *column_item = NULL; + if (OB_ISNULL(child_col_name)) { + LOG_WARN("failed to get column name of foreign key column", K(ret), K(q_name.col_name_)); + } else if (OB_FAIL(resolve_basic_column_item(table_item, child_col_name, parent_table_schema.is_oracle_tmp_table(), column_item))) { + LOG_WARN("resolve basic column item failed", K(i), K(q_name), K(ret)); + } else { + real_ref_expr = column_item->expr_; + } + } + + if (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && i < real_exprs.count(); ++i) { + if (OB_FAIL(ObRawExprUtils::replace_ref_column(real_ref_expr, columns.at(i).ref_expr_, real_exprs.at(i)))) { + LOG_WARN("failed to replace real expr", K(i), K(ret)); + } + } + if (OB_FAIL(ret)) { + // nothing to do + } else if (OB_FAIL(real_exprs.push_back(real_ref_expr))) { + LOG_WARN("failed to push back real ref exprs", K(ret)); + } else if OB_FAIL((ObRawExprUtils::replace_ref_column(expr, q_name.ref_expr_, real_ref_expr))) { + LOG_WARN("failed to replace real ref column", K(ret)); + } + } + } + } + return ret; +} + + //for recursively process columns item in resolve_partition_expr //just wrap columns process logic in resolve_partition_expr int ObDMLResolver::resolve_columns_for_partition_expr(ObRawExpr *&expr, @@ -5372,13 +5557,14 @@ int ObDMLResolver::resolve_columns_for_partition_expr(ObRawExpr *&expr, } return ret; } - int ObDMLResolver::resolve_partition_expr( const TableItem &table_item, const ObTableSchema &table_schema, const ObPartitionFuncType part_type, const ObString &part_str, - ObRawExpr *&expr) + ObRawExpr *&expr, + bool for_fk, + const ObForeignKeyInfo *fk_info) { int ret = OB_SUCCESS; ObArray columns; @@ -5489,7 +5675,14 @@ int ObDMLResolver::resolve_partition_expr( } } if (OB_SUCC(ret)) { - if (OB_FAIL(resolve_columns_for_partition_expr(expr, columns, table_item, table_schema.is_oracle_tmp_table()))) { + if (for_fk) { + if (OB_ISNULL(fk_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fk_info is nullptr when resolve foreign key part expr", K(ret)); + } else if (OB_FAIL(resolve_columns_for_fk_partition_expr(expr, columns, table_item, table_schema, fk_info))) { + LOG_WARN("resolve columns for parent table partition expr failed", K(ret)); + } + } else if (OB_FAIL(resolve_columns_for_partition_expr(expr, columns, table_item, table_schema.is_oracle_tmp_table()))) { LOG_WARN("resolve columns for partition expr failed", K(ret)); } } diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 1a7ee7a25a..0df8684092 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -256,6 +256,21 @@ public: JoinedTable* &joined_table); int resolve_table_partition_expr(const TableItem &table_item, const share::schema::ObTableSchema &table_schema); + int resolve_fk_table_partition_expr(const TableItem &table_item, const ObTableSchema &table_schema); + + int resolve_foreign_key_constraint(const TableItem *table_item); + + // map parent key column name to foreign key column name + int map_to_fk_column_name(const ObTableSchema &child_table_schema, + const ObTableSchema &parent_table_schema, + const ObForeignKeyInfo &fk_info, + const ObString &pk_col_name, + ObString &fk_col_name); + int resolve_columns_for_fk_partition_expr(ObRawExpr *&expr, + ObIArray &columns, + const TableItem &table_item, // table_item of dml table(child_table) + const ObTableSchema &parent_table_schema, + const ObForeignKeyInfo *fk_info); virtual int resolve_column_ref_expr(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr); int resolve_sql_expr(const ParseNode &node, ObRawExpr *&expr, ObArray *input_columns = NULL); @@ -263,7 +278,9 @@ public: const share::schema::ObTableSchema &table_schema, const share::schema::ObPartitionFuncType part_type, const common::ObString &part_str, - ObRawExpr *&expr); + ObRawExpr *&expr, + bool for_fk = false, + const ObForeignKeyInfo *fk_info = nullptr); static int resolve_special_expr_static(const ObTableSchema *table_schema, const ObSQLSessionInfo &session_info, ObRawExprFactory &expr_factory, diff --git a/src/sql/resolver/dml/ob_insert_resolver.cpp b/src/sql/resolver/dml/ob_insert_resolver.cpp index 40af1fa43c..f221959c17 100644 --- a/src/sql/resolver/dml/ob_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_insert_resolver.cpp @@ -424,6 +424,9 @@ int ObInsertResolver::resolve_insert_field(const ParseNode &insert_into, TableIt session_info_->set_table_name_hidden(old_flag); } OZ(column_namespace_checker_.add_reference_table(table_item)); + if (OB_SUCC(ret) && OB_FAIL(resolve_foreign_key_constraint(table_item))) { + LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); + } if (OB_SUCC(ret)) { current_scope_ = T_INSERT_SCOPE; const ObTableSchema *table_schema = NULL; diff --git a/src/sql/resolver/dml/ob_merge_resolver.cpp b/src/sql/resolver/dml/ob_merge_resolver.cpp index 7f2842edb2..a361d898a6 100644 --- a/src/sql/resolver/dml/ob_merge_resolver.cpp +++ b/src/sql/resolver/dml/ob_merge_resolver.cpp @@ -228,6 +228,8 @@ int ObMergeResolver::resolve_target_relation(const ParseNode *target_node) } else if (OB_ISNULL(table_item)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table_item is NULL", K(ret)); + } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { + LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); } else if (OB_FAIL(column_namespace_checker_.add_reference_table(table_item))) { LOG_WARN("add reference table to column namespace checker failed", K(ret)); } else if (table_item->is_generated_table()) { diff --git a/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp b/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp index 6481ef73a8..0b87189e63 100644 --- a/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp @@ -362,6 +362,8 @@ int ObMultiTableInsertResolver::resolve_insert_table_node(const ParseNode &inser LOG_WARN("a view is not appropriate here", K(ret)); } else if (OB_FAIL(column_namespace_checker_.add_reference_table(table_item))) { LOG_WARN("failed to resolve basic table"); + } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { + LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); } else { //提前设置好解析参数，对于多表插入会出现相同的表的场景，因此检查时需要提前设置好参数 if (OB_FAIL(generate_insert_all_table_info(*table_item, when_conds_idx, table_info))) { diff --git a/src/sql/resolver/dml/ob_update_resolver.cpp b/src/sql/resolver/dml/ob_update_resolver.cpp index 8d7ca292f4..5386e88f8d 100644 --- a/src/sql/resolver/dml/ob_update_resolver.cpp +++ b/src/sql/resolver/dml/ob_update_resolver.cpp @@ -443,6 +443,8 @@ int ObUpdateResolver::resolve_table_list(const ParseNode &parse_tree) LOG_WARN("failed to resolve table", K(ret)); //这里是为了兼容oracle的报错行为，对于直接向子查询更新数据时如果子查询中from项不为1项时，报错这种子查询是非法的， //其他情形同update view类似判断，这里不再重复解决 + } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { + LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); } else if (is_oracle_mode() && table_node->num_child_ == 2) { if (OB_ISNULL(table_item) || (!table_item->is_generated_table() && !table_item->is_temp_table()) || OB_ISNULL(ref_stmt = table_item->ref_query_)) { diff --git a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp index 2cbcb6683b..5643d29ecb 100644 --- a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp @@ -1090,9 +1090,9 @@ int ObMultiVersionMicroBlockRowScanner::inner_inner_get_next_row( sstable_->get_end_scn()); if (OB_FAIL(lock_for_read(lock_for_read_arg, - can_read, - trans_version, - is_determined_state))) { + can_read, + trans_version, + is_determined_state))) { STORAGE_LOG(WARN, "fail to check transaction status", K(ret), KPC(row_header), K_(macro_id)); } } @@ -1100,7 +1100,7 @@ int ObMultiVersionMicroBlockRowScanner::inner_inner_get_next_row( if (OB_SUCC(ret)) { ObStoreRowLockState lock_state; - if (param_->is_for_foreign_check_ && + if (context_->query_flag_.is_for_foreign_key_check() && OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint_for_sstable( acc_ctx.get_tx_table_guards(), acc_ctx.get_tx_id(), diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp index 2433c46782..c9be8eb0f1 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp @@ -14,6 +14,7 @@ #include "storage/memtable/mvcc/ob_mvcc_engine.h" #include "storage/memtable/mvcc/ob_mvcc_acc_ctx.h" #include "storage/memtable/ob_memtable.h" +#include "storage/memtable/ob_row_conflict_handler.h" #include "storage/tx_table/ob_tx_table.h" #include "storage/tx/ob_trans_define.h" #include "storage/access/ob_index_sstable_estimator.h" @@ -106,7 +107,8 @@ int ObMvccEngine::get(ObMvccAccessCtx &ctx, const ObQueryFlag &query_flag, const ObMemtableKey *parameter_key, ObMemtableKey *returned_key, - ObMvccValueIterator &value_iter) + ObMvccValueIterator &value_iter, + ObStoreRowLockState &lock_state) { int ret = OB_SUCCESS; ObMvccRow *value = NULL; @@ -128,6 +130,8 @@ int ObMvccEngine::get(ObMvccAccessCtx &ctx, if (OB_SUCCESS != (tmp_ret = try_compact_row_when_mvcc_read_(ctx.get_snapshot_version(), *value))) { TRANS_LOG(WARN, "fail to try to compact row", K(tmp_ret)); } + } else if (query_flag.is_for_foreign_key_check()) { + ret = ObRowConflictHandler::check_foreign_key_constraint_for_memtable(&ctx, value, lock_state); } else { // do nothing } @@ -240,8 +244,6 @@ int ObMvccEngine::check_row_locked(ObMvccAccessCtx &ctx, } } else if (OB_FAIL(value->check_row_locked(ctx, lock_state))) { TRANS_LOG(WARN, "check row locked fail", K(ret), KPC(value), K(ctx), K(lock_state)); - } else { - lock_state.mvcc_row_ = value; // use to re-check } return ret; diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.h b/src/storage/memtable/mvcc/ob_mvcc_engine.h index a2f71bfd7d..963073768d 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.h +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.h @@ -96,7 +96,8 @@ public: const ObQueryFlag &query_flag, const ObMemtableKey *parameter_key, ObMemtableKey *internal_key, - ObMvccValueIterator &value_iter); + ObMvccValueIterator &value_iter, + storage::ObStoreRowLockState &lock_state); int scan(ObMvccAccessCtx &ctx, const ObQueryFlag &query_flag, const ObMvccScanRange &range, diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp index bad8d1c016..39302733a4 100755 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp @@ -17,6 +17,7 @@ #include "storage/memtable/mvcc/ob_query_engine.h" #include "storage/memtable/ob_memtable_data.h" #include "storage/memtable/ob_memtable_context.h" +#include "storage/memtable/ob_row_conflict_handler.h" #include "storage/tx/ob_trans_ctx.h" #include "common/ob_clock_generator.h" @@ -343,8 +344,6 @@ int ObMvccValueIterator::check_row_locked(ObStoreRowLockState &lock_state) TRANS_LOG(WARN, "get value iter but mvcc row in it is null", K(ret)); } else if (OB_FAIL(value_->check_row_locked(*ctx_, lock_state))){ TRANS_LOG(WARN, "check row locked fail", K(ret), KPC(value_), KPC(ctx_), K(lock_state)); - } else { - lock_state.mvcc_row_ = value_; } return ret; } @@ -395,7 +394,8 @@ int ObMvccRowIterator::init( int ObMvccRowIterator::get_next_row( const ObMemtableKey *&key, ObMvccValueIterator *&value_iter, - uint8_t& iter_flag) + uint8_t& iter_flag, + ObStoreRowLockState &lock_state) { int ret = OB_SUCCESS; uint8_t read_partial_row = 0; @@ -419,22 +419,31 @@ int ObMvccRowIterator::get_next_row( } else if (NULL == (value = query_engine_iter_->get_value())) { TRANS_LOG(ERROR, "unexpected value null pointer", "ctx", *ctx_); ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(value_iter_.init(*ctx_, - tmp_key, - value, - query_flag_))) { - TRANS_LOG(WARN, "value iter init fail", K(ret), "ctx", *ctx_, KP(value), K(*value)); - } else if (!value_iter_.is_exist()) { - read_partial_row = (query_engine_iter_->get_iter_flag() & STORE_ITER_ROW_PARTIAL); - // continue - } else { - key = tmp_key; - value_iter = &value_iter_; - iter_flag = (query_engine_iter_->get_iter_flag() | read_partial_row); - break; + } else if (query_flag_.is_for_foreign_key_check()) { + if (OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint_for_memtable(ctx_, value, lock_state))) { + // we will throw error code if it's failed here, but we need to + // post lock with key outside, so we have to set it here. + key = tmp_key; + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(value_iter_.init(*ctx_, + tmp_key, + value, + query_flag_))) { + TRANS_LOG(WARN, "value iter init fail", K(ret), "ctx", *ctx_, KP(value), K(*value)); + } else if (!value_iter_.is_exist()) { + read_partial_row = (query_engine_iter_->get_iter_flag() & STORE_ITER_ROW_PARTIAL); + // continue + } else { + key = tmp_key; + value_iter = &value_iter_; + iter_flag = (query_engine_iter_->get_iter_flag() | read_partial_row); + break; + } } } - return ret; } diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.h b/src/storage/memtable/mvcc/ob_mvcc_iterator.h index 7d2e02472f..1653e9b648 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.h +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.h @@ -170,7 +170,8 @@ public: const ObQueryFlag &query_flag); int get_next_row(const ObMemtableKey *&key, ObMvccValueIterator *&value_iter, - uint8_t& iter_flag); + uint8_t& iter_flag, + storage::ObStoreRowLockState &lock_state); void reset(); int get_key_val(const ObMemtableKey*& key, ObMvccRow*& row); int try_purge(const transaction::ObTxSnapshot &snapshot_info, diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index c9edcb84f7..c2943d2246 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -1086,6 +1086,9 @@ int ObMvccRow::check_row_locked(ObMvccAccessCtx &ctx, ObStoreRowLockState &lock_ } } } + if (OB_SUCC(ret)) { + lock_state.mvcc_row_ = this; + } return ret; } diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 3fbe97763a..e88562e3bb 100755 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -576,6 +576,7 @@ int ObMemtable::exist( ObMemtableKey returned_mtk; ObMvccValueIterator value_iter; ObQueryFlag query_flag; + ObStoreRowLockState lock_state; query_flag.read_latest_ = true; query_flag.prewarm_ = false; //get_begin(context.store_ctx_->mvcc_acc_ctx_); @@ -594,8 +595,27 @@ int ObMemtable::exist( query_flag, ¶meter_mtk, &returned_mtk, - value_iter))) { + value_iter, + lock_state))) { TRANS_LOG(WARN, "get value iter fail, ", K(ret)); + if (OB_TRY_LOCK_ROW_CONFLICT == ret || OB_TRANSACTION_SET_VIOLATION == ret) { + if (!query_flag.is_for_foreign_key_check()) { + ret = OB_ERR_UNEXPECTED; // to prevent retrying casued by throwing 6005 + TRANS_LOG(WARN, "should not meet row conflict if it's not for foreign key check", + K(ret), K(query_flag)); + } else if (OB_TRY_LOCK_ROW_CONFLICT == ret) { + ObRowConflictHandler::post_row_read_conflict( + context.store_ctx_->mvcc_acc_ctx_, + *parameter_mtk.get_rowkey(), + lock_state, + key_.tablet_id_, + freezer_->get_ls_id(), + 0, 0 /* these two params get from mvcc_row, and for statistics, so we ignore them */, + lock_state.trans_scn_); + } + } else { + TRANS_LOG(WARN, "fail to do mvcc engine get", K(ret)); + } } else { const void *tnode = nullptr; const ObMemtableDataHeader *mtd = nullptr; @@ -710,20 +730,25 @@ int ObMemtable::get( context.query_flag_, ¶meter_mtk, &returned_mtk, - value_iter))) { - TRANS_LOG(WARN, "fail to do mvcc engine get", K(ret)); - } else if (param.is_for_foreign_check_ && - OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint_for_memtable(&value_iter, lock_state))) { - if (OB_TRY_LOCK_ROW_CONFLICT == ret) { - ObRowConflictHandler::post_row_read_conflict( - *value_iter.get_mvcc_acc_ctx(), - *parameter_mtk.get_rowkey(), - lock_state, - key_.tablet_id_, - freezer_->get_ls_id(), - value_iter.get_mvcc_row()->get_last_compact_cnt(), - value_iter.get_mvcc_row()->get_total_trans_node_cnt(), - lock_state.trans_scn_); + value_iter, + lock_state))) { + if (OB_TRY_LOCK_ROW_CONFLICT == ret || OB_TRANSACTION_SET_VIOLATION == ret) { + if (!context.query_flag_.is_for_foreign_key_check()) { + ret = OB_ERR_UNEXPECTED; // to prevent retrying casued by throwing 6005 + TRANS_LOG(WARN, "should not meet lock conflict if it's not for foreign key check", + K(ret), K(context.query_flag_)); + } else if (OB_TRY_LOCK_ROW_CONFLICT == ret){ + ObRowConflictHandler::post_row_read_conflict( + context.store_ctx_->mvcc_acc_ctx_, + *parameter_mtk.get_rowkey(), + lock_state, + key_.tablet_id_, + freezer_->get_ls_id(), + 0, 0 /* these two params get from mvcc_row, and for statistics, so we ignore them */, + lock_state.trans_scn_); + } + } else { + TRANS_LOG(WARN, "fail to do mvcc engine get", K(ret)); } } else { const int64_t request_cnt = read_info->get_request_count(); diff --git a/src/storage/memtable/ob_memtable_iterator.cpp b/src/storage/memtable/ob_memtable_iterator.cpp index 1bcf18afa9..66606e49e0 100755 --- a/src/storage/memtable/ob_memtable_iterator.cpp +++ b/src/storage/memtable/ob_memtable_iterator.cpp @@ -352,18 +352,34 @@ int ObMemtableScanIterator::inner_get_next_row(const ObDatumRow *&row) int ret = OB_SUCCESS; const ObMemtableKey *key = NULL; ObMvccValueIterator *value_iter = NULL; - + ObStoreRowLockState lock_state; if (IS_NOT_INIT) { TRANS_LOG(WARN, "not init", KP(this)); ret = OB_NOT_INIT; } else if (OB_FAIL(prepare_scan())) { TRANS_LOG(WARN, "prepare scan fail", K(ret)); - } else if (OB_FAIL(row_iter_.get_next_row(key, value_iter, iter_flag_)) + } else if (OB_FAIL(row_iter_.get_next_row(key, value_iter, iter_flag_, lock_state)) || NULL == key || NULL == value_iter) { - if (OB_ITER_END != ret) { + if (OB_TRY_LOCK_ROW_CONFLICT == ret || OB_TRANSACTION_SET_VIOLATION == ret) { + if (!context_->query_flag_.is_for_foreign_key_check()) { + ret = OB_ERR_UNEXPECTED; // to prevent retrying casued by throwing 6005 + TRANS_LOG(WARN, "should not meet row conflict if it's not for foreign key check", + K(ret), K(context_->query_flag_)); + } else if (OB_TRY_LOCK_ROW_CONFLICT == ret) { + const ObStoreRowkey *tmp_rowkey = nullptr; + key->get_rowkey(tmp_rowkey); + ObRowConflictHandler::post_row_read_conflict( + context_->store_ctx_->mvcc_acc_ctx_, + *tmp_rowkey, + lock_state, + context_->tablet_id_, + context_->ls_id_, + 0, 0 /* these two params get from mvcc_row, and for statistics, so we ignore them */, + lock_state.trans_scn_); + } + } else if (OB_ITER_END != ret) { TRANS_LOG(WARN, "row_iter_ get_next_row fail", K(ret), KP(key), KP(value_iter)); } - ret = (OB_SUCCESS == ret) ? OB_ERR_UNEXPECTED : ret; } else { TRANS_LOG(DEBUG, "chaser debug memtable next row", KPC(key), K(iter_flag_), K(bitmap_.get_nop_cnt())); const ObStoreRowkey *rowkey = NULL; @@ -373,21 +389,7 @@ int ObMemtableScanIterator::inner_get_next_row(const ObDatumRow *&row) concurrency_control::ObTransStatRow trans_stat_row; (void)value_iter->get_trans_stat_row(trans_stat_row); - ObStoreRowLockState lock_state; - if (param_->is_for_foreign_check_ && - OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint_for_memtable(value_iter, lock_state))) { - if (OB_TRY_LOCK_ROW_CONFLICT == ret) { - ObRowConflictHandler::post_row_read_conflict( - *value_iter->get_mvcc_acc_ctx(), - *rowkey, - lock_state, - context_->tablet_id_, - context_->ls_id_, - value_iter->get_mvcc_row()->get_last_compact_cnt(), - value_iter->get_mvcc_row()->get_total_trans_node_cnt(), - lock_state.trans_scn_); - } - } else if (OB_FAIL(ObReadRow::iterate_row(*read_info_, *rowkey, *(context_->allocator_), *value_iter, row_, bitmap_, row_scn))) { + if (OB_FAIL(ObReadRow::iterate_row(*read_info_, *rowkey, *(context_->allocator_), *value_iter, row_, bitmap_, row_scn))) { TRANS_LOG(WARN, "iterate_row fail", K(ret), K(*rowkey), KP(value_iter)); } else { STORAGE_LOG(DEBUG, "chaser debug memtable next row", K(row_)); diff --git a/src/storage/memtable/ob_row_conflict_handler.cpp b/src/storage/memtable/ob_row_conflict_handler.cpp index d3210fdbf7..488ffc4fee 100644 --- a/src/storage/memtable/ob_row_conflict_handler.cpp +++ b/src/storage/memtable/ob_row_conflict_handler.cpp @@ -20,17 +20,17 @@ using namespace common; using namespace memtable; using namespace transaction; namespace storage { -int ObRowConflictHandler::check_foreign_key_constraint_for_memtable(ObMvccValueIterator *value_iter, +int ObRowConflictHandler::check_foreign_key_constraint_for_memtable(ObMvccAccessCtx *ctx, + ObMvccRow *row, ObStoreRowLockState &lock_state) { int ret = OB_SUCCESS; - if (OB_ISNULL(value_iter)) { + if (OB_ISNULL(row)) { ret = OB_BAD_NULL_ERROR; TRANS_LOG(ERROR, "the ObMvccValueIterator is null", K(ret)); - } else if (OB_FAIL(value_iter->check_row_locked(lock_state))) { + } else if (OB_FAIL(row->check_row_locked(*ctx, lock_state))) { TRANS_LOG(WARN, "check row locked fail", K(ret), K(lock_state)); } else { - const ObMvccAccessCtx *ctx = value_iter->get_mvcc_acc_ctx(); const ObTransID my_tx_id = ctx->get_tx_id(); const share::SCN snapshot_version = ctx->get_snapshot_version(); if (lock_state.is_locked_ && my_tx_id != lock_state.lock_trans_id_) { @@ -62,7 +62,7 @@ int ObRowConflictHandler::check_foreign_key_constraint_for_sstable(ObTxTableGuar if (!data_trans_id.is_valid()) { if (trans_version > snapshot_version) { ret = OB_TRANSACTION_SET_VIOLATION; - TRANS_LOG(WARN, "meet tsc on sstable", K(ret), K(lock_state.trans_version_), K(snapshot_version)); + TRANS_LOG(WARN, "meet tsc on sstable", K(ret), K(trans_version), K(snapshot_version)); } } else { ObTxTable *tx_table = nullptr; diff --git a/src/storage/memtable/ob_row_conflict_handler.h b/src/storage/memtable/ob_row_conflict_handler.h index 9b58af6371..ebbb6acd2e 100644 --- a/src/storage/memtable/ob_row_conflict_handler.h +++ b/src/storage/memtable/ob_row_conflict_handler.h @@ -17,7 +17,7 @@ namespace oceanbase { namespace memtable { class ObMvccAccessCtx; -class ObMvccValueIterator; +class ObMvccRow; } namespace transaction { class ObTransID; @@ -26,6 +26,7 @@ class ObTxSEQ; namespace common { class ObTabletID; class ObStoreRowkey; +class ObQueryFlag; } namespace share { class ObLSID; @@ -46,7 +47,8 @@ public: // snapshot_version of current transaction, it will cause tsc. // If the check meet this case, we should return error code to sql layer, and it will // choose to retry or throw an exception according to the isolation level. - static int check_foreign_key_constraint_for_memtable(memtable::ObMvccValueIterator *value_iter, + static int check_foreign_key_constraint_for_memtable(memtable::ObMvccAccessCtx *ctx, + memtable::ObMvccRow *row, storage::ObStoreRowLockState &lock_state); static int check_foreign_key_constraint_for_sstable(storage::ObTxTableGuards &tx_table_guards, const transaction::ObTransID &read_trans_id, diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index b3391ccace..b3cc9c8047 100755 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -224,10 +224,6 @@ int ObAccessService::table_scan( ObLS *ls = nullptr; ObLSTabletService *tablet_service = nullptr; ObTableScanParam ¶m = static_cast(vparam); - // TODO(yichang): maybe we need to move this scan_flag_ setting to sql layer? - if (param.is_for_foreign_check_) { - param.scan_flag_.set_iter_uncommitted_row(); - } ObStoreAccessType access_type = param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; SCN user_specified_snapshot_scn;