diff --git a/src/libtable/src/ob_hkv_table.cpp b/src/libtable/src/ob_hkv_table.cpp index b114d59b0c..2362edd11a 100644 --- a/src/libtable/src/ob_hkv_table.cpp +++ b/src/libtable/src/ob_hkv_table.cpp @@ -141,12 +141,12 @@ int ObHKVTable::Entity::get_rowkey_value(int64_t idx, ObObj &value) const return ret; } -ObRowkey ObHKVTable::Entity::get_rowkey() +ObRowkey ObHKVTable::Entity::get_rowkey() const { - ObRowkey rk(key_objs_, 3); - (void)get_rowkey_value(0, key_objs_[0]); - (void)get_rowkey_value(1, key_objs_[1]); - (void)get_rowkey_value(2, key_objs_[2]); + ObRowkey rk(const_cast(key_objs_), 3); + (void)get_rowkey_value(0, const_cast(key_objs_[0])); + (void)get_rowkey_value(1, const_cast(key_objs_[1])); + (void)get_rowkey_value(2, const_cast(key_objs_[2])); return rk; } diff --git a/src/libtable/src/ob_hkv_table.h b/src/libtable/src/ob_hkv_table.h index 9616ee2bed..6576158a38 100644 --- a/src/libtable/src/ob_hkv_table.h +++ b/src/libtable/src/ob_hkv_table.h @@ -51,7 +51,7 @@ public: virtual int add_rowkey_value(const ObObj &value) override; virtual int64_t get_rowkey_size() const override { return 3; } virtual int get_rowkey_value(int64_t idx, ObObj &value) const override; - virtual ObRowkey get_rowkey() override; + virtual ObRowkey get_rowkey() const override; virtual int64_t hash_rowkey() const override; virtual int get_property(const ObString &prop_name, ObObj &prop_value) const override; virtual int set_property(const ObString &prop_name, const ObObj &prop_value) override; diff --git a/src/libtable/test/ob_batch_execute_test.cpp b/src/libtable/test/ob_batch_execute_test.cpp index 93325c138d..9e926fdaae 100644 --- a/src/libtable/test/ob_batch_execute_test.cpp +++ b/src/libtable/test/ob_batch_execute_test.cpp @@ -29,9 +29,12 @@ using namespace oceanbase::common; using namespace oceanbase::table; -const char* host = "100.88.11.91"; -int32_t sql_port = 60809; -int32_t rpc_port = 60808; +// const char* host = "100.88.11.91"; +// int32_t sql_port = 60809; +// int32_t rpc_port = 60808; +const char* host = "11.158.97.240"; +int32_t sql_port = 41101; +int32_t rpc_port = 41100; const char* tenant = "sys"; const char* user_name = "root"; const char* passwd = ""; @@ -90,27 +93,6 @@ void TestBatchExecute::SetUp() void TestBatchExecute::TearDown() { - if (NULL != table_) { - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - for (int64_t i = 0; i < BATCH_SIZE*2; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE*2, result.count()); - } service_client_->free_table(table_); table_ = NULL; ObTableServiceClient::free_client(service_client_); @@ -121,6 +103,95 @@ void TestBatchExecute::TearDown() ObString C1 = ObString::make_string("C1"); ObString C2 = ObString::make_string("C2"); ObString C3 = ObString::make_string("C3"); +ObString K = ObString::make_string("K"); +ObString Q = ObString::make_string("Q"); +ObString T = ObString::make_string("T"); +ObString V = ObString::make_string("V"); +ObString PK1 = ObString::make_string("PK1"); +ObString PK2 = ObString::make_string("PK2"); + +void TestBatchExecute::generate_get(ObTableQuery &query, ObObj pk_objs_start[], ObObj pk_objs_end[], const char* rowkey) +{ + ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); + + pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); + pk_objs_start[1].set_min_value(); + pk_objs_start[2].set_min_value(); + pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); + pk_objs_end[1].set_max_value(); + pk_objs_end[2].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 3); + range.end_key_.assign(pk_objs_end, 3); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); +} + +void TestBatchExecute::prepare_data(ObTable *the_table) +{ + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_SIZE]; + ASSERT_TRUE(NULL != rows); + static constexpr int64_t VERSIONS_COUNT = 10; + static constexpr int64_t COLUMNS_SIZE = 10; + char qualifier[COLUMNS_SIZE][128]; + for (int i = 0; i < COLUMNS_SIZE; ++i) + { + sprintf(qualifier[i], "cq%d", i); + } // end for + ObObj key1, key2, key3; + ObObj value; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + sprintf(rows[i], "row%ld", i); + key1.set_varbinary(ObString::make_string(rows[i])); + for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { + key2.set_varbinary(ObString::make_string(qualifier[j])); + for (int64_t k = 0; k < VERSIONS_COUNT; ++k) + { + key3.set_int(k); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + switch (i % 4) { + case 0: + value.set_varbinary(ObString::make_string("string2")); + break; + case 1: + value.set_varbinary(ObString::make_string("string3")); + break; + case 2: // row50 + value.set_varbinary(ObString::make_string("string0")); + break; + case 3: + value.set_varbinary(ObString::make_string("string1")); + break; + default: + ASSERT_TRUE(0); + } + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } // end for + } // end for + } // end for + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(1, result.count()); + ASSERT_EQ(OB_SUCCESS, result.at(0).get_errno()); + delete [] rows; +} TEST_F(TestBatchExecute, entity_factory) @@ -174,1241 +245,300 @@ TEST_F(TestBatchExecute, serialize_batch_result) ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, result2.at(0).type()); } -//TEST_F(TestBatchExecute, all_single_operation) -//{ - //OB_LOG(INFO, "begin all_single_operation"); - //// insert C2 - //ObTableEntityFactory entity_factory; - //ObITableEntity *entity = NULL; - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //int64_t key_key = 139107; - //ObObj key; - //key.set_int(key_key); - //int64_t value_value = 33521; - //ObObj value; - //value.set_int(value_value); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObString c3_value = ObString::make_string("hello world"); - //ObTableOperation table_operation = ObTableOperation::insert(*entity); - //ObTableOperationResult r; - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(NULL != result_entity); - //ASSERT_TRUE(result_entity->is_empty()); - //// insert again: fail - //{ - //ObTableOperation table_operation = ObTableOperation::insert(*entity); - //ObTableOperationResult r; - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ASSERT_TRUE(value.is_null()); - //} - //// update C3 - //{ - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ObTableOperation table_operation = ObTableOperation::update(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ASSERT_TRUE(str == c3_value); - //} - //// update C3 not exist key - //{ - //entity->reset(); - //ObObj not_exist_key; - //not_exist_key.set_int(key_key+1); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(not_exist_key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ObTableOperation table_operation = ObTableOperation::update(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// replace C3 - //{ - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ObTableOperation table_operation = ObTableOperation::replace(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(2, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_TRUE(value.is_null()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ASSERT_TRUE(str == c3_value); - //} - //// insert_or_update C2: update - //{ - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(value_value); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ASSERT_TRUE(str == c3_value); - //} - //// delete not exist row - //{ - //entity->reset(); - //ObObj not_exist_key; - //not_exist_key.set_int(key_key+1); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(not_exist_key)); - //ObTableOperation table_operation = ObTableOperation::del(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::DEL, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// delete - //{ - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ObTableOperation table_operation = ObTableOperation::del(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::DEL, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// get again - //{ - //ObObj null_obj; - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// insert_or_update C2: insert - //{ - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(value_value); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ASSERT_TRUE(value.is_null()); - //} - //// delete & cleanup - //{ - //entity->reset(); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ObTableOperation table_operation = ObTableOperation::del(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::DEL, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} -//} -//TEST_F(TestBatchExecute, multi_insert_or_update_AND_multi_get) -//{ - //OB_LOG(INFO, "begin multi_insert_or_update"); - //ObTableEntityFactory entity_factory; - //ObTableBatchOperation batch_operation; - //ObITableEntity *entity = NULL; - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ObObj value; - //value.set_int(100+i); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - //} - //ASSERT_TRUE(!batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ObTableBatchOperationResult result; - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) - //{ - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} // end for - - //// get and verify - //const ObITableEntity *result_entity = NULL; - //{ - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //ObObj null_obj; - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - //} - //ASSERT_TRUE(batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ObObj value; - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(100+i, value.get_int()); - //} - //} - //// multi-get case 2 - //{ - //ObObj null_obj; - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //for (int64_t i = BATCH_SIZE-1; i >= 0; --i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C2)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - //} - //ASSERT_TRUE(batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //result_entity = NULL; - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //if (i % 2 == 1) { - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ObObj value; - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ////fprintf(stderr, "get value i=%ld v=%s\n", i, S(value)); - //ASSERT_EQ(100+(BATCH_SIZE-1-i)/2, value.get_int()); - //} else { - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //} - //} -//} - -TEST_F(TestBatchExecute, multi_insert) +TEST_F(TestBatchExecute, all_single_operation) { - OB_LOG(INFO, "begin multi_insert"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("all_single_operation_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + OB_LOG(INFO, "begin all_single_operation"); + // insert C2 ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; ObITableEntity *entity = NULL; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ObObj value; - value.set_int(100+i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + int64_t key_key = 139107; + ObObj key; + key.set_int(key_key); + int64_t value_value = 33521; + ObObj value; + value.set_int(value_value); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("hello world"); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + // insert again: fail { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - ASSERT_EQ(1, r.get_affected_rows()); - const ObITableEntity *result_entity = NULL; ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); ASSERT_TRUE(result_entity->is_empty()); } - // get and verify + // get { - batch_operation.reset(); - entity_factory.free_and_reuse(); ObObj null_obj; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - ObObj value; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(100+i, value.get_int()); - } - } - // insert again - { - batch_operation.reset(); - entity_factory.free_and_reuse(); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ObObj value; - value.set_int(100+i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, r.get_errno()); - ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_TRUE(result_entity->is_empty()); - } - } -} - -TEST_F(TestBatchExecute, multi_replace) -{ - OB_LOG(INFO, "begin multi_replace"); - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ObObj value; - value.set_int(100+i); + entity->reset(); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - const ObITableEntity *result_entity = NULL; + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_TRUE(result_entity->is_empty()); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_TRUE(value.is_null()); } - // get and verify + // update C3 { - batch_operation.reset(); - entity_factory.free_and_reuse(); - ObObj null_obj; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - ObObj value; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(100+i, value.get_int()); - } - } - // replace again - { - batch_operation.reset(); - entity_factory.free_and_reuse(); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ObObj value; - value.set_int(200+i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(2, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_TRUE(result_entity->is_empty()); - } - } - // get and verify - { - batch_operation.reset(); - entity_factory.free_and_reuse(); - ObObj null_obj; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - ObObj value; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(200+i, value.get_int()); - } - } -} - -//TEST_F(TestBatchExecute, multi_update) -//{ - //OB_LOG(INFO, "begin multi_update"); - //ObTableEntityFactory entity_factory; - //ObTableBatchOperation batch_operation; - //// prepare data - //ObITableEntity *entity = NULL; - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ObObj value; - //value.set_int(100+i); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - //} - //ASSERT_TRUE(!batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ObTableBatchOperationResult result; - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) - //{ - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - //ASSERT_EQ(1, r.get_affected_rows()); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //ObString c3_value = "c3_value"; - //// update C3 - //{ - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ObObj value; - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); - //} - //ASSERT_TRUE(!batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) - //{ - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //} - //} - //// get and verify - //{ - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //ObObj null_obj; - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - //} - //ASSERT_TRUE(batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ObObj value; - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(100+i, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ASSERT_TRUE(str == c3_value); - //} - //} - //// update half of the rows - //{ - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i); - //ObObj value; - //value.set_int(200+i); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); - //} - //ASSERT_TRUE(!batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(result_entity->is_empty()); - //if (i % 2 == 0) { - //ASSERT_EQ(1, r.get_affected_rows()); - //} else { - //ASSERT_EQ(0, r.get_affected_rows()); - //} - //} - //} - //// get and verify - //{ - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //ObObj null_obj; - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - //} - //ASSERT_TRUE(batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ObObj value; - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //if (i < BATCH_SIZE/2) { - //ASSERT_EQ(200+i*2, value.get_int()); - //} else { - //ASSERT_EQ(100+i, value.get_int()); - //} - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ASSERT_TRUE(str == c3_value); - //} - //} -//} - - -TEST_F(TestBatchExecute, multi_delete) -{ - OB_LOG(INFO, "begin multi_delete"); - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - // prepare data - ObITableEntity *entity = NULL; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ObObj value; - value.set_int(100+i); + entity->reset(); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) - { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - ASSERT_EQ(1, r.get_affected_rows()); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_TRUE(result_entity->is_empty()); - } - - // delete half of the rows - { - batch_operation.reset(); - entity_factory.free_and_reuse(); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(ObTableOperationType::DEL, r.type()); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_TRUE(result_entity->is_empty()); - if (i % 2 == 0) { - ASSERT_EQ(1, r.get_affected_rows()); - } else { - ASSERT_EQ(0, r.get_affected_rows()); - } - } - } - // get and verify - { - batch_operation.reset(); - entity_factory.free_and_reuse(); - ObObj null_obj; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(BATCH_SIZE, result.count()); - ObObj value; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - if (i < BATCH_SIZE/2) { - ASSERT_TRUE(result_entity->is_empty()); - } else { - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(100+i, value.get_int()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - ASSERT_TRUE(value.is_null()); - } - } - } -} - - -TEST_F(TestBatchExecute, multi_get_complex) -{ - OB_LOG(INFO, "begin multi_get complex"); - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - // prepare data - ObITableEntity *entity = NULL; - ObString c3_value = "c3_value"; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ObObj value; - value.set_int(100+i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); value.set_varchar(c3_value); value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - const ObITableEntity *result_entity = NULL; - for (int64_t i = 0; i < BATCH_SIZE; ++i) - { - const ObTableOperationResult &r = result.at(i); + ObTableOperation table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); ASSERT_EQ(OB_SUCCESS, r.get_errno()); ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); ASSERT_TRUE(result_entity->is_empty()); } - // complex batch get and verify + // get { - batch_operation.reset(); - entity_factory.free_and_reuse(); ObObj null_obj; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i*2); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - if (i % 2 == 1) { - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - } else { - ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - } - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(!batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - ObObj value; - ObString str; - if (i % 2 == 1) { - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_TRUE(OB_SUCCESS != result_entity->get_property(C3, value)); - ASSERT_EQ(100+i, value.get_int()); - } else { - ASSERT_TRUE(OB_SUCCESS != result_entity->get_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - ASSERT_TRUE(str == c3_value); - } - } - } -} - -TEST_F(TestBatchExecute, complex_batch_execute) -{ - OB_LOG(INFO, "begin complex_batch_execute"); - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - // prepare data - ObITableEntity *entity = NULL; - ObString c3_value = "c3_value"; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i); - ObObj value; - value.set_int(100+i); + entity->reset(); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + // update C3 not exist key + { + entity->reset(); + ObObj not_exist_key; + not_exist_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(not_exist_key)); value.set_varchar(c3_value); - value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); + ObTableOperation table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - const ObITableEntity *result_entity = NULL; + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); ASSERT_TRUE(result_entity->is_empty()); } - + // replace C3 { - // complex batch execute with hybrid operations types - batch_operation.reset(); - entity_factory.free_and_reuse(); - ObObj null_obj; - ObObj value; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - switch (i % 6) { - case 0: // get - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - break; - case 1: // insert - value.set_int(200+i); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); - break; - case 2: // delete - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - break; - case 3: // update - value.set_int(300+i); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); - break; - case 4: // insert_or_update - value.set_int(400+i); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - break; - case 5: // replace - value.set_int(500+i); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); - break; - default: - ASSERT_TRUE(0); - break; - } - } - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(!batch_operation.is_same_type()); - ASSERT_TRUE(!batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - const ObITableEntity *result_entity = NULL; - ObString str; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - switch (i % 6) { - case 0: // get - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(100+i, value.get_int()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - ASSERT_TRUE(str == c3_value); - break; - case 1: // insert - ASSERT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, r.get_errno()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - ASSERT_TRUE(result_entity->is_empty()); - break; - case 2: // delete - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::DEL, r.type()); - ASSERT_TRUE(result_entity->is_empty()); - break; - case 3: // update - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); - ASSERT_TRUE(result_entity->is_empty()); - break; - case 4: // insert_or_update - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); - ASSERT_TRUE(result_entity->is_empty()); - break; - case 5: // replace - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(2, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); - ASSERT_TRUE(result_entity->is_empty()); - break; - default: - ASSERT_TRUE(0); - break; - } - } + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::replace(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(2, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); } + // get { - // get and verify - batch_operation.reset(); - entity_factory.free_and_reuse(); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - key.set_int(i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C2)); - ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); const ObITableEntity *result_entity = NULL; - ASSERT_EQ(BATCH_SIZE, result.count()); - ObObj value; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(value.is_null()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); ObString str; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - //fprintf(stderr, "result %ld\n", i); - switch (i % 6) { - case 0: // get - case 1: // insert - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(100+i, value.get_int()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - ASSERT_TRUE(str == c3_value); - break; - case 2: // delete - // entry not exist - ASSERT_TRUE(result_entity->is_empty()); - break; - case 3: // update - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(300+i, value.get_int()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - ASSERT_TRUE(str == c3_value); - break; - case 4: // insert_or_update - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(400+i, value.get_int()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - // verify the semantic of PUT - // https://aone.alibaba-inc.com/project/81079/issue/14554345 - ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - ASSERT_TRUE(str == c3_value); - break; - case 5: // replace - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - ASSERT_EQ(500+i, value.get_int()); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - ASSERT_TRUE(value.is_null()); - break; - default: - ASSERT_TRUE(0); - break; - } - } + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + // insert_or_update C2: update + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(value_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + // delete not exist row + { + entity->reset(); + ObObj not_exist_key; + not_exist_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(not_exist_key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // delete + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // get again + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // insert_or_update C2: insert + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(value_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_TRUE(value.is_null()); + } + // delete & cleanup + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); } } -// the table should be single partition for this case: -// create table if not exists batch_operation_with_same_keys_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) -TEST_F(TestBatchExecute, batch_operation_with_same_keys) +TEST_F(TestBatchExecute, multi_insert_or_update_AND_multi_get) { - OB_LOG(INFO, "begin batch_operation_with_duplicated_keys"); - // setup + OB_LOG(INFO, "begin multi_insert_or_update"); ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("batch_operation_with_same_keys_test"), the_table); + int ret = service_client_->alloc_table(ObString::make_string("batch_execute_test"), the_table); ASSERT_EQ(OB_SUCCESS, ret); - // start case ObTableEntityFactory entity_factory; ObTableBatchOperation batch_operation; ObITableEntity *entity = NULL; @@ -1416,12 +546,11 @@ TEST_F(TestBatchExecute, batch_operation_with_same_keys) entity = entity_factory.alloc(); ASSERT_TRUE(NULL != entity); ObObj key; - key.set_int(i*2 % 10); + key.set_int(i*2); ObObj value; value.set_int(100+i); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //fprintf(stderr, "put key=%ld value=%ld i=%ld\n", (i*2%10), value.get_int(), i); ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); } ASSERT_TRUE(!batch_operation.is_readonly()); @@ -1452,7 +581,7 @@ TEST_F(TestBatchExecute, batch_operation_with_same_keys) entity = entity_factory.alloc(); ASSERT_TRUE(NULL != entity); ObObj key; - key.set_int(i*2%10); + key.set_int(i*2); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); @@ -1472,22 +601,21 @@ TEST_F(TestBatchExecute, batch_operation_with_same_keys) ASSERT_EQ(0, result_entity->get_rowkey_size()); ObObj value; ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //fprintf(stderr, "get C2 value=%ld i=%ld key=%ld\n", value.get_int(), i, i*2%10); - ASSERT_EQ(195+i%5, value.get_int()); + ASSERT_EQ(100+i, value.get_int()); } } + // multi-get case 2 { - // all the same key + ObObj null_obj; batch_operation.reset(); entity_factory.free_and_reuse(); - ObObj null_obj; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { + for (int64_t i = BATCH_SIZE-1; i >= 0; --i) { entity = entity_factory.alloc(); ASSERT_TRUE(NULL != entity); ObObj key; - key.set_int(0); + key.set_int(i); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C2)); ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); } ASSERT_TRUE(batch_operation.is_readonly()); @@ -1495,77 +623,28 @@ TEST_F(TestBatchExecute, batch_operation_with_same_keys) ASSERT_TRUE(batch_operation.is_same_properties_names()); ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); OB_LOG(INFO, "batch execute result", K(result)); + result_entity = NULL; ASSERT_EQ(BATCH_SIZE, result.count()); for (int64_t i = 0; i < BATCH_SIZE; ++i) { const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - ObObj value; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //fprintf(stderr, "get C2 value=%ld i=%ld\n", value.get_int(), i); - ASSERT_EQ(195, value.get_int()); - } - } - { - // pattern: empty empty exist exist empty empty exist exist ... - batch_operation.reset(); - entity_factory.free_and_reuse(); - ObObj null_obj; - int64_t key1_not_exist = 1; - int64_t key2_not_exist = 3; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ObObj key; - if (i < BATCH_SIZE/4) { - key.set_int(key1_not_exist); - } else if (i < BATCH_SIZE/4*2) { - key.set_int(0); - } else if (i < BATCH_SIZE/4*3) { - key.set_int(key2_not_exist); - } else { - key.set_int(2); - } - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - } - ASSERT_TRUE(batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE, result.count()); - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(ObTableOperationType::GET, r.type()); - ASSERT_EQ(0, r.get_affected_rows()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_EQ(0, result_entity->get_rowkey_size()); - if (i < BATCH_SIZE/4) { - ASSERT_TRUE(result_entity->is_empty()); - } else if (i < BATCH_SIZE/4*2) { + if (i % 2 == 1) { + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); ObObj value; ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //fprintf(stderr, "get C2 value=%ld i=%ld\n", value.get_int(), i); - ASSERT_EQ(195, value.get_int()); - } else if (i < BATCH_SIZE/4*3) { - ASSERT_TRUE(result_entity->is_empty()); + //fprintf(stderr, "get value i=%ld v=%s\n", i, S(value)); + ASSERT_EQ(100+(BATCH_SIZE-1-i)/2, value.get_int()); } else { - ObObj value; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //fprintf(stderr, "get C2 value=%ld i=%ld\n", value.get_int(), i); - ASSERT_EQ(196, value.get_int()); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); } - } // end for + } } - // teardown - service_client_->free_table(the_table); - the_table = NULL; } // create table type_check_test (pk1 bigint, pk2 varchar(10), ctinyint tinyint, csmallint smallint, cmediumint mediumint, cint int, cbigint bigint, utinyint tinyint unsigned, usmallint smallint unsigned, uint int unsigned, ubigint bigint unsigned, cfloat float, cdouble double, ufloat float unsigned, udouble double unsigned, cnumber decimal(10, 2), unumber decimal(10,2) unsigned, cvarchar varchar(10), cchar char(10), cbinary binary(10), cvarbinary varbinary(10), ctimestamp timestamp, cdatetime datetime, cyear year, cdate date, ctime time, ctext text, cblob blob, cbit bit(64), cnotnull bigint not null, primary key(pk1, pk2)); @@ -1773,8 +852,6 @@ TEST_F(TestBatchExecute, column_type_check) pk2.set_collation_type(CS_TYPE_BINARY); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk1)); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk2)); - value.set_int(100); - ASSERT_EQ(OB_SUCCESS, entity->set_property(CTINYINT, value)); ObTableOperation table_operation = ObTableOperation::retrieve(*entity); ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); ASSERT_EQ(OB_ERR_COLLATION_MISMATCH, r.get_errno()); @@ -1793,6 +870,22 @@ TEST_F(TestBatchExecute, column_type_check) ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); ASSERT_EQ(OB_ERR_COLLATION_MISMATCH, r.get_errno()); } + + { + // case: replace + mediumint out of range + entity->reset(); + pk1.set_int(pk1_value); + pk2.set_varchar(pk2_value); + pk2.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk2)); + value.set_mediumint(INT32_MAX); + ASSERT_EQ(OB_SUCCESS, entity->set_property(ObString::make_string("cmediumint"), value)); + ObTableOperation table_operation = ObTableOperation::replace(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_DATA_OUT_OF_RANGE, r.get_errno()); + } + { // case: insert_or_update + rowkey + collation entity->reset(); @@ -1807,6 +900,22 @@ TEST_F(TestBatchExecute, column_type_check) ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); ASSERT_EQ(OB_ERR_COLLATION_MISMATCH, r.get_errno()); } + + { + // case: insertup + mediumint out of range + entity->reset(); + pk1.set_int(pk1_value); + pk2.set_varchar(pk2_value); + pk2.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk2)); + value.set_mediumint(INT32_MAX); + ASSERT_EQ(OB_SUCCESS, entity->set_property(ObString::make_string("cmediumint"), value)); + ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_DATA_OUT_OF_RANGE, r.get_errno()); + } + { // case: delete + rowkey + collation entity->reset(); @@ -1835,6 +944,22 @@ TEST_F(TestBatchExecute, column_type_check) ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); ASSERT_EQ(OB_ERR_COLLATION_MISMATCH, r.get_errno()); } + + { + // case: update + mediumint out of range + entity->reset(); + pk1.set_int(pk1_value); + pk2.set_varchar(pk2_value); + pk2.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(pk2)); + value.set_mediumint(INT32_MAX); + ASSERT_EQ(OB_SUCCESS, entity->set_property(ObString::make_string("cmediumint"), value)); + ObTableOperation table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_DATA_OUT_OF_RANGE, r.get_errno()); + } + { // case: insert_or_update varbinary -> blob // TODO (luohongdi.lhd): Fix it when lob ready @@ -2093,226 +1218,637 @@ TEST_F(TestBatchExecute, partial_update) } - // create table if not exists append_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext not null); -//TEST_F(TestBatchExecute, append_lob) -//{ - //// setup - //ObTable *the_table = NULL; - //int ret = service_client_->alloc_table(ObString::make_string("append_lob_test"), the_table); - //ASSERT_EQ(OB_SUCCESS, ret); - //ObTableEntityFactory entity_factory; - //ObTableOperationResult r; - //ObITableEntity *entity = NULL; - //const ObITableEntity *result_entity = NULL; - //ObTableOperation table_operation; +TEST_F(TestBatchExecute, append_lob) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("append_lob_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableOperationResult r; + ObITableEntity *entity = NULL; + const ObITableEntity *result_entity = NULL; + ObTableOperation table_operation; - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //int64_t key_key = 139107; - //ObObj key; - //key.set_int(key_key); - //int64_t value_value = 33521; - //ObObj value; - //value.set_int(value_value); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + int64_t key_key = 139107; + ObObj key; + key.set_int(key_key); + int64_t value_value = 33521; + ObObj value; + value.set_int(value_value); - ////prepare data - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObString c3_value = ObString::make_string("hello world"); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //table_operation = ObTableOperation::insert(*entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + //prepare data + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //entity->reset(); - //key.set_int(key_key + 1); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(1235); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //table_operation = ObTableOperation::insert(*entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + entity->reset(); + key.set_int(key_key + 1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(1235); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ////append small - //{ - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //table_operation = ObTableOperation::append(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(true); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::APPEND, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(NULL != result_entity); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(1, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); - //ASSERT_EQ(key_key, value.get_int()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello worldhello world"); - //ASSERT_TRUE(str == c3_new_value); - //ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); - //} + //append small + { + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::append(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(true); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::APPEND, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(1, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); + ASSERT_EQ(key_key, value.get_int()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello worldhello world"); + ASSERT_TRUE(str == c3_new_value); + ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); + } - ////append big - //{ - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //const int32_t big_value_len = 3 * 1024 * 1024; - //char *big_value = (char*) malloc(big_value_len); - //ASSERT_TRUE(NULL != big_value); - //memset(big_value, 'A', big_value_len); - //ObString c3_big_value(big_value_len, big_value); + //append big + { + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + const int32_t big_value_len = 3 * 1024 * 1024; + char *big_value = (char*) malloc(big_value_len); + ASSERT_TRUE(NULL != big_value); + memset(big_value, 'A', big_value_len); + ObString c3_big_value(big_value_len, big_value); - //value.set_varchar(c3_big_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //table_operation = ObTableOperation::append(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(true); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::APPEND, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(1, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); - //ASSERT_EQ(key_key, value.get_int()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); + value.set_varchar(c3_big_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::append(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(true); + // TODO:@linjing concat还未支持大对象，待修复到master后patch到特性分支 + ASSERT_EQ(OB_SIZE_OVERFLOW, the_table->execute(table_operation, req_options, r)); + // ASSERT_EQ(OB_SUCCESS, r.get_errno()); + // ASSERT_EQ(1, r.get_affected_rows()); + // ASSERT_EQ(ObTableOperationType::APPEND, r.type()); + // ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + // ASSERT_TRUE(!result_entity->is_empty()); + // ASSERT_EQ(1, result_entity->get_rowkey_size()); + // ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); + // ASSERT_EQ(key_key, value.get_int()); + // ASSERT_EQ(1, result_entity->get_properties_count()); + // ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + // ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); - //if (NULL != big_value) { - //free(big_value); - //big_value = NULL; - //} - //} + if (NULL != big_value) { + free(big_value); + big_value = NULL; + } + } - //// teardown - //service_client_->free_table(the_table); - //the_table = NULL; -//} + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} +// for lob column +// drop table if exists all_lob_test; create table if not exists all_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext, index i1(c2) local) +// TEST_F(TestBatchExecute, lob_column_test) +// { +// // setup +// ObTable *the_table = NULL; +// int ret = service_client_->alloc_table(ObString::make_string("all_lob_test"), the_table); +// ASSERT_EQ(OB_SUCCESS, ret); +// ObTableEntityFactory entity_factory; +// ObTableOperationResult r; +// ObITableEntity *entity = NULL; +// const ObITableEntity *result_entity = NULL; +// ObTableOperation table_operation; +// // k,v +// ObObj key; +// ObObj value; +// int64_t key_key = 139107; +// int64_t value_value = 33521; +// // small value +// ObString c3_value = ObString::make_string("hello world"); +// // big value +// const int32_t big_value_len = 3 * 1024 * 1024; +// char *big_value = (char*) malloc(big_value_len); +// ASSERT_TRUE(NULL != big_value); +// memset(big_value, 'A', big_value_len); +// ObString c3_big_value(big_value_len, big_value); +// entity = entity_factory.alloc(); +// { +// // insert small value +// ASSERT_TRUE(NULL != entity); +// key.set_int(key_key); +// value.set_int(value_value); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); +// value.set_varchar(c3_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::insert(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::INSERT, r.type()); +// ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); +// // insert big value +// entity->reset(); +// key.set_int(key_key + 1); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// value.set_int(1235); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); +// value.set_varchar(c3_big_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::insert(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::INSERT, r.type()); +// ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); +// } +// { +// // delete small value +// entity->reset(); +// key.set_int(key_key); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// table_operation = ObTableOperation::del(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// const ObITableEntity *result_entity = NULL; +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::DEL, r.type()); +// // delete big value +// entity->reset(); +// key.set_int(key_key + 1); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// table_operation = ObTableOperation::del(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::DEL, r.type()); +// } +// { +// // replace small value +// entity->reset(); +// key.set_int(key_key); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// ObString c3_new_value = ObString::make_string("rere hello world"); +// value.set_varchar(c3_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::append(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::APPEND, r.type()); +// //replace big +// entity->reset(); +// key.set_int(key_key+1); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// value.set_varchar(c3_big_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::append(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::APPEND, r.type()); +// } +// { +// // update small value +// entity->reset(); +// key.set_int(key_key); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// ObString c3_new_value = ObString::make_string("upup hello world"); +// value.set_varchar(c3_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::update(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); +// //update big value +// entity->reset(); +// key.set_int(key_key+1); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// value.set_varchar(c3_big_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::update(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); +// } +// { +// // insert_up small value +// entity->reset(); +// key.set_int(key_key); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// value.set_int(value_value+1); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); +// ObString c3_new_value = ObString::make_string("final hello world"); +// value.set_varchar(c3_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::insert_or_update(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); +// //insert_up big value +// entity->reset(); +// key.set_int(key_key+1); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// value.set_varchar(c3_big_value); +// value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); +// table_operation = ObTableOperation::insert_or_update(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); +// } +// { +// // get small value +// ObObj null_obj; +// entity->reset(); +// key.set_int(key_key); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); +// table_operation = ObTableOperation::retrieve(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// result_entity = NULL; +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(0, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::GET, r.type()); +// ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); +// ASSERT_EQ(0, result_entity->get_rowkey_size()); +// ASSERT_EQ(2, result_entity->get_properties_count()); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); +// ASSERT_EQ(value_value+1,value.get_int()); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); +// ObString str; +// ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); +// ASSERT_TRUE(str.case_compare("final hello world")); +// // get big value +// entity->reset(); +// key.set_int(key_key+1); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); +// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); +// table_operation = ObTableOperation::retrieve(*entity); +// ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); +// result_entity = NULL; +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(0, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::GET, r.type()); +// ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); +// ASSERT_EQ(0, result_entity->get_rowkey_size()); +// ASSERT_EQ(1, result_entity->get_properties_count()); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); +// ObString str2; +// ASSERT_EQ(OB_SUCCESS, value.get_varchar(str2)); +// ASSERT_TRUE(c3_big_value == str2); +// } -// create table if not exists generate_col_test (C1 bigint primary key, C2 bigint, C3 varchar(100), C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2))); -//TEST_F(TestBatchExecute, generate_col_test) -//{ - //// setup - //ObTable *the_table = NULL; - //int ret = service_client_->alloc_table(ObString::make_string("generate_col_test"), the_table); - //ASSERT_EQ(OB_SUCCESS, ret); - //ObTableEntityFactory entity_factory; - //ObTableOperationResult r; - //ObITableEntity *entity = NULL; - //const ObITableEntity *result_entity = NULL; - //ObTableOperation table_operation; - //ObString C3_PREFIX = ObString::make_string("C3_PREFIX"); +// if (NULL != big_value) { +// free(big_value); +// big_value = NULL; +// } +// // teardown +// service_client_->free_table(the_table); +// the_table = NULL; +// } - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //int64_t key_key = 139107; - //ObObj key; - //key.set_int(key_key); - //int64_t value_value = 33521; - //ObObj value; - //value.set_int(value_value); +// create table if not exists virtual_generate_col_test +// (C1 bigint primary key, C2 bigint, C3 varchar(100), +// C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2))); +TEST_F(TestBatchExecute, virtual_generate_col_test) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("virtual_generate_col_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableOperationResult r; + ObITableEntity *entity = NULL; + const ObITableEntity *result_entity = NULL; + ObTableOperation table_operation; + ObString C3_PREFIX = ObString::make_string("C3_PREFIX"); - ////prepare data insert - //ObString c3_value = ObString::make_string("hello world"); - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(value_value); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //table_operation = ObTableOperation::insert(*entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + int64_t key_key = 139107; + ObObj key; + key.set_int(key_key); + int64_t value_value = 33521; + ObObj value; + value.set_int(value_value); - ////get and check - //ObObj null_obj; - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3_PREFIX, null_obj)); - //table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3_PREFIX, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("he"); - //ASSERT_TRUE(str == c3_new_value); + //prepare data insert + ObString c3_value = ObString::make_string("hello world"); + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(value_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ////update and check - //c3_value = ObString::make_string("test hello"); - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //table_operation = ObTableOperation::update(*entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + //get and check + ObObj null_obj; + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3_PREFIX, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3_PREFIX, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("he"); + ASSERT_TRUE(str == c3_new_value); - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3_PREFIX, null_obj)); - //table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3_PREFIX, value)); - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //c3_new_value = ObString::make_string("te"); - //ASSERT_TRUE(str == c3_new_value); + //update and check + c3_value = ObString::make_string("test hello"); + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3_PREFIX, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3_PREFIX, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + c3_new_value = ObString::make_string("te"); + ASSERT_TRUE(str == c3_new_value); - //// teardown - //service_client_->free_table(the_table); - //the_table = NULL; -//} + // delete & cleanup + { + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} +// create table if not exists store_generate_col_test +// (c1 bigint primary key, c2 varchar(10), c3 varchar(10), +// gen varchar(30) generated always as (concat(c2,c3)) stored) +TEST_F(TestBatchExecute, stored_generate_col_test) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("store_generate_col_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableOperationResult r; + ObITableEntity *entity = NULL; + const ObITableEntity *result_entity = NULL; + ObTableOperation table_operation; + ObString GEN = ObString::make_string("gen"); + + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + int64_t key_key = 1; + ObObj key; + ObObj value; + + //prepare data insert + ObString c2_value = ObString::make_string("hello"); + ObString c3_value = ObString::make_string("world"); + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c2_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + + //get and check + ObObj null_obj; + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(GEN, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(GEN, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString gen_new_value = ObString::make_string("helloworld"); + ASSERT_TRUE(str == gen_new_value); + + //update and check + c3_value = ObString::make_string("oceanbase"); + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(GEN, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(GEN, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + gen_new_value = ObString::make_string("hellooceanbase"); + ASSERT_TRUE(str == gen_new_value); + + // insert or update (insert) + c3_value = ObString::make_string("world"); + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c2_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + + //get and check + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(GEN, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(GEN, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + gen_new_value = ObString::make_string("helloworld"); + ASSERT_TRUE(str == gen_new_value); + + // insert or update (update) + c2_value = ObString::make_string("oceanbase"); + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c2_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + + //get and check + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(GEN, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(GEN, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + gen_new_value = ObString::make_string("oceanbaseworld"); + ASSERT_TRUE(str == gen_new_value); + + // delete & cleanup + { + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} // create table if not exists large_scan_test (C1 bigint primary key, C2 bigint, C3 varchar(100)); TEST_F(TestBatchExecute, large_scan) @@ -2396,7 +1932,7 @@ TEST_F(TestBatchExecute, large_scan) //large batch append { - int64_t append_batch_size = 1000; + int64_t append_batch_size = 10; ObTableBatchOperation batch_operation; for (int64_t i = 0; i < append_batch_size; ++i) { entity = entity_factory.alloc(); @@ -2513,8 +2049,8 @@ TEST_F(TestBatchExecute, uniq_replace) //@TODO: table_api::replace need to return affected_rows=2 here, //but table api not maintain local index here, so affected_rows is 1 //need to fix me - //ASSERT_EQ(2, r.get_affected_rows()); - ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(2, r.get_affected_rows()); + // ASSERT_EQ(1, r.get_affected_rows()); ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); @@ -2543,8 +2079,6 @@ TEST_F(TestBatchExecute, uniq_replace) // create table if not exists execute_query_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2), INDEX idx1(C1, C2)); -ObString PK1 = ObString::make_string("PK1"); -ObString PK2 = ObString::make_string("PK2"); TEST_F(TestBatchExecute, query_pk_prefix) { // setup @@ -2722,10 +2256,6 @@ TEST_F(TestBatchExecute, compare_cell) ASSERT_TRUE(ObHTableUtils::compare_cell(cell2, cell1, scan_order) > 0); } -ObString K = ObString::make_string("K"); -ObString Q = ObString::make_string("Q"); -ObString T = ObString::make_string("T"); -ObString V = ObString::make_string("V"); // create table htable1_cf1 (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) partition by key(K) partitions 16; TEST_F(TestBatchExecute, htable_scan_basic) { @@ -3269,6 +2799,2415 @@ fprintf(stderr, "Case 1: reverse scan set max version only\n"); } } + +TEST_F(TestBatchExecute, hcolumn_desc) +{ + ObHColumnDescriptor column_desc; + ObString str = ObString::make_string("{\"HColumnDescriptor\": {\"TimeToLive\": 3600}}"); + ASSERT_EQ(OB_SUCCESS, column_desc.from_string(str)); + fprintf(stderr, "ttl=%d\n", column_desc.get_time_to_live()); + ASSERT_EQ(3600, column_desc.get_time_to_live()); +} + + +TEST_F(TestBatchExecute, htable_ttl) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_ttl"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + static constexpr int64_t VERSIONS_COUNT = 10; + static constexpr int64_t COLUMNS_SIZE = 10; + char qualifier[COLUMNS_SIZE][128]; + char qualifier2[COLUMNS_SIZE][128]; + for (int i = 0; i < COLUMNS_SIZE; ++i) + { + sprintf(qualifier[i], "cq%d", i); + } // end for + char values[VERSIONS_COUNT][128]; + for (int i = 0; i < VERSIONS_COUNT; ++i) { + sprintf(values[i], "ob%d", i); + } + const char* rowkey = "row0"; + ObObj key1, key2, key3; + ObObj value; + key1.set_varbinary(ObString::make_string(rowkey)); + for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { + key2.set_varbinary(ObString::make_string(qualifier[j])); // cq0 ~ cq9 + key3.set_int(INT64_MAX); // now + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + value.set_varbinary(ObString::make_string(values[j])); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } // end for + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(1, result.count()); + fprintf(stderr, "sleep 3\n"); + sleep(3); // ttl is 5 + { + // verify + ObTableQuery query; + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); + + ObHTableFilter &htable_filter = query.htable_filter(); + htable_filter.set_valid(true); + htable_filter.clear_columns(); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + int cqids_sorted[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + char expected_str[128]; + ObString str; + key1.set_varbinary(ObString::make_string(rowkey)); + for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { + sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); + key2.set_varbinary(ObString::make_string(qualifier2[j])); + sprintf(expected_str, "ob%ld", j); + + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); + + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_TRUE(0 == str.compare(ObString::make_string(expected_str))); + } // end for + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + // insert new value for cq0, cq2, cq4, ... cq8 + entity_factory.free_and_reuse(); + batch_operation.reset(); + for (int64_t j = 0; j < COLUMNS_SIZE/2; ++j) { + key2.set_varbinary(ObString::make_string(qualifier[j*2])); + key3.set_int(INT64_MAX); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + value.set_varbinary(ObString::make_string(values[j*2])); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } // end for + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(1, result.count()); + + // sleep for 3 seconds + fprintf(stderr, "sleep 3\n"); + sleep(3); // ttl is 5 + { + // verify + ObTableQuery query; + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); + + ObHTableFilter &htable_filter = query.htable_filter(); + htable_filter.set_valid(true); + htable_filter.clear_columns(); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + int cqids_sorted[] = {0, 2, 4, 6, 8}; + char expected_str[128]; + ObString str; + key1.set_varbinary(ObString::make_string(rowkey)); + for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { + sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); + key2.set_varbinary(ObString::make_string(qualifier2[j])); + sprintf(expected_str, "ob%ld", j*2); + + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); + + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_TRUE(0 == str.compare(ObString::make_string(expected_str))); + } // end for + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} + +// create table if not exists secondary_index_test ( +// C1 bigint primary key, +// C2 bigint, +// C3 varchar(100), +// index i1(c2) local, +// index i2(c3) local, +// index i3(c2, c3) local); +TEST_F(TestBatchExecute, secondary_index) +{ + OB_LOG(INFO, "begin secondary_index"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("secondary_index_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + + // + // > select * from secondary_index_test limit 10; + // +----+------+------+ + // | C1 | C2 | C3 | + // +----+------+------+ + // | 0 | 0 | aaa | + // | 2 | 1 | xxx | + // | 4 | 2 | yyy | + // | 6 | 3 | zzz | + // | 8 | 0 | AAA | + // | 10 | 1 | XXX | + // | 12 | 2 | YYY | + // | 14 | 3 | ZZZ | + // | 16 | 0 | aaa | + // | 18 | 1 | xxx | + // +----+------+------+ + + const char *c3_values[] = {"aaa", "xxx", "yyy", "zzz", "AAA", "XXX", "YYY", "ZZZ"}; + const int64_t c3_values_count = ARRAYSIZEOF(c3_values); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(i%4); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(ObString::make_string(c3_values[i%c3_values_count])); + value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(1, r.get_affected_rows()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // query using index i1(c2) + { + ObTableQuery query; + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + // 扫描C2为1和2的所有数据 + ObObj pk_objs_start[1]; + pk_objs_start[0].set_int(1); + ObObj pk_objs_end[1]; + pk_objs_end[0].set_int(2); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 1); + range.end_key_.assign(pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + query.set_scan_index(ObString::make_string("i1")); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ObObj obj1, obj2, obj3; + ObString str; + for (int64_t i = 0; i < BATCH_SIZE/2; ++i) + { + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ASSERT_EQ(3, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); + //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); + ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); + if (i % 2 == 0) { + ASSERT_EQ(1 , obj2.get_int()); + ASSERT_EQ(2 + 4 * i, obj1.get_int()); // 2, 10, 18,... + } else { + ASSERT_EQ(2 , obj2.get_int()); + ASSERT_EQ(4 + (i - 1) * 4, obj1.get_int()); // 4, 12, 20,... + } + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + // query using index i2(c3) + { + ObTableQuery query; + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[1]; + pk_objs_start[0].set_varchar(ObString::make_string("xxx")); + pk_objs_start[0].set_collation_type(CS_TYPE_UTF8MB4_BIN); + ObObj pk_objs_end[1]; + pk_objs_end[0].set_varchar(ObString::make_string("xxx")); + pk_objs_end[0].set_collation_type(CS_TYPE_UTF8MB4_BIN); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 1); + range.end_key_.assign(pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + query.set_scan_index(ObString::make_string("i2")); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ObObj obj1, obj2, obj3; + ObString str; + for (int64_t i = 0; i < BATCH_SIZE/4; ++i) + { + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ASSERT_EQ(3, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); + //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); + ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); + ASSERT_EQ(2+i*8, obj1.get_int()); + ASSERT_EQ(1 , obj2.get_int()); + ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + // query using index i3(c2,c3) + { + ObTableQuery query; + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[2]; + pk_objs_start[0].set_int(1); + pk_objs_start[1].set_varchar(ObString::make_string("xxx")); + pk_objs_start[1].set_collation_type(CS_TYPE_UTF8MB4_BIN); + ObObj pk_objs_end[2]; + pk_objs_end[0].set_int(1); + pk_objs_end[1].set_varchar(ObString::make_string("xxx")); + pk_objs_end[1].set_collation_type(CS_TYPE_UTF8MB4_BIN); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 2); + range.end_key_.assign(pk_objs_end, 2); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + query.set_scan_index(ObString::make_string("i3")); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ObObj obj1, obj2, obj3; + ObString str; + for (int64_t i = 0; i < BATCH_SIZE/4; ++i) + { + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ASSERT_EQ(3, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); + //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); + ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); + ASSERT_EQ(2+i*8, obj1.get_int()); + ASSERT_EQ(1 , obj2.get_int()); + ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + // 删除后一半row + batch_operation.reset(); + result.reset(); + for (int64_t i = 0; i < BATCH_SIZE/2; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int((BATCH_SIZE/2+i)*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); + } + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE/2, result.count()); + // then update C2 (2->0) + batch_operation.reset(); + result.reset(); + for (int64_t i = 0; i < BATCH_SIZE/2; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + if (i % 4 == 2) { + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(0); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); + } + } + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE/8, result.count()); + // query using index i1(c2) again + { + ObTableQuery query; + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[1]; + pk_objs_start[0].set_int(1); + ObObj pk_objs_end[1]; + pk_objs_end[0].set_int(2); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 1); + range.end_key_.assign(pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + query.set_scan_index(ObString::make_string("i1")); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ObObj obj1, obj2, obj3; + ObString str; + for (int64_t i = 0; i < 13; ++i) + { + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ASSERT_EQ(3, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); + //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); + ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); + ASSERT_EQ(2+i*8, obj1.get_int()); + ASSERT_EQ(1 , obj2.get_int()); + ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} + + +TEST_F(TestBatchExecute, htable_empty_qualifier) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_empty_cq"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + static constexpr int64_t VERSIONS_COUNT = 10; + DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_SIZE]; + ASSERT_TRUE(NULL != rows); + ObObj key1, key2, key3; + ObObj value; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + sprintf(rows[i], "row%ld", i); + key1.set_varbinary(ObString::make_string(rows[i])); + key2.set_varbinary(ObString::make_string("")); // empty qualifier + for (int64_t k = 0; k < VERSIONS_COUNT; ++k) + { + key3.set_int(k); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + value.set_varbinary(ObString::make_string("value_string")); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } // end for + } // end for + + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE*VERSIONS_COUNT, result.count()); + //////////////////////////////////////////////////////////////// + ObTableQuery query; + ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); + ObObj pk_objs_start[3]; + pk_objs_start[0].set_varbinary(ObString::make_string("row50")); + pk_objs_start[1].set_min_value(); + pk_objs_start[2].set_min_value(); + ObObj pk_objs_end[3]; + pk_objs_end[0].set_varbinary(ObString::make_string("row59")); + pk_objs_end[1].set_max_value(); + pk_objs_end[2].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 3); + range.end_key_.assign(pk_objs_end, 3); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + + ObHTableFilter &htable_filter = query.htable_filter(); + ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(""))); + htable_filter.set_max_versions(2); + htable_filter.set_valid(true); + { + // verify put result + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + int64_t timestamps[] = {9, 8}; + for (int64_t i = 0; i < 10; ++i) { + // 10 rowkeys + sprintf(rows[i], "row%ld", i+50); + key1.set_varbinary(ObString::make_string(rows[i])); + key2.set_varbinary(ObString::make_string("")); + for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) + { + key3.set_int(timestamps[k]); + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_EQ(key3, ts); + } // end for + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + { + // delete row50 by qualifier and version + const char* rowkey = "row50"; + const char* cq = ""; + int64_t ts = 8; + batch_operation.reset(); + sprintf(rows[0], "%s", rowkey); + key1.set_varbinary(ObString::make_string(rows[0])); + key2.set_varbinary(ObString::make_string(cq)); + key3.set_int(ts); // delete the specified version + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); + + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + } + + { + // verify delete result + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + int64_t timestamps[] = {9, 8}; + for (int64_t i = 0; i < 10; ++i) { + // 10 rowkeys + sprintf(rows[i], "row%ld", i+50); + key1.set_varbinary(ObString::make_string(rows[i])); + key2.set_varbinary(ObString::make_string("")); + for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) + { + if (0 == i && k == 1) { + key3.set_int(7); + } else { + key3.set_int(timestamps[k]); + } + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + // fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + ASSERT_EQ(key3, ts); + } // end for + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; + delete [] rows; +} + +#define T_CASE_SUCC(SRC_TYPE, SRC_VAL, DELTA_TYPE, DELTA_VAL, RES_TYPE, RES_TYPE2, RES_VAL) \ + src.set_##SRC_TYPE(SRC_VAL);\ + delta.set_##DELTA_TYPE(DELTA_VAL);\ + target_type.set_type(src.get_type()); \ + target.set_null();\ + ASSERT_EQ(OB_SUCCESS, ObTableService::obj_increment(delta, src, target_type, target)); \ + ASSERT_EQ(Ob##RES_TYPE##Type, target.get_type()); \ + ASSERT_EQ(RES_VAL, target.get_##RES_TYPE2());\ + fprintf(stderr, "CASE: " #SRC_TYPE "(" #SRC_VAL ") + " #DELTA_TYPE "(" #DELTA_VAL ") = " #RES_TYPE2 "(" #RES_VAL ")\n"); + +#define T_CASE_FAIL(SRC_TYPE, SRC_VAL, DELTA_TYPE, DELTA_VAL, RES) \ + src.set_##SRC_TYPE(SRC_VAL);\ + delta.set_##DELTA_TYPE(DELTA_VAL);\ + target_type.set_type(src.get_type()); \ + target.set_null();\ + ASSERT_EQ(RES, ObTableService::obj_increment(delta, src, target_type, target)); \ + ASSERT_TRUE(target.is_null());\ + fprintf(stderr, "CASE: " #SRC_TYPE "(" #SRC_VAL ") + " #DELTA_TYPE "(" #DELTA_VAL ") = " #RES "\n"); + + +#define T_CASE_APP(SRC_VAL, SRC_CS_TYPE, DELTA_VAL, DELTA_CS_TYPE, RES_VAL, RES_CS_TYPE)\ + src.set_varchar(ObString::make_string(SRC_VAL));\ + src.set_collation_type(SRC_CS_TYPE);\ + delta.set_varchar(ObString::make_string(DELTA_VAL));\ + delta.set_collation_type(DELTA_CS_TYPE);\ + target_type.set_type(ObVarcharType);\ + target_type.set_collation_type(src.get_collation_type()); \ + target.set_null();\ + alloc.reset(); \ + ASSERT_EQ(OB_SUCCESS, ObTableService::obj_append(delta, src, target_type, alloc, target)); \ + ASSERT_EQ(ObVarcharType, target.get_type());\ + ASSERT_EQ(RES_CS_TYPE, target.get_collation_type());\ + ASSERT_TRUE(ObString::make_string(RES_VAL) == target.get_varchar());\ + fprintf(stderr, "CASE: " #SRC_CS_TYPE "(" #SRC_VAL ") || " #DELTA_CS_TYPE "(" #DELTA_VAL ") = " #RES_VAL "\n"); + +TEST(TestQueryResult, alloc_memory_if_need) +{ + int ret = OB_SUCCESS; + const int64_t alloc_size = 1024; + ObTableQueryResult query_result; + int64_t total_size = query_result.allocator_.total(); + while (OB_SUCC(ret)) { + ret = query_result.alloc_buf_if_need(alloc_size); + query_result.buf_.get_position() += alloc_size; + if (query_result.allocator_.total() != total_size) { + total_size = query_result.allocator_.total(); + printf("allocator: %ld, result_buf: %ld\n", query_result.allocator_.total(), query_result.buf_.get_capacity()); + } + } + ASSERT_EQ(query_result.buf_.get_capacity(), ObTableQueryResult::MAX_BUF_BLOCK_SIZE * 1); + ASSERT_GT(query_result.allocator_.total(), ObTableQueryResult::MAX_BUF_BLOCK_SIZE * 1); + ASSERT_LE(query_result.allocator_.total(), ObTableQueryResult::MAX_BUF_BLOCK_SIZE * 3); +} + +TEST_F(TestBatchExecute, update_table_with_index_by_lowercase_rowkey) +{ + + OB_LOG(INFO, "begin update_table_with_index_by_lowercase_rowkey"); + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("varchar_rowkey_update_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + // case for insert operation + ObTableEntityFactory entity_factory; + ObITableEntity *insert_entity = NULL; + ObITableEntity *update_entity = NULL; + ObTableOperationResult r; + { + // case: insert with rowkey + insert_entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != insert_entity); + + const char * rk_value = "TEST"; + int64_t v_value = 139107; + ObObj rk_obj; + rk_obj.set_varchar(rk_value); + rk_obj.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ObObj v_obj; + v_obj.set_int(v_value); + ASSERT_EQ(OB_SUCCESS, insert_entity->add_rowkey_value(rk_obj)); + ASSERT_EQ(OB_SUCCESS, insert_entity->set_property(T, v_obj)); + ObTableOperation table_operation = ObTableOperation::insert(*insert_entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + } + + { + // case: update with lowercase rowkey + update_entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != update_entity); + + const char * rk_value = "test"; + int64_t v_value = 1; + ObObj rk_obj; + rk_obj.set_varchar(rk_value); + rk_obj.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ObObj v_obj; + v_obj.set_int(v_value); + ASSERT_EQ(OB_SUCCESS, update_entity->add_rowkey_value(rk_obj)); + ASSERT_EQ(OB_SUCCESS, update_entity->set_property(T, v_obj)); + ObTableOperation table_operation = ObTableOperation::insert_or_update(*update_entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + } + { + // query with index + ObTableQuery query; + ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); + ObNewRange range; + range.set_whole_range(); + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + query.set_scan_index(ObString::make_string("idx_T")); + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + ObObj obj1, obj2, obj3; + ObString str; + for (int64_t i = 0; i < 1; ++i) + { + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, obj1)); + ASSERT_EQ(OB_SUCCESS, obj1.get_varchar(str)); + ASSERT_TRUE(str == ObString::make_string("test")); + } + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + } + + // todo@wenqu: mysqlproxy使用时报-1044，后面再调通。 +// ObISQLClient::ReadResult res; +// uint64_t tenant_id = service_client_->get_tenant_id(); +// ObString col_val; +// ret = service_client_->get_user_sql_client().read(res, tenant_id, +// "select /*+ index(varchar_rowkey_update_test idx_T)*/ K from test.varchar_rowkey_update_test"); +// ASSERT_EQ(OB_SUCCESS, ret) << "tenant_id: " << tenant_id << "\n"; +// sqlclient::ObMySQLResult *mysql_result = res.get_result(); +// ASSERT_TRUE(NULL != mysql_result); +// ASSERT_EQ(OB_SUCCESS, mysql_result->next()); +// ASSERT_EQ(OB_SUCCESS, mysql_result->get_varchar("K", col_val)); +// ASSERT_TRUE(col_val == ObString::make_string("TEST")); + + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} + +TEST_F(TestBatchExecute, single_get) +{ + OB_LOG(INFO, "begin single_get"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_get_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert (C1, C2, C3): (1234, 56.78, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1234; + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 56.78; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + const ObString default_c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get existed rowkey + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(c2_value, value.get_double()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + + // rowkey existed, but column not exist + ObString C4 = ObString::make_string("C4"); + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C4, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_NE(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get not existed rowkey + entity->reset(); + key.set_int(key_key + 1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + +} + +TEST_F(TestBatchExecute, multi_get) +{ + OB_LOG(INFO, "begin multi_get complex"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_get_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + // prepare data + ObITableEntity *entity = NULL; + ObString c3_value = "c3_value"; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + const ObITableEntity *result_entity = NULL; + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // case 1:complex batch get and verify --- ObTableProccessType::TABLE_API_BATCH_RETRIVE + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + if (i % 2 == 1) { + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + } else { + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + } + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(!batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ObString str; + if (i % 2 == 1) { + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(OB_SUCCESS != result_entity->get_property(C3, value)); + ASSERT_EQ(100+i, value.get_int()); + } else { + ASSERT_TRUE(OB_SUCCESS != result_entity->get_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + } + } + + // case 2: multi_get --- ObTableProccessType::TABLE_API_MULTI_GET; + { + ObObj null_obj; + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = BATCH_SIZE-1; i >= 0; --i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C2)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + if (i % 2 == 1) { + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+(BATCH_SIZE-1-i)/2, value.get_int()); + } else { + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + } + service_client_->free_table(the_table); +} + +// create table if not exists single_insert_test +// (C1 bigint primary key, C2 double, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, single_insert) +{ + OB_LOG(INFO, "begin single_insert"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_insert_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert (C1, C2, C3): (1234, 56.78, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1234; + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 56.78; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + const ObString default_c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get C1 == 1234 + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(c2_value, value.get_double()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + + // insert (C1, C3): (1235, "table api is delicious") + entity->reset(); + key.set_int(key_key + 1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get C1 == 1235 + entity->reset(); + key.set_int(key_key + 1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(value.is_null()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + + // insert (C1): (1236) + entity->reset(); + key.set_int(key_key + 2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get C1 == 1236 + entity->reset(); + key.set_int(key_key + 2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(value.is_null()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == default_c3_value); +} + +// create table if not exists single_insert_test +// (C1 bigint primary key, C2 double, C3 varchar(100), +// C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2))) +// PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, insert_generate) +{ + OB_LOG(INFO, "begin insert_generate_test"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("insert_generate_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert (C1, C2, C3): (1234, 56.78, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1234; + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 56.78; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + const ObString default_c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get C1 == 1234 + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(c2_value, value.get_double()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + + // insert (C1, C3): (1235, "table api is delicious") + entity->reset(); + key.set_int(key_key + 1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get C1 == 1235 + entity->reset(); + key.set_int(key_key + 1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(value.is_null()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + + // insert (C1): (1236) + entity->reset(); + key.set_int(key_key + 2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // get C1 == 1236 + entity->reset(); + key.set_int(key_key + 2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(value.is_null()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == default_c3_value); +} + +// create table if not exists single_update_test +// (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') +// PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, single_update) +{ + OB_LOG(INFO, "begin single_update"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_update_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert (C1, C2, C3): (1234, 56.78, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1234; + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 56.78; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // update C2 + { + entity->reset(); + result_entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_double(78.9); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(78.9, value.get_double()); + } + + // test for cache: update C2 + { + entity->reset(); + result_entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_double(96.0); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(96.0, value.get_double()); + } +} + +// create table if not exists update_generate_test +// (C1 bigint primary key, C2 varchar(100), C3 varchar(100), GEN varchar(100) GENERATED ALWAYS AS (concat(C2,c3)) stored) +// PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, update_generate) +{ + OB_LOG(INFO, "begin update_generate_test"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("update_generate_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert (C1, C2, C3): (1, 'hello ', "table api") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1; + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObString c2_value = ObString::make_string("hello "); + value.set_varchar(c2_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + + // update C2 + { + entity->reset(); + result_entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObString c2_update_value = ObString::make_string("hi "); + value.set_varchar(c2_update_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c2_value = ObString::make_string("hi "); + ASSERT_TRUE(str == c2_value); + } +} + +// create table if not exists single_delete_test +// (C1 bigint primary key, C2 double, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, single_delete) +{ + OB_LOG(INFO, "begin single_delete"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_delete_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + + // insert (C1, C2, C3): (1, 56.78, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1; + key.set_int(key_key); + { + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 56.78; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + const ObString default_c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + } + + // delete + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // delete not exist row + { + entity->reset(); + ObObj not_exist_key; + not_exist_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(not_exist_key)); + ObTableOperation table_operation = ObTableOperation::del(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } +} + +// create table if not exists single_replace_test +// (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, single_replace) +{ + OB_LOG(INFO, "begin single_replace"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_replace_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableOperationResult r; + ObITableEntity *result_entity = NULL; + + // insert (C1, C2, C3): (1, 1.1, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1; + key.set_int(key_key); + { + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 1.1; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + const ObString default_c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + } + + // replace C3 (Duplicate rowkey) + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 2.2; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("obkv is delicious"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::replace(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(2, r.get_affected_rows()); // delete + insert + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_value = ObString::make_string("obkv is delicious"); + ASSERT_TRUE(str == c3_value); + } + + // replace C3 (new rowkey) + { + entity->reset(); + ObObj new_key; + new_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(new_key)); + double c2_value = 3.3; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("obkv and tableapi are delicious"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::replace(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); // insert + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ObObj new_key; + new_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(new_key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_value = ObString::make_string("obkv and tableapi are delicious"); + ASSERT_TRUE(str == c3_value); + } +} + +// create table if not exists replace_unique_key_test +// (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', +// unique index i1(c2) local) PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, replace_unique_key) +{ + OB_LOG(INFO, "begin replace_unique_key_test"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("replace_unique_key_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableOperationResult r; + ObITableEntity *result_entity = NULL; + + // insert (C1, C2, C3): (1, 1.1, "table api is delicious") + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1; + key.set_int(key_key); + { + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 1.1; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("table api is delicious"); + const ObString default_c3_value = ObString::make_string("hello world"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(result_entity->is_empty()); + } + + // replace C3 (Duplicate unique key) + { + entity->reset(); + ObObj new_key; + new_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(new_key)); + double c2_value = 1.1; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObString c3_value = ObString::make_string("ob is delicious"); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::replace(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(2, r.get_affected_rows()); // delete + insert + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ObObj new_key; + new_key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(new_key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_value = ObString::make_string("ob is delicious"); + ASSERT_TRUE(str == c3_value); + } +} + +// create table if not exists single_insert_up_test +// (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, single_insert_up) +{ + OB_LOG(INFO, "begin single_insert_up"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_insert_up_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableOperationResult r; + ObITableEntity *result_entity = NULL; + + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + ObObj value; + int key_key = 1; + key.set_int(key_key); + + // insert_or_update C2: insert + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 1.1; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(1.1, value.get_double()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_value); + } + + // insert_or_update C2: update + { + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + double c2_value = 2.2; + value.set_double(c2_value); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::insert_or_update(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // get + { + ObObj null_obj; + entity->reset(); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(2.2, value.get_double()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_value); + } +} + +// CREATE TABLE IF NOT EXISTS `kv_query_test` ( +// C1 bigint, +// C2 bigint, +// C3 bigint, +// PRIMARY KEY(`C1`, `C2`), +// KEY idx_c2 (`C2`), +// KEY idx_c3 (`C3`), +// KEY idx_c2c3(`C2`, `C3`)); +// INSERT INTO kv_query_test VALUES (1,2,3),(4,5,6),(7,8,9),(10,11,12),(13,14,15); +TEST_F(TestBatchExecute, table_query_with_secondary_index) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("kv_query_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableOperationResult r; + const ObITableEntity *result_entity = NULL; + ObTableEntityIterator *iter = nullptr; + int expect_query_cnt = 5; + ObArray properties_values; + + // insert + { + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const int INERT_COUNT = 5; + for (int64_t i = 0; i < INERT_COUNT; i++) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key1; + key1.set_int(i * 3 + 1); + ObObj key2; + key2.set_int(i * 3 + 2); + ObObj value; + value.set_int(i * 3 + 3); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(INERT_COUNT, result.count()); + for (int64_t i = 0; i < INERT_COUNT; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(1, r.get_affected_rows()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + + //scan + ObTableQuery query; + { + // case 1: primary key + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[2]; + pk_objs_start[0].set_int(0); + pk_objs_start[1].set_min_value(); + ObObj pk_objs_end[2]; + pk_objs_end[0].set_max_value(); + pk_objs_end[1].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 2); + range.end_key_.assign(pk_objs_end, 2); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + int64_t result_cnt = 0; + int64_t i = 0; + while (OB_SUCC(iter->get_next_entity(result_entity))) { + result_cnt++; + ASSERT_EQ(OB_SUCCESS, result_entity->get_properties_values(properties_values)); + ASSERT_EQ(3, properties_values.count()); + ASSERT_EQ(++i, properties_values.at(0).get_int()); + ASSERT_EQ(++i, properties_values.at(1).get_int()); + ASSERT_EQ(++i, properties_values.at(2).get_int()); + properties_values.reset(); + } + ASSERT_EQ(OB_ITER_END, ret); + ASSERT_EQ(result_cnt, expect_query_cnt); + } + + { + // case 2: index is the subset of primary key + query.reset(); + + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start; + pk_objs_start.set_int(0); + ObObj pk_objs_end; + pk_objs_end.set_max_value(); + ObNewRange range; + range.start_key_.assign(&pk_objs_start, 1); + range.end_key_.assign(&pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c2"))); + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + int64_t result_cnt = 0; + int64_t i = 0; + while (OB_SUCC(iter->get_next_entity(result_entity))) { + result_cnt++; + ASSERT_EQ(OB_SUCCESS, result_entity->get_properties_values(properties_values)); + ASSERT_EQ(3, properties_values.count()); + ASSERT_EQ(++i, properties_values.at(0).get_int()); + ASSERT_EQ(++i, properties_values.at(1).get_int()); + ASSERT_EQ(++i, properties_values.at(2).get_int()); + properties_values.reset(); + } + ASSERT_EQ(OB_ITER_END, ret); + ASSERT_EQ(result_cnt, expect_query_cnt); + } + + { + // case 3: has itersection between primary key and index + query.reset(); + + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[2]; + pk_objs_start[0].set_int(0); + pk_objs_start[1].set_min_value(); + ObObj pk_objs_end[2]; + pk_objs_end[0].set_max_value(); + pk_objs_end[1].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 2); + range.end_key_.assign(pk_objs_end, 2); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c2c3"))); + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + int64_t result_cnt = 0; + int64_t i = 0; + while (OB_SUCC(iter->get_next_entity(result_entity))) { + result_cnt++; + ASSERT_EQ(OB_SUCCESS, result_entity->get_properties_values(properties_values)); + ASSERT_EQ(3, properties_values.count()); + ASSERT_EQ(++i, properties_values.at(0).get_int()); + ASSERT_EQ(++i, properties_values.at(1).get_int()); + ASSERT_EQ(++i, properties_values.at(2).get_int()); + properties_values.reset(); + } + ASSERT_EQ(OB_ITER_END, ret); + ASSERT_EQ(result_cnt, expect_query_cnt); + } + + { + // case 4: has no itersection between primary key and index + query.reset(); + + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start; + pk_objs_start.set_int(0); + ObObj pk_objs_end; + pk_objs_end.set_max_value(); + ObNewRange range; + range.start_key_.assign(&pk_objs_start, 1); + range.end_key_.assign(&pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c3"))); + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + int64_t result_cnt = 0; + int64_t i = 0; + while (OB_SUCC(iter->get_next_entity(result_entity))) { + result_cnt++; + ASSERT_EQ(OB_SUCCESS, result_entity->get_properties_values(properties_values)); + ASSERT_EQ(3, properties_values.count()); + ASSERT_EQ(++i, properties_values.at(0).get_int()); + ASSERT_EQ(++i, properties_values.at(1).get_int()); + ASSERT_EQ(++i, properties_values.at(2).get_int()); + properties_values.reset(); + } + ASSERT_EQ(OB_ITER_END, ret); + ASSERT_EQ(result_cnt, expect_query_cnt); + } + + // teardown + iter = nullptr; + service_client_->free_table(the_table); + the_table = NULL; +} + +// CREATE TABLE IF NOT EXISTS `check_scan_range_test` ( +// C1 bigint, +// C2 varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, +// C3 bigint, +// PRIMARY KEY(`C1`, `C2`), +// KEY idx_c3 (`C3`)); +// INSERT INTO kv_query_test VALUES (1,'hello',3),(4,'hello',6),(7,'hello',9),(10,'hello',12),(13,'hello',15); +TEST_F(TestBatchExecute, check_scan_range) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("check_scan_range_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableOperationResult r; + const ObITableEntity *result_entity = NULL; + ObTableEntityIterator *iter = nullptr; + int expect_query_cnt = 5; + ObArray properties_values; + + // insert + { + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const int INERT_COUNT = 5; + for (int64_t i = 0; i < INERT_COUNT; i++) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key1; + key1.set_int(i * 3 + 1); + ObObj key2; + key2.set_varchar("hello"); + key2.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ObObj value; + value.set_int(i * 3 + 3); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(INERT_COUNT, result.count()); + for (int64_t i = 0; i < INERT_COUNT; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(1, r.get_affected_rows()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + + // case 1: scan by primary key, but key objs count is invalid + ObTableQuery query; + { + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[1]; + pk_objs_start[0].set_int(0); + // pk_objs_start[1].set_min_value(); + ObObj pk_objs_end[1]; + pk_objs_end[0].set_max_value(); + // pk_objs_end[1].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 1); + range.end_key_.assign(pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); + ASSERT_EQ(OB_ERR_UNEXPECTED, the_table->execute_query(query, iter)); // wrong rowkey size + } + + // case 2: scan by primary key, but key objs type is invalid + { + query.reset(); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[2]; + pk_objs_start[0].set_double(3.14); // invalid type + pk_objs_start[1].set_min_value(); + ObObj pk_objs_end[2]; + pk_objs_end[0].set_max_value(); + pk_objs_end[1].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 2); + range.end_key_.assign(pk_objs_end, 2); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); + ASSERT_EQ(OB_OBJ_TYPE_ERROR, the_table->execute_query(query, iter)); // wrong rowkey type + } + + // case 3: scan by primary key, but collation type is invalid + { + query.reset(); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[2]; + pk_objs_start[0].set_min_value(); + pk_objs_start[1].set_varchar("hello"); + pk_objs_start[1].set_collation_type(ObCollationType::CS_TYPE_GBK_BIN); // invalid collation type + ObObj pk_objs_end[2]; + pk_objs_end[0].set_max_value(); + pk_objs_end[1].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 2); + range.end_key_.assign(pk_objs_end, 2); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); + ASSERT_EQ(OB_ERR_COLLATION_MISMATCH, the_table->execute_query(query, iter)); // wrong collation type + } + + // case 4: scan by primary key, but accuracy is invalid + { + query.reset(); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start[2]; + pk_objs_start[0].set_min_value(); + pk_objs_start[1].set_varchar("hello11111111111"); // invalid accuracy, too long + pk_objs_start[1].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + ObObj pk_objs_end[2]; + pk_objs_end[0].set_max_value(); + pk_objs_end[1].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 2); + range.end_key_.assign(pk_objs_end, 2); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); + ASSERT_EQ(OB_ERR_DATA_TOO_LONG, the_table->execute_query(query, iter)); // wrong accuracy + } + + // case 5: scan by second index, but range is incomplete, lack rowkey. + // server will complement rowkey range. + { + query.reset(); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); + ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); + ObObj pk_objs_start; + pk_objs_start.set_int(0); // lack rowkey + ObObj pk_objs_end; + pk_objs_end.set_max_value(); + ObNewRange range; + range.start_key_.assign(&pk_objs_start, 1); + range.end_key_.assign(&pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); + ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c3"))); + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + int64_t result_cnt = 0; + int64_t i = 1; + while (OB_SUCC(iter->get_next_entity(result_entity))) { + result_cnt++; + ASSERT_EQ(OB_SUCCESS, result_entity->get_properties_values(properties_values)); + ASSERT_EQ(3, properties_values.count()); + ASSERT_EQ(i, properties_values.at(0).get_int()); + ObString str; + ASSERT_EQ(OB_SUCCESS, properties_values.at(1).get_varchar(str)); + ASSERT_EQ(0, str.case_compare("hello")); + ASSERT_EQ(i + 2, properties_values.at(2).get_int()); + i += 3; + properties_values.reset(); + } + ASSERT_EQ(OB_ITER_END, ret); + ASSERT_EQ(result_cnt, expect_query_cnt); + } + + // teardown + iter = nullptr; + service_client_->free_table(the_table); + the_table = NULL; +} + +TEST_F(TestBatchExecute, multi_insert) +{ + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_insert_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + OB_LOG(INFO, "begin multi_insert"); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + // multi insert + { + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(1, r.get_affected_rows()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+i, value.get_int()); + } + } + // insert again + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE + 1; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + // 存在一个冲突，全部回滚 + ASSERT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, the_table->batch_execute(batch_operation, result)); + } +} + +TEST_F(TestBatchExecute, multi_delete) +{ + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_delete_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + OB_LOG(INFO, "begin multi_delete"); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + // prepare data + ObITableEntity *entity = NULL; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(1, r.get_affected_rows()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + // delete half of the rows + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + if (i % 2 == 0) { + ASSERT_EQ(1, r.get_affected_rows()); + } else { + ASSERT_EQ(0, r.get_affected_rows()); + } + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + ObObj value; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + if (i < BATCH_SIZE/2) { + ASSERT_TRUE(result_entity->is_empty()); + } else { + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_TRUE(value.is_null()); + } + } + } +} + TEST_F(TestBatchExecute, htable_scan_with_filter) { // setup @@ -4114,6 +6053,907 @@ TEST_F(TestBatchExecute, htable_scan_with_filter) delete [] rows; } +TEST_F(TestBatchExecute, single_increment_append) +{ + OB_LOG(INFO, "begin single_increment"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("single_increment_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + // insert C2 + ObTableEntityFactory entity_factory; + ObITableEntity *entity = NULL; + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + int64_t key_key = 10; + ObObj key; + key.set_int(key_key); + int64_t value_value = 1000; + ObObj c2_obj; + c2_obj.set_int(value_value); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, c2_obj)); + ObString c3_value = ObString::make_string("hello world"); + ObObj c3_obj; + c3_obj.set_varchar(c3_value); + c3_obj.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, c3_obj)); + ObTableOperation table_operation = ObTableOperation::insert(*entity); + ObTableOperationResult r; + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + // insert another row (11, 1000, null) + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, c2_obj)); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + // insert another row (12, null, "hello world") + entity->reset(); + key.set_int(key_key+2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, c3_obj)); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ObObj value; + // append C3 + { + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::append(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(true); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::APPEND, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(NULL != result_entity); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(1, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); + ASSERT_EQ(key_key, value.get_int()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello worldhello world"); + ASSERT_TRUE(str == c3_new_value); + ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); + } + // increment C2 + { + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(111); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::increment(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(false); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value+111, value.get_int()); + } + + // get + { + ObObj null_obj; + entity->reset(); + key.set_int(key_key); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value+111, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello worldhello world"); + ASSERT_TRUE(str == c3_new_value); + } + // append to null column + { + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::append(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(true); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::APPEND, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(1, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); + ASSERT_EQ(key_key+1, value.get_int()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_new_value); + ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); + } + // get + { + ObObj null_obj; + entity->reset(); + key.set_int(key_key+1); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(value_value, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_new_value); + } + // increment null value + { + entity->reset(); + key.set_int(key_key+2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(111); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::increment(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(false); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(111, value.get_int()); + } + // get + { + ObObj null_obj; + entity->reset(); + key.set_int(key_key+2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(111, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_new_value); + } + + // increment row not exist + { + entity->reset(); + key.set_int(key_key+3); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_int(111); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ObTableOperation table_operation = ObTableOperation::increment(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(false); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(111, value.get_int()); + } + // get + { + ObObj null_obj; + entity->reset(); + key.set_int(key_key+3); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(111, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_TRUE(value.is_null()); + } + // append to row not exist + { + entity->reset(); + key.set_int(key_key+4); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ObTableOperation table_operation = ObTableOperation::append(*entity); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(true); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, req_options, r)); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::APPEND, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(!result_entity->is_empty()); + ASSERT_EQ(1, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); + ASSERT_EQ(key_key+4, value.get_int()); + ASSERT_EQ(1, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_new_value); + ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); + } + // get + { + ObObj null_obj; + entity->reset(); + key.set_int(key_key+4); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ObTableOperation table_operation = ObTableOperation::retrieve(*entity); + ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(2, result_entity->get_properties_count()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_TRUE(value.is_null()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ObString c3_new_value = ObString::make_string("hello world"); + ASSERT_TRUE(str == c3_new_value); + } + service_client_->free_table(the_table); +} + +// create table if not exists multi_update_test +// (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') +// PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, multi_update) +{ + OB_LOG(INFO, "begin multi_update"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_update_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + // multi insert + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_EQ(1, r.get_affected_rows()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(2*i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+i, value.get_int()); + } + } + + ObString c3_value = "c3_value"; + // update C3 + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(2*i); + ObObj value; + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(2*i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + } + // update half of the rows + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ObObj value; + value.set_int(200+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + if (i % 2 == 0) { + ASSERT_EQ(1, r.get_affected_rows()); + } else { + ASSERT_EQ(0, r.get_affected_rows()); + } + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + if (i < BATCH_SIZE/2) { + ASSERT_EQ(200+i*2, value.get_int()); + } else { + ASSERT_EQ(100+i, value.get_int()); + } + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + } +} + +TEST_F(TestBatchExecute, multi_insert_or_update) +{ + OB_LOG(INFO, "begin multi_insert_or_update"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_insert_or_update_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + // multi insert + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ObObj value; + value.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } // end for + + // get and verify + const ObITableEntity *result_entity = NULL; + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(i, value.get_int()); + } + } + + // multi update + ObString c3_value = "c3_value"; + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ObObj value1; + value1.set_int(i+1); + ObObj value2; + value2.set_varchar(c3_value); + value2.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value1)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value2)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(i+1, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ObString str; + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + } + } +} + +// create table if not exists multi_replace_test +// (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world') +// PARTITION BY KEY(C1) PARTITIONS 16 +TEST_F(TestBatchExecute, multi_replace) +{ + OB_LOG(INFO, "begin multi_replace_test"); + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_replace_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const int64_t SIZE = 10; + for (int64_t i = 0; i < SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ObObj value; + value.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(SIZE, result.count()); + for (int64_t i = 0; i < SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(SIZE, result.count()); + for (int64_t i = 0; i < SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(i, value.get_int()); + } + } + // replace again + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ObObj value; + value.set_int(200+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(SIZE, result.count()); + for (int64_t i = 0; i < SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(2, r.get_affected_rows()); // delete + insert + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(SIZE, result.count()); + for (int64_t i = 0; i < SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(200+i, value.get_int()); + } + } + // replace again (unique key C2 dup) + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(100+i); + ObObj value; + value.set_int(200+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(SIZE, result.count()); + for (int64_t i = 0; i < SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(2, r.get_affected_rows()); // delete + insert + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + } + // get and verify + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(SIZE, result.count()); + for (int64_t i = 0; i < SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(200+i, value.get_int()); + } + } +} + TEST_F(TestBatchExecute, htable_delete) { // setup @@ -4534,6 +7374,549 @@ TEST_F(TestBatchExecute, htable_delete) delete [] rows; } +TEST_F(TestBatchExecute, complex_batch_execute) +{ + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("complex_batch_execute_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + OB_LOG(INFO, "begin complex_batch_execute"); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + // prepare data + ObITableEntity *entity = NULL; + ObString c3_value = "c3_value"; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } + + { + // complex batch execute with hybrid operations types + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + ObObj value; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + switch (i % 6) { + case 0: // get + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + break; + case 1: // insert + value.set_int(200+i); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); + break; + case 2: // delete + ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); + break; + case 3: // update + value.set_int(300+i); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); + break; + case 4: // insert_or_update + value.set_int(400+i); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + break; + case 5: // replace + value.set_int(500+i); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.replace(*entity)); + break; + default: + ASSERT_TRUE(0); + break; + } + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(!batch_operation.is_same_type()); + ASSERT_TRUE(!batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + const ObITableEntity *result_entity = NULL; + ObString str; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + switch (i % 6) { + case 0: // get + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + break; + case 1: // insert + ASSERT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, r.get_errno()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT, r.type()); + ASSERT_TRUE(result_entity->is_empty()); + break; + case 2: // delete + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::DEL, r.type()); + ASSERT_TRUE(result_entity->is_empty()); + break; + case 3: // update + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::UPDATE, r.type()); + ASSERT_TRUE(result_entity->is_empty()); + break; + case 4: // insert_or_update + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + ASSERT_TRUE(result_entity->is_empty()); + break; + case 5: // replace + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(2, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::REPLACE, r.type()); + ASSERT_TRUE(result_entity->is_empty()); + break; + default: + ASSERT_TRUE(0); + break; + } + } + } + { + // get and verify + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C2)); + ASSERT_EQ(OB_SUCCESS, entity->add_retrieve_property(C3)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(BATCH_SIZE, result.count()); + ObObj value; + ObString str; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + //fprintf(stderr, "result %ld\n", i); + switch (i % 6) { + case 0: // get + case 1: // insert + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(100+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + break; + case 2: // delete + // entry not exist + ASSERT_TRUE(result_entity->is_empty()); + break; + case 3: // update + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(300+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + break; + case 4: // insert_or_update + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(400+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + // verify the semantic of PUT + // https://aone.alibaba-inc.com/project/81079/issue/14554345 + ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); + ASSERT_TRUE(str == c3_value); + break; + case 5: // replace + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + ASSERT_EQ(500+i, value.get_int()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); + ASSERT_TRUE(value.is_null()); + break; + default: + ASSERT_TRUE(0); + break; + } + } + } + service_client_->free_table(the_table); +} + +TEST_F(TestBatchExecute, increment_and_append_batch) +{ + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("multi_increment_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + // prepare + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + ObString c3_value = ObString::make_string("hello world"); + ObString c3_new_value = ObString::make_string("hello worldhello world"); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + // case + batch_operation.reset(); + entity_factory.free_and_reuse(); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ObObj value; + if (0 == i % 2) { + value.set_int(i); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.increment(*entity)); + } else if (1 == i % 2) { + value.set_varchar(c3_value); + value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.append(*entity)); + } + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(!batch_operation.is_same_type()); + ASSERT_TRUE(!batch_operation.is_same_properties_names()); + ObTableRequestOptions req_options; + req_options.set_returning_affected_entity(true); + req_options.set_returning_rowkey(true); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, req_options, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + if (0 == i % 2) { + ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); + } else { + ASSERT_EQ(ObTableOperationType::APPEND, r.type()); + } + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(!result_entity->is_empty()); + ObObj value; + ASSERT_EQ(1, result_entity->get_rowkey_size()); + ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); + ASSERT_EQ(i*2, value.get_int()); + + ObObj c2_obj, c3_obj; + ObObj val2, val3; + if (0 == i % 2) { + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, c2_obj)); + ASSERT_EQ(OB_SEARCH_NOT_FOUND, result_entity->get_property(C3, c3_obj)); + val2.set_int(100+2*i); + ASSERT_EQ(val2, c2_obj); + } else { + ASSERT_EQ(OB_SEARCH_NOT_FOUND, result_entity->get_property(C2, c2_obj)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, c3_obj)); + val3.set_varchar(c3_new_value); + val3.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(val3, c3_obj); + } + } // end for + + + // get and verify + const ObITableEntity *result_entity = NULL; + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj c2_obj, c3_obj; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, c2_obj)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, c3_obj)); + //fprintf(stderr, "%ld (%s,%s)\n", i, S(c2_obj), S(c3_obj)); + ObObj val2, val3; + if (0 == i % 2) { + val2.set_int(100+2*i); + val3.set_varchar(c3_value); + val3.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + } else { + val2.set_int(100+i); + val3.set_varchar(c3_new_value); + val3.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + } + ASSERT_EQ(val2, c2_obj); + ASSERT_EQ(val3, c3_obj); + } + } +} + +TEST_F(TestBatchExecute, htable_put) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_put"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + const int64_t BATCH_PUT_SIZE = 10; + DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_PUT_SIZE]; + ASSERT_TRUE(NULL != rows); + static constexpr int64_t VERSIONS_COUNT = 10; + static constexpr int64_t COLUMNS_SIZE = 10; + char qualifier[COLUMNS_SIZE][128]; + char qualifier2[COLUMNS_SIZE][128]; + for (int i = 0; i < COLUMNS_SIZE; ++i) + { + sprintf(qualifier[i], "cq%d", i); + } // end for + ObObj key1, key2, key3; + ObObj value; + for (int64_t i = 0; i < BATCH_PUT_SIZE; ++i) { + sprintf(rows[i], "row%ld", i); + key1.set_varbinary(ObString::make_string(rows[i])); + for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { + key2.set_varbinary(ObString::make_string(qualifier[j])); + for (int64_t k = 0; k < VERSIONS_COUNT; ++k) + { + key3.set_int(k); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + switch (i % 4) { + case 0: + value.set_varbinary(ObString::make_string("string2")); + break; + case 1: + value.set_varbinary(ObString::make_string("string3")); + break; + case 2: // row50 + value.set_varbinary(ObString::make_string("string0")); + break; + case 3: + value.set_varbinary(ObString::make_string("string1")); + break; + default: + ASSERT_TRUE(0); + } + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); // insert + } // end for + } // end for + } // end for + + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(1, result.count()); + const ObTableOperationResult &r = result.at(0); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(BATCH_PUT_SIZE * COLUMNS_SIZE * VERSIONS_COUNT, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + + // teardown + service_client_->free_table(the_table); + the_table = NULL; + delete [] rows; +} + +TEST_F(TestBatchExecute, htable_mutations) +{ + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_mutate"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + the_table->set_entity_type(ObTableEntityType::ET_HKV); // important + ASSERT_NO_FATAL_FAILURE(prepare_data(the_table)); + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + //////////////////////////////////////////////////////////////// + // row_mutation (cq0->haha, delete cq3, cq5, cq5->kaka) + ObObj key1, key2, key3, value; + key1.set_varbinary(ObString::make_string("row0")); + key2.set_varbinary(ObString::make_string("cq0")); + key3.set_int(INT64_MAX); + value.set_varbinary(ObString::make_string("haha")); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + + static constexpr int64_t COLUMNS_SIZE = 10; + key1.set_varbinary(ObString::make_string("row0")); + int cqids[] = {3, 5}; + char qualifier2[COLUMNS_SIZE][128]; + for (int64_t j = 0; j < ARRAYSIZEOF(cqids); ++j) { + sprintf(qualifier2[j], "cq%d", cqids[j]); + key2.set_varbinary(ObString::make_string(qualifier2[j])); + key3.set_int(-INT64_MAX); // delete all version + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); + } + + key1.set_varbinary(ObString::make_string("row0")); + key2.set_varbinary(ObString::make_string("cq5")); + key3.set_int(INT64_MAX); + value.set_varbinary(ObString::make_string("kaka")); + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + ASSERT_EQ(1, result.count()); + ASSERT_EQ(OB_SUCCESS, result.at(0).get_errno()); + //////////////////////////////////////////////////////////////// + // verify + ObTableQuery query; + ObObj pk_objs_start[3]; + ObObj pk_objs_end[3]; + ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, "row0")); + ObHTableFilter &htable_filter = query.htable_filter(); + htable_filter.set_valid(true); + + ObTableEntityIterator *iter = nullptr; + ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + const ObITableEntity *result_entity = NULL; + int cqids_sorted[] = { 0, 1, 2, 4, 5, 6, 7, 8, 9}; + int64_t timestamps[] = {-1, 9, 9, 9, -1, 9, 9, 9, 9}; + const char* values[] = {"haha", "string2", "string2", "string2", + "kaka", "string2", "string2", "string2", "string2"}; + for (int64_t i = 0; i < ARRAYSIZEOF(cqids_sorted); ++i) { + key1.set_varbinary(ObString::make_string("row0")); + sprintf(qualifier2[i], "cq%d", cqids_sorted[i]); + key2.set_varbinary(ObString::make_string(qualifier2[i])); + key3.set_int(timestamps[i]); + value.set_varbinary(ObString::make_string(values[i])); + ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + ObObj rk, cq, ts, val; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + // fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); + ASSERT_EQ(key1, rk); + ASSERT_EQ(key2, cq); + if (timestamps[i] >= 0) { + ASSERT_EQ(key3, ts); + } + ASSERT_EQ(value, val); + } // end for + ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + //////////////////////////////////////////////////////////////// + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} + TEST_F(TestBatchExecute, htable_query_and_mutate) { // setup @@ -4785,7 +8168,6 @@ TEST_F(TestBatchExecute, htable_query_and_mutate) } else { ASSERT_EQ(OB_NOT_SUPPORTED, ret); } - fprintf(stderr, "query_and_mutate affected_rows=%ld\n", affected_rows); } // end for // scan row50 ~ row57 @@ -4945,27 +8327,6 @@ TEST_F(TestBatchExecute, htable_query_and_mutate) delete [] rows; } -void TestBatchExecute::generate_get(ObTableQuery &query, ObObj pk_objs_start[], ObObj pk_objs_end[], const char* rowkey) -{ - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - - pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); -} - TEST_F(TestBatchExecute, htable_increment) { // setup @@ -4985,14 +8346,14 @@ TEST_F(TestBatchExecute, htable_increment) } // end for char values[VERSIONS_COUNT][8]; for (int i = 0; i < VERSIONS_COUNT; ++i) { - ObHTableUtils::int64_to_java_bytes(i*2, values[i]); + ObHTableUtils::int64_to_java_bytes(i, values[i]); } const char* rowkey = "row0"; ObObj key1, key2, key3; ObObj value; key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { - key2.set_varbinary(ObString::make_string(qualifier[j])); // cq0, cq1, cq2, ... cq9 + for (int64_t j = 0; j < COLUMNS_SIZE/2; ++j) { + key2.set_varbinary(ObString::make_string(qualifier[j*2])); // cq0, cq2, cq4, ... cq8 for (int64_t k = 0; k < VERSIONS_COUNT; ++k) { key3.set_int(k); @@ -5001,7 +8362,7 @@ TEST_F(TestBatchExecute, htable_increment) ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ObString str(8, values[j]); + ObString str(8, values[j*2]); value.set_varbinary(str); ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); @@ -5009,7 +8370,7 @@ TEST_F(TestBatchExecute, htable_increment) } // end for ObTableBatchOperationResult result; ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - ASSERT_EQ(1*COLUMNS_SIZE*VERSIONS_COUNT, result.count()); + ASSERT_EQ(1*COLUMNS_SIZE/2*VERSIONS_COUNT, result.count()); //////////////////////////////////////////////////////////////// // case ObTableQueryAndMutate query_and_mutate; @@ -5041,7 +8402,7 @@ TEST_F(TestBatchExecute, htable_increment) ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); ObString v_str(8, values[j]); value.set_varbinary(v_str); - // 8 + 0, 7 + 2, ... 1 + 14 + // 8 + 0, 7 + 1, ... 1 + 7 ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); ASSERT_EQ(OB_SUCCESS, mutations.increment(*entity)); } @@ -5059,9 +8420,13 @@ TEST_F(TestBatchExecute, htable_increment) ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); ASSERT_EQ(OB_SUCCESS, ObHTableUtils::java_bytes_to_int64(str, new_int)); - //fprintf(stderr, "%s | %ld\n", S(*result_entity), new_int); + // fprintf(stderr, "%s | %ld\n", S(*result_entity), new_int); num++; - ASSERT_EQ(16, new_int); + if (num % 2 == 0) { + ASSERT_EQ(8, new_int); + } else { + ASSERT_EQ(8-num, new_int); + } } ASSERT_EQ(num, ARRAYSIZEOF(cqids)); } @@ -5070,7 +8435,7 @@ TEST_F(TestBatchExecute, htable_increment) ObTableEntityIterator *iter = nullptr; ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); const ObITableEntity *result_entity = NULL; - int cqids_sorted[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + int cqids_sorted[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; key1.set_varbinary(ObString::make_string(rowkey)); int64_t expected_int = 0; ObString str; @@ -5078,9 +8443,22 @@ TEST_F(TestBatchExecute, htable_increment) for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); key2.set_varbinary(ObString::make_string(qualifier2[j])); - expected_int = 16; - // version is now() - key3.set_null(); + if (j == 0 || j == 8) { + expected_int = j; + if (j == 8) { + // version is now() + key3.set_null(); + } else { + key3.set_int(9); + } + } else if (j % 2==0) { + expected_int = 8; + // version is now() + key3.set_null(); + } else { + expected_int = 8-j; // first write + key3.set_int(VERSIONS_COUNT); + } ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); ObObj rk, cq, ts, val; ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); @@ -5089,16 +8467,14 @@ TEST_F(TestBatchExecute, htable_increment) ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); ASSERT_EQ(OB_SUCCESS, ObHTableUtils::java_bytes_to_int64(str, read_int)); - //fprintf(stderr, "(%s,%s,%s,%s(%ld))\n", S(rk), S(cq), S(ts), S(val), read_int); + // fprintf(stderr, "(%s,%s,%s,%s(%ld))\n", S(rk), S(cq), S(ts), S(val), read_int); + UNUSED(expected_int); ASSERT_EQ(key1, rk); ASSERT_EQ(key2, cq); - if (j == 0) { - ASSERT_EQ(read_int, 0); - } else if (j == 9) { - ASSERT_EQ(read_int, 18); - } else { - ASSERT_EQ(read_int, expected_int); + if (!key3.is_null()) { + ASSERT_EQ(key3, ts); } + ASSERT_EQ(read_int, expected_int); } // end for ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); @@ -5112,7 +8488,7 @@ TEST_F(TestBatchExecute, htable_increment_empty) { // setup ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_increment"), the_table); + int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_increment_empty"), the_table); ASSERT_EQ(OB_SUCCESS, ret); the_table->set_entity_type(ObTableEntityType::ET_HKV); // important ObTableEntityFactory entity_factory; @@ -5168,22 +8544,23 @@ TEST_F(TestBatchExecute, htable_increment_empty) } ObTableQueryAndMutateResult inc_result; int64_t &affected_rows = inc_result.affected_rows_; - ASSERT_EQ(OB_NOT_SUPPORTED, the_table->execute_query_and_mutate(query_and_mutate, inc_result)); + ASSERT_EQ(OB_SUCCESS, the_table->execute_query_and_mutate(query_and_mutate, inc_result)); + fprintf(stderr, "affected_rows=%ld\n", affected_rows); { - // int64_t num = 0; - // int64_t new_int = 0; - // ObObj val; - // ObString str; - // const ObITableEntity *result_entity = NULL; - // while(OB_SUCC(inc_result.affected_entity_.get_next_entity(result_entity))) { - // ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - // ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); - // ASSERT_EQ(OB_SUCCESS, ObHTableUtils::java_bytes_to_int64(str, new_int)); - // //fprintf(stderr, "%s | %ld\n", S(*result_entity), new_int); - // num++; - // ASSERT_EQ(8-num, new_int); - // } - // ASSERT_EQ(num, ARRAYSIZEOF(cqids)); + int64_t num = 0; + int64_t new_int = 0; + ObObj val; + ObString str; + const ObITableEntity *result_entity = NULL; + while(OB_SUCC(inc_result.affected_entity_.get_next_entity(result_entity))) { + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + ASSERT_EQ(OB_SUCCESS, ObHTableUtils::java_bytes_to_int64(str, new_int)); + //fprintf(stderr, "%s | %ld\n", S(*result_entity), new_int); + num++; + ASSERT_EQ(8-num, new_int); + } + ASSERT_EQ(num, ARRAYSIZEOF(cqids)); } //////////////////////////////////////////////////////////////// // teardown @@ -5191,128 +8568,6 @@ TEST_F(TestBatchExecute, htable_increment_empty) the_table = NULL; } -// todo@dazhi: for update is not work in master, use htable lock to fix it -// TEST_F(TestBatchExecute, htable_increment_multi_thread) -// { -// // setup -// ObTable *the_table = NULL; -// int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_increment"), the_table); -// ASSERT_EQ(OB_SUCCESS, ret); -// the_table->set_entity_type(ObTableEntityType::ET_HKV); // important -// ObTableEntityFactory entity_factory; -// ObTableBatchOperation batch_operation; -// ObITableEntity *entity = NULL; - -// // 1. insert cell: row1 cq1 0 0 -// const char* rowkey = "row2"; -// const char *qualifier = "cq1"; -// char old_val[8]; -// ObHTableUtils::int64_to_java_bytes(0, old_val); -// ObString str(8, old_val); -// int version = 0; -// ObObj key1, key2, key3, value; -// key1.set_varbinary(ObString::make_string(rowkey)); -// key2.set_varbinary(ObString::make_string(qualifier)); // cq0, cq2, cq4, ... cq8 -// key3.set_int(version); -// value.set_varbinary(str); -// entity = entity_factory.alloc(); -// ASSERT_TRUE(NULL != entity); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); -// ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); -// ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); -// ObTableBatchOperationResult result; -// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); -// ASSERT_EQ(1, result.count()); -// // after insertion -// // row2 cq0 0 0 - -// //////////////////////////////////////////////////////////////// -// // 2. fill query and mutations -// // query: query with row0 and htable filter for column cq0 -// // mutation: row2, cq0, 1, 1 -// ObTableQueryAndMutate query_and_mutate; -// ObTableQuery &query = query_and_mutate.get_query(); -// ObTableBatchOperation &mutations = query_and_mutate.get_mutations(); -// ObObj pk_objs_start[3]; -// ObObj pk_objs_end[3]; -// ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); -// ObHTableFilter &htable_filter = query.htable_filter(); -// const char *qualifier2 = qualifier; -// htable_filter.add_column(ObString::make_string(qualifier2)); -// ASSERT_EQ(1, htable_filter.get_max_versions()); -// htable_filter.set_valid(true); - -// mutations.reset(); -// key1.set_varbinary(ObString::make_string(rowkey)); -// key2.set_varbinary(ObString::make_string(qualifier2)); -// key3.set_int(version + 1); -// entity = entity_factory.alloc(); -// ASSERT_TRUE(NULL != entity); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); -// char inc_value[8]; -// ObHTableUtils::int64_to_java_bytes(1, inc_value); -// ObString inc_str(8, inc_value); -// value.set_varbinary(inc_str); -// ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); -// ASSERT_EQ(OB_SUCCESS, mutations.increment(*entity)); - -// // 3. execute increment and check result -// auto task = [&](ObTable* one_table, const ObTableQueryAndMutate &query_and_mutate, uint64_t inc_times) { -// ObTableQueryAndMutateResult inc_result; -// for (int i = 0; i < inc_times; i++) { -// ASSERT_EQ(OB_SUCCESS, one_table->execute_query_and_mutate(query_and_mutate, inc_result)); -// ASSERT_EQ(1, inc_result.affected_rows_); -// } -// }; -// constexpr uint64_t thread_num = 30; -// constexpr uint64_t inc_times = 50; -// std::vector threads; -// time_t start_time = time(NULL); -// printf("begin to run tasks, thread_num: %ld\n", thread_num); -// for (uint64_t i = 0; i < thread_num; ++i) { -// std::thread t(task, the_table, query_and_mutate, inc_times); -// threads.push_back(std::move(t)); -// } -// for (uint64_t i = 0; i < thread_num; ++i) { -// threads.at(i).join(); -// } -// printf("time elapsed during query process: %lfs\n", double(time(NULL) - start_time)); - - -// // 4. execute query and verify result -// htable_filter.clear_columns(); -// ObTableEntityIterator *iter = nullptr; -// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); -// const ObITableEntity *result_entity = NULL; -// key1.set_varbinary(ObString::make_string(rowkey)); -// key2.set_varbinary(ObString::make_string(qualifier2)); -// key3.set_int(version); -// int64_t read_int = 0; -// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); -// ObObj rk, cq, ts, val; -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); -// ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); -// ASSERT_EQ(OB_SUCCESS, ObHTableUtils::java_bytes_to_int64(str, read_int)); -// ASSERT_EQ(key1, rk); -// ASSERT_EQ(key2, cq); -// // ASSERT_EQ(key3, ts); -// std::cout << "key3: " << ts.get_int() << std::endl; -// ASSERT_EQ(read_int, thread_num * inc_times); -// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - -// //////////////////////////////////////////////////////////////// -// // teardown -// service_client_->free_table(the_table); -// the_table = NULL; -// } - TEST_F(TestBatchExecute, htable_append) { // setup @@ -5403,7 +8658,7 @@ TEST_F(TestBatchExecute, htable_append) while(OB_SUCC(inc_result.affected_entity_.get_next_entity(result_entity))) { ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); - fprintf(stderr, "%s\n", S(*result_entity)); + // fprintf(stderr, "%s\n", S(*result_entity)); num++; // if (num % 2 == 0) { // ASSERT_EQ(8, new_int); @@ -5443,7 +8698,7 @@ TEST_F(TestBatchExecute, htable_append) ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); - fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); + // fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); ASSERT_EQ(key1, rk); ASSERT_EQ(key2, cq); @@ -5460,1519 +8715,8 @@ TEST_F(TestBatchExecute, htable_append) the_table = NULL; } -void TestBatchExecute::prepare_data(ObTable *the_table) -{ - the_table->set_entity_type(ObTableEntityType::ET_HKV); // important - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_SIZE]; - ASSERT_TRUE(NULL != rows); - static constexpr int64_t VERSIONS_COUNT = 10; - static constexpr int64_t COLUMNS_SIZE = 10; - char qualifier[COLUMNS_SIZE][128]; - for (int i = 0; i < COLUMNS_SIZE; ++i) - { - sprintf(qualifier[i], "cq%d", i); - } // end for - ObObj key1, key2, key3; - ObObj value; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - sprintf(rows[i], "row%ld", i); - key1.set_varbinary(ObString::make_string(rows[i])); - for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { - key2.set_varbinary(ObString::make_string(qualifier[j])); - for (int64_t k = 0; k < VERSIONS_COUNT; ++k) - { - key3.set_int(k); - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - switch (i % 4) { - case 0: - value.set_varbinary(ObString::make_string("string2")); - break; - case 1: - value.set_varbinary(ObString::make_string("string3")); - break; - case 2: // row50 - value.set_varbinary(ObString::make_string("string0")); - break; - case 3: - value.set_varbinary(ObString::make_string("string1")); - break; - default: - ASSERT_TRUE(0); - } - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - } // end for - } // end for - } // end for - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(1, result.count()); - ASSERT_EQ(OB_SUCCESS, result.at(0).get_errno()); - delete [] rows; -} - -TEST_F(TestBatchExecute, htable_mutations) -{ - // setup - ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_mutate"), the_table); - ASSERT_EQ(OB_SUCCESS, ret); - the_table->set_entity_type(ObTableEntityType::ET_HKV); // important - ASSERT_NO_FATAL_FAILURE(prepare_data(the_table)); - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - //////////////////////////////////////////////////////////////// - // row_mutation (cq0->haha, delete cq3, cq5, cq5->kaka) - ObObj key1, key2, key3, value; - key1.set_varbinary(ObString::make_string("row0")); - key2.set_varbinary(ObString::make_string("cq0")); - key3.set_int(INT64_MAX); - value.set_varbinary(ObString::make_string("haha")); - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - - static constexpr int64_t COLUMNS_SIZE = 10; - key1.set_varbinary(ObString::make_string("row0")); - int cqids[] = {3, 5}; - char qualifier2[COLUMNS_SIZE][128]; - for (int64_t j = 0; j < ARRAYSIZEOF(cqids); ++j) { - sprintf(qualifier2[j], "cq%d", cqids[j]); - key2.set_varbinary(ObString::make_string(qualifier2[j])); - key3.set_int(-INT64_MAX); // delete all version - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - } - - key1.set_varbinary(ObString::make_string("row0")); - key2.set_varbinary(ObString::make_string("cq5")); - key3.set_int(INT64_MAX); - value.set_varbinary(ObString::make_string("kaka")); - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - ASSERT_EQ(1, result.count()); - ASSERT_EQ(OB_SUCCESS, result.at(0).get_errno()); - //////////////////////////////////////////////////////////////// - // verify - ObTableQuery query; - ObObj pk_objs_start[3]; - ObObj pk_objs_end[3]; - ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, "row0")); - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int cqids_sorted[] = { 0, 1, 2, 4, 5, 6, 7, 8, 9}; - int64_t timestamps[] = {-1, 9, 9, 9, -1, 9, 9, 9, 9}; - const char* values[] = {"haha", "string2", "string2", "string2", - "kaka", "string2", "string2", "string2", "string2"}; - for (int64_t i = 0; i < ARRAYSIZEOF(cqids_sorted); ++i) { - key1.set_varbinary(ObString::make_string("row0")); - sprintf(qualifier2[i], "cq%d", cqids_sorted[i]); - key2.set_varbinary(ObString::make_string(qualifier2[i])); - key3.set_int(timestamps[i]); - value.set_varbinary(ObString::make_string(values[i])); - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - if (timestamps[i] >= 0) { - ASSERT_EQ(key3, ts); - } - ASSERT_EQ(value, val); - } // end for - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - //////////////////////////////////////////////////////////////// - // teardown - service_client_->free_table(the_table); - the_table = NULL; -} - -TEST_F(TestBatchExecute, hcolumn_desc) -{ - ObHColumnDescriptor column_desc; - ObString str = ObString::make_string("{\"HColumnDescriptor\": {\"TimeToLive\": 3600}}"); - ASSERT_EQ(OB_SUCCESS, column_desc.from_string(str)); - fprintf(stderr, "ttl=%d\n", column_desc.get_time_to_live()); - ASSERT_EQ(3600, column_desc.get_time_to_live()); -} - - -TEST_F(TestBatchExecute, htable_ttl) -{ - // setup - ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_ttl"), the_table); - ASSERT_EQ(OB_SUCCESS, ret); - the_table->set_entity_type(ObTableEntityType::ET_HKV); // important - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - static constexpr int64_t VERSIONS_COUNT = 10; - static constexpr int64_t COLUMNS_SIZE = 10; - char qualifier[COLUMNS_SIZE][128]; - char qualifier2[COLUMNS_SIZE][128]; - for (int i = 0; i < COLUMNS_SIZE; ++i) - { - sprintf(qualifier[i], "cq%d", i); - } // end for - char values[VERSIONS_COUNT][128]; - for (int i = 0; i < VERSIONS_COUNT; ++i) { - sprintf(values[i], "ob%d", i); - } - const char* rowkey = "row0"; - ObObj key1, key2, key3; - ObObj value; - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { - key2.set_varbinary(ObString::make_string(qualifier[j])); // cq0 ~ cq9 - key3.set_int(INT64_MAX); // now - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - value.set_varbinary(ObString::make_string(values[j])); - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - } // end for - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - ASSERT_EQ(1, result.count()); - fprintf(stderr, "sleep 3\n"); - sleep(3); // ttl is 5 - { - // verify - ObTableQuery query; - ObObj pk_objs_start[3]; - ObObj pk_objs_end[3]; - ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); - - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - htable_filter.clear_columns(); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int cqids_sorted[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - char expected_str[128]; - ObString str; - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { - sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); - key2.set_varbinary(ObString::make_string(qualifier2[j])); - sprintf(expected_str, "ob%ld", j); - - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); - fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_TRUE(0 == str.compare(ObString::make_string(expected_str))); - } // end for - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - // insert new value for cq0, cq2, cq4, ... cq8 - entity_factory.free_and_reuse(); - batch_operation.reset(); - for (int64_t j = 0; j < COLUMNS_SIZE/2; ++j) { - key2.set_varbinary(ObString::make_string(qualifier[j*2])); - key3.set_int(INT64_MAX); - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - value.set_varbinary(ObString::make_string(values[j*2])); - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - } // end for - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - ASSERT_EQ(1, result.count()); - - // sleep for 3 seconds - fprintf(stderr, "sleep 3\n"); - sleep(3); // ttl is 5 - { - // verify - ObTableQuery query; - ObObj pk_objs_start[3]; - ObObj pk_objs_end[3]; - ASSERT_NO_FATAL_FAILURE(generate_get(query, pk_objs_start, pk_objs_end, rowkey)); - - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - htable_filter.clear_columns(); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int cqids_sorted[] = {0, 2, 4, 6, 8}; - char expected_str[128]; - ObString str; - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { - sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); - key2.set_varbinary(ObString::make_string(qualifier2[j])); - sprintf(expected_str, "ob%ld", j*2); - - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); - fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_TRUE(0 == str.compare(ObString::make_string(expected_str))); - } // end for - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - - //////////////////////////////////////////////////////////////// - // teardown - service_client_->free_table(the_table); - the_table = NULL; -} - -// TEST_F(TestBatchExecute, secondary_index) -// { -// OB_LOG(INFO, "begin secondary_index"); -// ObTable *the_table = NULL; -// int ret = service_client_->alloc_table(ObString::make_string("secondary_index_test"), the_table); -// ASSERT_EQ(OB_SUCCESS, ret); -// ObTableEntityFactory entity_factory; -// ObTableBatchOperation batch_operation; -// ObITableEntity *entity = NULL; -// -// // -// // > select * from secondary_index_test limit 10; -// // +----+------+------+ -// // | C1 | C2 | C3 | -// // +----+------+------+ -// // | 0 | 0 | aaa | -// // | 2 | 1 | xxx | -// // | 4 | 2 | yyy | -// // | 6 | 3 | zzz | -// // | 8 | 0 | AAA | -// // | 10 | 1 | XXX | -// // | 12 | 2 | YYY | -// // | 14 | 3 | ZZZ | -// // | 16 | 0 | aaa | -// // | 18 | 1 | xxx | -// // +----+------+------+ -// -// const char *c3_values[] = {"aaa", "xxx", "yyy", "zzz", "AAA", "XXX", "YYY", "ZZZ"}; -// const int64_t c3_values_count = ARRAYSIZEOF(c3_values); -// for (int64_t i = 0; i < BATCH_SIZE; ++i) { -// entity = entity_factory.alloc(); -// ASSERT_TRUE(NULL != entity); -// ObObj key; -// key.set_int(i*2); -// ObObj value; -// value.set_int(i%4); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); -// ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); -// value.set_varchar(ObString::make_string(c3_values[i%c3_values_count])); -// value.set_collation_type(CS_TYPE_UTF8MB4_BIN); -// ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); -// ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); -// } -// ASSERT_TRUE(!batch_operation.is_readonly()); -// ASSERT_TRUE(batch_operation.is_same_type()); -// ASSERT_TRUE(batch_operation.is_same_properties_names()); -// ObTableBatchOperationResult result; -// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); -// ASSERT_EQ(BATCH_SIZE, result.count()); -// for (int64_t i = 0; i < BATCH_SIZE; ++i) -// { -// const ObTableOperationResult &r = result.at(i); -// ASSERT_EQ(OB_SUCCESS, r.get_errno()); -// ASSERT_EQ(ObTableOperationType::INSERT, r.type()); -// ASSERT_EQ(1, r.get_affected_rows()); -// const ObITableEntity *result_entity = NULL; -// ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); -// ASSERT_TRUE(result_entity->is_empty()); -// } -// // query using index i1(c2) -// { -// ObTableQuery query; -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); -// ObObj pk_objs_start[1]; -// pk_objs_start[0].set_int(1); -// ObObj pk_objs_end[1]; -// pk_objs_end[0].set_int(2); -// ObNewRange range; -// range.start_key_.assign(pk_objs_start, 1); -// range.end_key_.assign(pk_objs_end, 1); -// range.border_flag_.set_inclusive_start(); -// range.border_flag_.set_inclusive_end(); -// -// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); -// query.set_scan_index(ObString::make_string("i1")); -// ObTableEntityIterator *iter = nullptr; -// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); -// const ObITableEntity *result_entity = NULL; -// ObObj obj1, obj2, obj3; -// ObString str; -// for (int64_t i = 0; i < BATCH_SIZE/2; ++i) -// { -// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); -// ASSERT_EQ(3, result_entity->get_properties_count()); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); -// //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); -// ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); -// if (i < BATCH_SIZE/4) { -// ASSERT_EQ(2+i*8, obj1.get_int()); -// ASSERT_EQ(1 , obj2.get_int()); -// ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); -// } else { -// ASSERT_EQ(4+(i-BATCH_SIZE/4)*8, obj1.get_int()); -// ASSERT_EQ(2 , obj2.get_int()); -// ASSERT_TRUE(str == ObString::make_string(c3_values[((i-BATCH_SIZE/4)%2==0)?2:6])); -// } -// } -// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); -// } -// // query using index i2(c3) -// { -// ObTableQuery query; -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); -// ObObj pk_objs_start[1]; -// pk_objs_start[0].set_varchar(ObString::make_string("xxx")); -// pk_objs_start[0].set_collation_type(CS_TYPE_UTF8MB4_BIN); -// ObObj pk_objs_end[1]; -// pk_objs_end[0].set_varchar(ObString::make_string("xxx")); -// pk_objs_end[0].set_collation_type(CS_TYPE_UTF8MB4_BIN); -// ObNewRange range; -// range.start_key_.assign(pk_objs_start, 1); -// range.end_key_.assign(pk_objs_end, 1); -// range.border_flag_.set_inclusive_start(); -// range.border_flag_.set_inclusive_end(); -// -// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); -// query.set_scan_index(ObString::make_string("i2")); -// ObTableEntityIterator *iter = nullptr; -// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); -// const ObITableEntity *result_entity = NULL; -// ObObj obj1, obj2, obj3; -// ObString str; -// for (int64_t i = 0; i < BATCH_SIZE/4; ++i) -// { -// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); -// ASSERT_EQ(3, result_entity->get_properties_count()); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); -// //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); -// ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); -// ASSERT_EQ(2+i*8, obj1.get_int()); -// ASSERT_EQ(1 , obj2.get_int()); -// ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); -// } -// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); -// } -// // query using index i3(c2,c3) -// { -// ObTableQuery query; -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); -// ObObj pk_objs_start[2]; -// pk_objs_start[0].set_int(1); -// pk_objs_start[1].set_varchar(ObString::make_string("xxx")); -// pk_objs_start[1].set_collation_type(CS_TYPE_UTF8MB4_BIN); -// ObObj pk_objs_end[2]; -// pk_objs_end[0].set_int(1); -// pk_objs_end[1].set_varchar(ObString::make_string("xxx")); -// pk_objs_end[1].set_collation_type(CS_TYPE_UTF8MB4_BIN); -// ObNewRange range; -// range.start_key_.assign(pk_objs_start, 2); -// range.end_key_.assign(pk_objs_end, 2); -// range.border_flag_.set_inclusive_start(); -// range.border_flag_.set_inclusive_end(); -// -// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); -// query.set_scan_index(ObString::make_string("i3")); -// ObTableEntityIterator *iter = nullptr; -// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); -// const ObITableEntity *result_entity = NULL; -// ObObj obj1, obj2, obj3; -// ObString str; -// for (int64_t i = 0; i < BATCH_SIZE/4; ++i) -// { -// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); -// ASSERT_EQ(3, result_entity->get_properties_count()); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); -// //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); -// ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); -// ASSERT_EQ(2+i*8, obj1.get_int()); -// ASSERT_EQ(1 , obj2.get_int()); -// ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); -// } -// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); -// } -// // 删除后一半row -// batch_operation.reset(); -// result.reset(); -// for (int64_t i = 0; i < BATCH_SIZE/2; ++i) { -// entity = entity_factory.alloc(); -// ASSERT_TRUE(NULL != entity); -// ObObj key; -// key.set_int((BATCH_SIZE/2+i)*2); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); -// ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); -// } -// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); -// ASSERT_EQ(BATCH_SIZE/2, result.count()); -// // then update C2 (2->0) -// batch_operation.reset(); -// result.reset(); -// for (int64_t i = 0; i < BATCH_SIZE/2; ++i) { -// entity = entity_factory.alloc(); -// ASSERT_TRUE(NULL != entity); -// if (i % 4 == 2) { -// ObObj key; -// key.set_int(i*2); -// ObObj value; -// value.set_int(0); -// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); -// ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); -// ASSERT_EQ(OB_SUCCESS, batch_operation.update(*entity)); -// } -// } -// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); -// ASSERT_EQ(BATCH_SIZE/8, result.count()); -// // query using index i1(c2) again -// { -// ObTableQuery query; -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); -// ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); -// ObObj pk_objs_start[1]; -// pk_objs_start[0].set_int(1); -// ObObj pk_objs_end[1]; -// pk_objs_end[0].set_int(2); -// ObNewRange range; -// range.start_key_.assign(pk_objs_start, 1); -// range.end_key_.assign(pk_objs_end, 1); -// range.border_flag_.set_inclusive_start(); -// range.border_flag_.set_inclusive_end(); -// -// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); -// query.set_scan_index(ObString::make_string("i1")); -// ObTableEntityIterator *iter = nullptr; -// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); -// const ObITableEntity *result_entity = NULL; -// ObObj obj1, obj2, obj3; -// ObString str; -// for (int64_t i = 0; i < 13; ++i) -// { -// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); -// ASSERT_EQ(3, result_entity->get_properties_count()); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, obj1)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, obj2)); -// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, obj3)); -// //fprintf(stderr, "%ld: (%s,%s,%s)\n", i, S(obj1), S(obj2), S(obj3)); -// ASSERT_EQ(OB_SUCCESS, obj3.get_varchar(str)); -// ASSERT_EQ(2+i*8, obj1.get_int()); -// ASSERT_EQ(1 , obj2.get_int()); -// ASSERT_TRUE(str == ObString::make_string(c3_values[(i%2==0)?1:5])); -// } -// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); -// } -// //////////////////////////////////////////////////////////////// -// // teardown -// service_client_->free_table(the_table); -// the_table = NULL; -// } - -TEST_F(TestBatchExecute, htable_empty_qualifier) -{ - // setup - ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_empty_cq"), the_table); - ASSERT_EQ(OB_SUCCESS, ret); - the_table->set_entity_type(ObTableEntityType::ET_HKV); // important - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - static constexpr int64_t VERSIONS_COUNT = 10; - DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_SIZE]; - ASSERT_TRUE(NULL != rows); - ObObj key1, key2, key3; - ObObj value; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - sprintf(rows[i], "row%ld", i); - key1.set_varbinary(ObString::make_string(rows[i])); - key2.set_varbinary(ObString::make_string("")); // empty qualifier - for (int64_t k = 0; k < VERSIONS_COUNT; ++k) - { - key3.set_int(k); - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - value.set_varbinary(ObString::make_string("value_string")); - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); - } // end for - } // end for - - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE*VERSIONS_COUNT, result.count()); - //////////////////////////////////////////////////////////////// - ObTableQuery query; - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - ObObj pk_objs_start[3]; - pk_objs_start[0].set_varbinary(ObString::make_string("row50")); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - ObObj pk_objs_end[3]; - pk_objs_end[0].set_varbinary(ObString::make_string("row59")); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - - ObHTableFilter &htable_filter = query.htable_filter(); - ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(""))); - htable_filter.set_max_versions(2); - htable_filter.set_valid(true); - { - // verify put result - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int64_t timestamps[] = {9, 8}; - for (int64_t i = 0; i < 10; ++i) { - // 10 rowkeys - sprintf(rows[i], "row%ld", i+50); - key1.set_varbinary(ObString::make_string(rows[i])); - key2.set_varbinary(ObString::make_string("")); - for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) - { - key3.set_int(timestamps[k]); - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_EQ(key3, ts); - } // end for - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - { - // delete row50 by qualifier and version - const char* rowkey = "row50"; - const char* cq = ""; - int64_t ts = 8; - batch_operation.reset(); - sprintf(rows[0], "%s", rowkey); - key1.set_varbinary(ObString::make_string(rows[0])); - key2.set_varbinary(ObString::make_string(cq)); - key3.set_int(ts); // delete the specified version - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - } - - { - // verify delete result - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int64_t timestamps[] = {9, 8}; - for (int64_t i = 0; i < 10; ++i) { - // 10 rowkeys - sprintf(rows[i], "row%ld", i+50); - key1.set_varbinary(ObString::make_string(rows[i])); - key2.set_varbinary(ObString::make_string("")); - for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) - { - if (0 == i && k == 1) { - key3.set_int(7); - } else { - key3.set_int(timestamps[k]); - } - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - // fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_EQ(key3, ts); - } // end for - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - //////////////////////////////////////////////////////////////// - // teardown - service_client_->free_table(the_table); - the_table = NULL; - delete [] rows; -} - -#define T_CASE_SUCC(SRC_TYPE, SRC_VAL, DELTA_TYPE, DELTA_VAL, RES_TYPE, RES_TYPE2, RES_VAL) \ - src.set_##SRC_TYPE(SRC_VAL);\ - delta.set_##DELTA_TYPE(DELTA_VAL);\ - target_type.set_type(src.get_type()); \ - target.set_null();\ - ASSERT_EQ(OB_SUCCESS, ObTableService::obj_increment(delta, src, target_type, target)); \ - ASSERT_EQ(Ob##RES_TYPE##Type, target.get_type()); \ - ASSERT_EQ(RES_VAL, target.get_##RES_TYPE2());\ - fprintf(stderr, "CASE: " #SRC_TYPE "(" #SRC_VAL ") + " #DELTA_TYPE "(" #DELTA_VAL ") = " #RES_TYPE2 "(" #RES_VAL ")\n"); - -#define T_CASE_FAIL(SRC_TYPE, SRC_VAL, DELTA_TYPE, DELTA_VAL, RES) \ - src.set_##SRC_TYPE(SRC_VAL);\ - delta.set_##DELTA_TYPE(DELTA_VAL);\ - target_type.set_type(src.get_type()); \ - target.set_null();\ - ASSERT_EQ(RES, ObTableService::obj_increment(delta, src, target_type, target)); \ - ASSERT_TRUE(target.is_null());\ - fprintf(stderr, "CASE: " #SRC_TYPE "(" #SRC_VAL ") + " #DELTA_TYPE "(" #DELTA_VAL ") = " #RES "\n"); - - -#define T_CASE_APP(SRC_VAL, SRC_CS_TYPE, DELTA_VAL, DELTA_CS_TYPE, RES_VAL, RES_CS_TYPE)\ - src.set_varchar(ObString::make_string(SRC_VAL));\ - src.set_collation_type(SRC_CS_TYPE);\ - delta.set_varchar(ObString::make_string(DELTA_VAL));\ - delta.set_collation_type(DELTA_CS_TYPE);\ - target_type.set_type(ObVarcharType);\ - target_type.set_collation_type(src.get_collation_type()); \ - target.set_null();\ - alloc.reset(); \ - ASSERT_EQ(OB_SUCCESS, ObTableService::obj_append(delta, src, target_type, alloc, target)); \ - ASSERT_EQ(ObVarcharType, target.get_type());\ - ASSERT_EQ(RES_CS_TYPE, target.get_collation_type());\ - ASSERT_TRUE(ObString::make_string(RES_VAL) == target.get_varchar());\ - fprintf(stderr, "CASE: " #SRC_CS_TYPE "(" #SRC_VAL ") || " #DELTA_CS_TYPE "(" #DELTA_VAL ") = " #RES_VAL "\n"); - - -//TEST_F(TestBatchExecute, increment_and_append) -//{ - //// insert C2 - //ObTableEntityFactory entity_factory; - //ObITableEntity *entity = NULL; - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //int64_t key_key = 10; - //ObObj key; - //key.set_int(key_key); - //int64_t value_value = 1000; - //ObObj c2_obj; - //c2_obj.set_int(value_value); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, c2_obj)); - //ObString c3_value = ObString::make_string("hello world"); - //ObObj c3_obj; - //c3_obj.set_varchar(c3_value); - //c3_obj.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, c3_obj)); - //ObTableOperation table_operation = ObTableOperation::insert(*entity); - //ObTableOperationResult r; - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //// insert another row (11, 1000, null) - //entity->reset(); - //key.set_int(key_key+1); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, c2_obj)); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //// insert another row (12, null, "hello world") - //entity->reset(); - //key.set_int(key_key+2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, c3_obj)); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ObObj value; - //// append C3 - //{ - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ObTableOperation table_operation = ObTableOperation::append(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(true); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::APPEND, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(NULL != result_entity); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(1, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); - //ASSERT_EQ(key_key, value.get_int()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello worldhello world"); - //ASSERT_TRUE(str == c3_new_value); - //ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); - //} - //// increment C2 - //{ - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(111); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObTableOperation table_operation = ObTableOperation::increment(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(false); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value+111, value.get_int()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //key.set_int(key_key); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value+111, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello worldhello world"); - //ASSERT_TRUE(str == c3_new_value); - //} - //// append to null column - //{ - //entity->reset(); - //key.set_int(key_key+1); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_BIN); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ObTableOperation table_operation = ObTableOperation::append(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(true); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::APPEND, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(1, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); - //ASSERT_EQ(key_key+1, value.get_int()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello world"); - //ASSERT_TRUE(str == c3_new_value); - //ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //key.set_int(key_key+1); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(value_value, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello world"); - //ASSERT_TRUE(str == c3_new_value); - //} - //// increment null value - //{ - //entity->reset(); - //key.set_int(key_key+2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(111); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObTableOperation table_operation = ObTableOperation::increment(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(false); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(111, value.get_int()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //key.set_int(key_key+2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(111, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello world"); - //ASSERT_TRUE(str == c3_new_value); - //} - //// increment row not exist - //{ - //entity->reset(); - //key.set_int(key_key+3); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_int(111); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ObTableOperation table_operation = ObTableOperation::increment(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(false); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(111, value.get_int()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //key.set_int(key_key+3); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_EQ(111, value.get_int()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ASSERT_TRUE(value.is_null()); - //} - //// append to row not exist - //{ - //entity->reset(); - //key.set_int(key_key+4); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_BIN); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ObTableOperation table_operation = ObTableOperation::append(*entity); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(true); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, req_options, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::APPEND, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ASSERT_EQ(1, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); - //ASSERT_EQ(key_key+4, value.get_int()); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello world"); - //ASSERT_TRUE(str == c3_new_value); - //ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, value.get_collation_type()); - //} - //// get - //{ - //ObObj null_obj; - //entity->reset(); - //key.set_int(key_key+4); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ObTableOperation table_operation = ObTableOperation::retrieve(*entity); - //ASSERT_EQ(OB_SUCCESS, table_->execute(table_operation, r)); - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ASSERT_EQ(2, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); - //ASSERT_TRUE(value.is_null()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, value)); - //ObString str; - //ASSERT_EQ(OB_SUCCESS, value.get_varchar(str)); - //ObString c3_new_value = ObString::make_string("hello world"); - //ASSERT_TRUE(str == c3_new_value); - //} -//} - -//TEST_F(TestBatchExecute, increment_and_append_batch) -//{ - //// prepare - //ObTableEntityFactory entity_factory; - //ObTableBatchOperation batch_operation; - //ObITableEntity *entity = NULL; - //ObString c3_value = ObString::make_string("hello world"); - //ObString c3_new_value = ObString::make_string("hello worldhello world"); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ObObj value; - //value.set_int(100+i); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); - //} - //ASSERT_TRUE(!batch_operation.is_readonly()); - //ASSERT_TRUE(batch_operation.is_same_type()); - //ASSERT_TRUE(batch_operation.is_same_properties_names()); - //ObTableBatchOperationResult result; - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //// case - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ObObj value; - //if (0 == i % 2) { - //value.set_int(i); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.increment(*entity)); - //} else if (1 == i % 2) { - //value.set_varchar(c3_value); - //value.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.append(*entity)); - //} - //} - //ASSERT_TRUE(!batch_operation.is_readonly()); - //ASSERT_TRUE(!batch_operation.is_same_type()); - //ASSERT_TRUE(!batch_operation.is_same_properties_names()); - //ObTableRequestOptions req_options; - //req_options.set_returning_affected_entity(true); - //req_options.set_returning_rowkey(true); - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, req_options, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) - //{ - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(1, r.get_affected_rows()); - //if (0 == i % 2) { - //ASSERT_EQ(ObTableOperationType::INCREMENT, r.type()); - //} else { - //ASSERT_EQ(ObTableOperationType::APPEND, r.type()); - //} - //const ObITableEntity *result_entity = NULL; - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_TRUE(!result_entity->is_empty()); - //ObObj value; - //ASSERT_EQ(1, result_entity->get_rowkey_size()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_rowkey_value(0, value)); - //ASSERT_EQ(i*2, value.get_int()); - - //ObObj c2_obj, c3_obj; - //ObObj val2, val3; - //if (0 == i % 2) { - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, c2_obj)); - //ASSERT_EQ(OB_HASH_NOT_EXIST, result_entity->get_property(C3, c3_obj)); - //val2.set_int(100+2*i); - //ASSERT_EQ(val2, c2_obj); - //} else { - //ASSERT_EQ(OB_HASH_NOT_EXIST, result_entity->get_property(C2, c2_obj)); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, c3_obj)); - //val3.set_varchar(c3_new_value); - //val3.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ASSERT_EQ(val3, c3_obj); - //} - //} // end for - - - //// get and verify - //const ObITableEntity *result_entity = NULL; - //{ - //batch_operation.reset(); - //entity_factory.free_and_reuse(); - //ObObj null_obj; - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != entity); - //ObObj key; - //key.set_int(i*2); - //ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); - //ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, null_obj)); - //ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); - //} - //ASSERT_EQ(OB_SUCCESS, table_->batch_execute(batch_operation, result)); - //OB_LOG(INFO, "batch execute result", K(result)); - //ASSERT_EQ(BATCH_SIZE, result.count()); - //for (int64_t i = 0; i < BATCH_SIZE; ++i) { - //const ObTableOperationResult &r = result.at(i); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //ASSERT_EQ(ObTableOperationType::GET, r.type()); - //ASSERT_EQ(0, r.get_affected_rows()); - //ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - //ASSERT_EQ(0, result_entity->get_rowkey_size()); - //ObObj c2_obj, c3_obj; - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, c2_obj)); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, c3_obj)); - ////fprintf(stderr, "%ld (%s,%s)\n", i, S(c2_obj), S(c3_obj)); - //ObObj val2, val3; - //if (0 == i % 2) { - //val2.set_int(100+2*i); - //val3.set_varchar(c3_value); - //val3.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //} else { - //val2.set_int(100+i); - //val3.set_varchar(c3_new_value); - //val3.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //} - //ASSERT_EQ(val2, c2_obj); - //ASSERT_EQ(val3, c3_obj); - //} - //} -//} - -TEST(TestQueryResult, alloc_memory_if_need) -{ - int ret = OB_SUCCESS; - const int64_t alloc_size = 1024; - ObTableQueryResult query_result; - int64_t total_size = query_result.allocator_.total(); - while (OB_SUCC(ret)) { - ret = query_result.alloc_buf_if_need(alloc_size); - query_result.buf_.get_position() += alloc_size; - if (query_result.allocator_.total() != total_size) { - total_size = query_result.allocator_.total(); - printf("allocator: %ld, result_buf: %ld\n", query_result.allocator_.total(), query_result.buf_.get_capacity()); - } - } - ASSERT_EQ(query_result.buf_.get_capacity(), ObTableQueryResult::MAX_BUF_BLOCK_SIZE * 1); - ASSERT_GT(query_result.allocator_.total(), ObTableQueryResult::MAX_BUF_BLOCK_SIZE * 1); - ASSERT_LE(query_result.allocator_.total(), ObTableQueryResult::MAX_BUF_BLOCK_SIZE * 3); -} - -//TEST_F(TestBatchExecute, update_table_with_index_by_lowercase_rowkey) -//{ - - //OB_LOG(INFO, "begin update_table_with_index_by_lowercase_rowkey"); - //// setup - //ObTable *the_table = NULL; - //int ret = service_client_->alloc_table(ObString::make_string("varchar_rowkey_update_test"), the_table); - //ASSERT_EQ(OB_SUCCESS, ret); - //// case for insert operation - //ObTableEntityFactory entity_factory; - //ObITableEntity *insert_entity = NULL; - //ObITableEntity *update_entity = NULL; - //ObTableOperationResult r; - //{ - //// case: insert with rowkey - //insert_entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != insert_entity); - - //const char * rk_value = "TEST"; - //int64_t v_value = 139107; - //ObObj rk_obj; - //rk_obj.set_varchar(rk_value); - //rk_obj.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ObObj v_obj; - //v_obj.set_int(v_value); - //ASSERT_EQ(OB_SUCCESS, insert_entity->add_rowkey_value(rk_obj)); - //ASSERT_EQ(OB_SUCCESS, insert_entity->set_property(T, v_obj)); - //ObTableOperation table_operation = ObTableOperation::insert(*insert_entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //} - - //{ - //// case: update with lowercase rowkey - //update_entity = entity_factory.alloc(); - //ASSERT_TRUE(NULL != update_entity); - - //const char * rk_value = "test"; - //int64_t v_value = 1; - //ObObj rk_obj; - //rk_obj.set_varchar(rk_value); - //rk_obj.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - //ObObj v_obj; - //v_obj.set_int(v_value); - //ASSERT_EQ(OB_SUCCESS, update_entity->add_rowkey_value(rk_obj)); - //ASSERT_EQ(OB_SUCCESS, update_entity->set_property(T, v_obj)); - //ObTableOperation table_operation = ObTableOperation::insert_or_update(*update_entity); - //ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - //ASSERT_EQ(OB_SUCCESS, r.get_errno()); - //} - //{ - //// query with index - //ObTableQuery query; - //ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - //ObNewRange range; - //range.set_whole_range(); - //ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - //query.set_scan_index(ObString::make_string("idx_T")); - //ObTableEntityIterator *iter = nullptr; - //ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - //const ObITableEntity *result_entity = NULL; - //ObObj obj1, obj2, obj3; - //ObString str; - //for (int64_t i = 0; i < 1; ++i) - //{ - //ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - //ASSERT_EQ(1, result_entity->get_properties_count()); - //ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, obj1)); - //ASSERT_EQ(OB_SUCCESS, obj1.get_varchar(str)); - //ASSERT_TRUE(str == ObString::make_string("test")); - //} - //ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - //} - - //// todo@wenqu: mysqlproxy使用时报-1044，后面再调通。 -//// ObISQLClient::ReadResult res; -//// uint64_t tenant_id = service_client_->get_tenant_id(); -//// ObString col_val; -//// ret = service_client_->get_user_sql_client().read(res, tenant_id, -//// "select /*+ index(varchar_rowkey_update_test idx_T)*/ K from test.varchar_rowkey_update_test"); -//// ASSERT_EQ(OB_SUCCESS, ret) << "tenant_id: " << tenant_id << "\n"; -//// sqlclient::ObMySQLResult *mysql_result = res.get_result(); -//// ASSERT_TRUE(NULL != mysql_result); -//// ASSERT_EQ(OB_SUCCESS, mysql_result->next()); -//// ASSERT_EQ(OB_SUCCESS, mysql_result->get_varchar("K", col_val)); -//// ASSERT_TRUE(col_val == ObString::make_string("TEST")); - - ////////////////////////////////////////////////////////////////// - //// teardown - //service_client_->free_table(the_table); - //the_table = NULL; -//} - -// CREATE TABLE IF NOT EXISTS `kv_query_test` ( -// C1 bigint, -// C2 bigint, -// C3 bigint, -// PRIMARY KEY(`C1`, `C2`), -// KEY idx_c2 (`C2`), -// KEY idx_c3 (`C3`), -// KEY idx_c2c3(`C2`, `C3`)); -TEST_F(TestBatchExecute, table_query_with_secondary_index) -{ - // setup - ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("kv_query_test"), the_table); - ASSERT_EQ(OB_SUCCESS, ret); - ObTableEntityFactory entity_factory; - ObTableOperationResult r; - ObITableEntity *entity = NULL; - const ObITableEntity *result_entity = NULL; - ObTableOperation table_operation; - ObTableEntityIterator *iter = nullptr; - - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - int64_t key_key = 1; - ObObj key; - key.set_int(key_key); - int64_t value_value = 1; - ObObj value; - value.set_int(value_value); - - - //prepare data - const int64_t batch_size = 100; - for (int64_t i = 0; i < batch_size; ++i) { - entity->reset(); - key.set_int(key_key + i); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); - - value.set_int(value_value); - ASSERT_EQ(OB_SUCCESS, entity->set_property(C3, value)); - - table_operation = ObTableOperation::insert(*entity); - ASSERT_EQ(OB_SUCCESS, the_table->execute(table_operation, r)); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - } - entity->reset(); - - //scan - ObTableQuery query; - - { - // case 1: primary key - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); - ObObj pk_objs_start[2]; - pk_objs_start[0].set_int(0); - pk_objs_start[1].set_min_value(); - ObObj pk_objs_end[2]; - pk_objs_end[0].set_max_value(); - pk_objs_end[1].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 2); - range.end_key_.assign(pk_objs_end, 2); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - int64_t result_cnt = 0; - while (OB_SUCC(iter->get_next_entity(result_entity))) { - result_cnt++; - } - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(result_cnt, batch_size); - } - - { - // case 2: index is the subset of primary key - query.reset(); - - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); - ObObj pk_objs_start; - pk_objs_start.set_int(0); - ObObj pk_objs_end; - pk_objs_end.set_max_value(); - ObNewRange range; - range.start_key_.assign(&pk_objs_start, 1); - range.end_key_.assign(&pk_objs_end, 1); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c2"))); - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - int64_t result_cnt = 0; - while (OB_SUCC(iter->get_next_entity(result_entity))) { - result_cnt++; - } - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(result_cnt, batch_size); - } - - { - // case 3: has itersection between primary key and index - query.reset(); - - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); - ObObj pk_objs_start[2]; - pk_objs_start[0].set_int(0); - pk_objs_start[1].set_min_value(); - ObObj pk_objs_end[2]; - pk_objs_end[0].set_max_value(); - pk_objs_end[1].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 2); - range.end_key_.assign(pk_objs_end, 2); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c2c3"))); - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - int64_t result_cnt = 0; - while (OB_SUCC(iter->get_next_entity(result_entity))) { - result_cnt++; - } - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(result_cnt, batch_size); - } - - { - // case 4: has no itersection between primary key and index - query.reset(); - - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C1)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C2)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(C3)); - ObObj pk_objs_start; - pk_objs_start.set_int(0); - ObObj pk_objs_end; - pk_objs_end.set_max_value(); - ObNewRange range; - range.start_key_.assign(&pk_objs_start, 1); - range.end_key_.assign(&pk_objs_end, 1); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("idx_c3"))); - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - int64_t result_cnt = 0; - while (OB_SUCC(iter->get_next_entity(result_entity))) { - result_cnt++; - } - ASSERT_EQ(OB_ITER_END, ret); - ASSERT_EQ(result_cnt, batch_size); - } - - // teardown - iter = nullptr; - service_client_->free_table(the_table); - the_table = NULL; -} - // create table if not exists query_sync_multi_batch_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2), INDEX idx1(C1, C2)); -/*TEST_F(TestBatchExecute, query_sync_multi_batch) +TEST_F(TestBatchExecute, query_sync_multi_batch) { // setup ObTable *the_table = NULL; @@ -7050,7 +8794,7 @@ TEST_F(TestBatchExecute, table_query_with_secondary_index) int result_cnt = 0; for (int64_t i = 5; i < 15; ++i) { - printf("start to get entity from result iterator, i: %ld, result_cnt: %d\n", i, result_cnt); + // printf("start to get entity from result iterator, i: %ld, result_cnt: %d\n", i, result_cnt); ASSERT_EQ(1, iter->get_row_count()); ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); ASSERT_EQ(4, result_entity->get_properties_count()); @@ -7158,7 +8902,7 @@ TEST_F(TestBatchExecute, large_scan_query_sync) ASSERT_EQ(OB_SUCCESS, the_table->query_start(query, iter)); do { - printf("iterator row count: %ld\n", iter->get_row_count()); + // printf("iterator row count: %ld\n", iter->get_row_count()); while (OB_SUCC(iter->get_next_entity(result_entity))) { ++result_cnt; } @@ -7172,7 +8916,7 @@ TEST_F(TestBatchExecute, large_scan_query_sync) ASSERT_EQ(OB_SUCCESS, the_table->query_start(query, iter)); result_cnt = 0; do { - printf("iterator row count: %ld\n", iter->get_row_count()); + // printf("iterator row count: %ld\n", iter->get_row_count()); while (OB_SUCC(iter->get_next_entity(result_entity))) { ++result_cnt; } @@ -7185,7 +8929,9 @@ TEST_F(TestBatchExecute, large_scan_query_sync) the_table = NULL; } -// create table if not exists query_sync_with_index_test (C1 bigint, C2 bigint, C3 bigint, primary key(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3)); +// create table if not exists query_sync_with_index_test +// (C1 bigint, C2 bigint, C3 bigint, +// primary key(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3)); TEST_F(TestBatchExecute, query_sync_with_index) { // setup @@ -7367,12 +9113,13 @@ TEST_F(TestBatchExecute, query_sync_with_index) the_table = NULL; } -// create table if not exists query_sync_multi_task_test (C1 bigint primary key, C2 bigint, C3 varchar(100)); +// create table if not exists query_sync_multi_task_test +// (C1 bigint primary key, C2 bigint, C3 varchar(100)); TEST_F(TestBatchExecute, query_sync_multi_task) { // setup - constexpr int64_t thread_num = 32; - const int64_t large_batch_size = 100000; + constexpr int64_t thread_num = 10; + const int64_t large_batch_size = 1000; int64_t query_round = 10; const int64_t query_batch_size = large_batch_size / query_round; ObVector tables; @@ -7510,8 +9257,176 @@ TEST_F(TestBatchExecute, query_sync_multi_task) for (uint64_t i = 0; i < N; ++i) { service_client_->free_table(tables.at(i)); } -}*/ +} +// the table should be single partition for this case: +// create table if not exists batch_operation_with_same_keys_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) +TEST_F(TestBatchExecute, batch_operation_with_same_keys) +{ + OB_LOG(INFO, "begin batch_operation_with_duplicated_keys"); + // setup + ObTable *the_table = NULL; + int ret = service_client_->alloc_table(ObString::make_string("batch_operation_with_same_keys_test"), the_table); + ASSERT_EQ(OB_SUCCESS, ret); + // start case + ObTableEntityFactory entity_factory; + ObTableBatchOperation batch_operation; + ObITableEntity *entity = NULL; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2 % 10); + ObObj value; + value.set_int(100+i); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, value)); + //fprintf(stderr, "put key=%ld value=%ld i=%ld\n", (i*2%10), value.get_int(), i); + ASSERT_EQ(OB_SUCCESS, batch_operation.insert_or_update(*entity)); + } + ASSERT_TRUE(!batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ObTableBatchOperationResult result; + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) + { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(1, r.get_affected_rows()); + ASSERT_EQ(ObTableOperationType::INSERT_OR_UPDATE, r.type()); + const ObITableEntity *result_entity = NULL; + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_TRUE(result_entity->is_empty()); + } // end for + + // get and verify + const ObITableEntity *result_entity = NULL; + { + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(i*2%10); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + //fprintf(stderr, "get C2 value=%ld i=%ld key=%ld\n", value.get_int(), i, i*2%10); + ASSERT_EQ(195+i%5, value.get_int()); + } + } + { + // all the same key + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + key.set_int(0); + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + //fprintf(stderr, "get C2 value=%ld i=%ld\n", value.get_int(), i); + ASSERT_EQ(195, value.get_int()); + } + } + { + // pattern: empty empty exist exist empty empty exist exist ... + batch_operation.reset(); + entity_factory.free_and_reuse(); + ObObj null_obj; + int64_t key1_not_exist = 1; + int64_t key2_not_exist = 3; + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + entity = entity_factory.alloc(); + ASSERT_TRUE(NULL != entity); + ObObj key; + if (i < BATCH_SIZE/4) { + key.set_int(key1_not_exist); + } else if (i < BATCH_SIZE/4*2) { + key.set_int(0); + } else if (i < BATCH_SIZE/4*3) { + key.set_int(key2_not_exist); + } else { + key.set_int(2); + } + ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key)); + ASSERT_EQ(OB_SUCCESS, entity->set_property(C2, null_obj)); + ASSERT_EQ(OB_SUCCESS, batch_operation.retrieve(*entity)); + } + ASSERT_TRUE(batch_operation.is_readonly()); + ASSERT_TRUE(batch_operation.is_same_type()); + ASSERT_TRUE(batch_operation.is_same_properties_names()); + ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); + OB_LOG(INFO, "batch execute result", K(result)); + ASSERT_EQ(BATCH_SIZE, result.count()); + for (int64_t i = 0; i < BATCH_SIZE; ++i) { + const ObTableOperationResult &r = result.at(i); + ASSERT_EQ(OB_SUCCESS, r.get_errno()); + ASSERT_EQ(ObTableOperationType::GET, r.type()); + ASSERT_EQ(0, r.get_affected_rows()); + ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); + ASSERT_EQ(0, result_entity->get_rowkey_size()); + if (i < BATCH_SIZE/4) { + ASSERT_TRUE(result_entity->is_empty()); + } else if (i < BATCH_SIZE/4*2) { + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + //fprintf(stderr, "get C2 value=%ld i=%ld\n", value.get_int(), i); + ASSERT_EQ(195, value.get_int()); + } else if (i < BATCH_SIZE/4*3) { + ASSERT_TRUE(result_entity->is_empty()); + } else { + ObObj value; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C2, value)); + //fprintf(stderr, "get C2 value=%ld i=%ld\n", value.get_int(), i); + ASSERT_EQ(196, value.get_int()); + } + } // end for + } + // teardown + service_client_->free_table(the_table); + the_table = NULL; +} int main(int argc, char **argv) { diff --git a/src/libtable/test/run_test_table_api.sh b/src/libtable/test/run_test_table_api.sh index 9a2b9aaef7..aa30e1960f 100755 --- a/src/libtable/test/run_test_table_api.sh +++ b/src/libtable/test/run_test_table_api.sh @@ -4,6 +4,11 @@ HOST=${1:-11.166.82.163} PORT=${2:-46905} RPCPORT=${3:-46904} + +# HOST=${1:-11.158.97.240} +# PORT=${2:-41101} +# RPCPORT=${3:-41100} + #HOST=100.88.11.96 #PORT=60803 #RPCPORT=60802 @@ -23,76 +28,115 @@ echo run test... rm -f libobtable.log # table api mysql -h $HOST -P $PORT -u $user -e "drop table if exists batch_execute_test; create table if not exists batch_execute_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists complex_batch_execute_test; create table if not exists complex_batch_execute_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists all_single_operation_test; create table if not exists all_single_operation_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists batch_operation_with_same_keys_test; create table if not exists batch_operation_with_same_keys_test (C1 bigint primary key, C2 bigint, C3 varchar(100))" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists type_check_test; create table type_check_test (pk1 bigint, pk2 varchar(10), ctinyint tinyint, csmallint smallint, cmediumint mediumint, cint int, cbigint bigint, utinyint tinyint unsigned, usmallint smallint unsigned, umediumint mediumint unsigned, uint int unsigned, ubigint bigint unsigned, cfloat float, cdouble double, ufloat float unsigned, udouble double unsigned, cnumber decimal(10, 2), unumber decimal(10,2) unsigned, cvarchar varchar(10), cchar char(10), cbinary binary(10), cvarbinary varbinary(10), ctimestamp timestamp, cdatetime datetime, cyear year, cdate date, ctime time, ctext text, cblob blob, cbit bit(64), cnotnull bigint not null default 111, primary key(pk1, pk2));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists type_check_test; create table type_check_test (pk1 bigint, pk2 varchar(10), ctinyint tinyint, csmallint smallint, cmediumint mediumint, cint int, cbigint bigint, utinyint tinyint unsigned, usmallint smallint unsigned, umediumint mediumint unsigned, uint int unsigned, ubigint bigint unsigned, cfloat float, cdouble double, ufloat float unsigned, udouble double unsigned, cnumber decimal(10, 2), unumber decimal(10,2) unsigned, cvarchar varchar(10), cchar char(10), cbinary binary(10), cvarbinary varbinary(10), ctimestamp timestamp, cdatetime datetime, cyear year, cdate date, ctime time, ctext text, cblob blob, cbit bit(64), cnotnull bigint not null default 111, PRIMARY KEY(pk1, pk2));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists column_default_value; create table column_default_value (C1 bigint primary key, C2 bigint default 1, C3 varchar(100) default 'abc') PARTITION BY KEY(C1) PARTITIONS 16" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists partial_update_test; create table if not exists partial_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100) not null) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists partial_update_test; create table if not exists partial_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists append_lob_test; create table if not exists append_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext)" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists uniq_replace_test; create table if not exists uniq_replace_test (C1 bigint primary key, C2 bigint, C3 varchar(100), unique key C2_uniq(C2))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists all_lob_test; create table if not exists all_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext)" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists large_scan_test; create table if not exists large_scan_test (C1 bigint primary key, C2 bigint, C3 varchar(100))" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists generate_col_test; create table if not exists generate_col_test (C1 bigint primary key, C2 bigint, C3 varchar(100), C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2)))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists uniq_replace_test; create table if not exists uniq_replace_test (C1 bigint primary key, C2 bigint, C3 varchar(100), unique key C2_uniq(C2))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists varchar_rowkey_update_test; create table if not exists varchar_rowkey_update_test (K varchar(512) primary key, T bigint, KEY idx_T(T) LOCAL);" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_increment_test; create table if not exists multi_increment_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_increment_test; create table if not exists single_increment_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_get_test; create table if not exists single_get_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_get_test; create table if not exists multi_get_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_insert_test; create table if not exists single_insert_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists insert_generate_test; create table if not exists insert_generate_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2))) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_update_test; create table if not exists single_update_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_update_test; create table if not exists multi_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists update_generate_test; create table if not exists update_generate_test (C1 bigint primary key, C2 varchar(100), C3 varchar(100), GEN varchar(100) GENERATED ALWAYS AS (concat(C2,c3)) stored) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_delete_test; create table if not exists single_delete_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_replace_test; create table if not exists single_replace_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists replace_unique_key_test; create table if not exists replace_unique_key_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', unique index i1(c2) local)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_replace_test; create table if not exists multi_replace_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world', unique index i1(c2) local)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_insert_up_test; create table if not exists single_insert_up_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_insert_or_update_test; create table if not exists multi_insert_or_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world') PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists kv_query_test; create table if not exists kv_query_test (C1 bigint, C2 bigint, C3 bigint, PRIMARY KEY(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists virtual_generate_col_test; create table if not exists virtual_generate_col_test (C1 bigint primary key, C2 bigint, C3 varchar(100), C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2)))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists store_generate_col_test; create table if not exists store_generate_col_test (C1 bigint primary key, C2 varchar(10), C3 varchar(10), GEN varchar(30) generated always as (concat(C2,C3)) stored)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists check_scan_range_test; create table if not exists check_scan_range_test (C1 bigint, C2 varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, C3 bigint, PRIMARY KEY(C1, C2), KEY idx_c3 (C3));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_insert_test; create table if not exists multi_insert_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_delete_test; create table if not exists multi_delete_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists query_sync_multi_batch_test; create table if not exists query_sync_multi_batch_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2), INDEX idx1(C1, C2));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists large_scan_query_sync_test; create table if not exists large_scan_query_sync_test (C1 bigint primary key, C2 bigint, C3 varchar(100));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists query_sync_with_index_test; create table if not exists query_sync_with_index_test (C1 bigint, C2 bigint, C3 bigint, primary key(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists query_sync_multi_task_test; create table if not exists query_sync_multi_task_test (C1 bigint primary key, C2 bigint, C3 varchar(100));" $db -# , INDEX idx1(C1, C2) +# INDEX idx1(C1, C2) mysql -h $HOST -P $PORT -u $user -e "drop table if exists execute_query_test; create table if not exists execute_query_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists secondary_index_test; create table if not exists secondary_index_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3) local)" $db # hbase api mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1; create table if not exists htable1_cf1 (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substr(K,1,32)) STORED, primary key(K, Q, T));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_reverse; create table if not exists htable1_cf1_reverse like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_filter; create table if not exists htable1_cf1_filter like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_query_and_mutate; create table if not exists htable1_cf1_query_and_mutate like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_delete; create table if not exists htable1_cf1_delete like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_mutate; create table if not exists htable1_cf1_mutate like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment; create table if not exists htable1_cf1_increment like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_append; create table if not exists htable1_cf1_append like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_ttl; create table if not exists htable1_cf1_ttl (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) comment='{\"HColumnDescriptor\": {\"TimeToLive\": 5}}'" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_filter; create table if not exists htable1_cf1_filter like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_delete; create table if not exists htable1_cf1_delete like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_put; create table if not exists htable1_cf1_put like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_mutate; create table if not exists htable1_cf1_mutate like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_query_and_mutate; create table if not exists htable1_cf1_query_and_mutate like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment; create table if not exists htable1_cf1_increment like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment_empty; create table if not exists htable1_cf1_increment_empty like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_append; create table if not exists htable1_cf1_append like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_empty_cq; create table if not exists htable1_cf1_empty_cq (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T));" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists varchar_rowkey_update_test; create table if not exists varchar_rowkey_update_test (K varchar(512) primary key, T bigint, KEY idx_T(T) LOCAL);" $db -mysql -h $HOST -P $PORT -u $user -e "DROP TABLE IF EXISTS kv_query_test; CREATE TABLE IF NOT EXISTS kv_query_test (C1 bigint, C2 bigint, C3 bigint, PRIMARY KEY(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3));" $db -sleep 3 +# run ./test_table_api "$HOST" "$PORT" "$tenant_name" "$user_name" "$passwd" "$db" "$table_name" $RPCPORT -#./test_table_api "$HOST" "$PORT" sys root '' test batch_execute_test $RPCPORT --gtest_filter=TestBatchExecute.column_default_value -#./test_table_api "$HOST" "$PORT" sys root '' test batch_execute_test $RPCPORT --gtest_filter=TestBatchExecute.secondary_index -#./test_table_api "$HOST" "$PORT" sys root '' test batch_execute_test $RPCPORT --gtest_filter=TestBatchExecute.multi_insert_or_update_AND_multi_get -#./test_table_api "$HOST" "$PORT" sys root '' test batch_execute_test $RPCPORT --gtest_filter=TestBatchExecute.compare_cell -#./test_table_api "$HOST" "$PORT" sys root '' test batch_execute_test $RPCPORT --gtest_filter=TestBatchExecute.htable_scan_basic -#./test_table_api "$HOST" "$PORT" sys root '' test batch_execute_test $RPCPORT --gtest_filter=TestBatchExecute.htable_empty_qualifier -#exit 0 -echo "round 2 with index" -# table api -mysql -h $HOST -P $PORT -u $user -e "drop table if exists batch_execute_test; create table if not exists batch_execute_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3) local) PARTITION BY KEY(C1) PARTITIONS 16" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists batch_operation_with_same_keys_test; create table if not exists batch_operation_with_same_keys_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2), index i2(c3), index i3(c2, c3))" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists type_check_test; create table type_check_test (pk1 bigint, pk2 varchar(10), ctinyint tinyint, csmallint smallint, cmediumint mediumint, cint int, cbigint bigint, utinyint tinyint unsigned, usmallint smallint unsigned, umediumint mediumint unsigned, uint int unsigned, ubigint bigint unsigned, cfloat float, cdouble double, ufloat float unsigned, udouble double unsigned, cnumber decimal(10, 2), unumber decimal(10,2) unsigned, cvarchar varchar(10), cchar char(10), cbinary binary(10), cvarbinary varbinary(10), ctimestamp timestamp, cdatetime datetime, cyear year, cdate date, ctime time, ctext text, cblob blob, cbit bit(64), cnotnull bigint not null default 111, primary key(pk1, pk2), index i1(utinyint, cvarbinary));" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists column_default_value; create table column_default_value (C1 bigint primary key, C2 bigint default 1, C3 varchar(100) default 'abc') PARTITION BY KEY(C1) PARTITIONS 16" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists partial_update_test; create table if not exists partial_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100) not null) PARTITION BY KEY(C1) PARTITIONS 16" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists append_lob_test; create table if not exists append_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext)" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists uniq_replace_test; create table if not exists uniq_replace_test (C1 bigint primary key, C2 bigint, C3 varchar(100), unique key C2_uniq(C2))" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists large_scan_test; create table if not exists large_scan_test (C1 bigint primary key, C2 bigint, C3 varchar(100))" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists generate_col_test; create table if not exists generate_col_test (C1 bigint primary key, C2 bigint, C3 varchar(100), C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2)))" $db +# round2 with index +mysql -h $HOST -P $PORT -u $user -e "drop table if exists batch_execute_test; create table if not exists batch_execute_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists complex_batch_execute_test; create table if not exists complex_batch_execute_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists all_single_operation_test; create table if not exists all_single_operation_test (C1 bigint primary key, C2 bigint, C3 varchar(100)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists batch_operation_with_same_keys_test; create table if not exists batch_operation_with_same_keys_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists type_check_test; create table type_check_test (pk1 bigint, pk2 varchar(10), ctinyint tinyint, csmallint smallint, cmediumint mediumint, cint int, cbigint bigint, utinyint tinyint unsigned, usmallint smallint unsigned, umediumint mediumint unsigned, uint int unsigned, ubigint bigint unsigned, cfloat float, cdouble double, ufloat float unsigned, udouble double unsigned, cnumber decimal(10, 2), unumber decimal(10,2) unsigned, cvarchar varchar(10), cchar char(10), cbinary binary(10), cvarbinary varbinary(10), ctimestamp timestamp, cdatetime datetime, cyear year, cdate date, ctime time, ctext text, cblob blob, cbit bit(64), cnotnull bigint not null default 111, PRIMARY KEY(pk1, pk2));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists column_default_value; create table column_default_value (C1 bigint primary key, C2 bigint default 1, C3 varchar(100) default 'abc', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists partial_update_test; create table if not exists partial_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists append_lob_test; create table if not exists append_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext, index i1(c2) local)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists all_lob_test; create table if not exists all_lob_test (C1 bigint primary key, C2 bigint, C3 mediumtext, index i1(c2) local)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists large_scan_test; create table if not exists large_scan_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists uniq_replace_test; create table if not exists uniq_replace_test (C1 bigint primary key, C2 bigint, C3 varchar(100), unique key C2_uniq(C2), index i2(c3) local)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists varchar_rowkey_update_test; create table if not exists varchar_rowkey_update_test (K varchar(512) primary key, T bigint, KEY idx_T(T) LOCAL);" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_increment_test; create table if not exists multi_increment_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_get_test; create table if not exists single_get_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_get_test; create table if not exists multi_get_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_increment_test; create table if not exists single_increment_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_insert_test; create table if not exists single_insert_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_update_test; create table if not exists single_update_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_update_test; create table if not exists multi_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists insert_generate_test; create table if not exists insert_generate_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2))) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists update_generate_test; create table if not exists update_generate_test (C1 bigint primary key, C2 varchar(100), C3 varchar(100), GEN varchar(100) GENERATED ALWAYS AS (concat(C2,c3)) stored) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_delete_test; create table if not exists single_delete_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_replace_test; create table if not exists single_replace_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world',index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists replace_unique_key_test; create table if not exists replace_unique_key_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', unique index i1(c2) local)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists single_insert_up_test; create table if not exists single_insert_up_test (C1 bigint primary key, C2 double, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_insert_or_update_test; create table if not exists multi_insert_or_update_test (C1 bigint primary key, C2 bigint, C3 varchar(100) default 'hello world', index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists kv_query_test; create table if not exists kv_query_test (C1 bigint, C2 bigint, C3 bigint, PRIMARY KEY(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists check_scan_range_test; create table if not exists check_scan_range_test (C1 bigint, C2 varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, C3 bigint, PRIMARY KEY(C1, C2), KEY idx_c3 (C3));" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists virtual_generate_col_test; create table if not exists virtual_generate_col_test (C1 bigint primary key, C2 bigint, C3 varchar(100), C3_PREFIX varchar(10) GENERATED ALWAYS AS (substr(C3,1,2)))" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists store_generate_col_test; create table if not exists store_generate_col_test (C1 bigint primary key, C2 varchar(10), C3 varchar(10), GEN varchar(30) generated always as (concat(C2,C3)) stored)" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_insert_test; create table if not exists multi_insert_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists multi_delete_test; create table if not exists multi_delete_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3)) PARTITION BY KEY(C1) PARTITIONS 16" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists query_sync_multi_batch_test; create table if not exists query_sync_multi_batch_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2), INDEX idx1(C1, C2));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists large_scan_query_sync_test; create table if not exists large_scan_query_sync_test (C1 bigint primary key, C2 bigint, C3 varchar(100));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists query_sync_with_index_test; create table if not exists query_sync_with_index_test (C1 bigint, C2 bigint, C3 bigint, primary key(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists query_sync_multi_task_test; create table if not exists query_sync_multi_task_test (C1 bigint primary key, C2 bigint, C3 varchar(100));" $db - - -mysql -h $HOST -P $PORT -u $user -e "drop table if exists execute_query_test; create table if not exists execute_query_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2), index i1(c1, c2));" $db +# INDEX idx1(C1, C2) +mysql -h $HOST -P $PORT -u $user -e "drop table if exists execute_query_test; create table if not exists execute_query_test (PK1 bigint, PK2 bigint, C1 bigint, C2 varchar(100), C3 bigint, PRIMARY KEY(PK1, PK2));" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists secondary_index_test; create table if not exists secondary_index_test (C1 bigint primary key, C2 bigint, C3 varchar(100), index i1(c2) local, index i2(c3) local, index i3(c2, c3) local)" $db # hbase api mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1; create table if not exists htable1_cf1 (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substr(K,1,32)) STORED, primary key(K, Q, T));" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_reverse; create table if not exists htable1_cf1_reverse like htable1_cf1;" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_filter; create table if not exists htable1_cf1_filter like htable1_cf1" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_query_and_mutate; create table if not exists htable1_cf1_query_and_mutate like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_delete; create table if not exists htable1_cf1_delete like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_put; create table if not exists htable1_cf1_put like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_mutate; create table if not exists htable1_cf1_mutate like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_query_and_mutate; create table if not exists htable1_cf1_query_and_mutate like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment; create table if not exists htable1_cf1_increment like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_increment_empty; create table if not exists htable1_cf1_increment_empty like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_append; create table if not exists htable1_cf1_append like htable1_cf1" $db +mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_reverse; create table if not exists htable1_cf1_reverse like htable1_cf1" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_ttl; create table if not exists htable1_cf1_ttl (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T)) comment='{\"HColumnDescriptor\": {\"TimeToLive\": 5}}'" $db mysql -h $HOST -P $PORT -u $user -e "drop table if exists htable1_cf1_empty_cq; create table if not exists htable1_cf1_empty_cq (K varbinary(1024), Q varbinary(256), T bigint, V varbinary(1024), primary key(K, Q, T));" $db -mysql -h $HOST -P $PORT -u $user -e "drop table if exists varchar_rowkey_update_test; create table if not exists varchar_rowkey_update_test (K varchar(512) primary key, T bigint, KEY idx_T(T) LOCAL);" $db -mysql -h $HOST -P $PORT -u $user -e "DROP TABLE IF EXISTS kv_query_test; CREATE TABLE IF NOT EXISTS kv_query_test (C1 bigint, C2 bigint, C3 bigint, PRIMARY KEY(C1, C2), KEY idx_c2 (C2), KEY idx_c3 (C3), KEY idx_c2c3(C2, C3));" $db -sleep 3 + +# run ./test_table_api "$HOST" "$PORT" "$tenant_name" "$user_name" "$passwd" "$db" "$table_name" $RPCPORT diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index c3ce0e9780..96dcddf28a 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -125,7 +125,6 @@ ob_set_subtarget(ob_server table table/ob_htable_filter_parser.cpp table/ob_htable_filters.cpp table/ob_htable_utils.cpp - table/ob_table_api_row_iterator.cpp table/ob_table_batch_execute_processor.cpp table/ob_table_end_trans_cb.cpp table/ob_table_execute_processor.cpp @@ -134,6 +133,21 @@ ob_set_subtarget(ob_server table table/ob_table_rpc_processor.cpp table/ob_table_query_sync_processor.cpp table/ob_table_service.cpp + table/ob_table_context.cpp + table/ob_table_executor.cpp + table/ob_table_delete_executor.cpp + table/ob_table_insert_executor.cpp + table/ob_table_insert_up_executor.cpp + table/ob_table_lock_executor.cpp + table/ob_table_modify_executor.cpp + table/ob_table_replace_executor.cpp + table/ob_table_scan_executor.cpp + table/ob_table_update_executor.cpp + table/ob_table_executor_factory.cpp + table/ob_table_cg_service.cpp + table/ob_table_cache.cpp + table/ob_table_session_pool.cpp + table/ob_table_op_wrapper.cpp ) ob_set_subtarget(ob_server virtual_table diff --git a/src/observer/table/ob_htable_filter_operator.cpp b/src/observer/table/ob_htable_filter_operator.cpp index da504291bd..a652637048 100644 --- a/src/observer/table/ob_htable_filter_operator.cpp +++ b/src/observer/table/ob_htable_filter_operator.cpp @@ -130,7 +130,7 @@ int ObHTableExplicitColumnTracker::init(const table::ObHTableFilter &htable_filt std::sort(&columns_.at(0), end, ColumnCountComparator()); } } - + if (OB_SUCC(ret)) { // check duplicated qualifiers for (int64_t i = 0; OB_SUCCESS == ret && i < N - 1; ++i) @@ -286,7 +286,7 @@ int ObHTableExplicitColumnTracker::get_next_column_or_row(const ObHTableCell &ce //////////////////////////////////////////////////////////////// ObHTableWildcardColumnTracker::ObHTableWildcardColumnTracker() - :allocator_(ObModIds::TABLE_PROC), + :allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), current_qualifier_(), current_count_(0) {} @@ -326,7 +326,7 @@ int ObHTableWildcardColumnTracker::reset_cell(const ObHTableCell &cell) int ObHTableWildcardColumnTracker::check_versions(const ObHTableCell &cell, ObHTableMatchCode &match_code) { int ret = OB_SUCCESS; - int cmp_ret; + int cmp_ret; if (current_qualifier_.empty()) { // first iteration ret = reset_cell(cell); @@ -337,7 +337,7 @@ int ObHTableWildcardColumnTracker::check_versions(const ObHTableCell &cell, ObHT if (common::ObQueryFlag::Reverse == tracker_scan_order_) { cmp_ret = ObHTableUtils::compare_qualifier(current_qualifier_, cell.get_qualifier()); } else { - cmp_ret = ObHTableUtils::compare_qualifier(cell.get_qualifier(),current_qualifier_); + cmp_ret = ObHTableUtils::compare_qualifier(cell.get_qualifier(),current_qualifier_); } if (0 == cmp_ret) { match_code = check_version(cell.get_timestamp()); @@ -399,7 +399,7 @@ ObHTableScanMatcher::ObHTableScanMatcher(const table::ObHTableFilter &htable_fil :time_range_(-htable_filter.get_max_stamp(), -htable_filter.get_min_stamp()), column_tracker_(column_tracker), hfilter_(NULL), - allocator_(ObModIds::TABLE_PROC), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), curr_row_() { } @@ -512,9 +512,9 @@ ObHTableMatchCode ObHTableScanMatcher::merge_filter_return_code(const ObHTableCe } else { if (ObHTableMatchCode::INCLUDE == match_code) { ret_code = ObHTableMatchCode::INCLUDE_AND_SEEK_NEXT_COL; - } + } } - + break; case Filter::ReturnCode::INCLUDE_AND_SEEK_NEXT_ROW: ret_code = ObHTableMatchCode::INCLUDE_AND_SEEK_NEXT_ROW; @@ -624,7 +624,7 @@ ObHTableRowIterator::ObHTableRowIterator(const ObTableQuery &query) batch_size_(query.get_batch()), time_to_live_(0), curr_cell_(), - allocator_(ObModIds::TABLE_PROC), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), column_tracker_(NULL), matcher_(NULL), column_tracker_wildcard_(), @@ -660,11 +660,11 @@ int ObHTableRowIterator::reverse_next_cell(ObIArray &same_kq_c { ObNewRow *ob_row = NULL; int ret = child_op_->get_next_row(ob_row); - if ((ObQueryFlag::Reverse == scan_order_ && OB_ITER_END == ret) || - (ObQueryFlag::Reverse == scan_order_ && OB_SUCCESS == ret && + if ((ObQueryFlag::Reverse == scan_order_ && OB_ITER_END == ret) || + (ObQueryFlag::Reverse == scan_order_ && OB_SUCCESS == ret && NULL != hfilter_ && hfilter_->filter_all_remaining())) { ret = add_same_kq_to_res(same_kq_cells, out_result); - same_kq_cells.reset(); + same_kq_cells.reset(); has_more_cells_ = false; curr_cell_.set_ob_row(NULL); matcher_->clear_curr_row(); @@ -674,7 +674,7 @@ int ObHTableRowIterator::reverse_next_cell(ObIArray &same_kq_c LOG_DEBUG("[sldebug] curr cell", K_(curr_cell)); } else if (OB_SUCCESS != ret && OB_ITER_END != ret){ LOG_WARN("the ret doesn't match what we had been expected",K(ret)); - } + } return ret; } @@ -687,14 +687,14 @@ int ObHTableRowIterator::add_same_kq_to_res(ObIArray &same_kq_cells, O ObNewRow &tmp = same_kq_cells.at(i); ObHTableCellEntity tmp_cell; tmp_cell.set_ob_row(&tmp); - + //make timestamp as positive int64_t timestamp = 0; if (OB_FAIL(tmp.get_cell(ObHTableConstants::COL_IDX_T).get_int(timestamp))) { LOG_WARN("failed to get timestamp",K(ret)); } else { tmp.get_cell(ObHTableConstants::COL_IDX_T).set_int(-timestamp); - } + } if (OB_SUCC(ret)) { ObHTableMatchCode match_code = ObHTableMatchCode::INCLUDE; if (NULL != hfilter_ && ObHTableMatchCode::INCLUDE == match_code) { @@ -710,7 +710,7 @@ int ObHTableRowIterator::add_same_kq_to_res(ObIArray &same_kq_cells, O if (OB_SUCC(ret) && NULL != hfilter_ && hfilter_->filter_all_remaining()) { match_code = ObHTableMatchCode::DONE_SCAN; - has_more_cells_ = false; + has_more_cells_ = false; } if (OB_SUCC(ret) && ObHTableMatchCode::INCLUDE == match_code) { @@ -723,7 +723,7 @@ int ObHTableRowIterator::add_same_kq_to_res(ObIArray &same_kq_cells, O } } } - + return ret; } @@ -736,9 +736,9 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) ObHTableMatchCode match_code = ObHTableMatchCode::DONE_SCAN; // initialize if (ObQueryFlag::Reverse == scan_order_ && (-1 != limit_per_row_per_cf_ || 0 != offset_per_row_per_cf_)) { ret = OB_NOT_SUPPORTED; - LOG_WARN("server don't support set limit_per_row_per_cf_ and offset_per_row_per_cf_ in reverse scan yet", + LOG_WARN("server don't support set limit_per_row_per_cf_ and offset_per_row_per_cf_ in reverse scan yet", K(ret), K(scan_order_), K(limit_per_row_per_cf_), K(offset_per_row_per_cf_)); - } + } if (OB_SUCC(ret) && NULL == column_tracker_) { // first iteration if (htable_filter_.get_columns().count() <= 0) { @@ -812,10 +812,10 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) matcher_->clear_curr_row(); if (ObQueryFlag::Reverse == scan_order_) { ret = add_same_kq_to_res(same_kq_cells, out_result); - same_kq_cells.reset(); + same_kq_cells.reset(); } else { ret = seek_or_skip_to_next_row(curr_cell_); - } + } loop = false; LOG_DEBUG("[yzfdebug] reach per row limit", K(ret), K_(offset_per_row_per_cf), K_(limit_per_row_per_cf), K_(count_per_row)); break; @@ -823,8 +823,8 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) // whether skip offset if (count_per_row_ > offset_per_row_per_cf_) { if(OB_SUCC(ret) && ObQueryFlag::Reverse == scan_order_) { - // reverse scan if the current cell has the same key and qualifier with the last, match_code = INCLUDE; else match_code = DONE_REVERSE_SCAN; - // INCELUDE: put current cell into result vector + // reverse scan if the current cell has the same key and qualifier with the last, match_code = INCLUDE; else match_code = DONE_REVERSE_SCAN; + // INCELUDE: put current cell into result vector // DONE_REVERSE_SCAN: end this round scan, choose cell to return ObString pre_key; ObString pre_qualifier; @@ -839,12 +839,12 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) match_code = ObHTableMatchCode::DONE_REVERSE_SCAN; } else { match_code = ObHTableMatchCode::INCLUDE; - } + } } } } else { - match_code = ObHTableMatchCode::INCLUDE; - } + match_code = ObHTableMatchCode::INCLUDE; + } } else { int64_t timestamp = 0; if (OB_FAIL(curr_cell_.get_ob_row()->get_cell(ObHTableConstants::COL_IDX_T).get_int(timestamp))) { @@ -852,7 +852,7 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) } else { const_cast(curr_cell_.get_ob_row())->get_cell(ObHTableConstants::COL_IDX_T).set_int(-timestamp); } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret)) { if (OB_FAIL(out_result->add_row(*(curr_cell_.get_ob_row())))) { LOG_WARN("failed to add row to result", K(ret)); } else { @@ -871,7 +871,7 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) ret = seek_or_skip_to_next_col(curr_cell_); } else if (ObHTableMatchCode::DONE_REVERSE_SCAN == match_code) { ret = add_same_kq_to_res(same_kq_cells, out_result); - same_kq_cells.reset(); + same_kq_cells.reset(); ObNewRow new_row; if (OB_SUCC(ret) && OB_FAIL(ob_write_row(allocator_, *(curr_cell_.get_ob_row()), new_row))) { LOG_WARN("failed to copy row", K(ret)); @@ -912,7 +912,7 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) // done current row if (ObQueryFlag::Reverse == scan_order_) { ret = add_same_kq_to_res(same_kq_cells, out_result); - same_kq_cells.reset(); + same_kq_cells.reset(); } matcher_->clear_curr_row(); loop = false; @@ -931,7 +931,7 @@ int ObHTableRowIterator::get_next_result(ObTableQueryResult *&out_result) break; case ObHTableMatchCode::SEEK_NEXT_COL: if (ObQueryFlag::Reverse == scan_order_) { - ret = reverse_next_cell(same_kq_cells, out_result); + ret = reverse_next_cell(same_kq_cells, out_result); } else { ret = seek_or_skip_to_next_col(curr_cell_); } diff --git a/src/observer/table/ob_htable_filter_operator.h b/src/observer/table/ob_htable_filter_operator.h index 3e832a6c73..1b2b6ba0b8 100644 --- a/src/observer/table/ob_htable_filter_operator.h +++ b/src/observer/table/ob_htable_filter_operator.h @@ -20,6 +20,7 @@ #include "ob_htable_utils.h" #include "ob_htable_filter_parser.h" #include "ob_htable_filters.h" +#include "ob_table_scan_executor.h" #include namespace oceanbase @@ -185,7 +186,7 @@ public: virtual int get_next_result(ObTableQueryResult *&one_row) override; int seek(const ObHTableCell &key); - void set_scan_result(common::ObNewRowIterator *scan_result) { child_op_ = scan_result; } + void set_scan_result(table::ObTableApiScanRowIterator *scan_result) { child_op_ = scan_result; } bool has_more_result() const { return has_more_cells_; } void set_hfilter(table::hfilter::Filter *hfilter); void set_ttl(int32_t ttl_value); @@ -199,7 +200,7 @@ private: bool reach_batch_limit() const; bool reach_size_limit() const; private: - common::ObNewRowIterator *child_op_; + table::ObTableApiScanRowIterator *child_op_; const table::ObHTableFilter &htable_filter_; table::hfilter::Filter *hfilter_; int32_t limit_per_row_per_cf_; @@ -232,7 +233,7 @@ public: /// Fetch next batch result virtual int get_next_result(ObTableQueryResult *&one_result) override; virtual bool has_more_result() const override { return row_iterator_.has_more_result(); } - void set_scan_result(common::ObNewRowIterator *scan_result) { row_iterator_.set_scan_result(scan_result); } + void set_scan_result(table::ObTableApiScanRowIterator *scan_result) { row_iterator_.set_scan_result(scan_result); } void set_ttl(int32_t ttl_value) { row_iterator_.set_ttl(ttl_value); } // parse the filter string int parse_filter_string(common::ObArenaAllocator* allocator); diff --git a/src/observer/table/ob_table_api_row_iterator.h b/src/observer/table/ob_table_api_row_iterator.h deleted file mode 100644 index ce41229091..0000000000 --- a/src/observer/table/ob_table_api_row_iterator.h +++ /dev/null @@ -1,261 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OB_TABLE_API_ROW_ITERATOR_H_ -#define OB_TABLE_API_ROW_ITERATOR_H_ - -#include "ob_table_service.h" -#include "common/row/ob_row_iterator.h" -#include "share/schema/ob_table_param.h" -#include "storage/tx_storage/ob_access_service.h" - -namespace oceanbase { -namespace sql { - struct ObTempExpr; -} -namespace observer { - -class ObTableApiRowIterator : public common::ObNewRowIterator { -public: - ObTableApiRowIterator(); - virtual ~ObTableApiRowIterator(); - int init( - storage::ObAccessService &access_service, - share::schema::ObMultiVersionSchemaService &schema_service, - ObTableServiceCtx &ctx); - virtual void reset(); - OB_INLINE common::ObIArray &get_column_ids() { return column_ids_; } - OB_INLINE common::ObIArray &get_properties() { return properties_; } - OB_INLINE int64_t get_schema_version() { return schema_version_; } - OB_INLINE int64_t get_rowkey_column_cnt() { return rowkey_column_cnt_; } -protected: - int check_row(common::ObNewRow &row); - int entity_to_row(const table::ObITableEntity &entity, common::ObIArray &row); - int cons_all_columns(const table::ObITableEntity &entity, const bool ignore_missing_column = false); - int cons_missing_columns(const table::ObITableEntity &entity); - int fill_get_param( - ObTableServiceCtx &ctx, - const table::ObTableOperationType::Type op_type, - ObRowkey &rowkey, - storage::ObTableScanParam &scan_param, - share::schema::ObTableParam &table_param); - int fill_multi_get_param( - ObTableServiceCtx &ctx, - const ObTableBatchOperation &batch_operation, - storage::ObTableScanParam &scan_param, - share::schema::ObTableParam &table_param); - int fill_generate_columns(common::ObNewRow &row); - virtual bool is_read() const { return false; } -private: - int check_table_supported(const share::schema::ObTableSchema *table_schema); - int check_column_type(const sql::ObExprResType &column_type, common::ObObj &obj); - int fill_range(const ObRowkey &rowkey, ObIArray &ranges); - int fill_flag(ObTableServiceCtx &ctx, storage::ObTableScanParam &scan_param); - int add_column_type(const share::schema::ObColumnSchemaV2 &column_schema); - int cons_column_type(const share::schema::ObColumnSchemaV2 &column_schema, sql::ObExprResType &column_type); -protected: - static const int64_t COMMON_COLUMN_NUM = 16; - storage::ObAccessService *access_service_; - share::schema::ObMultiVersionSchemaService *schema_service_; - ObTableServiceCtx *ctx_; - share::schema::ObSchemaGetterGuard schema_guard_; - const share::schema::ObTableSchema *table_schema_; - int64_t table_id_; - int64_t tenant_id_; - int64_t schema_version_; - int64_t rowkey_column_cnt_; - common::ObSEArray properties_; - common::ObSEArray column_ids_; - common::ObSEArray columns_type_; - common::ObSEArray column_descs_; - common::ObSEArray row_objs_; - common::ObSEArray missing_default_objs_; - common::ObSEArray generate_column_exprs_; - common::ObSEArray generate_column_idxs_; - common::ObExprCtx expr_ctx_; - common::ObNewRow row_; - common::ObArenaAllocator stmt_allocator_; - common::ObArenaAllocator row_allocator_; - const table::ObITableEntity *entity_; - bool has_generate_column_; - bool is_inited_; -}; - - -class ObTableApiInsertRowIterator : public ObTableApiRowIterator -{ -public: - ObTableApiInsertRowIterator(); - virtual ~ObTableApiInsertRowIterator(); - int open(const ObTableOperation &table_operation); - virtual int get_next_row(common::ObNewRow *&row); -protected: - int cons_row(const table::ObITableEntity &entity, common::ObNewRow *&row); - virtual bool is_read() const override { return false; } -}; - - -class ObTableApiMultiInsertRowIterator : public ObTableApiInsertRowIterator -{ -public: - ObTableApiMultiInsertRowIterator(); - virtual ~ObTableApiMultiInsertRowIterator(); - virtual void reset(); - int open(const ObTableBatchOperation &table_operation); - virtual int get_next_row(common::ObNewRow *&row); - OB_INLINE void continue_iter() { is_iter_pause_ = false; } -private: - const ObTableBatchOperation *batch_operation_; - int64_t row_idx_; - int64_t batch_cnt_; - bool is_iter_pause_; -}; - - -class ObTableApiUpdateRowIterator : public ObTableApiRowIterator -{ -public: - ObTableApiUpdateRowIterator(); - virtual ~ObTableApiUpdateRowIterator(); - virtual void reset(); - int open(const ObTableOperation &table_operation, - const ObRowkey &rowkey, bool need_update_rowkey = false); - virtual int get_next_row(common::ObNewRow *&row); - OB_INLINE common::ObIArray &get_update_column_ids() { return update_column_ids_; } - OB_INLINE common::ObNewRow *get_cur_new_row() { return new_row_; } -protected: - int cons_update_columns(bool need_update_rowkey); - int cons_new_row(const ObTableOperation &table_operation, common::ObNewRow *&row); - virtual bool is_read() const override { return false; } -private: - int obj_increment( - const common::ObObj &delta, - const common::ObObj &src, - const sql::ObExprResType target_type, - common::ObObj &target); - int obj_append( - const common::ObObj &delta, - const common::ObObj &src, - const sql::ObExprResType target_type, - common::ObObj &target); - int int_add_int_with_check( - int64_t old_int, - int64_t delta_int, - common::ObObjType result_type, - common::ObObj &result); - int uint_add_int_with_check( - uint64_t old_uint, - int64_t delta_int, - common::ObObjType result_type, - common::ObObj &result); -protected: - storage::ObTableScanParam scan_param_; - share::schema::ObTableParam table_param_; - common::ObSEArray update_column_ids_; - common::ObNewRowIterator *scan_iter_; - common::ObNewRow *old_row_; - common::ObNewRow *new_row_; - int64_t row_idx_; - bool need_update_rowkey_; -private: - const ObTableOperation *table_operation_; -}; - - -class ObTableApiMultiUpdateRowIterator : public ObTableApiUpdateRowIterator -{ -public: - ObTableApiMultiUpdateRowIterator(); - virtual ~ObTableApiMultiUpdateRowIterator(); - virtual void reset(); - int open(const ObTableBatchOperation &batch_operation); - virtual int get_next_row(common::ObNewRow *&row); - OB_INLINE void continue_iter() { is_iter_pause_ = false; } - OB_INLINE int64_t get_cur_update_idx() { return cur_update_idx_; } - OB_INLINE bool has_finished() { return batch_idx_ >= batch_cnt_; } -private: - const ObTableBatchOperation *batch_operation_; - int64_t batch_cnt_; - int64_t batch_idx_; - int64_t cur_update_idx_; - bool is_iter_pause_; -}; - - -class ObTableApiDeleteRowIterator : public ObTableApiRowIterator -{ -public: - ObTableApiDeleteRowIterator(); - virtual ~ObTableApiDeleteRowIterator(); - virtual void reset(); - int open(const ObTableOperation &table_operation); - virtual int get_next_row(common::ObNewRow *&row); - OB_INLINE common::ObIArray &get_delete_column_ids() { return column_ids_; } -protected: - virtual bool is_read() const override { return false; } -protected: - storage::ObTableScanParam scan_param_; - share::schema::ObTableParam table_param_; - common::ObNewRowIterator *scan_iter_; -}; - - -class ObTableApiMultiDeleteRowIterator : public ObTableApiDeleteRowIterator -{ -public: - ObTableApiMultiDeleteRowIterator(); - virtual ~ObTableApiMultiDeleteRowIterator(); - virtual void reset(); - int open(const ObTableBatchOperation &table_operation); - virtual int get_next_row(common::ObNewRow *&row); - OB_INLINE void continue_iter() { is_iter_pause_ = false; } - OB_INLINE int64_t get_cur_delete_idx() { return cur_delete_idx_; } - OB_INLINE bool has_finished() { return batch_idx_ >= batch_cnt_; } -private: - const ObTableBatchOperation *batch_operation_; - int64_t batch_cnt_; - int64_t batch_idx_; - int64_t cur_delete_idx_; - bool is_iter_pause_; -}; - - -class ObTableApiGetRowIterator : public ObTableApiRowIterator -{ -public: - ObTableApiGetRowIterator(); - virtual ~ObTableApiGetRowIterator(); - virtual void reset(); - int open(const ObTableOperation &table_operation); - virtual int get_next_row(common::ObNewRow *&row); -protected: - virtual bool is_read() const override { return true; } -protected: - storage::ObTableScanParam scan_param_; - share::schema::ObTableParam table_param_; - common::ObNewRowIterator *scan_iter_; -}; - - -class ObTableApiMultiGetRowIterator : public ObTableApiGetRowIterator -{ -public: - ObTableApiMultiGetRowIterator(); - virtual ~ObTableApiMultiGetRowIterator(); - int open(const ObTableBatchOperation &table_operation); -}; - - -} -} - -#endif /* OB_TABLE_API_ROW_ITERATOR_H_ */ diff --git a/src/observer/table/ob_table_batch_execute_processor.cpp b/src/observer/table/ob_table_batch_execute_processor.cpp index 9af4993f9c..fef9501ba5 100644 --- a/src/observer/table/ob_table_batch_execute_processor.cpp +++ b/src/observer/table/ob_table_batch_execute_processor.cpp @@ -19,6 +19,9 @@ #include "lib/stat/ob_diagnose_info.h" #include "lib/stat/ob_session_stat.h" #include "ob_htable_utils.h" +#include "ob_table_cg_service.h" +#include "observer/ob_req_time_service.h" + using namespace oceanbase::observer; using namespace oceanbase::common; using namespace oceanbase::table; @@ -27,8 +30,8 @@ using namespace oceanbase::sql; ObTableBatchExecuteP::ObTableBatchExecuteP(const ObGlobalContext &gctx) :ObTableRpcProcessor(gctx), - allocator_(ObModIds::TABLE_PROC), - table_service_ctx_(allocator_), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + tb_ctx_(allocator_), need_rollback_trans_(false) { } @@ -46,9 +49,9 @@ int ObTableBatchExecuteP::deserialize() { ObITableEntity *entity = nullptr; if (OB_FAIL(const_cast(arg_.batch_operation_.at(i)).get_entity(entity))) { - LOG_WARN("failed to get entity", K(ret), K(i)); + LOG_WARN("fail to get entity", K(ret), K(i)); } else if (OB_FAIL(ObTableRpcProcessorUtil::negate_htable_timestamp(*entity))) { - LOG_WARN("failed to negate timestamp value", K(ret)); + LOG_WARN("fail to negate timestamp value", K(ret)); } } // end for } @@ -122,9 +125,9 @@ int ObTableBatchExecuteP::response(const int retcode) { ObITableEntity *entity = nullptr; if (OB_FAIL(result_.at(i).get_entity(entity))) { - LOG_WARN("failed to get entity", K(ret), K(i)); + LOG_WARN("fail to get entity", K(ret), K(i)); } else if (OB_FAIL(ObTableRpcProcessorUtil::negate_htable_timestamp(*entity))) { - LOG_WARN("failed to negate timestamp value", K(ret)); + LOG_WARN("fail to negate timestamp value", K(ret)); } } // end for } @@ -137,7 +140,6 @@ int ObTableBatchExecuteP::response(const int retcode) void ObTableBatchExecuteP::reset_ctx() { - table_service_ctx_.reset_dml(); need_retry_in_queue_ = false; need_rollback_trans_ = false; result_.reset(); @@ -187,7 +189,7 @@ int ObTableBatchExecuteP::try_process() break; case ObTableOperationType::UPDATE: stat_event_type_ = ObTableProccessType::TABLE_API_MULTI_UPDATE; - ret = multi_update(); + ret = batch_execute(false); break; case ObTableOperationType::INSERT_OR_UPDATE: if (ObTableEntityType::ET_HKV == arg_.entity_type_) { @@ -195,7 +197,7 @@ int ObTableBatchExecuteP::try_process() ret = htable_put(); } else { stat_event_type_ = ObTableProccessType::TABLE_API_MULTI_INSERT_OR_UPDATE; - ret = multi_insert_or_update(); + ret = batch_execute(false); } break; case ObTableOperationType::REPLACE: @@ -249,7 +251,7 @@ ObTableAPITransCb *ObTableBatchExecuteP::new_callback(rpc::ObRequest *req) // @todo optimize to avoid this copy int ret = OB_SUCCESS; if (OB_FAIL(cb->assign_batch_execute_result(result_))) { - LOG_WARN("failed to assign result", K(ret)); + LOG_WARN("fail to assign result", K(ret)); cb->~ObTableBatchExecuteEndTransCb(); cb = NULL; } else { @@ -269,7 +271,7 @@ int ObTableBatchExecuteP::get_rowkeys(ObIArray &rowkeys) const ObTableOperation &table_op = batch_operation.at(i); ObRowkey rowkey = const_cast(table_op.entity()).get_rowkey(); if (OB_FAIL(rowkeys.push_back(rowkey))) { - LOG_WARN("failed to push back", K(ret)); + LOG_WARN("fail to push back", K(ret)); } } // end for return ret; @@ -282,107 +284,72 @@ int ObTableBatchExecuteP::get_tablet_ids(uint64_t table_id, ObIArray if (!tablet_id.is_valid()) { ObSEArray rowkeys; if (OB_FAIL(get_rowkeys(rowkeys))) { - LOG_WARN("failed to get rowkeys", K(ret)); + LOG_WARN("fail to get rowkeys", K(ret)); } else if (OB_FAIL(get_tablet_by_rowkey(table_id, rowkeys, tablet_ids))) { - LOG_WARN("failed to get partition", K(ret), K(rowkeys)); + LOG_WARN("fail to get partition", K(ret), K(rowkeys)); } } else { if (OB_FAIL(tablet_ids.push_back(tablet_id))) { - LOG_WARN("failed to push back", K(ret)); + LOG_WARN("fail to push back", K(ret)); } } return ret; } -int ObTableBatchExecuteP::multi_insert_or_update() -{ - int ret = OB_SUCCESS; - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_INSERT, consistency_level, - table_id, table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->multi_insert_or_update(table_service_ctx_, batch_operation, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert_or_update", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - int ObTableBatchExecuteP::htable_put() { - int ret = OB_NOT_SUPPORTED; + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + int64_t affected_rows = 0; const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t table_id = OB_INVALID_ID; - ObSEArray tablet_ids; - ObLSID ls_id; if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), ls_id))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_INSERT, consistency_level, - table_id, ls_id, get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("failed to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx_, + cache_guard_, + spec))) { + LOG_WARN("fail to get or create spec", K(ret)); } else { - int64_t affected_rows = 0; - SMART_VAR(ObHTablePutExecutor, put_executor, allocator_, - table_id, - tablet_ids.at(0), - ls_id, - get_timeout_ts(), - this, - table_service_) { - ret = put_executor.htable_put(batch_operation, affected_rows); - } - if (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_operation.count(); ++i) { + const ObTableOperation &table_operation = batch_operation.at(i); ObTableOperationResult single_op_result; - single_op_result.set_entity(result_entity_); - single_op_result.set_type(ObTableOperationType::INSERT_OR_UPDATE); - single_op_result.set_errno(ret); - single_op_result.set_affected_rows(affected_rows); - result_.reset(); - if (OB_FAIL(result_.push_back(single_op_result))) { - LOG_WARN("failed to add result", K(ret)); + tb_ctx_.set_entity(&table_operation.entity()); + if (OB_FAIL(ObTableOpWrapper::process_op_with_spec(tb_ctx_, spec, single_op_result))) { + LOG_WARN("fail to process op with spec", K(ret)); + } else { + affected_rows += single_op_result.get_affected_rows(); } } } + + if (OB_SUCC(ret)) { + ObTableOperationResult op_result; + op_result.set_type(ObTableOperationType::INSERT_OR_UPDATE); + op_result.set_entity(result_entity_); + op_result.set_errno(ret); + op_result.set_affected_rows(affected_rows); + result_.reset(); + if (OB_FAIL(result_.push_back(op_result))) { + LOG_WARN("failed to add result", K(ret)); + } + } + int tmp_ret = ret; if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); + LOG_WARN("fail to end trans"); } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; return ret; } @@ -390,117 +357,592 @@ int ObTableBatchExecuteP::htable_put() int ObTableBatchExecuteP::multi_get() { int ret = OB_SUCCESS; - need_rollback_trans_ = false; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - ObSEArray tablet_ids; - const bool is_readonly = true; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; + ObTableApiSpec *spec = nullptr; + const ObTableBatchOperation &batch_operation = arg_.batch_operation_; + if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_SELECT, consistency_level, table_id, - table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start readonly transaction", K(ret)); - } else if (OB_FAIL(table_service_->multi_get(table_service_ctx_, arg_.batch_operation_, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute get", K(ret), K(table_id)); + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(init_read_trans(arg_.consistency_level_, + tb_ctx_.get_ls_id(), + tb_ctx_.get_timeout_ts()))) { + LOG_WARN("fail to init wead read trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx_, + cache_guard_, + spec))) { + LOG_WARN("fail to get or create spec", K(ret)); + } else { + const ObTableSchema *table_schema = tb_ctx_.get_table_schema(); + for (int64_t i = 0; OB_SUCC(ret) && i < batch_operation.count(); ++i) { + const ObTableOperation &table_operation = batch_operation.at(i); + tb_ctx_.set_entity(&table_operation.entity()); + ObTableOperationResult op_result; + ObITableEntity *result_entity = result_.get_entity_factory()->alloc(); + ObNewRow *row = nullptr; + if (OB_FAIL(ObTableOpWrapper::process_get_with_spec(tb_ctx_, spec, row))) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to process get with spec", K(ret), K(i)); + } + } else { + // fill result entity + ObArray properties; + const ObITableEntity *request_entity = tb_ctx_.get_entity(); + if (OB_FAIL(request_entity->get_properties_names(properties))) { + LOG_WARN("fail to get entity properties", K(ret), K(i)); + } else if (OB_FAIL(ObTableApiUtil::construct_entity_from_row(row, + table_schema, + properties, + result_entity))) { + LOG_WARN("fail to fill result entity", K(ret), K(i)); + } + + } + op_result.set_entity(*result_entity); + op_result.set_errno(ret); + op_result.set_type(tb_ctx_.get_opertion_type()); + if (OB_FAIL(result_.push_back(op_result))) { + LOG_WARN("fail to push back op result", K(ret), K(i)); + } } - } else {} - need_rollback_trans_ = (OB_SUCCESS != ret); - int tmp_ret = ret; - if (OB_FAIL(end_trans(need_rollback_trans_, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret), "rollback", need_rollback_trans_); } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + release_read_trans(); + return ret; } int ObTableBatchExecuteP::multi_delete() { int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - ObSEArray tablet_ids; + if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_DELETE, consistency_level, - table_id, table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->multi_delete(table_service_ctx_, batch_operation, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_delete", K(ret), K(table_id)); + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (ObTableOpWrapper::get_or_create_spec(tb_ctx_, cache_guard_, spec)) { + LOG_WARN("fail to get or create spec", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_operation.count(); ++i) { + const ObTableOperation &table_operation = batch_operation.at(i); + tb_ctx_.set_entity(&table_operation.entity()); + ObTableOperationResult op_result; + ObTableApiExecutor *executor = nullptr; + ObITableEntity *result_entity = result_.get_entity_factory()->alloc(); + if (OB_ISNULL(result_entity)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memroy for result_entity", K(ret)); + } else if (FALSE_IT(op_result.set_entity(*result_entity))) { + } else if (OB_FAIL(ObTableOpWrapper::process_op_with_spec(tb_ctx_, spec, op_result))) { + LOG_WARN("fail to process insert with spec", K(ret), K(i)); + } else if (OB_FAIL(result_.push_back(op_result))) { + LOG_WARN("fail to push back result", K(ret)); + } } } + int tmp_ret = ret; if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); + LOG_WARN("fail to end trans"); } ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; return ret; } - int ObTableBatchExecuteP::htable_delete() { int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + int64_t affected_rows = 0; const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; + tb_ctx_.set_batch_operation(&batch_operation); + + if (OB_FAIL(check_arg2())) { + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("failed to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx_, cache_guard_, spec))) { + LOG_WARN("fail to get or create spec", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx_, executor))) { + LOG_WARN("fail to create executor", K(ret)); + } else { + ObHTableDeleteExecutor delete_executor(tb_ctx_, static_cast(executor)); + if (OB_FAIL(delete_executor.open())) { + LOG_WARN("fail to open htable delete executor", K(ret)); + } else if (OB_FAIL(delete_executor.get_next_row())) { + LOG_WARN("fail to call htable delete get_next_row", K(ret)); + } else if (FALSE_IT(affected_rows = delete_executor.get_affected_rows())) { + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = delete_executor.close())) { + LOG_WARN("fail to close htable delete executor", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + + if (OB_NOT_NULL(spec)) { + spec->destroy_executor(executor); + tb_ctx_.set_expr_info(nullptr); + } + + if (OB_SUCC(ret)) { + ObTableOperationResult single_op_result; + single_op_result.set_entity(result_entity_); + single_op_result.set_type(ObTableOperationType::DEL); + single_op_result.set_errno(ret); + single_op_result.set_affected_rows(affected_rows); + result_.reset(); + if (OB_FAIL(result_.push_back(single_op_result))) { + LOG_WARN("failed to add result", K(ret)); + } + } + + int tmp_ret = ret; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + LOG_WARN("fail to end trans"); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + return ret; +} + +int ObTableBatchExecuteP::multi_insert() +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + const ObTableBatchOperation &batch_operation = arg_.batch_operation_; + + if (OB_FAIL(check_arg2())) { + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (ObTableOpWrapper::get_or_create_spec(tb_ctx_, cache_guard_, spec)) { + LOG_WARN("fail to get or create spec", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_operation.count(); ++i) { + const ObTableOperation &table_operation = batch_operation.at(i); + tb_ctx_.set_entity(&table_operation.entity()); + ObTableOperationResult op_result; + ObITableEntity *result_entity = result_.get_entity_factory()->alloc(); + if (OB_ISNULL(result_entity)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc entity", K(ret), K(i)); + } else if (FALSE_IT(op_result.set_entity(*result_entity))) { + } else if (OB_FAIL(ObTableOpWrapper::process_op_with_spec(tb_ctx_, spec, op_result))) { + LOG_WARN("fail to process insert with spec", K(ret), K(i)); + } else if (OB_FAIL(result_.push_back(op_result))) { + LOG_WARN("fail to push back result", K(ret)); + } + } + } + + int tmp_ret = ret; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + LOG_WARN("fail to end trans"); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + return ret; +} + +int ObTableBatchExecuteP::multi_replace() +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + const ObTableBatchOperation &batch_operation = arg_.batch_operation_; + + if (OB_FAIL(check_arg2())) { + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_single_op_tb_ctx(tb_ctx_, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (ObTableOpWrapper::get_or_create_spec(tb_ctx_, cache_guard_, spec)) { + LOG_WARN("fail to get or create spec", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_operation.count(); ++i) { + const ObTableOperation &table_operation = batch_operation.at(i); + tb_ctx_.set_entity(&table_operation.entity()); + ObTableOperationResult op_result; + ObITableEntity *result_entity = result_.get_entity_factory()->alloc(); + if (OB_ISNULL(result_entity)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc entity", K(ret), K(i)); + } else if (FALSE_IT(op_result.set_entity(*result_entity))) { + } else if (OB_FAIL(ObTableOpWrapper::process_op_with_spec(tb_ctx_, spec, op_result))) { + LOG_WARN("fail to process insert with spec", K(ret), K(i)); + } else if (OB_FAIL(result_.push_back(op_result))) { + LOG_WARN("fail to push back result", K(ret)); + } + } + } + + int tmp_ret = ret; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + LOG_WARN("fail to end trans"); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + return ret; +} + +int ObTableBatchExecuteP::batch_execute(bool is_readonly) +{ + int ret = OB_SUCCESS; + uint64_t table_id = OB_INVALID_ID; + ObSEArray tablet_ids; + ObLSID ls_id; + if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { + LOG_WARN("fail to get table id", K(ret)); + } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { + LOG_WARN("fail to get tablet id", K(ret)); + } else if (1 != tablet_ids.count()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); + } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), ls_id))) { + LOG_WARN("fail to get ls id", K(ret)); + } else if (OB_FAIL(start_trans(is_readonly, /* is_readonly */ + (is_readonly ? sql::stmt::T_SELECT : sql::stmt::T_UPDATE), + arg_.consistency_level_, + table_id, + ls_id, + get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret)); + } else if (OB_FAIL(batch_execute_internal(arg_.batch_operation_, result_))) { + LOG_WARN("fail to execute batch", K(ret)); + } + + int tmp_ret = ret; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + LOG_WARN("fail to end trans"); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + return ret; +} + +int ObTableBatchExecuteP::batch_execute_internal(const ObTableBatchOperation &batch_operation, + ObTableBatchOperationResult &result) +{ + int ret = OB_SUCCESS; + // loop: process each op + for (int64_t i = 0; OB_SUCC(ret) && i < batch_operation.count(); ++i) { + const ObTableOperation &table_operation = batch_operation.at(i); + ObTableOperationResult op_result; + ObITableEntity *result_entity = result.get_entity_factory()->alloc(); + + SMART_VAR(table::ObTableCtx, op_tb_ctx, allocator_) { + if (OB_FAIL(init_single_op_tb_ctx(op_tb_ctx, table_operation))) { + LOG_WARN("fail to init table ctx for single operation", K(ret)); + } else if (OB_FAIL(op_tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(op_tb_ctx)); + } else if (OB_ISNULL(result_entity)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memroy for result_entity", K(ret)); + } else { + op_result.set_entity(*result_entity); + } + if (OB_SUCC(ret)) { + switch(table_operation.type()) { + case ObTableOperationType::GET: + ret = process_get(op_tb_ctx, op_result); + break; + case ObTableOperationType::INSERT: + ret = ObTableOpWrapper::process_op(op_tb_ctx, op_result); + break; + case ObTableOperationType::DEL: + ret = ObTableOpWrapper::process_op(op_tb_ctx, op_result); + break; + case ObTableOperationType::UPDATE: + ret = ObTableOpWrapper::process_op(op_tb_ctx, op_result); + break; + case ObTableOperationType::INSERT_OR_UPDATE: + ret = ObTableOpWrapper::process_op(op_tb_ctx, op_result); + break; + case ObTableOperationType::REPLACE: + ret = ObTableOpWrapper::process_op(op_tb_ctx, op_result); + break; + case ObTableOperationType::APPEND: + case ObTableOperationType::INCREMENT: + ret = ObTableOpWrapper::process_op(op_tb_ctx, op_result); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected operation type", "type", table_operation.type()); + break; + } + ObTableRpcProcessorUtil::replace_ret_code(ret); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(result.push_back(op_result))) { + LOG_WARN("fail to push back result", K(ret)); + } + } else { + LOG_WARN("fail to execute batch operation, ", K(ret), K(table_operation.type()), K(i)); + } + } // end for + return ret; +} + +int ObTableBatchExecuteP::init_single_op_tb_ctx(table::ObTableCtx &ctx, + const ObTableOperation &table_operation) +{ + int ret = OB_SUCCESS; + ctx.set_entity(&table_operation.entity()); + ctx.set_entity_type(arg_.entity_type_); + ctx.set_operation_type(table_operation.type()); + ObExprFrameInfo *expr_frame_info = nullptr; + if (ctx.is_init()) { + LOG_INFO("tb ctx has been inited", K(ctx)); + } else if (OB_FAIL(ctx.init_common(credential_, + arg_.tablet_id_, + arg_.table_name_, + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), K(arg_.table_name_)); + } else { + ObTableOperationType::Type op_type = table_operation.type(); + switch (op_type) { + case ObTableOperationType::GET: { + if (OB_FAIL(ctx.init_get())) { + LOG_WARN("fail to init get ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::INSERT: { + if (OB_FAIL(ctx.init_insert())) { + LOG_WARN("fail to init insert ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::DEL: { + if (OB_FAIL(ctx.init_delete())) { + LOG_WARN("fail to init delete ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::UPDATE: { + if (OB_FAIL(ctx.init_update())) { + LOG_WARN("fail to init update ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::INSERT_OR_UPDATE: { + if (OB_FAIL(ctx.init_insert_up())) { + LOG_WARN("fail to init insert up ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::REPLACE: { + if (OB_FAIL(ctx.init_replace())) { + LOG_WARN("fail to init replace ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::APPEND: { + if (OB_FAIL(ctx.init_append(arg_.returning_affected_entity_, + arg_.returning_rowkey_))) { + LOG_WARN("fail to init append ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::INCREMENT: { + if (OB_FAIL(ctx.init_increment(arg_.returning_affected_entity_, + arg_.returning_rowkey_))) { + LOG_WARN("fail to init increment ctx", K(ret), K(ctx)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected operation type", "type", op_type); + break; + } + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(ctx.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(ctx)); + } + + return ret; +} + +int ObTableBatchExecuteP::process_get(table::ObTableCtx &op_tb_ctx, + ObTableOperationResult &result) +{ + int ret = OB_SUCCESS; + ObNewRow *row; + ObITableEntity *result_entity = nullptr; + const ObTableSchema *table_schema = op_tb_ctx.get_table_schema(); + const ObTableEntity *request_entity = static_cast(op_tb_ctx.get_entity()); + const ObIArray &cnames = request_entity->get_properties_names(); + if (OB_FAIL(ObTableOpWrapper::process_get(op_tb_ctx, row))) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to process get", K(ret)); + } + } else if (OB_FAIL(result.get_entity(result_entity))) { + LOG_WARN("fail to get result entity", K(ret)); + } else if (OB_FAIL(ObTableApiUtil::construct_entity_from_row(row, + table_schema, + cnames, + result_entity))) { + LOG_WARN("fail to cosntruct result entity", K(ret)); + } + result.set_errno(ret); + result.set_type(op_tb_ctx.get_opertion_type()); + return ret; +} + +int ObTableBatchExecuteP::htable_mutate_row() +{ + int ret = OB_SUCCESS; const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; uint64_t table_id = OB_INVALID_ID; ObSEArray tablet_ids; ObLSID ls_id; if (OB_FAIL(check_arg2())) { + LOG_WARN("fail to check arg", K(ret)); } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); + LOG_WARN("fail to get table id", K(ret)); } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); + LOG_WARN("fail to get tablet id", K(ret)); } else if (1 != tablet_ids.count()) { ret = OB_NOT_SUPPORTED; LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), ls_id))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_DELETE, consistency_level, - table_id, ls_id, get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); + LOG_WARN("fail to get ls id", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_DELETE, + consistency_level, + table_id, + ls_id, + get_timeout_ts()))) { + LOG_WARN("fail to start transaction", K(ret)); } else { + const ObTableBatchOperation &batch_operation = arg_.batch_operation_; + int64_t N = batch_operation.count(); + for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) + { + // execute each mutation one by one + const ObTableOperation &table_operation = batch_operation.at(i); + ObTableBatchOperation batch_ops; + if (OB_FAIL(batch_ops.add(table_operation))) { + LOG_WARN("failed to add operation", K(ret), K(table_operation)); + } else { + switch (table_operation.type()) { + case ObTableOperationType::INSERT_OR_UPDATE: { + if (OB_FAIL(execute_htable_put(batch_ops))) { + LOG_WARN("fail to execute htable put", K(ret), K(i), K(N), K(table_operation)); + } + break; + } + + case ObTableOperationType::DEL: { + if (OB_FAIL(execute_htable_delete(batch_ops))) { + LOG_WARN("fail to execute htable delete", K(ret), K(i), K(N), K(table_operation)); + } + break; + } + + default: { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported mutation type", K(ret), K(table_operation)); + break; + } + } + } + } + } + + int tmp_ret = ret; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + LOG_WARN("failed to end trans"); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + + return ret; +} + +int ObTableBatchExecuteP::execute_htable_delete(const ObTableBatchOperation &batch_operation) +{ + int ret = OB_SUCCESS; + + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableApiCacheGuard cache_guard; int64_t affected_rows = 0; - SMART_VAR(ObHTableDeleteExecutor, delete_executor, allocator_, - table_id, - tablet_ids.at(0), - ls_id, - get_timeout_ts(), - this, - table_service_, - access_service_) { - ret = delete_executor.htable_delete(batch_operation, affected_rows); + tb_ctx.set_batch_operation(&batch_operation); + + if (OB_FAIL(init_single_op_tb_ctx(tb_ctx, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx, cache_guard, spec))) { + LOG_WARN("fail to get or create spec", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx, executor))) { + LOG_WARN("fail to create executor", K(ret)); + } else { + ObHTableDeleteExecutor delete_executor(tb_ctx, static_cast(executor)); + if (OB_FAIL(delete_executor.open())) { + LOG_WARN("fail to open htable delete executor", K(ret)); + } else if (OB_FAIL(delete_executor.get_next_row())) { + LOG_WARN("fail to call htable delete get_next_row", K(ret)); + } else if (FALSE_IT(affected_rows = delete_executor.get_affected_rows())) { + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = delete_executor.close())) { + LOG_WARN("fail to close htable delete executor", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + if (OB_NOT_NULL(spec)) { + spec->destroy_executor(executor); + tb_ctx.set_expr_info(nullptr); } if (OB_SUCC(ret)) { @@ -515,245 +957,31 @@ int ObTableBatchExecuteP::htable_delete() } } } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + return ret; } -int ObTableBatchExecuteP::multi_insert() +int ObTableBatchExecuteP::execute_htable_put(const ObTableBatchOperation &batch_operation) { int ret = OB_SUCCESS; - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_INSERT, consistency_level, - table_id, table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->multi_insert(table_service_ctx_, batch_operation, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_insert", K(ret), K(table_id)); + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + int64_t affected_rows = 0; + tb_ctx.set_batch_operation(&batch_operation); + ObTableOperationResult single_op_result; + single_op_result.set_entity(result_entity_); + if (OB_FAIL(init_single_op_tb_ctx(tb_ctx, batch_operation.at(0)))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else if (OB_FAIL(ObTableOpWrapper::process_op(tb_ctx, single_op_result))) { + LOG_WARN("fail to process insertup op", K(ret)); + } else if (FALSE_IT(result_.reset())) { + } else if (OB_FAIL(result_.push_back(single_op_result))) { + LOG_WARN("fail to push add result", K(ret)); } } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} -int ObTableBatchExecuteP::multi_replace() -{ - int ret = OB_SUCCESS; - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_REPLACE, consistency_level, - table_id, table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->multi_replace(table_service_ctx_, batch_operation, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_replace", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; return ret; -} - -int ObTableBatchExecuteP::multi_update() -{ - int ret = OB_SUCCESS; - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_/*important*/); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_UPDATE, consistency_level, - table_id, table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->multi_update(table_service_ctx_, batch_operation, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_update", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - -int ObTableBatchExecuteP::batch_execute(bool is_readonly) -{ - int ret = OB_SUCCESS; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_, - arg_.returning_affected_entity_, - arg_.returning_rowkey_); - ObSEArray tablet_ids; - if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, (is_readonly ? sql::stmt::T_SELECT : sql::stmt::T_UPDATE), - consistency_level, table_id, table_service_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->batch_execute(table_service_ctx_, batch_operation, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute batch", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - -int ObTableBatchExecuteP::htable_mutate_row() -{ - int ret = OB_SUCCESS; - const ObTableBatchOperation &batch_operation = arg_.batch_operation_; - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - uint64_t table_id = OB_INVALID_ID; - ObSEArray tablet_ids; - ObLSID ls_id; - int64_t now_ms = -ObHTableUtils::current_time_millis(); - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), ls_id))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_DELETE, consistency_level, - table_id, ls_id, get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else { - int64_t N = batch_operation.count(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) - { - // execute each mutation one by one - const ObTableOperation &table_operation = batch_operation.at(i); - ObTableBatchOperation batch_ops; - if (OB_FAIL(batch_ops.add(table_operation))) { - LOG_WARN("failed to add", K(ret)); - break; - } - switch(table_operation.type()) { - case ObTableOperationType::INSERT_OR_UPDATE: - { - int64_t affected_rows = 0; - SMART_VAR(ObHTablePutExecutor, put_executor, allocator_, - table_id, - tablet_ids.at(0), - ls_id, - get_timeout_ts(), - this, - table_service_) { - ret = put_executor.htable_put(batch_ops, affected_rows, now_ms); - } - } - break; - case ObTableOperationType::DEL: - { - int64_t affected_rows = 0; - SMART_VAR(ObHTableDeleteExecutor, delete_executor, allocator_, - table_id, - tablet_ids.at(0), - ls_id, - get_timeout_ts(), - this, - table_service_, - access_service_) { - ret = delete_executor.htable_delete(batch_ops, affected_rows); - } - } - break; - default: - ret = OB_NOT_SUPPORTED; - LOG_WARN("not supported mutation type", K(ret), K(table_operation)); - break; - } // end switch - } // end for - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans"); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} +} \ No newline at end of file diff --git a/src/observer/table/ob_table_batch_execute_processor.h b/src/observer/table/ob_table_batch_execute_processor.h index 43c8441498..f96d4b2bd7 100644 --- a/src/observer/table/ob_table_batch_execute_processor.h +++ b/src/observer/table/ob_table_batch_execute_processor.h @@ -17,6 +17,19 @@ #include "share/table/ob_table_rpc_proxy.h" #include "ob_table_rpc_processor.h" #include "ob_table_service.h" +#include "ob_table_context.h" +#include "ob_table_executor.h" +#include "ob_table_cache.h" +#include "sql/plan_cache/ob_cache_object_factory.h" +#include "sql/plan_cache/ob_plan_cache.h" +#include "ob_table_update_executor.h" +#include "ob_table_insert_executor.h" +#include "ob_table_delete_executor.h" +#include "ob_table_replace_executor.h" +#include "ob_table_insert_up_executor.h" +#include "ob_table_op_wrapper.h" + + namespace oceanbase { namespace observer @@ -43,22 +56,31 @@ private: int check_arg2() const; int get_rowkeys(common::ObIArray &rowkeys); int get_tablet_ids(uint64_t table_id, ObIArray &tablet_ids); - int multi_insert_or_update(); int multi_get(); int multi_delete(); int multi_insert(); int multi_replace(); - int multi_update(); - int batch_execute(bool is_readonly); int htable_delete(); int htable_put(); int htable_mutate_row(); + + // for batch execute + int batch_execute(bool is_readonly); + int batch_execute_internal(const ObTableBatchOperation &batch_operation, + ObTableBatchOperationResult &result); + int init_single_op_tb_ctx(table::ObTableCtx &ctx, + const ObTableOperation &table_operation); + int process_get(table::ObTableCtx &op_tb_ctx, ObTableOperationResult &result); + int execute_htable_delete(const ObTableBatchOperation &batch_operation); + int execute_htable_put(const ObTableBatchOperation &batch_operation); + private: static const int64_t COMMON_COLUMN_NUM = 16; table::ObTableEntityFactory default_entity_factory_; table::ObTableEntity result_entity_; common::ObArenaAllocator allocator_; - ObTableServiceGetCtx table_service_ctx_; + table::ObTableCtx tb_ctx_; + table::ObTableApiCacheGuard cache_guard_; bool need_rollback_trans_; }; diff --git a/src/observer/table/ob_table_cache.cpp b/src/observer/table/ob_table_cache.cpp new file mode 100644 index 0000000000..bd1ad9c415 --- /dev/null +++ b/src/observer/table/ob_table_cache.cpp @@ -0,0 +1,259 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_cache.h" +#include "ob_table_cg_service.h" + +namespace oceanbase +{ + +namespace table +{ + +int ObTableApiCacheKey::deep_copy(common::ObIAllocator &allocator, const ObILibCacheKey &other) +{ + int ret = OB_SUCCESS; + const ObTableApiCacheKey &table_key = static_cast(other); + table_id_ = table_key.table_id_; + index_table_id_ = table_key.index_table_id_; + schema_version_ = table_key.schema_version_; + operation_type_ = table_key.operation_type_; + namespace_ = table_key.namespace_; + for (int64_t i = 0; OB_SUCC(ret) && i < table_key.op_column_ids_.count(); i++) { + if(OB_FAIL(op_column_ids_.push_back(table_key.op_column_ids_.at(i)))) { + LOG_WARN("fail to push back column id ", K(ret)); + } + } + return ret; +} + +uint64_t ObTableApiCacheKey::hash() const +{ + uint64_t hash_val = 0; + hash_val = murmurhash(&table_id_, sizeof(table_id_), hash_val); + hash_val = murmurhash(&index_table_id_, sizeof(index_table_id_), hash_val); + hash_val = murmurhash(&schema_version_, sizeof(schema_version_), hash_val); + hash_val = murmurhash(&operation_type_, sizeof(operation_type_), hash_val); + for (int64_t i = 0; i < op_column_ids_.count(); i++) { + hash_val = murmurhash(&(op_column_ids_.at(i)), sizeof(uint64_t), hash_val); + } + return hash_val; +} + +bool ObTableApiCacheKey::is_equal(const ObILibCacheKey &other) const +{ + const ObTableApiCacheKey &table_key = static_cast(other); + bool cmp_ret = table_id_ == table_key.table_id_ && + index_table_id_ == table_key.index_table_id_ && + schema_version_ == table_key.schema_version_ && + operation_type_ == table_key.operation_type_ && + namespace_ == table_key.namespace_ && + op_column_ids_.count() == table_key.op_column_ids_.count(); + for (int64_t i = 0; (cmp_ret == true) && i < op_column_ids_.count(); i++) { + if (op_column_ids_.at(i) != table_key.op_column_ids_.at(i)) { + cmp_ret = false; + } + } + return cmp_ret; +} + +void ObTableApiCacheKey::reset() +{ + table_id_ = common::OB_INVALID_ID; + index_table_id_ = common::OB_INVALID_ID; + schema_version_ = -1; + operation_type_ = ObTableOperationType::Type::INVALID; + namespace_ = ObLibCacheNameSpace::NS_TABLEAPI; + op_column_ids_.reset(); +} + +int ObTableApiCacheNode::inner_get_cache_obj(ObILibCacheCtx &ctx, + ObILibCacheKey *key, + ObILibCacheObject *&cache_obj) +{ + UNUSED(ctx); + UNUSED(key); + int ret = OB_SUCCESS; + if (OB_ISNULL(cache_obj_)) { + ret = OB_SQL_PC_NOT_EXIST; + LOG_WARN("fail to get cache obj", K(ret)); + } else { + cache_obj = cache_obj_; + } + return ret; +} + +int ObTableApiCacheNode::inner_add_cache_obj(ObILibCacheCtx &ctx, + ObILibCacheKey *key, + ObILibCacheObject *cache_obj) +{ + UNUSED(ctx); + UNUSED(key); + int ret = OB_SUCCESS; + if (OB_ISNULL(cache_obj)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache obj is null", K(ret)); + } else { + cache_obj_ = cache_obj; + } + return ret; +} + +int ObTableApiCacheGuard::init(ObTableCtx *tb_ctx) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = tb_ctx->get_tenant_id(); + if (OB_ISNULL(lib_cache_ = ObCacheObjectFactory::get_plan_cache(tenant_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get lib cache", K(ret)); + } else if (OB_FAIL(create_cache_key(tb_ctx))) { + LOG_WARN("fail to create cache key", K(ret)); + } else if (OB_FAIL(get_or_create_cache_obj())) { + LOG_WARN("fail to get or generate cache obj", K(ret), K(is_use_cache_)); + } + return ret; +} + +int ObTableApiCacheGuard::create_cache_key(ObTableCtx *tb_ctx) +{ + int ret = OB_SUCCESS; + cache_key_.table_id_ = tb_ctx->get_table_id(); + cache_key_.index_table_id_ = tb_ctx->get_index_table_id(); + cache_key_.schema_version_ = tb_ctx->get_table_schema()->get_schema_version(); + ObTableOperationType::Type operation_type = tb_ctx->get_opertion_type(); + cache_key_.operation_type_ = operation_type; + if (operation_type == ObTableOperationType::Type::UPDATE + || operation_type == ObTableOperationType::Type::INSERT_OR_UPDATE + || operation_type == ObTableOperationType::Type::INCREMENT + || operation_type == ObTableOperationType::Type::APPEND) { + const ObTableCtx::ObAssignIds &assign_ids = tb_ctx->get_assign_ids(); + for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids.count(); i++) { + if (OB_FAIL(cache_key_.op_column_ids_.push_back(assign_ids.at(i).column_id_))) { + LOG_WARN("fail to add assign column id", K(ret), K(i)); + } + } + } else if (tb_ctx->is_get() || tb_ctx->is_scan()) { + const ObIArray &select_col_ids = tb_ctx->get_select_col_ids(); + for (int64_t i = 0; OB_SUCC(ret) && i < select_col_ids.count(); i++) { + if (OB_FAIL(cache_key_.op_column_ids_.push_back(select_col_ids.at(i)))) { + LOG_WARN("fail to add select column id", K(ret), K(i)); + } + } + } + return ret; +} + +int ObTableApiCacheGuard::get_or_create_cache_obj() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(lib_cache_->get_cache_obj(cache_ctx_, &cache_key_, cache_guard_))) { + if (ret == OB_SQL_PC_NOT_EXIST) { + is_use_cache_ = false; + if (OB_FAIL(ObCacheObjectFactory::alloc(cache_guard_, + ObLibCacheNameSpace::NS_TABLEAPI, + lib_cache_->get_tenant_id()))) { + LOG_WARN("fail to alloc new cache obj", K(ret)); + } + } else { + LOG_WARN("fail to get cache obj", K(ret), K(cache_key_)); + } + } else { + is_use_cache_ = true; + } + return ret; +} + +int ObTableApiCacheGuard::get_expr_info(ObTableCtx *tb_ctx, ObExprFrameInfo *&expr_frame_info) +{ + int ret = OB_SUCCESS; + ObTableApiCacheObj *cache_obj = static_cast(cache_guard_.get_cache_obj()); + if (OB_ISNULL(cache_obj)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get cache obj", K(ret), K(is_use_cache_)); + } else { + ObIAllocator &allocator = cache_obj->get_allocator(); + expr_frame_info = cache_obj->get_expr_frame_info(); + if (!is_use_cache_ && OB_FAIL(ObTableExprCgService::generate_exprs(*tb_ctx, + allocator, + *expr_frame_info))) { + LOG_WARN("fail to generate expr", K(ret)); + } else if (!is_use_cache_ && OB_FAIL(tb_ctx->classify_scan_exprs())) { + LOG_WARN("fail to classify scan exprs", K(ret)); + } else { + expr_frame_info = cache_obj->get_expr_frame_info(); + } + } + return ret; +} + +template +int ObTableApiCacheGuard::get_spec(ObTableCtx *tb_ctx, ObTableApiSpec *&spec) +{ + int ret = OB_SUCCESS; + ObTableApiCacheObj *cache_obj = nullptr; + ObTableApiSpec *tmp_spec = nullptr; + if (OB_ISNULL(cache_obj = static_cast(cache_guard_.get_cache_obj()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache obj is null", K(ret)); + } else if (OB_ISNULL(tmp_spec = cache_obj->get_spec())) { + if (OB_FAIL(ObTableSpecCgService::generate(cache_obj->get_allocator(), + *tb_ctx, + tmp_spec))) { + LOG_WARN("fail to generate spec", K(ret)); + } else { + cache_obj->set_spec(tmp_spec); + if (OB_FAIL(lib_cache_->add_cache_obj(cache_ctx_, &cache_key_, cache_obj))) { + // spec生成后直接加入的lib cache + LOG_WARN("fail to add cache obj to lib cache", K(ret), K(cache_key_)); + } + } + } + spec = tmp_spec; + return ret; +} + +void ObTableApiCacheGuard::reset() +{ + int ret = cache_guard_.force_early_release(lib_cache_); + if (OB_FAIL(ret)) { + LOG_WARN("fail to release cache guard", K(ret)); + } + cache_key_.reset(); + is_use_cache_ = false; + lib_cache_ = nullptr; +} + +int ObTableApiCacheGuard::append_column_ids(const ObITableEntity *entity, + const ObTableSchema *table_schema, + common::ObArray &op_column_ids) +{ + int ret = OB_SUCCESS; + ObArray properties_names; + if (OB_FAIL(entity->get_properties_names(properties_names))) { + LOG_WARN("fail to get columns name", K(ret)); + } else { + const ObColumnSchemaV2 *col_schema = nullptr; + for (int64_t i = 0; i < properties_names.count(); i++) { + if (OB_ISNULL(col_schema = table_schema->get_column_schema(properties_names.at(i)))) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("fail to get column schema", K(ret), K(i), K(properties_names.at(i))); + } else if (OB_FAIL(op_column_ids.push_back(col_schema->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret), K(i), K(col_schema->get_column_id())); + } + } + } + return ret; +} + +} // end namespace table +} // end namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_cache.h b/src/observer/table/ob_table_cache.h new file mode 100644 index 0000000000..0bdd87d7bb --- /dev/null +++ b/src/observer/table/ob_table_cache.h @@ -0,0 +1,142 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_CACHE_H_ +#define OCEANBASE_OBSERVER_OB_TABLE_CACHE_H_ +#include "ob_table_executor.h" +#include "sql/plan_cache/ob_i_lib_cache_key.h" +#include "sql/plan_cache/ob_i_lib_cache_node.h" +#include "sql/plan_cache/ob_i_lib_cache_object.h" +#include "sql/plan_cache/ob_i_lib_cache_context.h" +#include "lib/utility/utility.h" +#include "sql/plan_cache/ob_lib_cache_register.h" +#include "sql/plan_cache/ob_plan_cache.h" + +namespace oceanbase +{ + +namespace table +{ + +struct ObTableApiCacheKey: public ObILibCacheKey +{ + ObTableApiCacheKey() + : ObILibCacheKey(ObLibCacheNameSpace::NS_TABLEAPI), + table_id_(common::OB_INVALID_ID), + index_table_id_(common::OB_INVALID_ID), + schema_version_(-1), + operation_type_(ObTableOperationType::Type::INVALID) + {} + ObTableApiCacheKey(common::ObTableID table_id, + common::ObTableID index_table_id, + int64_t schema_version, + ObTableOperationType::Type operation_type, + uint8_t extend_type = 0) + : ObILibCacheKey(ObLibCacheNameSpace::NS_TABLEAPI), + table_id_(table_id), + index_table_id_(index_table_id), + schema_version_(schema_version), + operation_type_(operation_type) + {} + void reset(); + virtual int deep_copy(common::ObIAllocator &allocator, const ObILibCacheKey &other); + virtual uint64_t hash() const; + virtual bool is_equal(const ObILibCacheKey &other) const; + + TO_STRING_KV(K_(table_id), + K_(schema_version), + K_(operation_type), + K_(index_table_id), + K_(op_column_ids), + K_(namespace)); + + common::ObTableID table_id_; + common::ObTableID index_table_id_; + int64_t schema_version_; + ObTableOperationType::Type operation_type_; + common::ObArray op_column_ids_; +}; + +class ObTableApiCacheNode: public ObILibCacheNode +{ +public: + ObTableApiCacheNode(ObPlanCache *lib_cache, lib::MemoryContext &mem_context) + : ObILibCacheNode(lib_cache, mem_context), + cache_obj_(nullptr) {} + virtual ~ObTableApiCacheNode() {} + virtual int inner_get_cache_obj(ObILibCacheCtx &ctx, + ObILibCacheKey *key, + ObILibCacheObject *&cache_obj) override; + virtual int inner_add_cache_obj(ObILibCacheCtx &ctx, + ObILibCacheKey *key, + ObILibCacheObject *cache_obj) override; +private: + ObILibCacheObject *cache_obj_; +}; + +class ObTableApiCacheObj: public ObILibCacheObject +{ +public: + ObTableApiCacheObj(lib::MemoryContext &mem_context) + : ObILibCacheObject(ObLibCacheNameSpace::NS_TABLEAPI, mem_context), + expr_info_(allocator_), + spec_(NULL) {} + + virtual ~ObTableApiCacheObj() + { + if (OB_NOT_NULL(spec_)) { + spec_->~ObTableApiSpec(); + } + } + OB_INLINE ObTableApiSpec* get_spec() { return spec_; } + OB_INLINE void set_spec(ObTableApiSpec* spec) { spec_ = spec; } + OB_INLINE sql::ObExprFrameInfo* get_expr_frame_info() { return &expr_info_; } + OB_INLINE const sql::ObExprFrameInfo* get_expr_frame_info() const { return &expr_info_; } +private: + sql::ObExprFrameInfo expr_info_; + ObTableApiSpec *spec_; +}; + +class ObTableApiCacheGuard +{ +public: + ObTableApiCacheGuard() + : is_use_cache_(false), + lib_cache_(nullptr), + cache_guard_(CacheRefHandleID::TABLEAPI_NODE_HANDLE) {} + ~ObTableApiCacheGuard() {} + int init(ObTableCtx *tb_ctx); + void reset(); + int get_expr_info(ObTableCtx *tb_ctx, ObExprFrameInfo *&exp_frame_info); + template + int get_spec(ObTableCtx *tb_ctx, ObTableApiSpec *&spec); + OB_INLINE bool is_use_cache() { return is_use_cache_; } +private: + int create_cache_key(ObTableCtx *tb_ctx); + int append_column_ids(const ObITableEntity *entity, + const ObTableSchema *table_schema, + common::ObArray &op_column_ids); + int get_or_create_cache_obj(); + +private: + bool is_use_cache_; + // 使用lib cache资源都需要用到 + observer::ObReqTimeGuard req_timeinfo_guard; + sql::ObPlanCache *lib_cache_; + ObTableApiCacheKey cache_key_; + ObCacheObjGuard cache_guard_; + ObILibCacheCtx cache_ctx_; +}; + +} // end namespace table +} // end namespace oceanbase +#endif /* OCEANBASE_OBSERVER_OB_TABLE_CACHE_H_ */ \ No newline at end of file diff --git a/src/observer/table/ob_table_cg_service.cpp b/src/observer/table/ob_table_cg_service.cpp new file mode 100644 index 0000000000..810be6eb1a --- /dev/null +++ b/src/observer/table/ob_table_cg_service.cpp @@ -0,0 +1,2235 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_cg_service.h" +#include "share/datum/ob_datum_util.h" +#include "ob_table_executor_factory.h" +#include "sql/code_generator/ob_static_engine_cg.h" +#include "share/system_variable/ob_system_variable.h" // for ObBinlogRowImage::FULL + +namespace oceanbase +{ +namespace table +{ +ObRawExpr* ObTableExprCgService::get_ref_raw_expr(const ObIArray &all_exprs, + const ObString &col_name) +{ + bool found = false; + ObColumnRefRawExpr *expr = nullptr; + for (int64_t i = 0; i < all_exprs.count() && !found; i++) { + if (T_REF_COLUMN == all_exprs.at(i)->get_expr_type()) { + expr = static_cast(all_exprs.at(i)); + if (0 == col_name.case_compare(expr->get_column_name())) { + found = true; + } + } + } + + return expr; +} + +int ObTableExprCgService::build_generated_column_expr(ObTableCtx &ctx, + ObColumnRefRawExpr &col_expr, + const ObString &expr_str, + const ObIArray &exprs) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret), K(ctx)); + } else { + ObArray columns; + ObSchemaChecker schema_checker; + ObSchemaGetterGuard &schema_guard = ctx.get_schema_guard(); + ObRawExprFactory &expr_factory = ctx.get_expr_factory(); + ObSQLSessionInfo &sess_info = ctx.get_session_info(); + ObRawExpr *gen_expr = nullptr; + if (OB_FAIL(schema_checker.init(schema_guard))) { + LOG_WARN("fail to init schema checker", K(ret)); + } else if(OB_FAIL(ObRawExprUtils::build_generated_column_expr(expr_str, + expr_factory, + sess_info, + gen_expr, + columns, + table_schema, + false, /* allow_sequence */ + nullptr, + &schema_checker))) { + LOG_WARN("fail to build generated expr", K(ret), K(expr_str), K(ctx)); + } else if (OB_ISNULL(gen_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("generated expr is null", K(ret)); + } else { + ObRawExpr *real_ref_expr = nullptr; + bool is_inc_or_append_replace = !ctx.get_delta_exprs().empty(); + if (is_inc_or_append_replace && 2 != columns.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column for increment or append", K(ret), K(columns.count())); + } + const ObIArray *src_exprs = &exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { + if (1 == i && is_inc_or_append_replace) { + src_exprs = &ctx.get_delta_exprs(); + } + if (OB_ISNULL(real_ref_expr = get_ref_raw_expr(*src_exprs, + columns.at(i).col_name_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get ref expr", K(ret), K(columns.at(i).ref_expr_->get_column_id())); + } else if (OB_FAIL(ObRawExprUtils::replace_ref_column(gen_expr, + columns.at(i).ref_expr_, + real_ref_expr))) { + LOG_WARN("fail to replace column reference expr", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(gen_expr->formalize(&sess_info))) { + LOG_WARN("fail to formailize column reference expr", K(ret)); + } else if (ObRawExprUtils::need_column_conv(col_expr.get_result_type(), *gen_expr)) { + if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(expr_factory, + ctx.get_allocator(), + col_expr, + gen_expr, + &sess_info))) { + LOG_WARN("fail to build column convert expr", K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + gen_expr->set_for_generated_column(); + col_expr.set_dependant_expr(gen_expr); + } + } + } + + return ret; +} + +int ObTableExprCgService::resolve_generated_column_expr(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + const ObIArray &exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); i++) { + if (T_REF_COLUMN == exprs.at(i)->get_expr_type()) { + ObColumnRefRawExpr *ref_expr = static_cast(exprs.at(i)); + if (ref_expr->is_generated_column()) { + const ObColumnSchemaV2 *gen_col_schema = nullptr; + if (OB_ISNULL(gen_col_schema = ctx.get_table_schema()->get_column_schema(ref_expr->get_column_id()))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get generated column schema", K(ret), K(ref_expr->get_column_id())); + } else if (OB_FAIL(build_generated_column_expr(ctx, + *ref_expr, + gen_col_schema->get_cur_default_value().get_string(), + exprs))) { + LOG_WARN("fail to build generated raw expr", K(ret), K(*ref_expr)); + } + } + } + } + + return ret; +} + +int ObTableExprCgService::generate_column_ref_raw_expr(ObTableCtx &ctx, + const ObColumnSchemaV2 &col_schema, + ObColumnRefRawExpr *&col_ref_expr) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObRawExprUtils::build_column_expr(ctx.get_expr_factory(), + col_schema, + col_ref_expr))) { + LOG_WARN("fail to build column expr", K(ret), K(col_schema)); + } else if (OB_ISNULL(col_ref_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } + + return ret; +} + +int ObTableExprCgService::generate_exprs(ObTableCtx &ctx, + oceanbase::common::ObIAllocator &allocator, + oceanbase::sql::ObExprFrameInfo &expr_frame_info) +{ + int ret = OB_SUCCESS; + if (ctx.is_for_update() || ctx.is_for_insertup()) { + if (OB_FAIL(generate_update_raw_exprs(ctx))) { + LOG_WARN("fail to generate update raw exprs", K(ret), K(ctx)); + } + } else if (OB_FAIL(generate_column_raw_exprs(ctx))) { + LOG_WARN("fail to generate column raw exprs", K(ret), K(ctx)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(generate_expr_frame_info(ctx, allocator, expr_frame_info))) { + LOG_WARN("fail to generate expr frame info", K(ret), K(ctx)); + } + } + return ret; +} + +int ObTableExprCgService::generate_column_raw_exprs(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObIArray *exprs = (ctx.is_for_update() || ctx.is_for_insertup())? + &ctx.get_old_row_exprs() : const_cast *>(&ctx.get_all_exprs().get_expr_array()); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret), K(ctx)); + } else { + ObColumnRefRawExpr *col_ref_expr = nullptr; + const ObColumnSchemaV2 *column_schema = nullptr; + const uint64_t column_cnt = table_schema->get_column_count(); + for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; i++) { + if (OB_ISNULL(column_schema = table_schema->get_column_schema_by_idx(i))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret)); + } else if (OB_FAIL(generate_column_ref_raw_expr(ctx, *column_schema, col_ref_expr))) { + LOG_WARN("fail to generate column ref raw expr", K(ret), K(*column_schema)); + } else if (OB_FAIL(exprs->push_back(col_ref_expr))) { + LOG_WARN("fail to push back column ref raw expr", K(ret)); + } + } + } + + if (OB_SUCC(ret) && table_schema->has_generated_column()) { + if (OB_FAIL(resolve_generated_column_expr(ctx))) { + LOG_WARN("fail to resolver generated column expr", K(ret)); + } + } + + return ret; +} + +int ObTableExprCgService::generate_full_assign_raw_exprs(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObIArray &full_assign_exprs = ctx.get_full_assign_exprs(); + bool is_inc_or_append = ctx.is_inc_or_append(); + bool has_stored_gen_col = false; + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret), K(ctx)); + } else { + ObColumnRefRawExpr *col_ref_expr = nullptr; + const ObColumnSchemaV2 *column_schema = nullptr; + const uint64_t column_cnt = table_schema->get_column_count(); + for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; i++) { + bool is_virtual_col = false; + if (OB_ISNULL(column_schema = table_schema->get_column_schema_by_idx(i))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(i)); + } else if (column_schema->is_virtual_generated_column()) { + is_virtual_col = true; + } else if (column_schema->is_stored_generated_column()) { + has_stored_gen_col = true; + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(!is_virtual_col && generate_column_ref_raw_expr(ctx, *column_schema, col_ref_expr))) { + LOG_WARN("fail to generate column ref raw expr", K(ret), K(*column_schema)); + } else if (OB_FAIL(!is_virtual_col && full_assign_exprs.push_back(col_ref_expr))) { + LOG_WARN("fail to push back column ref raw expr", K(ret)); + } + } + } + + if (OB_SUCC(ret) && has_stored_gen_col) { + for (int64_t i = 0; OB_SUCC(ret) && i < full_assign_exprs.count(); i++) { + if (T_REF_COLUMN == full_assign_exprs.at(i)->get_expr_type()) { + ObColumnRefRawExpr *ref_expr = static_cast(full_assign_exprs.at(i)); + if (ref_expr->is_stored_generated_column()) { + const ObColumnSchemaV2 *gen_col_schema = nullptr; + if (OB_ISNULL(gen_col_schema = ctx.get_table_schema()->get_column_schema(ref_expr->get_column_id()))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get generated column schema", K(ret), K(ref_expr->get_column_id())); + } else if (OB_FAIL(build_generated_column_expr(ctx, + *ref_expr, + gen_col_schema->get_cur_default_value().get_string(), + full_assign_exprs))) { + LOG_WARN("fail to build generated column expr", K(ret), K(*ref_expr)); + } + } + } + } + } + + // 构造delta expr，作为生成列表达式的参数表达式 + // 针对increment和append场景，将对应的列表达式设置为stored生成列表达式，并且构造生成列 + if (OB_SUCC(ret) && is_inc_or_append) { + const ObColumnSchemaV2 *column_schema = nullptr; + ObColumnRefRawExpr *col_ref_expr = nullptr; + ObIArray &delta_exprs = ctx.get_delta_exprs(); + const ObIArray &expr_strs = ctx.get_expr_strs(); + const ObTableCtx::ObAssignIds &assign_ids = ctx.get_assign_ids(); + const int64_t N = assign_ids.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + uint64_t idx = assign_ids.at(i).idx_; + uint64_t column_id = assign_ids.at(i).column_id_; + if (OB_ISNULL(column_schema = table_schema->get_column_schema(column_id))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(column_id)); + } else if (OB_FAIL(generate_column_ref_raw_expr(ctx, *column_schema, col_ref_expr))) { + LOG_WARN("fail to generate column ref raw expr", K(ret), K(*column_schema)); + } else if (OB_FAIL(delta_exprs.push_back(col_ref_expr))) { + LOG_WARN("fail to push back column ref raw expr", K(ret)); + } else { + ObColumnRefRawExpr *ref_expr = static_cast(full_assign_exprs.at(idx)); + ref_expr->set_column_flags(ref_expr->get_column_flags() | STORED_GENERATED_COLUMN_FLAG); + if (OB_FAIL(build_generated_column_expr(ctx, + *ref_expr, + expr_strs.at(i), + ctx.get_old_row_exprs()))) { + LOG_WARN("fail to build generated column expr", K(ret), K(*ref_expr), K(expr_strs.at(i))); + } + } + } + } + + return ret; +} + +int ObTableExprCgService::generate_assign_exprs(ObTableCtx &ctx, + const ObTableCtx::ObAssignIds &assign_ids, + common::ObIArray &assign_exprs) +{ + int ret = OB_SUCCESS; + const ObIArray &full_exprs = ctx.get_full_assign_exprs(); + + if (assign_ids.count() > full_exprs.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid assign ids count", K(ret), K(assign_ids.count()), K(full_exprs.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids.count(); i++) { + const ObTableCtx::ObAssignId &assign_id = assign_ids.at(i); + if (OB_FAIL(assign_exprs.push_back(full_exprs.at(assign_id.idx_)))) { + LOG_WARN("fail to push back assign expr", K(ret), K(i), K(assign_id)); + } + } + } + + return ret; +} + +int ObTableExprCgService::generate_update_raw_exprs(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_column_raw_exprs(ctx))) { + LOG_WARN("fail to generate column raw exprs", K(ret), K(ctx)); + } else if (OB_FAIL(generate_full_assign_raw_exprs(ctx))) { + LOG_WARN("fail to generate assign raw exprs", K(ret), K(ctx)); + } else { + ObRawExprUniqueSet &all_exprs = ctx.get_all_exprs(); + ObIArray &old_row_exprs = ctx.get_old_row_exprs(); + ObIArray &assign_exprs = ctx.get_full_assign_exprs(); + ObIArray &delta_exprs = ctx.get_delta_exprs(); + + // 将 old row exprs加到all exprs里面 + for (int64_t i = 0; OB_SUCC(ret) && i < old_row_exprs.count(); i++) { + if (OB_FAIL(all_exprs.append(old_row_exprs.at(i)))) { + LOG_WARN("fail to append old row expr to all exprs", K(ret), K(i)); + } + } + + // 将 assign row exprs加到all exprs里面 + for (int64_t i = 0; OB_SUCC(ret) && i < assign_exprs.count(); i++) { + if (OB_FAIL(all_exprs.append(assign_exprs.at(i)))) { + LOG_WARN("fail to append assign expr to all exprs", K(ret), K(i)); + } + } + + // 将 delta exprs加到all exprs里面 + for (int64_t i = 0; OB_SUCC(ret) && i < delta_exprs.count(); i++) { + if (OB_FAIL(all_exprs.append(delta_exprs.at(i)))) { + LOG_WARN("fail to append delta expr to all exprs", K(ret), K(i)); + } + } + } + + return ret; +} + +int ObTableExprCgService::generate_expr_frame_info(ObTableCtx &ctx, + common::ObIAllocator &allocator, + ObExprFrameInfo &expr_frame_info) +{ + int ret = OB_SUCCESS; + ObStaticEngineExprCG expr_cg(allocator, + &ctx.get_session_info(), + &ctx.get_schema_guard(), + 0, + 0); + if (OB_FAIL(expr_cg.generate(ctx.get_all_exprs(), expr_frame_info))) { + LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); + } + return ret; +} + +int ObTableExprCgService::alloc_exprs_memory(ObTableCtx &ctx, ObExprFrameInfo &expr_frame_info) +{ + int ret = OB_SUCCESS; + ObExecContext &exec_ctx = ctx.get_exec_ctx(); + uint64_t frame_cnt = 0; + char **frames = NULL; + common::ObArray param_frame_ptrs; + + if (OB_FAIL(expr_frame_info.alloc_frame(ctx.get_allocator(), + param_frame_ptrs, + frame_cnt, + frames))) { + LOG_WARN("fail to alloc frame", K(ret), K(expr_frame_info)); + } else { + exec_ctx.set_frame_cnt(frame_cnt); + exec_ctx.set_frames(frames); + } + + return ret; +} + +int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiScanSpec &spec) +{ + int ret = OB_SUCCESS; + // init tsc_ctdef_ + if (OB_FAIL(ObTableTscCgService::generate_tsc_ctdef(ctx, alloc, spec.get_ctdef()))) { + LOG_WARN("fail to generate table scan ctdef", K(ret), K(ctx)); + } + + return ret; +} + +/* + table_loc_id_ ref_table_id_ + 主表: 主表table_id 主表table_id + 索引表: 主表table_id 索引表table_id + 回表: 主表table_id 主表table_id +*/ +int ObTableLocCgService::generate_table_loc_meta(const ObTableCtx &ctx, + ObDASTableLocMeta &loc_meta, + bool is_lookup) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + const ObTableSchema *index_schema = ctx.get_index_schema(); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (ctx.is_index_scan() && OB_ISNULL(index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("index schema is null", K(ret)); + } else { + loc_meta.reset(); + // is_lookup 有什么用？好像都是 false + loc_meta.ref_table_id_ = is_lookup ? ctx.get_ref_table_id() : ctx.get_index_table_id(); + loc_meta.table_loc_id_ = ctx.get_ref_table_id(); + if (is_lookup) { + loc_meta.is_dup_table_ = table_schema->is_duplicate_table(); + } else { + loc_meta.is_dup_table_ = ctx.is_index_scan() ? index_schema->is_duplicate_table() + : table_schema->is_duplicate_table(); + } + if (ctx.is_weak_read()) { + loc_meta.is_weak_read_ = 1; + loc_meta.select_leader_ = 0; + } else if (loc_meta.is_dup_table_) { + loc_meta.select_leader_ = 0; + loc_meta.is_weak_read_ = 0; + } else { + //strong consistency read policy is used by default + loc_meta.select_leader_ = 1; + loc_meta.is_weak_read_ = 0; + } + } + + if (OB_SUCC(ret)) { + const ObIArray &related_index_ids = ctx.get_related_index_ids(); + loc_meta.related_table_ids_.set_capacity(related_index_ids.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < related_index_ids.count(); i++) { + if (OB_FAIL(loc_meta.related_table_ids_.push_back(related_index_ids.at(i)))) { + LOG_WARN("fail to store related table id", K(ret), K(i)); + } + } + } + + return ret; +} + +int ObTableExprCgService::refresh_update_exprs_frame(ObTableCtx &ctx, + const ObIArray &old_row, + const ObIArray &new_row, + const ObIArray &full_assign_row, + const ObTableCtx::ObAssignIds &assign_ids, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(refresh_assign_exprs_frame(ctx, + old_row, + new_row, + full_assign_row, + assign_ids, + entity))) { + LOG_WARN("fail to refresh assign exprs frame", K(ret), K(assign_ids)); + } + + return ret; +} + +int ObTableExprCgService::refresh_insert_up_exprs_frame(ObTableCtx &ctx, + const ObIArray &ins_new_row, + const ObIArray &delta_exprs, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + const ObIArray &rowkey = entity.get_rowkey_objs(); + + if (OB_FAIL(refresh_rowkey_exprs_frame(ctx, ins_new_row, rowkey))) { + LOG_WARN("fail to init rowkey exprs frame", K(ret), K(ctx), K(rowkey)); + } else if (OB_FAIL(refresh_properties_exprs_frame(ctx, ins_new_row, entity))) { + LOG_WARN("fail to init properties exprs frame", K(ret), K(ctx)); + } else if (ctx.is_inc_or_append() && OB_FAIL(refresh_delta_exprs_frame(ctx, delta_exprs, entity))) { + LOG_WARN("fail to init delta exprs frame", K(ret), K(ctx)); + } + + return ret; +} + +int ObTableExprCgService::refresh_insert_exprs_frame(ObTableCtx &ctx, + const ObIArray &exprs, + const ObTableEntity &entity) +{ + return refresh_exprs_frame(ctx, exprs, entity); +} + +int ObTableExprCgService::refresh_replace_exprs_frame(ObTableCtx &ctx, + const ObIArray &exprs, + const ObTableEntity &entity) +{ + return refresh_exprs_frame(ctx, exprs, entity); +} + +// only for htable +int ObTableExprCgService::refresh_delete_exprs_frame(ObTableCtx &ctx, + const ObIArray &exprs, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + ObSEArray rowkey; + ObObj k_obj; + ObObj q_obj; + ObObj t_obj; + int64_t time = 0; + + // htable场景rowkey都在properties中，所以需要从properties中提取出rowkey + if (OB_FAIL(entity.get_property(ObHTableConstants::ROWKEY_CNAME_STR, k_obj))) { + LOG_WARN("fail to get K", K(ret)); + } else if (OB_FAIL(entity.get_property(ObHTableConstants::CQ_CNAME_STR, q_obj))) { + LOG_WARN("fail to get Q", K(ret)); + } else if (OB_FAIL(entity.get_property(ObHTableConstants::VERSION_CNAME_STR, t_obj))) { + LOG_WARN("fail to get T", K(ret)); + } else if (OB_FAIL(rowkey.push_back(k_obj))) { + LOG_WARN("fail to push back k_obj", K(ret), K(k_obj)); + } else if (OB_FAIL(rowkey.push_back(q_obj))) { + LOG_WARN("fail to push back q_obj", K(ret), K(q_obj)); + } else if (FALSE_IT(time = t_obj.get_int())) { + // do nothing + } else if (FALSE_IT(t_obj.set_int(-1 * time))) { + // do nothing + } else if (OB_FAIL(rowkey.push_back(t_obj))) { + LOG_WARN("fail to push back t_obj", K(ret), K(t_obj)); + } else if (OB_FAIL(refresh_rowkey_exprs_frame(ctx, exprs, rowkey))) { + LOG_WARN("fail to init rowkey exprs frame", K(ret), K(ctx), K(rowkey)); + } else if (OB_FAIL(refresh_properties_exprs_frame(ctx, exprs, entity))) { + LOG_WARN("fail to init properties exprs frame", K(ret), K(ctx)); + } + + return ret; +} + +int ObTableExprCgService::refresh_exprs_frame(ObTableCtx &ctx, + const ObIArray &exprs, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + const ObIArray &rowkey = entity.get_rowkey_objs(); + + if (OB_FAIL(refresh_rowkey_exprs_frame(ctx, exprs, rowkey))) { + LOG_WARN("fail to init rowkey exprs frame", K(ret), K(ctx), K(rowkey)); + } else if (OB_FAIL(refresh_properties_exprs_frame(ctx, exprs, entity))) { + LOG_WARN("fail to init properties exprs frame", K(ret), K(ctx)); + } + + return ret; +} + +// exprs必须是按照schema序的完整行 +int ObTableExprCgService::refresh_rowkey_exprs_frame(ObTableCtx &ctx, + const ObIArray &exprs, + const ObIArray &rowkey) +{ + int ret = OB_SUCCESS; + ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey.count(); i++) { + const ObExpr *expr = exprs.at(i); + expr->locate_datum_for_write(eval_ctx).from_obj(rowkey.at(i)); + expr->get_eval_info(eval_ctx).evaluated_ = true; + expr->get_eval_info(eval_ctx).projected_ = true; + } + + return ret; +} + +// exprs必须是按照schema序的完整行 +int ObTableExprCgService::refresh_properties_exprs_frame(ObTableCtx &ctx, + const ObIArray &exprs, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + ObObj prop_value; + ObObj *obj = nullptr; + const int64_t rowkey_column_cnt = table_schema->get_rowkey_column_num(); + for (int64_t i = rowkey_column_cnt; OB_SUCC(ret) && i < exprs.count(); i++) { + const ObExpr *expr = exprs.at(i); + const ObColumnSchemaV2 *col_schema = nullptr; + if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(i))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(i), K(*table_schema)); + } else if (col_schema->is_generated_column()) { // 生成列需要eval一次 + ObDatum *tmp_datum = nullptr; + if (OB_FAIL(expr->eval(eval_ctx, tmp_datum))) { + LOG_WARN("fail to eval generate expr", K(ret)); + } + } else { + const ObString &col_name = col_schema->get_column_name_str(); + // 这里使用schema的列名在entity中查找property，有可能出现本身entity中的prop_name是不对的，导致找不到 + bool use_default = (OB_SEARCH_NOT_FOUND == entity.get_property(col_name, prop_value)); + if (use_default) { + obj = const_cast(&col_schema->get_cur_default_value()); + } else { + obj = &prop_value; + } + expr->locate_datum_for_write(eval_ctx).from_obj(*obj); + expr->get_eval_info(eval_ctx).evaluated_ = true; + expr->get_eval_info(eval_ctx).projected_ = true; + } + } + } + + return ret; +} + +// 将生成列引用的，并且entity没有填充的列刷成旧值 +// eg: create table t(c1 int primary key, +// c2 varchar(10), +// c3 varchar(10), +// gen varchar(30) generated always as (concat(c2,c3)) stored); +// update t set c3 = ? where c1 = ?; +// full row: c1(old) | c2(old) | c3(old) | gen(old) concat(c1(old), c2(old)) +// c1(new) | c2(new) | c3(new) | gen(new) concat(c1(new), c2(new)) +// 这个时候，只是更新了c3的值，gen引用的是c2,c3的值，这里将c2(old)填到c2(new)中 +int ObTableExprCgService::refresh_generated_column_related_frame(ObTableCtx &ctx, + const ObIArray &old_row, + const ObIArray &full_assign_row, + const ObTableCtx::ObAssignIds &assign_ids, + const ObColumnSchemaV2 &col_schema) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + ObArray schema_column_ids; + ObArray ref_column_ids; + + if (!col_schema.is_stored_generated_column()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr", K(ret), K(col_schema)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table schema is null", K(ret)); + } else if (OB_FAIL(table_schema->get_column_ids(schema_column_ids))) { + LOG_WARN("fail to get schema column ids", K(ret)); + } else if (OB_FAIL(col_schema.get_cascaded_column_ids(ref_column_ids))) { + LOG_WARN("fail to get cascade column ids", K(ret)); + } else { + ObArray not_upd_col_ids; + // 查询生成列依赖的列是否都被更新，记下未被更新的列到not_upd_col_ids + for (int64_t i = 0; OB_SUCC(ret) && i < ref_column_ids.count(); i++) { + bool found = false; + uint64_t col_id = ref_column_ids.at(i); + for (int64_t j = 0; OB_SUCC(ret) && j < assign_ids.count() && !found; j++) { + if (col_id == assign_ids.at(j).column_id_) { + found = true; + } + } + if (!found && OB_FAIL(not_upd_col_ids.push_back(col_id))) { + LOG_WARN("fail to push back not update column id", K(ret), K(col_id)); + } + } + + // 通过column id找到old row对应的列 + for (int64_t i = 0; OB_SUCC(ret) && i < not_upd_col_ids.count(); i++) { + for (int64_t j = 0; OB_SUCC(ret) && j < schema_column_ids.count(); j++) { + if (schema_column_ids.at(j) == not_upd_col_ids.at(i)) { + if (j >= old_row.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid column id", K(ret), K(j), K(old_row.count())); + } else if (j >= full_assign_row.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid column id", K(ret), K(j), K(full_assign_row.count())); + } else { + ObExpr *old_expr = old_row.at(j); + ObExpr *new_expr = full_assign_row.at(j); + new_expr->locate_expr_datum(eval_ctx) = old_expr->locate_expr_datum(eval_ctx); + new_expr->get_eval_info(eval_ctx).evaluated_ = true; + new_expr->get_eval_info(eval_ctx).projected_ = true; + } + } + } + } + } + + return ret; +} + +int ObTableExprCgService::refresh_assign_exprs_frame(ObTableCtx &ctx, + const ObIArray &old_row, + const ObIArray &new_row, + const ObIArray &full_assign_row, + const ObTableCtx::ObAssignIds &assign_ids, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + ObObj prop_value; + const int64_t N = assign_ids.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + const ObColumnSchemaV2 *col_schema = nullptr; + uint64_t assign_id = assign_ids.at(i).idx_; + if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(assign_id))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(assign_id), K(*table_schema)); + } else if (col_schema->is_virtual_generated_column()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should not have virtual generated expr", K(ret)); + } else if (col_schema->is_stored_generated_column()) { + if (OB_FAIL(refresh_generated_column_related_frame(ctx, + old_row, + full_assign_row, + assign_ids, + *col_schema))) { + LOG_WARN("fail to refresh generated column related frame", K(ret)); + } + } else if (OB_FAIL(entity.get_property(col_schema->get_column_name_str(), prop_value))) { + LOG_WARN("fail to get assign propertity value", K(ret), K(col_schema->get_column_name_str())); + } else { + if (assign_id >= new_row.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid assign idx", K(ret), K(assign_id), K(new_row.count())); + } else { + const ObExpr *expr = new_row.at(assign_id); + expr->locate_datum_for_write(eval_ctx).from_obj(prop_value); + expr->get_eval_info(eval_ctx).evaluated_ = true; + expr->get_eval_info(eval_ctx).projected_ = true; + } + } + } + } + + return ret; +} + +int ObTableExprCgService::refresh_delta_exprs_frame(ObTableCtx &ctx, + const ObIArray &delta_exprs, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + + if (!ctx.is_inc_or_append()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid operation type", K(ret), K(ctx)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + ObObj prop_value; + const ObTableCtx::ObAssignIds &assign_ids = ctx.get_assign_ids(); + const int64_t N = assign_ids.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + const ObColumnSchemaV2 *col_schema = nullptr; + uint64_t column_id = assign_ids.at(i).column_id_; + if (OB_ISNULL(col_schema = table_schema->get_column_schema(column_id))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(column_id), K(*table_schema)); + } else if (OB_FAIL(entity.get_property(col_schema->get_column_name_str(), prop_value))) { + LOG_WARN("fail to get assign propertity value", K(ret), K(col_schema->get_column_name_str())); + } else { + const ObExpr *expr = delta_exprs.at(i); + expr->locate_datum_for_write(eval_ctx).from_obj(prop_value); + expr->get_eval_info(eval_ctx).evaluated_ = true; + expr->get_eval_info(eval_ctx).projected_ = true; + } + } + } + + return ret; +} + +int ObTableDmlCgService::generate_insert_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableInsCtDef &ins_ctdef) +{ + int ret = OB_SUCCESS; + ObSEArray old_row; + ObSEArray new_row; + const ObIArray &exprs = ctx.is_for_insertup() ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + + if (OB_FAIL(new_row.assign(exprs))) { + LOG_WARN("fail to assign new row", K(ret)); + } else if (OB_FAIL(generate_base_ctdef(ctx, + ins_ctdef, + old_row, + new_row))) { + LOG_WARN("fail to generate dml base ctdef", K(ret)); + } else if (OB_FAIL(generate_das_ins_ctdef(ctx, + ctx.get_ref_table_id(), + ins_ctdef.das_ctdef_, + new_row))) { + LOG_WARN("fail to generate das insert ctdef", K(ret)); + } else if (OB_FAIL(generate_related_ins_ctdef(ctx, + allocator, + new_row, + ins_ctdef.related_ctdefs_))) { + LOG_WARN("fail to generate related ins ctdef", K(ret)); + } + + return ret; +} + +int ObTableDmlCgService::generate_update_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObTableCtx::ObAssignIds &assign_ids, + ObTableUpdCtDef &upd_ctdef) +{ + int ret = OB_SUCCESS; + ObSEArray old_row; + ObSEArray new_row; + ObSEArray full_row; + ObSEArray assign_exprs; + ObStaticEngineCG cg; + + if (OB_FAIL(old_row.assign(ctx.get_old_row_exprs()))) { + LOG_WARN("fail to assign old row expr", K(ret)); + } else if (OB_FAIL(new_row.assign(old_row))) { + LOG_WARN("fail to assign new row", K(ret)); + } else if (OB_FAIL(append(full_row, old_row))) { + LOG_WARN("fail to append old row expr to full row", K(ret), K(old_row)); + } else if (OB_FAIL(ObTableExprCgService::generate_assign_exprs(ctx, assign_ids, assign_exprs))) { + LOG_WARN("fail to generate assign exprs", K(ret), K(ctx), K(assign_ids)); + } else { + ObColumnRefRawExpr *assign_col_expr = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < assign_exprs.count(); i++) { + if (OB_FAIL(full_row.push_back(assign_exprs.at(i)))) { + LOG_WARN("fail to add assign expr to full row", K(ret), K(i)); + } else if (T_REF_COLUMN != assign_exprs.at(i)->get_expr_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(assign_exprs.at(i)->get_expr_type())); + } else if (FALSE_IT(assign_col_expr = static_cast(assign_exprs.at(i)))) { + } else { + ObColumnRefRawExpr *old_col_expr = nullptr; + for (int64_t j = 0; OB_SUCC(ret) && j < old_row.count(); j++) { + if (T_REF_COLUMN != old_row.at(i)->get_expr_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(old_row.at(i)->get_expr_type())); + } else if (FALSE_IT(old_col_expr = static_cast(old_row.at(j)))) { + } else if (old_col_expr->get_column_id() == assign_col_expr->get_column_id()) { + new_row.at(j) = assign_col_expr; + } + } + } + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(generate_base_ctdef(ctx, + upd_ctdef, + old_row, + new_row))) { + LOG_WARN("fail to generate dml base ctdef", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(full_row, upd_ctdef.full_row_))) { + LOG_WARN("fail to generate dml update full row exprs", K(ret), K(full_row)); + } else if (OB_FAIL(cg.generate_rt_exprs(ctx.get_full_assign_exprs(), upd_ctdef.full_assign_row_))) { + LOG_WARN("fail to generate full assign row exprs", K(ret)); + } else if (ctx.is_inc_or_append() && OB_FAIL(cg.generate_rt_exprs(ctx.get_delta_exprs(), upd_ctdef.delta_exprs_))) { + LOG_WARN("fail to generate delta exprs", K(ret)); + } else if (OB_FAIL(generate_das_upd_ctdef(ctx, + ctx.get_ref_table_id(), + assign_exprs, + upd_ctdef.das_ctdef_, + old_row, + new_row, + full_row))) { + LOG_WARN("fail to generate das upd ctdef", K(ret)); + } else if (OB_FAIL(generate_related_upd_ctdef(ctx, + allocator, + assign_exprs, + old_row, + new_row, + full_row, + upd_ctdef.related_ctdefs_))) { + LOG_WARN("fail to generate related upd ctdef", K(ret)); + } else if (OB_FAIL(generate_upd_assign_infos(ctx, allocator, assign_exprs, upd_ctdef))) { + LOG_WARN("fail to generate related upd assign info", K(ret)); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (ctx.is_for_insertup()) { + ObDMLCtDefAllocator ddel_allocator(allocator); + ObDMLCtDefAllocator dins_allocator(allocator); + if (OB_ISNULL(upd_ctdef.ddel_ctdef_ = ddel_allocator.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate das del ctdef", K(ret)); + } else if (OB_ISNULL(upd_ctdef.dins_ctdef_ = dins_allocator.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate das ins ctdef", K(ret)); + } else if (OB_FAIL(generate_das_del_ctdef(ctx, + ctx.get_ref_table_id(), + *upd_ctdef.ddel_ctdef_, + old_row))) { + LOG_WARN("fail to generate das delete ctdef for update", K(ret)); + } else if (OB_FAIL(generate_related_del_ctdef(ctx, + allocator, + old_row, + upd_ctdef.related_del_ctdefs_))) { + LOG_WARN("fail to generate related del ctdef", K(ret)); + } else if (OB_FAIL(generate_das_ins_ctdef(ctx, + ctx.get_ref_table_id(), + *upd_ctdef.dins_ctdef_, + new_row))) { + LOG_WARN("fail to generate das insert ctdef for update", K(ret)); + } else if (OB_FAIL(generate_related_ins_ctdef(ctx, + allocator, + new_row, + upd_ctdef.related_ins_ctdefs_))) { + LOG_WARN("fail to generate related ins ctdef", K(ret)); + } + } + + return ret; +} + +int ObTableDmlCgService::generate_das_upd_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + const ObIArray &assign_exprs, + ObDASUpdCtDef &das_upd_ctdef, + const ObIArray &old_row, + const ObIArray &new_row, + const ObIArray &full_row) +{ + int ret = OB_SUCCESS; + ObArray dml_column_ids; + + if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_upd_ctdef))) { + LOG_WARN("fail to generate das dml ctdef", K(ret)); + } else if (OB_FAIL(generate_updated_column_ids(assign_exprs, + das_upd_ctdef.column_ids_, + das_upd_ctdef.updated_column_ids_))) { + LOG_WARN("fail to add updated column ids", K(ret)); + } else if (OB_FAIL(generate_column_ids(ctx.get_old_row_exprs(), + dml_column_ids))) { + LOG_WARN("fail to generate dml column ids", K(ret)); + } else if (OB_FAIL(generate_projector(dml_column_ids, // new row and old row's columns id + das_upd_ctdef.column_ids_, // schmea column ids for given index_tid + old_row, + new_row, + full_row, + das_upd_ctdef))) { + LOG_WARN("fail to generate projector", K(ret), K(full_row)); + } + + return ret; +} + +int ObTableDmlCgService::generate_updated_column_ids(const ObIArray &assign_exprs, + const ObIArray &column_ids, + ObIArray &updated_column_ids) +{ + int ret = OB_SUCCESS; + updated_column_ids.reset(); + + if (!assign_exprs.empty()) { + if (OB_FAIL(updated_column_ids.reserve(assign_exprs.count()))) { + LOG_WARN("fail to init updated column ids array", K(ret), K(assign_exprs.count())); + } else { + ObColumnRefRawExpr *col_expr = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < assign_exprs.count(); i++) { + if (T_REF_COLUMN != assign_exprs.at(i)->get_expr_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(assign_exprs.at(i)->get_expr_type())); + } else if (FALSE_IT(col_expr = static_cast(assign_exprs.at(i)))) { + } else if (!has_exist_in_array(column_ids, col_expr->get_column_id())) { + //not found in column ids, ignore it + } else if (OB_FAIL(updated_column_ids.push_back(col_expr->get_column_id()))) { + LOG_WARN("fail to add updated column id", K(ret), K(col_expr->get_column_id())); + } + } + } + } + + return ret; +} + +int ObTableDmlCgService::generate_upd_assign_infos(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObIArray &assign_exprs, + ObTableUpdCtDef &udp_ctdef) +{ + int ret = OB_SUCCESS; + int64_t assign_cnt = assign_exprs.count(); + ColContentFixedArray &assign_infos = udp_ctdef.assign_columns_; + + if (OB_FAIL(assign_infos.init(assign_cnt))) { + LOG_WARN("fail to init assign info array", K(ret), K(assign_cnt)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < assign_cnt; ++i) { + ColumnContent column_content; + int64_t idx = 0; + ObColumnRefRawExpr *col = static_cast(assign_exprs.at(i)); + column_content.auto_filled_timestamp_ = col->get_result_type().has_result_flag(ON_UPDATE_NOW_FLAG); + column_content.is_nullable_ = !col->get_result_type().is_not_null_for_write(); + column_content.is_predicate_column_ = false; + column_content.is_implicit_ = false; + if (OB_FAIL(ob_write_string(allocator, + col->get_column_name(), + column_content.column_name_))) { + LOG_WARN("fail to copy column name", K(ret), K(col->get_column_name())); + } else if (!has_exist_in_array(udp_ctdef.das_ctdef_.column_ids_, col->get_column_id(), &idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column not exists in old column", K(ret), KPC(col)); + } else if (FALSE_IT(column_content.projector_index_ = static_cast(idx))) { + //do nothing + } else if (OB_FAIL(assign_infos.push_back(column_content))) { + LOG_WARN("fail to store colum content to assign infos", K(ret), K(column_content)); + } + } + return ret; +} + +int ObTableDmlCgService::generate_delete_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableDelCtDef &del_ctdef) +{ + int ret = OB_SUCCESS; + ObSEArray old_row; + ObSEArray new_row; + + if (OB_FAIL(old_row.assign(ctx.get_all_exprs().get_expr_array()))) { + LOG_WARN("fail to assign old row expr", K(ret)); + } else if (OB_FAIL(generate_base_ctdef(ctx, + del_ctdef, + old_row, + new_row))) { + LOG_WARN("fail to generate dml base ctdef", K(ret)); + } else if (OB_FAIL(generate_das_del_ctdef(ctx, + ctx.get_ref_table_id(), + del_ctdef.das_ctdef_, + old_row))) { + LOG_WARN("fail to generate das delete ctdef", K(ret)); + } else if (OB_FAIL(generate_related_del_ctdef(ctx, + allocator, + old_row, + del_ctdef.related_ctdefs_))) { + LOG_WARN("fail to generate related del ctdef", K(ret)); + } + + return ret; +} + +int ObTableDmlCgService::generate_das_del_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + ObDASDelCtDef &das_del_ctdef, + const ObIArray &old_row) +{ + int ret = OB_SUCCESS; + ObArray dml_column_ids; + ObArray empty_new_row; + + if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_del_ctdef))) { + LOG_WARN("fail to generate das dml ctdef", K(ret)); + } else if (OB_FAIL(generate_column_ids(ctx.get_all_exprs().get_expr_array(), + dml_column_ids))) { + LOG_WARN("fail to generate dml column ids", K(ret)); + } else if (OB_FAIL(generate_projector(dml_column_ids, + das_del_ctdef.column_ids_, + old_row, + empty_new_row, + old_row, + das_del_ctdef))) { + LOG_WARN("fail to add old row projector", K(ret), K(old_row)); + } + + return ret; +} + +int ObTableDmlCgService::generate_related_del_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObIArray &old_row, + DASDelCtDefArray &del_ctdefs) +{ + int ret = OB_SUCCESS; + const ObIArray &related_index_tids = ctx.get_related_index_ids(); + del_ctdefs.set_capacity(related_index_tids.count()); + + for (int64_t i = 0; OB_SUCC(ret) && i < related_index_tids.count(); ++i) { + ObDMLCtDefAllocator das_alloc(allocator); + ObDASDelCtDef *related_das_ctdef = nullptr; + if (OB_ISNULL(related_das_ctdef = das_alloc.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate delete related das ctdef", K(ret)); + } else if (OB_FAIL(generate_das_del_ctdef(ctx, + related_index_tids.at(i), + *related_das_ctdef, + old_row))) { + LOG_WARN("fail to generate das del ctdef", K(ret)); + } else if (OB_FAIL(del_ctdefs.push_back(related_das_ctdef))) { + LOG_WARN("fail to store related ctdef", K(ret)); + } + } + + return ret; +} + +int ObTableDmlCgService::generate_replace_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableReplaceCtDef &replace_ctdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_insert_ctdef(ctx, allocator, replace_ctdef.ins_ctdef_))) { + LOG_WARN("fail to generate insert ctdef", K(ret), K(ctx)); + } else if (OB_FAIL(generate_table_rowkey_info(ctx, replace_ctdef.ins_ctdef_))) { + LOG_WARN("fail to generate table rowkey info", K(ret), K(ctx)); + } else if (OB_FAIL(generate_delete_ctdef(ctx, allocator, replace_ctdef.del_ctdef_))) { + LOG_WARN("fail to generate delete ctdef", K(ret), K(ctx)); + } + + return ret; +} + +int ObTableDmlCgService::generate_table_rowkey_info(ObTableCtx &ctx, + ObTableInsCtDef &ins_ctdef) +{ + int ret = OB_SUCCESS; + ObDASInsCtDef &das_ins_ctdef = ins_ctdef.das_ctdef_; + ObSEArray rowkey_column_ids; + ObSEArray rowkey_exprs; + ObSEArray rowkey_column_types; + + if (OB_FAIL(get_rowkey_exprs(ctx, rowkey_exprs))) { + LOG_WARN("fail to get rowkey exprs", K(ret), K(ctx)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_exprs.count(); ++i) { + ObRawExpr *expr = rowkey_exprs.at(i); + if (!expr->is_column_ref_expr()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr type is not column_ref expr", K(ret), KPC(expr)); + } else { + ObColumnRefRawExpr *col_expr = static_cast(expr); + if (OB_FAIL(rowkey_column_ids.push_back(col_expr->get_column_id()))) { + LOG_WARN("fail to push base column id", K(ret), K(col_expr->get_column_id())); + } else if (OB_FAIL(rowkey_column_types.push_back(col_expr->get_result_type()))) { + LOG_WARN("fail to push column type", K(ret), KPC(col_expr)); + } + } + } + + if (FAILEDx(das_ins_ctdef.table_rowkey_cids_.init(rowkey_column_ids.count()))) { + LOG_WARN("fail to init table rowkey column ids", K(ret), K(rowkey_column_ids.count())); + } else if (OB_FAIL(append(das_ins_ctdef.table_rowkey_cids_, rowkey_column_ids))) { + LOG_WARN("fail to append table rowkey column id", K(ret), K(rowkey_column_ids)); + } else if (OB_FAIL(das_ins_ctdef.table_rowkey_types_.init(rowkey_column_types.count()))) { + LOG_WARN("fail to init table_rowkey_types", K(ret), K(rowkey_column_types.count())); + } else if (OB_FAIL(append(das_ins_ctdef.table_rowkey_types_, rowkey_column_types))) { + LOG_WARN("fail to append table rowkey column type", K(ret), K(rowkey_column_types)); + } + + return ret; +} + +int ObTableDmlCgService::get_rowkey_exprs(ObTableCtx &ctx, + ObIArray &rowkey_exprs) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + const int rowkey_cnt = table_schema->get_rowkey_column_num(); + const ObIArray &all_exprs = ctx.is_for_insertup() ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; i++) { + if (OB_FAIL(rowkey_exprs.push_back(all_exprs.at(i)))) { + LOG_WARN("fail to push back rowkey expr", K(ret), K(i)); + } + } + } + + return ret; +} + +int ObTableDmlCgService::generate_tsc_ctdef(ObTableCtx &ctx, + ObDASScanCtDef &tsc_ctdef) +{ + int ret = OB_SUCCESS; + ObStaticEngineCG cg; + const ObIArray &all_exprs = ctx.is_for_insertup() ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + ObSEArray access_exprs; + const ObTableSchema *table_schema = ctx.get_table_schema(); + tsc_ctdef.ref_table_id_ = ctx.get_index_table_id(); + const uint64_t tenant_id = MTL_ID(); + + if (OB_FAIL(ctx.get_schema_guard().get_schema_version(TABLE_SCHEMA, + tenant_id, + tsc_ctdef.ref_table_id_, + tsc_ctdef.schema_version_))) { + LOG_WARN("fail to get schema version", K(ret), K(tenant_id), K(tsc_ctdef.ref_table_id_)); + } else if (OB_FAIL(access_exprs.assign(all_exprs))) { + LOG_WARN("fail to assign access exprs", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(access_exprs, + tsc_ctdef.pd_expr_spec_.access_exprs_))) { + LOG_WARN("fail to generate rt exprs ", K(ret)); + } else if (OB_FAIL(tsc_ctdef.access_column_ids_.init(access_exprs.count()))) { + LOG_WARN("fail to init access_column_ids_ ", K(ret)); + } else { + ARRAY_FOREACH(access_exprs, i) { + ObColumnRefRawExpr *ref_col_expr = static_cast(access_exprs.at(i)); + if (OB_ISNULL(ref_col_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null column ref expr", K(ret), K(i)); + } else if (OB_FAIL(tsc_ctdef.access_column_ids_.push_back(ref_col_expr->get_column_id()))) { + LOG_WARN("fail to add column id", K(ret), K(ref_col_expr->get_column_id())); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(tsc_ctdef.table_param_.convert(*table_schema, tsc_ctdef.access_column_ids_))) { + LOG_WARN("fail to convert table param", K(ret)); + } else if (OB_FAIL(ObTableTscCgService::generate_das_result_output(tsc_ctdef, + tsc_ctdef.access_column_ids_))) { + LOG_WARN("generate das result output failed", K(ret)); + } + } + + return ret; +} + +int ObTableDmlCgService::generate_single_constraint_info(ObTableCtx &ctx, + const ObTableSchema &index_schema, + const uint64_t table_id, + ObUniqueConstraintInfo &constraint_info) +{ + int ret = OB_SUCCESS; + + constraint_info.table_id_ = table_id; + constraint_info.index_tid_ = index_schema.get_table_id(); + if (!index_schema.is_index_table()) { + constraint_info.constraint_name_ = "PRIMARY"; + } else if (OB_FAIL(index_schema.get_index_name(constraint_info.constraint_name_))) { + LOG_WARN("fail to get index name", K(ret)); + } + + if (OB_SUCC(ret)) { + uint64_t rowkey_column_id = OB_INVALID_ID; + const ObIArray &all_exprs = ctx.is_for_insertup() ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + ObIArray &column_exprs = constraint_info.constraint_columns_; + const ObRowkeyInfo &rowkey_info = index_schema.get_rowkey_info(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_info.get_size(); ++i) { + if (OB_FAIL(rowkey_info.get_column_id(i, rowkey_column_id))) { + LOG_WARN("fail to get rowkey column id", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < all_exprs.count(); j++) { + if (T_REF_COLUMN == all_exprs.at(j)->get_expr_type()) { + ObColumnRefRawExpr *ref_expr = static_cast(all_exprs.at(j)); + if (ref_expr->get_column_id() == rowkey_column_id + && OB_FAIL(column_exprs.push_back(ref_expr))) { + LOG_WARN("fail to push back column ref expr to constraint columns", K(ret), K(j)); + } + } + } + } + } + } + + return ret; +} + +int ObTableDmlCgService::generate_constraint_infos(ObTableCtx &ctx, + ObIArray &cst_infos) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard &schema_suard = ctx.get_schema_guard(); + const ObTableSchema *table_schema = ctx.get_table_schema(); + const uint64_t ref_table_id = ctx.get_ref_table_id(); + ObUniqueConstraintInfo constraint_info; + ObSEArray index_infos; + + // 1. primary key + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (OB_FAIL(generate_single_constraint_info(ctx, + *table_schema, + ref_table_id, + constraint_info))) { + LOG_WARN("fail to generate primary key constraint info", K(ret), K(ref_table_id)); + } else if (OB_FAIL(cst_infos.push_back(constraint_info))) { + LOG_WARN("fail to push back constraint info", K(ret)); + } + + // 2. unique key + if (FAILEDx(table_schema->get_simple_index_infos(index_infos))) { + LOG_WARN("fail to get index infos", K(ret)); + } else { + const ObTableSchema *index_schema = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < index_infos.count(); i++) { + constraint_info.reset(); + if (OB_FAIL(schema_suard.get_table_schema(ctx.get_session_info().get_effective_tenant_id(), + index_infos.at(i).table_id_, + index_schema))) { + LOG_WARN("fail to get index schema", K(ret), K(index_infos.at(i).table_id_)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("index schema null", K(ret)); + } else if (!index_schema->is_final_invalid_index() && index_schema->is_unique_index()) { + if (OB_FAIL(generate_single_constraint_info(ctx, + *index_schema, + ref_table_id, + constraint_info))) { + LOG_WARN("fail to generate unique key constraint info", K(ret)); + } else if (OB_FAIL(cst_infos.push_back(constraint_info))) { + LOG_WARN("fail to push back constraint info", K(ret)); + } + } + } + } + + return ret; +} + +int ObTableDmlCgService::generate_constraint_ctdefs(ObTableCtx &ctx, + ObIAllocator &allocator, + sql::ObRowkeyCstCtdefArray &cst_ctdefs) +{ + int ret = OB_SUCCESS; + ObDMLCtDefAllocator cst_ctdef_allocator(allocator); + ObSEArray cst_infos; + ObRowkeyCstCtdef *rowkey_cst_ctdef = nullptr; + ObStaticEngineCG cg; + + if (OB_FAIL(generate_constraint_infos(ctx, cst_infos))) { + LOG_WARN("fail to generate constraint infos", K(ret), K(ctx)); + } else if (OB_FAIL(cst_ctdefs.init(cst_infos.count()))) { + LOG_WARN("fail to allocate conflict checker spec array", K(ret), K(cst_infos.count())); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < cst_infos.count(); i++) { + const ObIArray &cst_columns = cst_infos.at(i).constraint_columns_; + if (OB_ISNULL(rowkey_cst_ctdef = cst_ctdef_allocator.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc cst ctdef memory", K(ret)); + } else if (OB_FAIL(ob_write_string(allocator, + cst_infos.at(i).constraint_name_, + rowkey_cst_ctdef->constraint_name_))) { + LOG_WARN("fail to write string", K(ret), K(cst_infos.at(i).constraint_name_)); + } else if (OB_FAIL(rowkey_cst_ctdef->rowkey_expr_.init(cst_columns.count()))) { + LOG_WARN("fail to init rowkey", K(ret), K(cst_columns.count())); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < cst_columns.count(); ++j) { + ObExpr *expr = nullptr; + if (OB_FAIL(cg.generate_rt_expr(*cst_columns.at(j), expr))) { + LOG_WARN("fail to generate rt expr", K(ret)); + } else if (OB_FAIL(rowkey_cst_ctdef->rowkey_expr_.push_back(expr))) { + LOG_WARN("fail to push back rt expr", K(ret)); + } + } + + if (FAILEDx(cst_ctdefs.push_back(rowkey_cst_ctdef))) { + LOG_WARN("fail to push back rowkey constraint ctdef", K(ret)); + } + } + } + + return ret; +} + +int ObTableDmlCgService::generate_conflict_checker_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObConflictCheckerCtdef &conflict_checker_ctdef) +{ + int ret = OB_SUCCESS; + const ObIArray &exprs = ctx.is_for_insertup() ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + ObSEArray table_column_exprs; + ObSEArray rowkey_exprs; + ObStaticEngineCG cg; + + if (OB_FAIL(get_rowkey_exprs(ctx, rowkey_exprs))) { + LOG_WARN("fail to get table rowkey exprs", K(ret), K(ctx)); + } else if (OB_FAIL(generate_tsc_ctdef(ctx, conflict_checker_ctdef.das_scan_ctdef_))) { + LOG_WARN("fail to generate das_scan_ctdef", K(ret)); + } else if (OB_FAIL(generate_constraint_ctdefs(ctx, + allocator, + conflict_checker_ctdef.cst_ctdefs_))) { + LOG_WARN("fail to generate constraint infos", K(ret), K(ctx)); + } else if (OB_FAIL(cg.generate_rt_exprs(rowkey_exprs, + conflict_checker_ctdef.data_table_rowkey_expr_))) { + LOG_WARN("fail to generate data table rowkey expr", K(ret), K(rowkey_exprs)); + } else if (OB_FAIL(cg.generate_rt_exprs(exprs, conflict_checker_ctdef.table_column_exprs_))) { + LOG_WARN("fail to generate table columns rt exprs ", K(ret)); + } else { + conflict_checker_ctdef.rowkey_count_ = ctx.get_table_schema()->get_rowkey_column_num(); + } + + return ret; +} + +int ObTableDmlCgService::generate_insert_up_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObTableCtx::ObAssignIds &assign_ids, + ObTableInsUpdCtDef &ins_up_ctdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_insert_ctdef(ctx, allocator, ins_up_ctdef.ins_ctdef_))) { + LOG_WARN("fail to generate insert ctdef", K(ret), K(ctx)); + } else if (OB_FAIL(generate_table_rowkey_info(ctx, ins_up_ctdef.ins_ctdef_))) { + LOG_WARN("fail to generate table rowkey info", K(ret), K(ctx)); + } else if (OB_FAIL(generate_update_ctdef(ctx, + allocator, + assign_ids, + ins_up_ctdef.upd_ctdef_))) { + LOG_WARN("fail to generate delete ctdef", K(ret), K(ctx)); + } + + return ret; +} + +int ObTableDmlCgService::generate_lock_ctdef(ObTableCtx &ctx, + ObTableLockCtDef &lock_ctdef) +{ + int ret = OB_SUCCESS; + ObStaticEngineCG cg; + ObArray old_row; + + if (OB_FAIL(old_row.assign(ctx.get_all_exprs().get_expr_array()))) { + LOG_WARN("fail to assign old row expr", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(old_row, + lock_ctdef.old_row_))) { + LOG_WARN("fail to generate lock rt exprs", K(ret), K(old_row)); + } else if (OB_FAIL(generate_das_lock_ctdef(ctx, + ctx.get_ref_table_id(), + lock_ctdef.das_ctdef_, + old_row))) { + LOG_WARN("fail to generate das lock ctdef", K(ret)); + } + + return ret; +} + +int ObTableDmlCgService::generate_das_lock_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + ObDASLockCtDef &das_lock_ctdef, + const ObIArray &old_row) +{ + int ret = OB_SUCCESS; + ObArray dml_column_ids; + ObArray empty_new_row; + + if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_lock_ctdef))) { + LOG_WARN("fail to generate das dml ctdef", K(ret)); + } else if (OB_FAIL(generate_column_ids(ctx.get_all_exprs().get_expr_array(), + dml_column_ids))) { + LOG_WARN("fail to generate dml column ids", K(ret)); + } else if (OB_FAIL(generate_projector(dml_column_ids, + das_lock_ctdef.column_ids_, + old_row, + empty_new_row, + old_row, + das_lock_ctdef))) { + LOG_WARN("fail to add old row projector", K(ret), K(old_row)); + } + + return ret; +} + +int ObTableDmlCgService::generate_base_ctdef(ObTableCtx &ctx, + ObTableDmlBaseCtDef &base_ctdef, + ObIArray &old_row, + ObIArray &new_row) +{ + int ret = OB_SUCCESS; + ObStaticEngineCG cg; + const ObIArray &exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + + if (OB_FAIL(generate_column_ids(exprs, base_ctdef.column_ids_))) { + LOG_WARN("fail to generate dml column ids", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(old_row, base_ctdef.old_row_))) { + LOG_WARN("fail to generate old row exprs", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(new_row, base_ctdef.new_row_))) { + LOG_WARN("fail to generate new row exprs", K(ret)); + } + + return ret; +} + +int ObTableDmlCgService::generate_das_ins_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + ObDASInsCtDef &das_ins_ctdef, + const ObIArray &new_row) +{ + int ret = OB_SUCCESS; + ObArray dml_column_ids; + ObArray empty_old_row; + const ObIArray &exprs = ctx.is_for_insertup() ? + ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + + if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_ins_ctdef))) { + LOG_WARN("fail to generate das dml ctdef", K(ret)); + } else if (OB_FAIL(generate_column_ids(exprs, + dml_column_ids))) { + LOG_WARN("fail to generate dml column ids", K(ret)); + } else if (OB_FAIL(generate_projector(dml_column_ids, // new row and old row's columns id + das_ins_ctdef.column_ids_, // schmea column ids for given index_tid + empty_old_row, + new_row, + new_row, + das_ins_ctdef))) { + LOG_WARN("fail to add new row projector", K(ret), K(new_row)); + } + + return ret; +} + +int ObTableDmlCgService::generate_das_base_ctdef(uint64_t index_tid, + ObTableCtx &ctx, + ObDASDMLBaseCtDef &base_ctdef) +{ + int ret = OB_SUCCESS; + base_ctdef.table_id_ = ctx.get_ref_table_id(); + base_ctdef.index_tid_ = index_tid; + base_ctdef.is_ignore_ = false; // insert ignore + base_ctdef.is_batch_stmt_ = false; + int64_t binlog_row_image = share::ObBinlogRowImage::FULL; + ObSQLSessionInfo &session = ctx.get_session_info(); + + if (OB_FAIL(generate_column_info(index_tid, ctx, base_ctdef))) { + LOG_WARN("fail to generate column info", K(ret), K(index_tid), K(ctx)); + } else if (OB_FAIL(ctx.get_schema_guard().get_schema_version(TABLE_SCHEMA, + ctx.get_tenant_id(), + index_tid, + base_ctdef.schema_version_))) { + LOG_WARN("fail to get table schema version", K(ret)); + } else if (OB_FAIL(convert_table_param(ctx, base_ctdef))) { + LOG_WARN("fail to convert table dml param", K(ret)); + } else if (OB_FAIL(session.get_binlog_row_image(binlog_row_image))) { + LOG_WARN("fail to get binlog row image", K(ret)); + } else { + base_ctdef.tz_info_ = *session.get_tz_info_wrap().get_time_zone_info(); + base_ctdef.is_total_quantity_log_ = (share::ObBinlogRowImage::FULL == binlog_row_image); + base_ctdef.encrypt_meta_.reset(); + } + + return ret; +} + +// add column_ids, column_types, column_accuracys, rowkey_cnt, spk_cnt to ObDASDMLBaseCtDef +// according to table schema order +int ObTableDmlCgService::generate_column_info(ObTableID index_tid, + ObTableCtx &ctx, + ObDASDMLBaseCtDef &base_ctdef) +{ + int ret = OB_SUCCESS; + base_ctdef.column_ids_.reset(); + base_ctdef.column_types_.reset(); + const ObTableSchema *index_schema = nullptr; + + if (OB_FAIL(ctx.get_schema_guard().get_table_schema(ctx.get_tenant_id(), + index_tid, + index_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(index_tid)); + } else { + int64_t column_count = index_schema->get_column_count(); + base_ctdef.column_ids_.set_capacity(column_count); + base_ctdef.column_types_.set_capacity(column_count); + base_ctdef.column_accuracys_.set_capacity(column_count); + base_ctdef.rowkey_cnt_ = index_schema->get_rowkey_info().get_size(); + base_ctdef.spk_cnt_ = index_schema->get_shadow_rowkey_info().get_size(); + + // add rowkey column infos + const ObRowkeyInfo &rowkey_info = index_schema->get_rowkey_info(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_info.get_size(); ++i) { + const ObRowkeyColumn *rowkey_column = rowkey_info.get_column(i); + const ObColumnSchemaV2 *column = index_schema->get_column_schema(rowkey_column->column_id_); + ObObjMeta column_type; + column_type = column->get_meta_type(); + column_type.set_scale(column->get_accuracy().get_scale()); + if (OB_FAIL(base_ctdef.column_ids_.push_back(column->get_column_id()))) { + LOG_WARN("fail to add column id", K(ret)); + } else if (OB_FAIL(base_ctdef.column_types_.push_back(column_type))) { + LOG_WARN("fail to add column type", K(ret)); + } else if (OB_FAIL(base_ctdef.column_accuracys_.push_back(column->get_accuracy()))) { + LOG_WARN("fail to add column accuracys", K(ret)); + } + } + + // add normal column infos if need + if (OB_SUCC(ret)) { + ObTableSchema::const_column_iterator iter = index_schema->column_begin(); + for (; OB_SUCC(ret) && iter != index_schema->column_end(); ++iter) { + const ObColumnSchemaV2 *column = *iter; + if (column->is_rowkey_column() || column->is_virtual_generated_column()) { + // do nothing + } else { + ObObjMeta column_type; + column_type = column->get_meta_type(); + column_type.set_scale(column->get_accuracy().get_scale()); + if (OB_FAIL(base_ctdef.column_ids_.push_back(column->get_column_id()))) { + LOG_WARN("fail to add column id", K(ret)); + } else if (OB_FAIL(base_ctdef.column_types_.push_back(column_type))) { + LOG_WARN("fail to add column type", K(ret)); + } else if (OB_FAIL(base_ctdef.column_accuracys_.push_back(column->get_accuracy()))) { + LOG_WARN("fail to add column accuracys", K(ret)); + } + } + } + } + } + + return ret; +} + +int ObTableDmlCgService::convert_table_param(ObTableCtx &ctx, + ObDASDMLBaseCtDef &base_ctdef) +{ + int ret = OB_SUCCESS; + int64_t schema_version = OB_INVALID_VERSION; + const ObTableSchema *table_schema = NULL; + uint64_t tenant_id = ctx.get_tenant_id(); + + if (OB_FAIL(ctx.get_schema_guard().get_table_schema(tenant_id, + base_ctdef.index_tid_, + table_schema))) { + LOG_WARN("fail to get schema", K(ret), K(base_ctdef)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("table schema is NULL", K(ret)); + } else if (OB_FAIL(ctx.get_schema_guard().get_schema_version(tenant_id, schema_version))) { + LOG_WARN("fail to get tenant schema version", K(ret), K(tenant_id)); + } else if (OB_FAIL(base_ctdef.table_param_.convert(table_schema, + schema_version, + base_ctdef.column_ids_))) { + LOG_WARN("fail to convert table param", K(ret), K(base_ctdef)); + } + + return ret; +} + +int ObTableDmlCgService::generate_column_ids(const ObIArray &exprs, + ObIArray &column_ids) +{ + int ret = OB_SUCCESS; + column_ids.reset(); + + if (!exprs.empty()) { + if (OB_FAIL(column_ids.reserve(exprs.count()))) { + LOG_WARN("fail to reserve column ids capacity", K(ret), K(exprs.count())); + } else { + ARRAY_FOREACH(exprs, i) { + ObColumnRefRawExpr *col_expr = static_cast(exprs.at(i)); + if (OB_FAIL(column_ids.push_back(col_expr->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret), K(col_expr->get_column_id())); + } + } + } + } + + return ret; +} + +// 构造 das_ctdef 中的 old_row_projector 和 new_row_projector， +// 其中存储 storage column 对应表达式在 full row exprs 数组中下标 +int ObTableDmlCgService::generate_projector(const ObIArray &dml_column_ids, + const ObIArray &storage_column_ids, + const ObIArray &old_row, + const ObIArray &new_row, + const ObIArray &full_row, + ObDASDMLBaseCtDef &das_ctdef) +{ + int ret = OB_SUCCESS; + IntFixedArray &old_row_projector = das_ctdef.old_row_projector_; + IntFixedArray &new_row_projector = das_ctdef.new_row_projector_; + + // generate old row projector + // 查找old row expr 在full row expr中的位置（投影） + if (!old_row.empty()) { + if (OB_FAIL(old_row_projector.prepare_allocate(storage_column_ids.count()))) { + LOG_WARN("fail to init row projector array", K(ret), K(storage_column_ids.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < storage_column_ids.count(); ++i) { + uint64_t storage_cid = storage_column_ids.at(i); + uint64_t ref_cid = is_shadow_column(storage_cid) ? + storage_cid - OB_MIN_SHADOW_COLUMN_ID : + storage_cid; + int64_t column_idx = OB_INVALID_INDEX; + int64_t projector_idx = OB_INVALID_INDEX; + old_row_projector.at(i) = OB_INVALID_INDEX; + if (has_exist_in_array(dml_column_ids, ref_cid, &column_idx)) { + ObRawExpr *column_expr = old_row.at(column_idx); + if (!has_exist_in_array(full_row, column_expr, &projector_idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("row column not found in full row columns", K(ret), + K(column_idx), KPC(old_row.at(column_idx))); + } else { + old_row_projector.at(i) = projector_idx; + } + } + } + } + + // generate new row projector + // 查找new row expr 在full row expr中的位置（投影） + if (!new_row.empty() && OB_SUCC(ret)) { + if (OB_FAIL(new_row_projector.prepare_allocate(storage_column_ids.count()))) { + LOG_WARN("fail to init row projector array", K(ret), K(storage_column_ids.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < storage_column_ids.count(); ++i) { + uint64_t storage_cid = storage_column_ids.at(i); + uint64_t ref_cid = is_shadow_column(storage_cid) ? + storage_cid - OB_MIN_SHADOW_COLUMN_ID : + storage_cid; + int64_t column_idx = OB_INVALID_INDEX; + int64_t projector_idx = OB_INVALID_INDEX; + // 如果 projector[i] = j, 表达式按照 schema 顺序，第 i 个 column 在 full_row 中的下标为 j，如果在 new_row 中不存在，那么 j == -1 + new_row_projector.at(i) = OB_INVALID_INDEX; + if (has_exist_in_array(dml_column_ids, ref_cid, &column_idx)) { + ObRawExpr *column_expr = new_row.at(column_idx); + if (!has_exist_in_array(full_row, column_expr, &projector_idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("row column not found in full row columns", K(ret), + K(column_idx), KPC(new_row.at(column_idx))); + } else { + new_row_projector.at(i) = projector_idx; // projector_idx 为 column storage_column_ids[i] 在 full row expr 中的 index + } + } + } + } + return ret; +} + + +int ObTableDmlCgService::generate_related_ins_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObIArray &new_row, + DASInsCtDefArray &ins_ctdefs) +{ + int ret = OB_SUCCESS; + const ObIArray &related_index_tids = ctx.get_related_index_ids(); + ins_ctdefs.set_capacity(related_index_tids.count()); + + for (int64_t i = 0; OB_SUCC(ret) && i < related_index_tids.count(); ++i) { + ObDMLCtDefAllocator das_alloc(allocator); + ObDASInsCtDef *related_das_ctdef = nullptr; + if (OB_ISNULL(related_das_ctdef = das_alloc.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate insert related das ctdef", K(ret)); + } else if (OB_FAIL(generate_das_ins_ctdef(ctx, + related_index_tids.at(i), + *related_das_ctdef, + new_row))) { + LOG_WARN("fail to generate das ins ctdef", K(ret)); + } else if (OB_FAIL(ins_ctdefs.push_back(related_das_ctdef))) { + LOG_WARN("fail to store related ctdef", K(ret)); + } + } + + return ret; +} + +int ObTableDmlCgService::generate_related_upd_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObIArray &assign_exprs, + const ObIArray &old_row, + const ObIArray &new_row, + const ObIArray &full_row, + DASUpdCtDefArray &upd_ctdefs) +{ + int ret = OB_SUCCESS; + const ObIArray &related_index_tids = ctx.get_related_index_ids(); + upd_ctdefs.set_capacity(related_index_tids.count()); + + for (int64_t i = 0; OB_SUCC(ret) && i < related_index_tids.count(); ++i) { + ObDMLCtDefAllocator das_alloc(allocator); + ObDASUpdCtDef *related_das_ctdef = nullptr; + if (OB_ISNULL(related_das_ctdef = das_alloc.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate update related das ctdef", K(ret)); + } else if (OB_FAIL(generate_das_upd_ctdef(ctx, + related_index_tids.at(i), + assign_exprs, + *related_das_ctdef, + old_row, + new_row, + full_row))) { + LOG_WARN("fail to generate das update ctdef", K(ret)); + } else if (related_das_ctdef->updated_column_ids_.empty()) { + // ignore invalid update ctdef + } else if (OB_FAIL(upd_ctdefs.push_back(related_das_ctdef))) { + LOG_WARN("fail to store related ctdef", K(ret)); + } + } + + return ret; +} + +// generate spec tree witch a child node +template +int ObTableSpecCgService::generate_with_child(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec) +{ + int ret = OB_SUCCESS; + ObTableApiSpec *child_spec = nullptr; + if (FATHER_TYPE <= TABLE_API_EXEC_INVALID || FATHER_TYPE >= TABLE_API_EXEC_MAX || + CHILD_TYPE <= TABLE_API_EXEC_INVALID || CHILD_TYPE >= TABLE_API_EXEC_MAX) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid type", K(ret), K(FATHER_TYPE), K(CHILD_TYPE)); + } else if (OB_FAIL(ObTableExecutorFactory::generate_spec( + alloc, static_cast(CHILD_TYPE), ctx, child_spec))) { + LOG_WARN("fail to generate scan spec", K(ret)); + } else { + ObTableApiSpec *father_spec = nullptr; + if (OB_FAIL(ObTableExecutorFactory::generate_spec( + alloc, static_cast(FATHER_TYPE), ctx, father_spec))) { + LOG_WARN("fail to generate update spec", K(ret)); + } else { + father_spec->set_child(child_spec); + child_spec->set_parent(father_spec); + root_spec = father_spec; + } + } + + return ret; +} + +// NOTE: In default, generate spec tree with one node which type is input TYPE +template +int ObTableSpecCgService::generate(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec) +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + if (TYPE <= TABLE_API_EXEC_INVALID || TYPE >= TABLE_API_EXEC_MAX) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("input TYPE is invalid", K(ret), K(TYPE)); + } else if (TYPE == TABLE_API_EXEC_UPDATE) { + ret = ObTableSpecCgService::generate_with_child + (alloc, ctx, root_spec); + } else if (TYPE == TABLE_API_EXEC_DELETE) { + ret = ObTableSpecCgService::generate_with_child + (alloc, ctx, root_spec); + } else if (TYPE == TABLE_API_EXEC_LOCK) { + ret = ObTableSpecCgService::generate_with_child + (alloc, ctx, root_spec); + } else if (OB_FAIL(ObTableExecutorFactory::generate_spec(alloc, + static_cast(TYPE), + ctx, + spec))) { + LOG_WARN("fail to generate spec", K(ret)); + } else { + root_spec = spec; + } + + return ret; +} + +// NOTE: explicit instantiate here after adding a new executor, cannot put template define to +// the header file, it will cause circular dependent with executor factory +template int ObTableSpecCgService::generate(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec); +template int ObTableSpecCgService::generate(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec); +template int ObTableSpecCgService::generate(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec); +template int ObTableSpecCgService::generate(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec); + +int ObTableTscCgService::generate_rt_exprs(const ObTableCtx &ctx, + ObIAllocator &allocator, + const ObIArray &src, + ObIArray &dst) +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session_info = const_cast(&ctx.get_session_info()); + ObSchemaGetterGuard *schema_guard = const_cast(&ctx.get_schema_guard()); + ObStaticEngineExprCG expr_cg(allocator, + session_info, + schema_guard, + 0, + 0); + if (!src.empty()) { + if (OB_FAIL(dst.reserve(src.count()))) { + LOG_WARN("fail to init fixed array", K(ret), K(src.count())); + } else { + ObArray exprs; + for (int64_t i = 0; OB_SUCC(ret) && i < src.count(); i++) { + ObExpr *e = nullptr; + if (OB_FAIL(ObStaticEngineExprCG::generate_rt_expr(*src.at(i), exprs, e))) { + LOG_WARN("fail to generate rt expr", K(ret)); + } else if (OB_ISNULL(e)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else if (OB_FAIL(dst.push_back(e))) { + LOG_WARN("fail to push back rt expr", K(ret), K(i)); + } + } + } + } + + return ret; +} + +// 非索引扫描: access exprs = select exprs +// 索引表: access exprs = [index column exprs][rowkey expr] +// 索引回表: access expr = [rowkey expr][select without rowkey exprs] +int ObTableTscCgService::generate_access_ctdef(const ObTableCtx &ctx, + ObIAllocator &allocator, + ObDASScanCtDef &das_tsc_ctdef) +{ + int ret = OB_SUCCESS; + ObArray access_exprs; + const ObIArray &select_exprs = ctx.get_select_exprs(); + const ObIArray &rowkey_exprs = ctx.get_rowkey_exprs(); + const ObIArray &index_exprs = ctx.get_index_exprs(); + + if (!ctx.is_index_scan() && OB_FAIL(access_exprs.assign(select_exprs))) { // 非索引扫描 + LOG_WARN("fail to assign access exprs", K(ret)); + } else if (ctx.is_index_scan() && das_tsc_ctdef.ref_table_id_ == ctx.get_index_table_id()) { // 索引表 + if (OB_FAIL(access_exprs.assign(index_exprs))) { + LOG_WARN("fail to assign access exprs", K(ret), K(ctx.get_index_table_id())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_exprs.count(); i++) { + if (is_in_array(index_exprs, rowkey_exprs.at(i))) { + // 在index_exprs中的index expr不需要再次添加 + } else if (OB_FAIL(access_exprs.push_back(rowkey_exprs.at(i)))) { + LOG_WARN("fail to push back rowkey expr", K(ret), K(i)); + } + } + } + } else if (ctx.is_index_scan() && das_tsc_ctdef.ref_table_id_ == ctx.get_ref_table_id()) { // 索引回表 + if (OB_FAIL(access_exprs.assign(rowkey_exprs))) { + LOG_WARN("fail to assign access exprs", K(ret), K(ctx.get_ref_table_id())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); i++) { + if (is_in_array(rowkey_exprs, select_exprs.at(i))) { + // 已经在rowkey中，不需要再次添加 + } else if (OB_FAIL(access_exprs.push_back(select_exprs.at(i)))) { + LOG_WARN("fail to push back select expr", K(ret), K(i)); + } + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(generate_rt_exprs(ctx, allocator, access_exprs, das_tsc_ctdef.pd_expr_spec_.access_exprs_))) { + LOG_WARN("fail to generate access rt exprs", K(ret)); + } else if (OB_FAIL(das_tsc_ctdef.access_column_ids_.init(access_exprs.count()))) { + LOG_WARN("fail to init access column ids", K(ret), K(access_exprs.count())); + } else { + ObColumnRefRawExpr *col_expr = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); i++) { + if (T_REF_COLUMN != access_exprs.at(i)->get_expr_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(access_exprs.at(i)->get_expr_type())); + } else if (FALSE_IT(col_expr = static_cast(access_exprs.at(i)))) { + } else if (OB_FAIL(das_tsc_ctdef.access_column_ids_.push_back(col_expr->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret)); + } + } + } + } + + return ret; +} + +int ObTableTscCgService::generate_das_result_output(ObDASScanCtDef &das_tsc_ctdef, + const ObIArray &output_cids) +{ + int ret = OB_SUCCESS; + ExprFixedArray &access_exprs = das_tsc_ctdef.pd_expr_spec_.access_exprs_; + const ObIArray &access_cids = das_tsc_ctdef.access_column_ids_; + int64_t access_column_cnt = access_cids.count(); + int64_t access_expr_cnt = access_exprs.count(); + if (OB_UNLIKELY(access_column_cnt != access_expr_cnt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("access column count is invalid", K(ret), K(access_column_cnt), K(access_expr_cnt)); + } else if (OB_FAIL(das_tsc_ctdef.result_output_.init(output_cids.count() + 1))) { + LOG_WARN("fail to init result output", K(ret), K(output_cids.count())); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < output_cids.count(); i++) { + for (int64_t j = 0; OB_SUCC(ret) && j < access_column_cnt; j++) { + if (output_cids.at(i) == access_cids.at(j)) { + if (OB_FAIL(das_tsc_ctdef.result_output_.push_back(access_exprs.at(j)))) { + LOG_WARN("fail to push result output expr", K(ret), K(i), K(j)); + } + } + } + } + + return ret; +} + +// tsc_out_cols +// 主表/索引回表/索引扫描不需要回表: select column ids +// 索引表: rowkey column ids +int ObTableTscCgService::generate_table_param(const ObTableCtx &ctx, + ObDASScanCtDef &das_tsc_ctdef) +{ + int ret = OB_SUCCESS; + ObArray tsc_out_cols; + const ObTableSchema *table_schema = nullptr; + const ObIArray &select_exprs = ctx.get_select_exprs(); + + if (!ctx.is_index_scan() // 非索引扫描 + || (ctx.is_index_scan() && das_tsc_ctdef.ref_table_id_ == ctx.get_ref_table_id()) // 索引扫描回表 + || (ctx.is_index_scan() && !ctx.is_index_back())) { //索引扫描不需要回表 + ObColumnRefRawExpr *col_expr = NULL; + if (ctx.is_index_scan() && !ctx.is_index_back()) { + table_schema = ctx.get_index_schema(); + } else { + table_schema = ctx.get_table_schema(); + } + for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); i++) { + if (T_REF_COLUMN != select_exprs.at(i)->get_expr_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr type", K(ret), K(select_exprs.at(i)->get_expr_type())); + } else if (FALSE_IT(col_expr = static_cast(select_exprs.at(i)))) { + } else if (OB_FAIL(tsc_out_cols.push_back(col_expr->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret)); + } + } + } else if (ctx.is_index_scan() && das_tsc_ctdef.ref_table_id_ == ctx.get_index_table_id()) { // 索引表 + table_schema = ctx.get_index_schema(); + if (OB_FAIL(ctx.get_table_schema()->get_rowkey_column_ids(tsc_out_cols))) { + LOG_WARN("fail to get rowkey column ids", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(das_tsc_ctdef.table_param_.convert(*table_schema, + das_tsc_ctdef.access_column_ids_, + &tsc_out_cols))) { + LOG_WARN("fail to convert schema", K(ret), K(*table_schema)); + } else if (OB_FAIL(generate_das_result_output(das_tsc_ctdef, tsc_out_cols))) { + LOG_WARN("fail to generate das result outpur", K(ret), K(tsc_out_cols)); + } + + return ret; +} + +int ObTableTscCgService::generate_das_tsc_ctdef(const ObTableCtx &ctx, + ObIAllocator &allocator, + ObDASScanCtDef &das_tsc_ctdef) +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard &schema_guard = (const_cast(ctx)).get_schema_guard(); + das_tsc_ctdef.is_get_ = ctx.is_get(); + das_tsc_ctdef.schema_version_ = ctx.is_index_scan() ? ctx.get_index_schema()->get_schema_version() : + ctx.get_table_schema()->get_schema_version(); + + if (OB_FAIL(generate_access_ctdef(ctx, allocator, das_tsc_ctdef))) { // init access_column_ids_,pd_expr_spec_.access_exprs_ + LOG_WARN("fail to generate asccess ctdef", K(ret)); + } else if (OB_FAIL(generate_table_param(ctx, das_tsc_ctdef))) { // init table_param_, result_output_ + LOG_WARN("fail to generate table param", K(ret)); + } + + return ret; +} + +int ObTableTscCgService::generate_tsc_ctdef(const ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableApiScanCtDef &tsc_ctdef) +{ + int ret = OB_SUCCESS; + // init scan_ctdef_.ref_table_id_ + tsc_ctdef.scan_ctdef_.ref_table_id_ = ctx.get_index_table_id(); + // init scan_ctdef_ + if (OB_FAIL(generate_das_tsc_ctdef(ctx, allocator, tsc_ctdef.scan_ctdef_))) { + LOG_WARN("fail to generate das scan ctdef", K(ret)); + } else if (ctx.is_index_back()) { + // init lookup_ctdef_,lookup_loc_meta_ + void *lookup_buf = allocator.alloc(sizeof(ObDASScanCtDef)); + void *loc_meta_buf = allocator.alloc(sizeof(ObDASTableLocMeta)); + if (OB_ISNULL(lookup_buf) || OB_ISNULL(loc_meta_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate lookup ctdef buffer", K(ret), KP(lookup_buf), KP(loc_meta_buf)); + } else { + tsc_ctdef.lookup_ctdef_ = new(lookup_buf) ObDASScanCtDef(allocator); + tsc_ctdef.lookup_ctdef_->ref_table_id_ = ctx.get_ref_table_id(); + tsc_ctdef.lookup_loc_meta_ = new(loc_meta_buf) ObDASTableLocMeta(allocator); + if (OB_FAIL(generate_das_tsc_ctdef(ctx, allocator, *tsc_ctdef.lookup_ctdef_))) { + LOG_WARN("fail to generate das lookup scan ctdef", K(ret)); + } else if (OB_FAIL(ObTableLocCgService::generate_table_loc_meta(ctx, + *tsc_ctdef.lookup_loc_meta_, + true /* is_lookup */))) { + LOG_WARN("fail to generate table loc meta", K(ret)); + } + } + } + + return ret; +} + +int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiInsertSpec &spec) +{ + return ObTableDmlCgService::generate_insert_ctdef(ctx, alloc, spec.get_ctdef()); +} + +int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiUpdateSpec &spec) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableDmlCgService::generate_update_ctdef(ctx, + alloc, + ctx.get_assign_ids(), + spec.get_ctdef()))) { + LOG_WARN("fail to generate update ctdef", K(ret)); + } + + return ret; +} + +int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiDelSpec &spec) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableDmlCgService::generate_delete_ctdef(ctx, alloc, spec.get_ctdef()))) { + LOG_WARN("fail to generate delete ctdef", K(ret)); + } + + return ret; +} + +int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiReplaceSpec &spec) +{ + int ret = OB_SUCCESS; + ObTableReplaceCtDef &ctdef = spec.get_ctdef(); + + if (OB_FAIL(ObTableDmlCgService::generate_replace_ctdef(ctx, alloc, ctdef))) { + LOG_WARN("fail to generate replace ctdef", K(ret)); + } else if (ObTableDmlCgService::generate_conflict_checker_ctdef(ctx, alloc, spec.get_conflict_checker_ctdef())) { + LOG_WARN("fail to generate conflict checker ctdef", K(ret)); + } + + return ret; +} + +int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiInsertUpSpec &spec) +{ + int ret = OB_SUCCESS; + ObTableInsUpdCtDef &ctdef = spec.get_ctdef(); + + if (OB_FAIL(ObTableDmlCgService::generate_insert_up_ctdef(ctx, + alloc, + ctx.get_assign_ids(), + ctdef))) { + LOG_WARN("fail to generate insert up ctdef", K(ret)); + } else { + const ObIArray &exprs = ctx.get_old_row_exprs(); + ObStaticEngineCG cg; + if (ObTableDmlCgService::generate_conflict_checker_ctdef(ctx, + alloc, + spec.get_conflict_checker_ctdef())) { + LOG_WARN("fail to generate conflict checker ctdef", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(exprs, + spec.get_all_saved_exprs()))) { + LOG_WARN("fail to generate rt exprs ", K(ret)); + } + } + + return ret; +} + +int ObTableSpecCgService::generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiLockSpec &spec) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableDmlCgService::generate_lock_ctdef(ctx, spec.get_ctdef()))) { + LOG_WARN("fail to generate lock ctdef", K(ret)); + } + + return ret; +} +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_cg_service.h b/src/observer/table/ob_table_cg_service.h new file mode 100644 index 0000000000..17709fddae --- /dev/null +++ b/src/observer/table/ob_table_cg_service.h @@ -0,0 +1,324 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_CG_SERVICE_H_ +#define OCEANBASE_OBSERVER_OB_TABLE_CG_SERVICE_H_ +#include "ob_table_scan_executor.h" +#include "ob_table_insert_executor.h" +#include "ob_table_delete_executor.h" +#include "ob_table_update_executor.h" +#include "ob_table_insert_up_executor.h" +#include "ob_table_replace_executor.h" +#include "ob_table_lock_executor.h" +#include "ob_table_context.h" +#include "share/table/ob_table.h" // for ObTableQuery +#include "sql/engine/dml/ob_conflict_checker.h" // for ObConflictCheckerCtdef +#include "ob_table_cache.h" + +namespace oceanbase +{ +namespace table +{ +// 构造表达式的静态类 +class ObTableExprCgService +{ +public: + // 构造表达式 + static int generate_exprs(ObTableCtx &ctx, + common::ObIAllocator &allocator, + ObExprFrameInfo &expr_frame_info); + // 基于table schema构造全表列原生列引用表达式和生成列表达式 + static int generate_column_raw_exprs(ObTableCtx &ctx); + // 构造更新需要的表达式 + static int generate_update_raw_exprs(ObTableCtx &ctx); + // 基于原生表达式生成表达式内存布局 + static int generate_expr_frame_info(ObTableCtx &ctx, + common::ObIAllocator &allocator, + ObExprFrameInfo &expr_frame_info); + // 基于内存表达式内存布局申请内存(这里只是申请了frame内存，dml场景还需要初始化) + static int alloc_exprs_memory(ObTableCtx &ctx, ObExprFrameInfo &expr_frame_info); + static int refresh_insert_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &exprs, + const ObTableEntity &entity); + static int refresh_delete_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &exprs, + const ObTableEntity &entity); + static int refresh_replace_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &exprs, + const ObTableEntity &entity); + static int refresh_update_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &old_row, + const common::ObIArray &new_row, + const common::ObIArray &full_assign_row, + const ObTableCtx::ObAssignIds &assign_ids, + const ObTableEntity &entity); + static int refresh_insert_up_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &ins_new_row, + const common::ObIArray &delta_exprs, + const ObTableEntity &entity); + static int refresh_generated_column_related_frame(ObTableCtx &ctx, + const common::ObIArray &old_row, + const common::ObIArray &full_assign_row, + const ObTableCtx::ObAssignIds &assign_ids, + const ObColumnSchemaV2 &col_schema); + static int generate_assign_exprs(ObTableCtx &ctx, + const ObTableCtx::ObAssignIds &assign_ids, + common::ObIArray &assign_exprs); +private: + static int init_datum_param_store(ObTableCtx &ctx, + int64_t capacity); + static int refresh_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &exprs, + const ObTableEntity &entity); + static int refresh_rowkey_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &exprs, + const common::ObIArray &rowkey); + static int refresh_properties_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &exprs, + const ObTableEntity &entity); + static int refresh_assign_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &old_rows, + const common::ObIArray &new_rows, + const common::ObIArray &full_assign_rows, + const ObTableCtx::ObAssignIds &assign_ids, + const ObTableEntity &entity); + static int refresh_delta_exprs_frame(ObTableCtx &ctx, + const common::ObIArray &delta_exprs, + const ObTableEntity &entity); + static int generate_full_assign_raw_exprs(ObTableCtx &ctx); +private: + // 通过column_name在表达式数组获取列引用表达式 + static ObRawExpr* get_ref_raw_expr(const common::ObIArray &all_exprs, + const common::ObString &col_name); + // 构造生成列表达式 + static int build_generated_column_expr(ObTableCtx &ctx, + sql::ObColumnRefRawExpr &col_expr, + const common::ObString &expr_str, + const common::ObIArray &exprs); + // 处理生成列表达式 + static int resolve_generated_column_expr(ObTableCtx &ctx); + // 构造列引用原生表达式 + static int generate_column_ref_raw_expr(ObTableCtx &ctx, + const ObColumnSchemaV2 &col_schema, + sql::ObColumnRefRawExpr *&col_ref_expr); +private: + DISALLOW_COPY_AND_ASSIGN(ObTableExprCgService); +}; + +class ObTableLocCgService +{ +public: + static int generate_table_loc_meta(const ObTableCtx &ctx, + sql::ObDASTableLocMeta &loc_meta, + bool is_lookup = false); +private: + DISALLOW_COPY_AND_ASSIGN(ObTableLocCgService); +}; + +class ObTableDmlCgService +{ +public: + static int generate_insert_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableInsCtDef &ins_ctdef); + static int generate_update_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObTableCtx::ObAssignIds &assign_ids, + ObTableUpdCtDef &upd_ctdef); + static int generate_delete_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableDelCtDef &del_ctdef); + static int generate_replace_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableReplaceCtDef &replace_ctdef); + static int generate_insert_up_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const ObTableCtx::ObAssignIds &assign_ids, + ObTableInsUpdCtDef &ins_up_ctdef); + static int generate_lock_ctdef(ObTableCtx &ctx, + ObTableLockCtDef &lock_ctdef); + static int generate_conflict_checker_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + sql::ObConflictCheckerCtdef &conflict_checker_ctdef); +private: + static int generate_base_ctdef(ObTableCtx &ctx, + ObTableDmlBaseCtDef &base_ctdef, + common::ObIArray &old_row, + common::ObIArray &new_row); + static int generate_column_ids(const common::ObIArray &exprs, + common::ObIArray &column_ids); + static int generate_das_ins_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + sql::ObDASInsCtDef &das_ins_ctdef, + const common::ObIArray &new_row); + static int generate_das_upd_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + const common::ObIArray &assign_exprs, + sql::ObDASUpdCtDef &das_upd_ctdef, + const common::ObIArray &old_row, + const common::ObIArray &new_row, + const common::ObIArray &full_row); + static int generate_das_del_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + sql::ObDASDelCtDef &das_del_ctdef, + const common::ObIArray &old_row); + static int generate_das_lock_ctdef(ObTableCtx &ctx, + uint64_t index_tid, + sql::ObDASLockCtDef &das_lock_ctdef, + const common::ObIArray &old_row); + static int generate_updated_column_ids(const common::ObIArray &assign_exprs, + const common::ObIArray &column_ids, + common::ObIArray &updated_column_ids); + static int generate_upd_assign_infos(ObTableCtx &ctx, + ObIAllocator &allocator, + const common::ObIArray &assign_exprs, + ObTableUpdCtDef &udp_ctdef); + static int generate_das_base_ctdef(uint64_t index_tid, + ObTableCtx &ctx, + sql::ObDASDMLBaseCtDef &base_ctdef); + static int generate_column_info(ObTableID index_tid, + ObTableCtx &ctx, + sql::ObDASDMLBaseCtDef &base_ctdef); + static int convert_table_param(ObTableCtx &ctx, + sql::ObDASDMLBaseCtDef &base_ctdef); + static int generate_projector(const common::ObIArray &dml_column_ids, + const common::ObIArray &storage_column_ids, + const common::ObIArray &old_row, + const common::ObIArray &new_row, + const common::ObIArray &full_row, + sql::ObDASDMLBaseCtDef &das_ctdef); + static int generate_related_ins_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const common::ObIArray &new_row, + sql::DASInsCtDefArray &ins_ctdefs); + static int generate_related_upd_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const common::ObIArray &assign_exprs, + const common::ObIArray &old_row, + const common::ObIArray &new_row, + const common::ObIArray &full_row, + sql::DASUpdCtDefArray &upd_ctdefs); + static int generate_related_del_ctdef(ObTableCtx &ctx, + ObIAllocator &allocator, + const common::ObIArray &old_row, + sql::DASDelCtDefArray &del_ctdefs); + static int get_rowkey_exprs(ObTableCtx &ctx, + common::ObIArray &rowkey_exprs); + static int generate_table_rowkey_info(ObTableCtx &ctx, + ObTableInsCtDef &ins_ctdef); + static int generate_tsc_ctdef(ObTableCtx &ctx, + sql::ObDASScanCtDef &tsc_ctdef); + static int generate_single_constraint_info(ObTableCtx &ctx, + const share::schema::ObTableSchema &index_schema, + const uint64_t table_id, + sql::ObUniqueConstraintInfo &constraint_info); + static int generate_constraint_infos(ObTableCtx &ctx, + common::ObIArray &cst_infos); + static int generate_constraint_ctdefs(ObTableCtx &ctx, + ObIAllocator &allocator, + sql::ObRowkeyCstCtdefArray &cst_ctdefs); +private: + DISALLOW_COPY_AND_ASSIGN(ObTableDmlCgService); +}; + +class ObTableSpecCgService +{ +public: + // given operation type, generate spec tree + template + static int generate(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec); +public: + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiScanSpec &spec); + + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiInsertSpec &spec); + + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiUpdateSpec &spec); + + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiDelSpec &spec); + + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiInsertUpSpec &spec); + + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiReplaceSpec &spec); + + static int generate_spec(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiLockSpec &spec); + +private: + template + static int generate_with_child(common::ObIAllocator &alloc, + ObTableCtx &ctx, + ObTableApiSpec *&root_spec); + +private: + DISALLOW_COPY_AND_ASSIGN(ObTableSpecCgService); +}; + +class ObTableTscCgService +{ +public: + ObTableTscCgService() {} + virtual ~ObTableTscCgService() {} + static int generate_tsc_ctdef(const ObTableCtx &ctx, + ObIAllocator &allocator, + ObTableApiScanCtDef &tsc_ctdef); + static int generate_table_loc_meta(const ObTableCtx &ctx, + sql::ObDASTableLocMeta &loc_meta, + bool is_lookup); + static int generate_das_result_output(sql::ObDASScanCtDef &das_tsc_ctdef, + const common::ObIArray &output_cids); +private: + static int generate_das_tsc_ctdef(const ObTableCtx &ctx, + ObIAllocator &allocator, + sql::ObDASScanCtDef &das_tsc_ctdef); + static int generate_access_ctdef(const ObTableCtx &ctx, + ObIAllocator &allocator, + sql::ObDASScanCtDef &das_tsc_ctdef); + static int generate_table_param(const ObTableCtx &ctx, + sql::ObDASScanCtDef &das_tsc_ctdef); + static OB_INLINE bool is_in_array(const common::ObIArray &array, + const sql::ObRawExpr *expr) + { + bool is_in = false; + for (int64_t i = 0; i < array.count() && !is_in; i++) { + if (array.at(i) == expr) { + is_in = true; + } + } + return is_in; + } + static int generate_rt_exprs(const ObTableCtx &ctx, + ObIAllocator &allocator, + const common::ObIArray &src, + common::ObIArray &dst); +private: + DISALLOW_COPY_AND_ASSIGN(ObTableTscCgService); +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_CG_SERVICE_H_ */ \ No newline at end of file diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp new file mode 100644 index 0000000000..44aa63648e --- /dev/null +++ b/src/observer/table/ob_table_context.cpp @@ -0,0 +1,1035 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SERVER +#include "ob_table_context.h" +#include "ob_table_cg_service.h" // for generate_table_loc_meta +#include "sql/das/ob_das_define.h" // for ObDASTableLocMeta +#include "lib/utility/utility.h" +#include "ob_table_service.h" +namespace oceanbase +{ +namespace table +{ + +int ObTableCtx::get_tablet_by_rowkey(const ObRowkey &rowkey, + ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + ObSEArray rowkeys; + ObSEArray tablet_ids; + ObSEArray part_ids; + SMART_VAR(sql::ObTableLocation, location_calc) { + const uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(rowkeys.push_back(rowkey))) { + LOG_WARN("fail to push back rowkey", K(ret), K(rowkey)); + } else if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (OB_FAIL(location_calc.calculate_partition_ids_by_rowkey(get_session_info(), + schema_guard_, + table_schema_->get_table_id(), + rowkeys, + tablet_ids, + part_ids))) { + LOG_WARN("fail to calc partition id", K(ret)); + } else if (1 != tablet_ids.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("should have one tablet", K(ret), K(tablet_ids)); + } else { + tablet_id = tablet_ids.at(0); + } + } + return ret; +} + +int ObTableCtx::init_sess_info(uint64_t tenant_id, const ObString &tenant_name, uint64_t user_id) +{ + int ret = OB_SUCCESS; + + // try get session from session pool + if (OB_FAIL(GCTX.table_service_->get_sess_mgr().get_sess_info(tenant_id, + user_id, + sess_guard_))) { + LOG_WARN("fail to get session info", K(ret), K(tenant_id), K(user_id)); + } else if (OB_ISNULL(sess_guard_.get_sess_node_val())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info is null", K(ret), K(user_id)); + } + + return ret; +} + +int ObTableCtx::init_common(const ObTableApiCredential &credential, + const common::ObTabletID &arg_tablet_id, + const common::ObString &arg_table_name, + const int64_t &timeout_ts) +{ + int ret = OB_SUCCESS; + bool is_cache_hit = false; + const ObTenantSchema *tenant_schema = nullptr; + const uint64_t tenant_id = credential.tenant_id_; + const uint64_t database_id = credential.database_id_; + const uint64_t user_id = credential.user_id_; + ObTabletID tablet_id = arg_tablet_id; + + if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard_))) { + LOG_WARN("fail to get schema guard", K(ret), K(tenant_id), K(arg_table_name)); + } else if (OB_FAIL(schema_guard_.get_table_schema(tenant_id, + database_id, + arg_table_name, + false, /* is_index */ + table_schema_))) { + LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(database_id), K(arg_table_name)); + } else if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNKNOWN_TABLE; + LOG_WARN("fail get table schema by table name", K(ret), K(tenant_id), K(database_id), K(arg_table_name)); + } else if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id, tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K(tenant_id)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("tenant schema is null", K(ret)); + } else if (OB_FAIL(init_sess_info(tenant_id, tenant_schema->get_tenant_name_str(), user_id))) { + LOG_WARN("fail to init session info", K(ret), K(tenant_id), K(user_id)); + } else if (!arg_tablet_id.is_valid()) { + if (is_scan_) { // 扫描场景使用table_schema上的tablet id,客户端已经做了路由分发 + if (table_schema_->is_partitioned_table()) { // 不支持分区表 + ret = OB_NOT_SUPPORTED; + LOG_WARN("partitioned table not supported", K(ret), K(arg_table_name)); + } else { + tablet_id = table_schema_->get_tablet_id(); + } + } else { // dml场景使用rowkey计算出tablet id + if (!table_schema_->is_partitioned_table()) { + tablet_id = table_schema_->get_tablet_id(); + } else if (OB_ISNULL(entity_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else if (OB_FAIL(get_tablet_by_rowkey(entity_->get_rowkey(), tablet_id))) { + LOG_WARN("fail to get tablet id by rowkey", K(ret), K(entity_->get_rowkey())); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(GCTX.location_service_->get(tenant_id, + tablet_id, + 0, /* expire_renew_time */ + is_cache_hit, + ls_id_))) { + LOG_WARN("fail to get ls id", K(ret), K(tablet_id), K(arg_table_name)); + } else if (!is_scan_ && OB_FAIL(check_entity())) { + LOG_WARN("fail to check entity", K(ret)); + } else { + tenant_id_ = tenant_id; + database_id_ = database_id; + table_name_ = arg_table_name; + ref_table_id_ = table_schema_->get_table_id(); + index_table_id_ = ref_table_id_; + tablet_id_ = tablet_id; + index_tablet_id_ = tablet_id_; + tenant_schema_version_ = tenant_schema->get_schema_version(); + timeout_ts_ = timeout_ts; + } + + return ret; +} + +// 获取rowkey或者二级索引的columns type +int ObTableCtx::generate_columns_type(ObIArray &columns_type) +{ + int ret = OB_SUCCESS; + ObExprResType tmp_column_type; + const ObColumnSchemaV2 *column_schema = nullptr; + + if (is_index_scan_) { + const int64_t N = index_col_ids_.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + if (OB_ISNULL(column_schema = index_schema_->get_column_schema(index_col_ids_.at(i)))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(index_col_ids_.at(i))); + } else if (OB_FAIL(cons_column_type(*column_schema, tmp_column_type))) { + LOG_WARN("fail to cons column type", K(ret)); + } else if (OB_FAIL(columns_type.push_back(tmp_column_type))) { + LOG_WARN("fail to push back column type", K(ret), K(tmp_column_type)); + } + } + } else { // primary key + const ObRowkeyInfo &rowkey_info = table_schema_->get_rowkey_info(); + const int64_t N = rowkey_info.get_size(); + uint64_t column_id = OB_INVALID_ID; + for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { + LOG_WARN("failed to get column id", K(ret), K(i)); + } else if (OB_ISNULL(column_schema = table_schema_->get_column_schema(column_id))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(column_id)); + } else if (OB_FAIL(cons_column_type(*column_schema, tmp_column_type))) { + LOG_WARN("fail to cons column type", K(ret)); + } else if (OB_FAIL(columns_type.push_back(tmp_column_type))) { + LOG_WARN("fail to push back column type", K(ret), K(tmp_column_type)); + } + } + } + + return ret; +} + +int ObTableCtx::cons_column_type(const ObColumnSchemaV2 &column_schema, + ObExprResType &column_type) +{ + int ret = OB_SUCCESS; + column_type.set_type(column_schema.get_data_type()); + column_type.set_result_flag(ObRawExprUtils::calc_column_result_flag(column_schema)); + + if (ob_is_string_type(column_schema.get_data_type()) || ob_is_json(column_schema.get_data_type())) { + column_type.set_collation_type(column_schema.get_collation_type()); + column_type.set_collation_level(CS_LEVEL_IMPLICIT); + } else { + column_type.set_collation_type(CS_TYPE_BINARY); + column_type.set_collation_level(CS_LEVEL_NUMERIC); + } + const ObAccuracy &accuracy = column_schema.get_accuracy(); + column_type.set_accuracy(accuracy); + const bool is_zerofill = column_type.has_result_flag(ZEROFILL_FLAG); + if (is_zerofill) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("modifing column with ZEROFILL flag is not supported", K(ret), K(column_schema)); + } + + return ret; +} + +int ObTableCtx::check_column_type(const ObExprResType &column_type, + ObObj &obj) +{ + int ret = OB_SUCCESS; + const bool is_not_nullable = column_type.is_not_null_for_read(); + const ObCollationType cs_type = column_type.get_collation_type(); + + // 1. check nullable + if (is_not_nullable && obj.is_null()) { + ret = OB_BAD_NULL_ERROR; + } else if (obj.is_null()) { + // continue + } else if (column_type.get_type() != obj.get_type() + && !(ob_is_string_type(column_type.get_type()) && ob_is_string_type(obj.get_type()))) { + // 2. data type mismatch + ret = OB_OBJ_TYPE_ERROR; + LOG_WARN("object type mismatch with column type", K(ret), K(column_type), K(obj)); + } else { + // 3. check collation + if (!ob_is_string_type(obj.get_type())) { + // not string type, continue + } else { + if (cs_type == obj.get_collation_type()) { + // same collation type + } else if (cs_type == CS_TYPE_BINARY) { + // any collation type can be compatible with cs_type_binary + obj.set_collation_type(cs_type); + } else if (ObCharset::charset_type_by_coll(cs_type) == ObCharset::charset_type_by_coll(obj.get_collation_type())) { + // same charset, convert it + obj.set_collation_type(cs_type); + } else { + ret = OB_ERR_COLLATION_MISMATCH; + LOG_WARN("collation type mismatch with column", K(ret), K(column_type), K(obj)); + } + if (OB_SUCC(ret)) { + // convert obj type to the column type (char, varchar or text) + obj.set_type(column_type.get_type()); + } + } + // 4. check accuracy + if (OB_SUCC(ret)) { + if (OB_FAIL(ob_obj_accuracy_check_only(column_type.get_accuracy(), cs_type, obj))) { + LOG_WARN("accuracy check failed", K(ret), K(obj), K(column_type)); + } + } + } + + return ret; +} + +int ObTableCtx::check_rowkey(ObRowkey &rowkey) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + if (rowkey.get_obj_cnt() != table_schema_->get_rowkey_column_num()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity rowkey count mismatch table schema rowkey count", + K(ret), K(rowkey.get_obj_cnt()), K(table_schema_->get_rowkey_column_num())); + } else { + ObExprResType column_type; + const ObRowkeyInfo &rowkey_info = table_schema_->get_rowkey_info(); + const ObColumnSchemaV2 *col_schema = nullptr; + uint64_t column_id = OB_INVALID_ID; + ObObj *obj_ptr = rowkey.get_obj_ptr(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey.get_obj_cnt(); i++) { + if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { + LOG_WARN("fail to get column id", K(ret), K(i)); + } else if (OB_ISNULL(col_schema = table_schema_->get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column schema", K(ret), K(column_id)); + } else if (OB_FAIL(cons_column_type(*col_schema, column_type))) { + LOG_WARN("fali to construct column type", K(ret), K(column_id)); + } else if (OB_FAIL(check_column_type(column_type, obj_ptr[i]))) { + LOG_WARN("fali to check rowkey column type", K(ret), K(i), K(obj_ptr[i])); + } + } + } + } + + return ret; +} + +int ObTableCtx::check_properties(ObIArray> &properties) +{ + int ret = OB_SUCCESS; + bool is_get = (ObTableOperationType::Type::GET == operation_type_); + if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + const ObColumnSchemaV2 *col_schema = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < properties.count(); i++) { + ObString &col_name = properties.at(i).first; + ObObj &prop_obj = properties.at(i).second; + if (OB_ISNULL(col_schema = table_schema_->get_column_schema(col_name))) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("fail to get column schema", K(ret), K(col_name)); + } else if (is_get) { + // do nothing + } else { + ObExprResType column_type; + if (OB_FAIL(cons_column_type(*col_schema, column_type))) { + LOG_WARN("fail to construct column type", K(ret), K(*col_schema)); + } else if (OB_FAIL(check_column_type(column_type, prop_obj))) { + LOG_WARN("fail to check rowkey column type", K(ret), K(prop_obj)); + } + } + } + } + + return ret; +} + +int ObTableCtx::check_entity() +{ + int ret = OB_SUCCESS; + ObRowkey rowkey; + ObSEArray, 16> properties; + + if (OB_ISNULL(entity_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else if (FALSE_IT(rowkey = entity_->get_rowkey())) { + } else if (!is_htable() && OB_FAIL(check_rowkey(rowkey))) { + LOG_WARN("fail to check rowkey", K(ret)); + } else if (OB_FAIL(entity_->get_properties(properties))) { + LOG_WARN("fail to get properties name", K(ret)); + } else if (OB_FAIL(check_properties(properties))) { + LOG_WARN("fail to check properties", K(ret)); + } + + return ret; +} + +int ObTableCtx::generate_key_range(const ObIArray &scan_ranges) +{ + int ret = OB_SUCCESS; + int64_t padding_num = -1; + ObArray columns_type; + int64_t N = scan_ranges.count(); + + if (OB_FAIL(generate_columns_type(columns_type))) { + LOG_WARN("fail to generate columns type", K(ret)); + } else if (is_index_scan_) { + // 索引扫描场景下用户可能没有填写rowkey的key_range，需要加上 + padding_num = index_schema_->get_rowkey_column_num() - index_col_ids_.count(); + } + // check obj type in ranges + for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { // foreach range + const ObNewRange &range = scan_ranges.at(i); + // check column type + for (int64_t j = 0; OB_SUCCESS == ret && j < 2; ++j) { + const ObRowkey *p_key = nullptr; + if (0 == j) { + p_key = &range.get_start_key(); + } else { + p_key = &range.get_end_key(); + } + if (p_key->is_min_row() || p_key->is_max_row()) { + // do nothing + } else { + if (p_key->get_obj_cnt() != columns_type.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("wrong rowkey size", K(ret), K(i), K(j), K(*p_key), K(columns_type)); + } else { + const int64_t M = p_key->get_obj_cnt(); + for (int64_t k = 0; OB_SUCCESS == ret && k < M; ++k) { + ObObj &obj = const_cast(p_key->get_obj_ptr()[k]); + if (obj.is_min_value() || obj.is_max_value()) { + // do nothing + } else if (OB_FAIL(check_column_type(columns_type.at(k), obj))) { + LOG_WARN("fail to check column type", K(ret), K(columns_type.at(k)), K(obj)); + } + } + } + } + } + if (OB_UNLIKELY(padding_num > 0)) { + // index scan need fill primary key object + ObNewRange index_key_range = range; + for (int64_t j = 0; OB_SUCCESS == ret && j < 2; ++j) { + const ObRowkey *p_key = nullptr; + if (0 == j) { + p_key = &range.get_start_key(); + } else { + p_key = &range.get_end_key(); + } + if (p_key->is_min_row() || p_key->is_max_row()) { + // do nothing + } else { + const int64_t old_objs_num = p_key->get_obj_cnt(); + const int64_t new_objs_num = old_objs_num + padding_num; + ObObj *new_objs = static_cast(allocator_.alloc(sizeof(ObObj)*new_objs_num)); + if (OB_ISNULL(new_objs)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc new objs", K(ret)); + } else { + const ObObj *old_objs = p_key->get_obj_ptr(); + for (int64_t k = 0; k < old_objs_num; ++k) { + new_objs[k] = old_objs[k]; // shallow copy + } + if (0 == j) { // padding for startkey + for (int64_t k = 0; k < padding_num; ++k) { + new_objs[k+old_objs_num] = ObObj::make_min_obj(); + } + index_key_range.start_key_.assign(new_objs, new_objs_num); + } else { // padding for endkey + for (int64_t k = 0; k < padding_num; ++k) { + new_objs[k+old_objs_num] = ObObj::make_max_obj(); + } + index_key_range.end_key_.assign(new_objs, new_objs_num); + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(key_ranges_.push_back(index_key_range))) { + LOG_WARN("fail to push back key range", K(ret), K(index_key_range)); + } + } + } else { + if (OB_SUCC(ret)) { + if (OB_FAIL(key_ranges_.push_back(range))) { + LOG_WARN("fail to push back key range", K(ret), K(range)); + } + } + } + } + + return ret; +} + +int ObTableCtx::init_scan(const ObTableQuery &query, + const bool &is_wead_read) +{ + int ret = OB_SUCCESS; + const ObString &index_name = query.get_index_name(); + const ObIArray &select_columns = query.get_select_columns(); + const ObColumnSchemaV2 *column_schema = nullptr; + operation_type_ = ObTableOperationType::Type::SCAN; + // init is_weak_read_,scan_order_ + is_weak_read_ = is_wead_read; + scan_order_ = query.get_scan_order(); + // init limit_,offset_ + limit_ = query.get_limit(); + offset_ = query.get_offset(); + // init is_index_scan_,is_index_back_ + if (index_name.empty() || 0 == index_name.case_compare(ObIndexHint::PRIMARY_KEY)) { // scan with primary key + index_table_id_ = ref_table_id_; + is_index_back_ = false; + } else { + is_index_scan_ = true; + // init index_table_id_,index_schema_ + if (OB_FAIL(init_index_info(index_name))) { + LOG_WARN("fail to init index info", K(ret), K(index_name)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < select_columns.count() && !is_index_back_; i++) { + if (OB_ISNULL(column_schema = index_schema_->get_column_schema(select_columns.at(i)))) { + is_index_back_ = true; + } + } + // init index_col_ids_ + if (OB_SUCC(ret)) { + const common::ObIndexInfo &index_info = index_schema_->get_index_info(); + if (OB_FAIL(index_info.get_column_ids(index_col_ids_))) { + LOG_WARN("fail to get index column ids", K(ret), K(index_info)); + } + } + } + } + + // init key_ranges_ + if (OB_SUCC(ret)) { + if (OB_FAIL(generate_key_range(query.get_scan_ranges()))) { + LOG_WARN("fail to generate key ranges", K(ret)); + } else { + // select_col_ids用schema序 + for (ObTableSchema::const_column_iterator iter = table_schema_->column_begin(); + OB_SUCC(ret) && iter != table_schema_->column_end(); + ++iter) { + const ObColumnSchemaV2 *column_schema = *iter; + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is NULL", K(ret)); + } else if (has_exist_in_array(select_columns, column_schema->get_column_name_str())) { + if (OB_FAIL(select_col_ids_.push_back(column_schema->get_column_id()))) { + LOG_WARN("fail to add column id", K(ret)); + } else if (OB_FAIL(select_metas_.push_back(column_schema->get_meta_type()))) { + LOG_WARN("fail to add column meta", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (select_col_ids_.count() != select_columns.count() || + select_metas_.count() != select_columns.count()) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("select_col_ids or select_metas count is not equal to select_columns", + K(select_columns), + K(select_col_ids_), + K(select_metas_)); + } else { + // init query_col_ids_ + for (int64_t i = 0; OB_SUCC(ret) && i < select_columns.count(); i++) { + if (OB_ISNULL(column_schema = table_schema_->get_column_schema(select_columns.at(i)))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("select column not found in schema", K(ret), K(select_columns.at(i))); + } else if (OB_FAIL(query_col_ids_.push_back(column_schema->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret), K(column_schema->get_column_id())); + } + } + } + } + } + } + + return ret; +} + +int ObTableCtx::init_insert() +{ + return init_dml_related_tid(); +} + +int ObTableCtx::init_assign_ids(ObAssignIds &assign_ids, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + ObArray storage_col_ids; + + if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (OB_FAIL(table_schema_->get_column_ids(storage_col_ids))) { + LOG_WARN("fail to get column ids", K(ret)); + } else if (OB_FAIL(assign_ids.init(storage_col_ids.count()))) { + LOG_WARN("fail to init assign_ids capacity", K(ret), K(storage_col_ids.count())); + } else { + ObObj prop_value; // useless + const share::schema::ObColumnSchemaV2 *col_schema = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < storage_col_ids.count(); i++) { + ObString cname; + ObAssignId assign_id; + if (OB_ISNULL(col_schema = table_schema_->get_column_schema(storage_col_ids.at(i)))) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("column not exists", K(ret), K(cname)); + } else if (col_schema->is_stored_generated_column()) { + assign_id.column_id_ = storage_col_ids.at(i); + assign_id.idx_ = i; + if (OB_FAIL(assign_ids.push_back(assign_id))) { + LOG_WARN("fail to push back assign id", K(ret), K(assign_id)); + } + } else if (FALSE_IT(cname = col_schema->get_column_name_str())) { + // do nothing + } else if (OB_SUCCESS != entity.get_property(cname, prop_value)) { + // 这一列没有被更新，跳过 + } else { + assign_id.column_id_ = storage_col_ids.at(i); + assign_id.idx_ = i; + if (OB_FAIL(assign_ids.push_back(assign_id))) { + LOG_WARN("fail to push back assign id", K(ret), K(assign_id)); + } + } + } + } + + return ret; +} + +int ObTableCtx::init_update() +{ + int ret = OB_SUCCESS; + is_for_update_ = true; + const bool is_batch = OB_ISNULL(batch_op_) ? false : true; + const ObTableEntity *entity = nullptr; + ObAssignIds assign_ids(allocator_); + // 1. init assign_ids_ + if (OB_FAIL(init_assign_ids(assign_ids_, static_cast(*entity_)))) { + LOG_WARN("fail to init assign ids", K(ret), K(*entity_)); + } + + // 2. init scan + if (OB_SUCC(ret)) { + index_table_id_ = ref_table_id_; + is_index_scan_ = false; + is_index_back_ = false; + is_get_ = true; + scan_order_ = ObQueryFlag::Forward; + if (OB_FAIL(table_schema_->get_column_ids(select_col_ids_))) { // init select_col_ids_ + LOG_WARN("fail to get column ids", K(ret)); + } + } + + // 3. init related index table id + if (OB_SUCC(ret) && OB_FAIL(init_dml_related_tid())) { + LOG_WARN("fail to init dml related table ids", K(ret)); + } + + return ret; +} + +int ObTableCtx::init_delete() +{ + int ret = OB_SUCCESS; + // 1. init scan + const int64_t column_num = table_schema_->get_column_count(); + index_table_id_ = ref_table_id_; + is_index_scan_ = false; + is_index_back_ = false; + is_get_ = is_htable() ? false : true; + scan_order_ = ObQueryFlag::Forward; + + // 2. init select_col_ids_, select_metas_ + const ObColumnSchemaV2 *column_schema = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < column_num; i++) { + if (OB_ISNULL(column_schema = table_schema_->get_column_schema_by_idx(i))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema bu idx", K(ret), K(i)); + } else if (column_schema->is_generated_column()) { + // skip + } else if (OB_FAIL(select_col_ids_.push_back(column_schema->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret), K(column_schema->get_column_id())); + } else if (is_htable() && OB_FAIL(select_metas_.push_back(column_schema->get_meta_type()))) { + LOG_WARN("fail to push back meta type", K(ret), K(column_schema->get_meta_type())); + } + } + + // 3. init related index table id + if (OB_SUCC(ret) && OB_FAIL(init_dml_related_tid())) { + LOG_WARN("fail to init dml related table ids", K(ret)); + } + + return ret; +} + +int ObTableCtx::init_replace() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_dml_related_tid())) { + LOG_WARN("fail to init dml related tids", K(ret)); + } else { + void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); + } else { + ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); + phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 + phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); + exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); + } + } + + return ret; +} + +int ObTableCtx::init_insert_up() +{ + int ret = OB_SUCCESS; + is_for_insertup_ = true; + + if (OB_FAIL(init_update())) { + LOG_WARN("fail to init update", K(ret)); + } else { + void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); + } else { + ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); + phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 + phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); + exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); + } + } + + // reset for update flag + is_for_update_ = false; + return ret; +} + +int ObTableCtx::init_get() +{ + int ret = OB_SUCCESS; + // init scan + index_table_id_ = ref_table_id_; + is_index_scan_ = false; + is_index_back_ = false; + is_get_ = true; + scan_order_ = ObQueryFlag::Forward; + if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table scheam is null", K(ret)); + } else { + // init select_col_ids, select_metas_ + for (ObTableSchema::const_column_iterator iter = table_schema_->column_begin(); + OB_SUCC(ret) && iter != table_schema_->column_end(); + ++iter) { + const ObColumnSchemaV2 *column_schema = *iter; + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is NULL", K(ret)); + } else if (OB_FAIL(select_col_ids_.push_back(column_schema->get_column_id()))) { + LOG_WARN("fail to add column id", K(ret)); + } else if (OB_FAIL(select_metas_.push_back(column_schema->get_meta_type()))) { + LOG_WARN("fail to add column meta", K(ret)); + } else { /*do nothing*/ } + } + } + return ret; +} + +int ObTableCtx::init_append(bool return_affected_entity, bool return_rowkey) +{ + int ret = OB_SUCCESS; + return_affected_entity_ = return_affected_entity; + return_rowkey_ = return_rowkey; + ObSEArray, 8> properties; + + if (OB_FAIL(init_insert_up())) { + LOG_WARN("fail to init insert up", K(ret)); + } else if (OB_FAIL(entity_->get_properties(properties))) { + LOG_WARN("fail to get properties", K(ret)); + } else { + const int64_t N = properties.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + ObObj &delta = properties.at(i).second; + if (delta.is_null()) { + ret = OB_OBJ_TYPE_ERROR; + LOG_WARN("append NULL is illegal", K(ret), K(delta)); + } else if (OB_UNLIKELY(ObVarcharType != delta.get_type())) { + ret = OB_OBJ_TYPE_ERROR; + LOG_WARN("can only append varchar/varbinary type", K(ret), K(delta)); + } + } + } + + if (OB_SUCC(ret)) { + // 构造生成列表达式字符串"concat_ws('', `column_name`, `column_name`)" + const share::schema::ObColumnSchemaV2 *col_schema = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids_.count(); i++) { + uint64_t column_id = assign_ids_.at(i).column_id_; + ObString column_name; + if (OB_ISNULL(col_schema = table_schema_->get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column schema", K(ret), K(column_id)); + } else if (FALSE_IT(column_name = col_schema->get_column_name_str())) { + } else { + const int64_t total_len = 22 + column_name.length() + column_name.length(); + int64_t actual_len = -1; + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator_.alloc(total_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(total_len)); + } else if ((actual_len = snprintf(buf, total_len, "concat_ws('', `%s`, `%s`)", + column_name.ptr(), column_name.ptr())) < 0) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("fail to construct concat_ws expr string", K(ret), K(total_len)); + } else { + ObString expr_str(actual_len, buf); + if (OB_FAIL(expr_strs_.push_back(expr_str))) { + LOG_WARN("fail to push back expr str", K(ret)); + } + } + } + } + } + return ret; +} + +int ObTableCtx::init_increment(bool return_affected_entity, bool return_rowkey) +{ + int ret = OB_SUCCESS; + return_affected_entity_ = return_affected_entity; + return_rowkey_ = return_rowkey; + ObSEArray, 8> properties; + + if (OB_FAIL(init_insert_up())) { + LOG_WARN("fail to init insert up", K(ret)); + } else if (OB_FAIL(entity_->get_properties(properties))) { + LOG_WARN("fail to get properties", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < properties.count(); i++) { + ObObj &delta = properties.at(i).second; + if (!ob_is_int_tc(delta.get_type())) { + ret = OB_OBJ_TYPE_ERROR; + LOG_WARN("delta should only be signed integer type", K(ret), K(delta)); + } + } + } + + if (OB_SUCC(ret)) { + // 构造生成列表达式字符串"ifnull(C1, 0) + C1" + const share::schema::ObColumnSchemaV2 *col_schema = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids_.count(); i++) { + uint64_t column_id = assign_ids_.at(i).column_id_; + ObString column_name; + if (OB_ISNULL(col_schema = table_schema_->get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column schema", K(ret), K(column_id)); + } else if (FALSE_IT(column_name = col_schema->get_column_name_str())) { + } else { + const int64_t total_len = column_name.length() + column_name.length() + 16; + int64_t actual_len = -1; + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator_.alloc(total_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc memory", K(ret), K(total_len)); + } else if ((actual_len = snprintf(buf, total_len, "IFNULL(%s, 0) + %s", + column_name.ptr(), column_name.ptr())) < 0) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("fail to construct increment expr string", K(ret), K(total_len)); + } else { + ObString expr_str(actual_len, buf); + if (OB_FAIL(expr_strs_.push_back(expr_str))) { + LOG_WARN("fail to push back expr str", K(ret)); + } + } + } + } + } + return ret; +} + +int ObTableCtx::classify_scan_exprs() +{ + int ret = OB_SUCCESS; + const ObIArray *exprs = is_for_update_ ? &old_row_exprs_ : &all_exprs_.get_expr_array(); + int64_t exprs_cnt = exprs->count(); + + if (0 == exprs_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exprs is empty", K(ret)); + } else if (!select_exprs_.empty()) { + // had classify, do nothing + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < exprs_cnt; i++) { + if (T_REF_COLUMN == exprs->at(i)->get_expr_type()) { + ObColumnRefRawExpr *expr = static_cast(exprs->at(i)); + if (expr->is_rowkey_column() && OB_FAIL(rowkey_exprs_.push_back(expr))) { + LOG_WARN("fail to push back rowkey expr", K(ret)); + } else if (has_exist_in_array(select_col_ids_, expr->get_column_id()) + && OB_FAIL(select_exprs_.push_back(expr))) { + LOG_WARN("fail to push back select expr", K(ret)); + } else if (is_index_scan_ + && has_exist_in_array(index_col_ids_, expr->get_column_id()) + && OB_FAIL(index_exprs_.push_back(expr))) { + LOG_WARN("fail to push back index column expr", K(ret)); + } + } + } + } + + return ret; +} + +int ObTableCtx::init_exec_ctx() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_context(exec_ctx_.get_das_ctx()))) { + LOG_WARN("fail to init das context", K(ret)); + } else { + // init exec_ctx_.my_session_ + exec_ctx_.set_my_session(&get_session_info()); + } + return ret; +} + +// 初始化 das context 中的 table_loc 和 tablet_loc +int ObTableCtx::init_das_context(ObDASCtx &das_ctx) +{ + int ret = OB_SUCCESS; + ObDASTableLoc *table_loc = nullptr; + ObDASTabletLoc *tablet_loc = nullptr; + int64_t part_idx = OB_INVALID_ID; + int64_t subpart_idx = OB_INVALID_ID; + + if (OB_FAIL(ObTableLocCgService::generate_table_loc_meta(*this, loc_meta_))) { + LOG_WARN("fail to generate table location meta", K(ret)); + } else if (OB_FAIL(exec_ctx_.get_das_ctx().extended_table_loc(loc_meta_, table_loc))) { + LOG_WARN("fail to extend table loc", K(ret), K(loc_meta_)); + } else if (OB_ISNULL(table_schema_)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("table schema is null", K(ret)); + } else if (table_schema_->is_partitioned_table() && + OB_FAIL(table_schema_->get_part_idx_by_tablet(tablet_id_, part_idx, subpart_idx))) { + LOG_WARN("fail to get part idx by tablet", K(ret), K_(tablet_id)); + } else { + DASRelatedTabletMap &related_tablet_map = das_ctx.get_related_tablet_map(); + for (int64_t i = 0; OB_SUCC(ret) && i < related_index_ids_.count(); i++) { + ObTableID related_table_id = related_index_ids_.at(i); + const ObSimpleTableSchemaV2 *relative_table_schema = nullptr; + ObObjectID related_part_id = OB_INVALID_ID; + ObTabletID related_tablet_id; + if (OB_FAIL(schema_guard_.get_simple_table_schema(tenant_id_, + related_table_id, + relative_table_schema))) { + LOG_WARN("get_table_schema fail", K(ret), K(tenant_id_), K(related_table_id)); + } else if (OB_ISNULL(relative_table_schema)) { + ret = OB_SCHEMA_EAGAIN; + LOG_WARN("fail to get table schema", KR(ret), K(related_table_id)); + } else if (OB_FAIL(relative_table_schema->get_part_id_and_tablet_id_by_idx(part_idx, + subpart_idx, + related_part_id, + related_tablet_id))) { + LOG_WARN("get part by idx failed", K(ret), K(part_idx), K(subpart_idx), K(related_table_id)); + } else if (OB_FAIL(related_tablet_map.add_related_tablet_id(tablet_id_, + related_table_id, + related_tablet_id, + related_part_id))) { + LOG_WARN("fail to add related tablet id", K(ret), + K(tablet_id_), K(related_table_id), K(related_part_id), K(related_tablet_id)); + } + } + } + + if (OB_SUCC(ret) && OB_FAIL(exec_ctx_.get_das_ctx().extended_tablet_loc(*table_loc, + index_tablet_id_, + tablet_loc))) { + LOG_WARN("fail to extend tablet loc", K(ret), K(index_tablet_id_)); + } + + return ret; +} + +int ObTableCtx::init_trans(transaction::ObTxDesc *trans_desc, + const transaction::ObTxReadSnapshot &tx_snapshot) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(trans_desc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("trans desc is null", K(ret)); + } else { + get_session_info().get_tx_desc() = trans_desc; + // init tx_snapshot_ + tx_snapshot_ = tx_snapshot; + exec_ctx_.get_das_ctx().set_snapshot(tx_snapshot_); + } + + return ret; +} + +// for query +int ObTableCtx::init_index_info(const ObString &index_name) +{ + int ret = OB_SUCCESS; + uint64_t tids[OB_MAX_INDEX_PER_TABLE]; + int64_t index_cnt = OB_MAX_INDEX_PER_TABLE; + + if (OB_FAIL(schema_guard_.get_can_read_index_array(tenant_id_, + ref_table_id_, + tids, + index_cnt, + false))) { + LOG_WARN("fail to get can read index", K(ret), K_(tenant_id), K_(ref_table_id)); + } else if (index_cnt > OB_MAX_INDEX_PER_TABLE) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("index count is bigger than OB_MAX_INDEX_PER_TABLE", K(ret), K(index_cnt)); + } else { + const share::schema::ObTableSchema *index_schema = nullptr; + ObString this_index_name; + bool is_found = false; + for (int64_t i = 0; OB_SUCC(ret) && i < index_cnt && !is_found; i++) { + if (OB_FAIL(schema_guard_.get_table_schema(tenant_id_, tids[i], index_schema))) { + LOG_WARN("fail to get index schema", K(ret), K_(tenant_id), K(tids[i])); + } else if (OB_ISNULL(index_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("index schema is null", K(ret), K_(tenant_id), K(tids[i])); + } else if (OB_FAIL(index_schema->get_index_name(this_index_name))) { + LOG_WARN("fail to get index name", K(ret)); + } else if (0 == this_index_name.case_compare(index_name)) { + is_found = true; + index_table_id_ = tids[i]; + index_schema_ = index_schema; + index_tablet_id_ = index_schema->get_tablet_id(); + } + } + + if (OB_SUCC(ret) && !is_found) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid index name", K(ret), K(index_name)); + } + } + + return ret; +} + +int ObTableCtx::init_dml_related_tid() +{ + int ret = OB_SUCCESS; + uint64_t tids[OB_MAX_INDEX_PER_TABLE]; + int64_t index_cnt = OB_MAX_INDEX_PER_TABLE; + const ObTableSchema *index_schema = nullptr; + + if (OB_FAIL(schema_guard_.get_can_write_index_array(tenant_id_, + ref_table_id_, + tids, + index_cnt, + false /*only global*/))) { + LOG_WARN("fail to get can write index array", K(ret), K_(ref_table_id)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < index_cnt; i++) { + if (OB_FAIL(schema_guard_.get_table_schema(tenant_id_, tids[i], index_schema))) { + LOG_WARN("fail to get index schema", K(ret), K(tids[i]), K(i), K(index_cnt)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("null index schema", K(ret)); + } else if (index_schema->is_index_local_storage()) { + if (OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { + LOG_WARN("fail to add related index ids", K(ret), K(index_schema->get_table_id())); + } + } + } + } + + return ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_context.h b/src/observer/table/ob_table_context.h new file mode 100644 index 0000000000..bf8aeb6fec --- /dev/null +++ b/src/observer/table/ob_table_context.h @@ -0,0 +1,643 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_CONTEXT_H_ +#define OCEANBASE_OBSERVER_OB_TABLE_CONTEXT_H_ + +#include "sql/resolver/expr/ob_raw_expr.h" +#include "sql/resolver/expr/ob_raw_expr_util.h" +#include "sql/engine/dml/ob_dml_ctx_define.h" +#include "sql/das/ob_das_scan_op.h" // for ObDASScanRtDef +#include "share/table/ob_table.h" +#include "ob_table_session_pool.h" + +namespace oceanbase +{ +namespace table +{ + +enum ObTableExecutorType +{ + TABLE_API_EXEC_INVALID = 0, + TABLE_API_EXEC_SCAN = 1, + TABLE_API_EXEC_INSERT = 2, + TABLE_API_EXEC_DELETE = 3, + TABLE_API_EXEC_UPDATE = 4, + TABLE_API_EXEC_INSERT_UP = 5, + TABLE_API_EXEC_REPLACE = 6, + TABLE_API_EXEC_LOCK = 7, + // append new executor type here + TABLE_API_EXEC_MAX = 8 +}; + +// 1.用于存放整个process过程中需要的通用上下文信息 +// 2.在try_process()中进行初始化 +class ObTableCtx +{ +public: + struct ObAssignId { + ObAssignId() + : idx_(OB_INVALID_ID), + column_id_(OB_INVALID_ID) + {} + TO_STRING_KV("index", idx_, + "column_id", column_id_); + uint64_t idx_; + uint64_t column_id_; + }; + typedef common::ObFixedArray ObAssignIds; +public: + explicit ObTableCtx(common::ObIAllocator &allocator) + : allocator_(allocator), + expr_info_(nullptr), + exec_ctx_(allocator_), + expr_factory_(allocator_), + all_exprs_(false), + loc_meta_(allocator_), + assign_ids_(allocator_) + { + // common + is_init_ = false; + tenant_id_ = common::OB_INVALID_TENANT_ID; + database_id_ = common::OB_INVALID_ID; + ref_table_id_ = common::OB_INVALID_ID; + index_table_id_ = common::OB_INVALID_ID; + tablet_id_ = ObTabletID::INVALID_TABLET_ID; + index_tablet_id_ = ObTabletID::INVALID_TABLET_ID; + ls_id_ = share::ObLSID::INVALID_LS_ID; + timeout_ts_ = 0; + table_schema_ = nullptr; + // scan + is_scan_ = false; + is_index_scan_ = false; + is_index_back_ = false; + is_weak_read_ = false; + is_get_ = false; + index_schema_ = nullptr; + limit_ = -1; + offset_ = 0; + tenant_schema_version_ = -1; + is_for_update_ = false; + is_for_insertup_ = false; + entity_type_ = ObTableEntityType::ET_DYNAMIC; + entity_ = nullptr; + batch_op_ = nullptr; + return_affected_entity_ = false; + return_rowkey_ = false; + } + virtual ~ObTableCtx() + {} + TO_STRING_KV("is_init", is_init_, + "tenant_id", tenant_id_, + "database_id", database_id_, + "table_name", table_name_, + "ref_table_id", ref_table_id_, + "index_table_id", index_table_id_, + "tablet_id", tablet_id_, + "index_tablet_id", index_tablet_id_, + "ls_id", ls_id_, + "tenant_schema_version", tenant_schema_version_, + // scan to string + "is_scan", is_scan_, + "is_index_scan", is_index_scan_, + "is_index_back", is_index_back_, + "is_weak_read", is_weak_read_, + "is_get_", is_get_, + "limit", limit_, + "offset", offset_, + // update to string + "is_update", is_for_update_, + // insert up to string + "is_for_insertup", is_for_insertup_, + "entity_type", entity_type_); +public: + //////////////////////////////////////// getter //////////////////////////////////////////////// + // for common + OB_INLINE common::ObIAllocator& get_allocator() { return allocator_; } + OB_INLINE common::ObIAllocator& get_allocator() const { return allocator_; } + OB_INLINE uint64_t get_tenant_id() const { return tenant_id_; } + OB_INLINE uint64_t get_database_id() const { return database_id_; } + OB_INLINE const common::ObString& get_table_name() const { return table_name_; } + OB_INLINE common::ObTableID &get_table_id() { return index_table_id_; } + OB_INLINE common::ObTableID get_ref_table_id() const { return ref_table_id_; } + OB_INLINE common::ObTableID get_index_table_id() const { return index_table_id_; } + OB_INLINE common::ObTabletID get_tablet_id() const { return tablet_id_; } + OB_INLINE common::ObTabletID& get_tablet_id() { return tablet_id_; } + OB_INLINE common::ObTabletID get_index_tablet_id() const { return index_tablet_id_; } + OB_INLINE share::ObLSID get_ls_id() const { return ls_id_; } + OB_INLINE share::ObLSID& get_ls_id() { return ls_id_; } + OB_INLINE int64_t get_timeout_ts() const { return timeout_ts_; } + OB_INLINE const share::schema::ObTableSchema* get_table_schema() const { return table_schema_; } + OB_INLINE const share::schema::ObSchemaGetterGuard& get_schema_guard() const { return schema_guard_; } + OB_INLINE share::schema::ObSchemaGetterGuard& get_schema_guard() { return schema_guard_; } + OB_INLINE sql::ObExprFrameInfo* get_expr_frame_info() { return expr_info_; } + OB_INLINE const sql::ObExprFrameInfo* get_expr_frame_info() const { return expr_info_; } + OB_INLINE sql::ObExecContext& get_exec_ctx() { return exec_ctx_; } + OB_INLINE sql::ObRawExprFactory& get_expr_factory() { return expr_factory_; } + OB_INLINE sql::ObRawExprUniqueSet& get_all_exprs() { return all_exprs_; } + OB_INLINE sql::ObSQLSessionInfo& get_session_info() + { return sess_guard_.get_sess_info();} + OB_INLINE const sql::ObSQLSessionInfo& get_session_info() const + { return sess_guard_.get_sess_info(); } + OB_INLINE int64_t get_tenant_schema_version() const { return tenant_schema_version_; } + OB_INLINE const transaction::ObTxReadSnapshot& get_tx_snapshot() const { return tx_snapshot_; } + OB_INLINE transaction::ObTxReadSnapshot& get_tx_snapshot() { return tx_snapshot_; } + OB_INLINE ObTableOperationType::Type get_opertion_type() const { return operation_type_; } + OB_INLINE bool is_init() const { return is_init_; } + // for scan + OB_INLINE bool is_scan() const { return is_scan_; } + OB_INLINE bool is_index_scan() const { return is_index_scan_; } + OB_INLINE bool is_weak_read() const { return is_weak_read_; } + OB_INLINE bool is_index_back() const { return is_index_back_; } + OB_INLINE bool is_get() const { return is_get_; } + OB_INLINE common::ObQueryFlag::ScanOrder get_scan_order() const { return scan_order_; } + OB_INLINE const ObIArray& get_select_metas() const { return select_metas_; } + OB_INLINE const ObIArray& get_select_exprs() const { return select_exprs_; } + OB_INLINE const ObIArray& get_rowkey_exprs() const { return rowkey_exprs_; } + OB_INLINE const ObIArray& get_index_exprs() const { return index_exprs_; } + OB_INLINE const share::schema::ObTableSchema* get_index_schema() const { return index_schema_; } + OB_INLINE int64_t get_limit() const { return limit_; } + OB_INLINE int64_t get_offset() const { return offset_; } + OB_INLINE const common::ObIArray& get_key_ranges() const { return key_ranges_; } + OB_INLINE common::ObIArray& get_key_ranges() { return key_ranges_; } + OB_INLINE const common::ObIArray& get_select_col_ids() const { return select_col_ids_; } + OB_INLINE const common::ObIArray& get_query_col_ids() const { return query_col_ids_; } + // for update + OB_INLINE bool is_for_update() const { return is_for_update_; } + OB_INLINE const common::ObIArray& get_expr_strs() const { return expr_strs_; } + OB_INLINE bool is_inc_or_append() const + { + return ObTableOperationType::Type::APPEND == operation_type_ + || ObTableOperationType::Type::INCREMENT == operation_type_; + } + OB_INLINE ObIArray& get_old_row_exprs() { return old_row_exprs_; } + OB_INLINE const ObIArray& get_old_row_exprs() const { return old_row_exprs_; } + OB_INLINE ObIArray& get_full_assign_exprs() { return full_assign_exprs_; } + OB_INLINE ObIArray& get_delta_exprs() { return delta_exprs_; } + OB_INLINE const ObAssignIds& get_assign_ids() const { return assign_ids_; } + // for dml + OB_INLINE const ObIArray& get_related_index_ids() const { return related_index_ids_; } + OB_INLINE bool is_for_insertup() const { return is_for_insertup_; } + OB_INLINE const ObITableEntity* get_entity() const { return entity_; } + OB_INLINE ObTableEntityType get_entity_type() const { return entity_type_; } + OB_INLINE bool is_htable() const { return ObTableEntityType::ET_HKV == entity_type_; } + // for htable + OB_INLINE const ObTableBatchOperation* get_batch_operation() const { return batch_op_; } + // for increment/append + OB_INLINE bool return_affected_entity() const { return return_affected_entity_;} + OB_INLINE bool return_rowkey() const { return return_rowkey_;} + + //////////////////////////////////////// setter //////////////////////////////////////////////// + // for common + OB_INLINE void set_init_flag(bool is_init) { is_init_ = is_init; } + OB_INLINE void set_tenant_id(const uint64_t &tenant_id) { tenant_id_ = tenant_id; } + OB_INLINE void set_database_id(const uint64_t &database_id) { database_id_ = database_id; } + OB_INLINE void set_table_name(const common::ObString &table_name) { table_name_ = table_name; } + OB_INLINE void set_ref_table_id(const common::ObTableID &table_id) { ref_table_id_ = table_id; } + OB_INLINE void set_index_table_id(const common::ObTableID &index_tid) { index_table_id_ = index_tid; } + OB_INLINE void set_tablet_id(const common::ObTabletID &tablet_id) { tablet_id_ = tablet_id; } + OB_INLINE void set_index_tablet_id(const common::ObTabletID &tablet_id) { index_tablet_id_ = tablet_id; } + OB_INLINE void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } + OB_INLINE void set_timeout_ts(const int64_t &timeout_ts) { timeout_ts_ = timeout_ts; } + OB_INLINE void set_tenant_schema_version(const int64_t &version) { tenant_schema_version_ = version; } + OB_INLINE void set_tx_snapshot(const transaction::ObTxReadSnapshot &snapshot) { tx_snapshot_ = snapshot; } + OB_INLINE void set_expr_info(ObExprFrameInfo *expr_info) { expr_info_ = expr_info; } + // for scan + OB_INLINE void set_scan(const bool &is_scan) { is_scan_ = is_scan; } + OB_INLINE void set_index_scan(const bool &is_index_scan) { is_index_scan_ = is_index_scan; } + OB_INLINE void set_weak_read(const bool &is_weak_read) { is_weak_read_ = is_weak_read; } + OB_INLINE void set_index_back(const bool &is_index_back) { is_index_back_ = is_index_back; } + OB_INLINE void set_get(const bool &is_get) { is_get_ = is_get; } + OB_INLINE void set_scan_order(const common::ObQueryFlag::ScanOrder &scan_order) { scan_order_ = scan_order; } + OB_INLINE void set_limit(const int64_t &limit) { limit_ = limit; } + OB_INLINE void set_offset(const int64_t &offset) { offset_ = offset; } + // for dml + OB_INLINE void set_for_update(const bool &is_for_update) { is_for_update_ = is_for_update; } + OB_INLINE void set_entity(const ObITableEntity *entity) { entity_ = entity; } + OB_INLINE void set_entity_type(const ObTableEntityType &type) { entity_type_ = type; } + OB_INLINE void set_operation_type(const ObTableOperationType::Type op_type) { operation_type_ = op_type; } + // for htable + OB_INLINE void set_batch_operation(const ObTableBatchOperation *batch_op) { batch_op_ = batch_op; } + +public: + // 初始化common部分(不包括expr_info_, exec_ctx_, all_exprs_) + int init_common(const ObTableApiCredential &credential, + const common::ObTabletID &arg_tablet_id, + const common::ObString &arg_table_name, + const int64_t &timeout_ts); + // 初始化 insert 相关 + int init_insert(); + // 初始化scan相关(不包括表达分类) + int init_scan(const ObTableQuery &query, + const bool &is_wead_read); + // 初始化update相关 + int init_update(); + // 初始化delete相关 + int init_delete(); + // 初始化replace相关 + int init_replace(); + // 初始化insert_up相关 + int init_insert_up(); + // 初始化get相关 + int init_get(); + // 初始化increment相关 + int init_increment(bool return_affected_entity, bool return_rowkey); + // 初始化append相关 + int init_append(bool return_affected_entity, bool return_rowkey); + // 分类扫描相关表达式 + int classify_scan_exprs(); + // 初始化exec_ctx_和exec_ctx_.das_ctx_ + int init_exec_ctx(); + // init exec_ctx_.my_session_.tx_desc_ + int init_trans(transaction::ObTxDesc *trans_desc, + const transaction::ObTxReadSnapshot &tx_snapshot); + // insert 表达式 + int classify_insert_exprs(); + int init_das_context(ObDASCtx &das_ctx); +private: + // for common + int get_tablet_by_rowkey(const common::ObRowkey &rowkey, + common::ObTabletID &tablet_id); + int init_sess_info(uint64_t tenant_id, const common::ObString &tenant_name, uint64_t user_id); + // for scan + int init_index_info(const common::ObString &index_name); + int generate_columns_type(common::ObIArray &columns_type); + int generate_key_range(const common::ObIArray &scan_ranges); + // for dml + int init_dml_related_tid(); + // for update + int init_assign_ids(ObAssignIds &assign_ids, + const ObTableEntity &entity); +private: + int cons_column_type(const share::schema::ObColumnSchemaV2 &column_schema, + sql::ObExprResType &column_type); + int check_column_type(const ObExprResType &column_type, ObObj &obj); + int check_rowkey(ObRowkey &rowkey); + int check_properties(ObIArray> &properties); + int check_entity(); +private: + bool is_init_; + common::ObIAllocator &allocator_; + uint64_t tenant_id_; + uint64_t database_id_; + common::ObString table_name_; + common::ObTableID ref_table_id_; + common::ObTableID index_table_id_; + common::ObTabletID tablet_id_; + common::ObTabletID index_tablet_id_; + share::ObLSID ls_id_; + int64_t timeout_ts_; + const share::schema::ObTableSchema *table_schema_; + share::schema::ObSchemaGetterGuard schema_guard_; + sql::ObExprFrameInfo *expr_info_; + sql::ObExecContext exec_ctx_; + sql::ObRawExprFactory expr_factory_; + sql::ObRawExprUniqueSet all_exprs_; + ObTableApiSessGuard sess_guard_; + sql::ObDASTableLocMeta loc_meta_; + int64_t tenant_schema_version_; + transaction::ObTxReadSnapshot tx_snapshot_; + // for scan + bool is_scan_; + bool is_index_scan_; + bool is_index_back_; + bool is_weak_read_; + bool is_get_; + common::ObQueryFlag::ScanOrder scan_order_; + common::ObArray select_exprs_; + common::ObArray rowkey_exprs_; + common::ObArray index_exprs_; + common::ObArray select_metas_; + common::ObArray select_col_ids_; // 基于schema序的select column id + common::ObArray query_col_ids_; // 用户查询的select column id + common::ObArray index_col_ids_; + const share::schema::ObTableSchema *index_schema_; + int64_t offset_; + int64_t limit_; + common::ObSEArray key_ranges_; + // for update + bool is_for_update_; + ObTableOperationType::Type operation_type_; + common::ObArray old_row_exprs_; + common::ObArray full_assign_exprs_; + ObAssignIds assign_ids_; + // for increment/append + common::ObSEArray expr_strs_; + common::ObArray delta_exprs_; // for increment/append + bool return_affected_entity_; + bool return_rowkey_; + // for dml + common::ObSEArray related_index_ids_; + bool is_for_insertup_; + ObTableEntityType entity_type_; + const ObITableEntity *entity_; + // for htable + const ObTableBatchOperation *batch_op_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableCtx); +}; + +struct ObTableDmlBaseCtDef +{ +public: + virtual ~ObTableDmlBaseCtDef() = default; + VIRTUAL_TO_STRING_KV(K_(column_ids), + K_(old_row), + K_(new_row)); + + UIntFixedArray column_ids_; + ExprFixedArray old_row_; + ExprFixedArray new_row_; +protected: + ObTableDmlBaseCtDef(common::ObIAllocator &alloc) + : column_ids_(alloc), + old_row_(alloc), + new_row_(alloc) + { + } +}; + +struct ObTableApiScanCtDef +{ +public: + ObTableApiScanCtDef(common::ObIAllocator &allocator) + : scan_ctdef_(allocator), + lookup_ctdef_(nullptr), + lookup_loc_meta_(nullptr), + allocator_(allocator) + { + } + const sql::ExprFixedArray &get_das_output_exprs() const + { + return lookup_ctdef_ != nullptr ? lookup_ctdef_->result_output_ : scan_ctdef_.result_output_; + } + const sql::UIntFixedArray &get_full_acccess_cids() const + { + return lookup_ctdef_ != nullptr ? + lookup_ctdef_->access_column_ids_ : + scan_ctdef_.access_column_ids_; + } + TO_STRING_KV(K_(scan_ctdef), + KPC_(lookup_ctdef), + KPC_(lookup_loc_meta)); + sql::ObDASScanCtDef scan_ctdef_; + sql::ObDASScanCtDef *lookup_ctdef_; + sql::ObDASTableLocMeta *lookup_loc_meta_; + common::ObIAllocator &allocator_; +}; + +struct ObTableApiScanRtDef +{ + ObTableApiScanRtDef(common::ObIAllocator &allocator) + : scan_rtdef_(), + lookup_rtdef_(nullptr) + { + } + TO_STRING_KV(K_(scan_rtdef), + KPC_(lookup_rtdef)); + sql::ObDASScanRtDef scan_rtdef_; + sql::ObDASScanRtDef *lookup_rtdef_; +}; + +struct ObTableDmlBaseRtDef +{ + virtual ~ObTableDmlBaseRtDef() = default; + VIRTUAL_TO_STRING_KV(K_(cur_row_num)); + int64_t cur_row_num_; +protected: + ObTableDmlBaseRtDef() + : cur_row_num_(0) + { + } +}; + +struct ObTableInsCtDef : ObTableDmlBaseCtDef +{ +public: + ObTableInsCtDef(common::ObIAllocator &alloc) + : ObTableDmlBaseCtDef(alloc), + das_ctdef_(alloc), + related_ctdefs_(alloc), + column_infos_(alloc), + alloc_(alloc) + { + } + TO_STRING_KV(K_(das_ctdef), + K_(related_ctdefs)); + ObDASInsCtDef das_ctdef_; + sql::DASInsCtDefArray related_ctdefs_; + ColContentFixedArray column_infos_; + common::ObIAllocator &alloc_; +}; + +struct ObTableInsRtDef : ObTableDmlBaseRtDef +{ + ObTableInsRtDef() + : ObTableDmlBaseRtDef(), + das_rtdef_(), + related_rtdefs_() + { + } + TO_STRING_KV(K_(das_rtdef), + K_(related_rtdefs)); + ObDASInsRtDef das_rtdef_; + sql::DASInsRtDefArray related_rtdefs_; +}; + +struct ObTableUpdCtDef : ObTableDmlBaseCtDef +{ +public: + ObTableUpdCtDef(common::ObIAllocator &alloc) + : ObTableDmlBaseCtDef(alloc), + full_row_(alloc), + full_assign_row_(alloc), + delta_exprs_(alloc), + das_ctdef_(alloc), + assign_columns_(alloc), + related_ctdefs_(alloc), + ddel_ctdef_(nullptr), + dins_ctdef_(nullptr), + related_del_ctdefs_(alloc), + related_ins_ctdefs_(alloc), + alloc_(alloc) + { + } + TO_STRING_KV(K_(full_row), + K_(full_assign_row), + K_(delta_exprs), + K_(das_ctdef), + K_(assign_columns), + K_(related_ctdefs)); + ExprFixedArray full_row_; + ExprFixedArray full_assign_row_; + ExprFixedArray delta_exprs_; // for increment/append + ObDASUpdCtDef das_ctdef_; + ColContentFixedArray assign_columns_; + DASUpdCtDefArray related_ctdefs_; + // for insert up begin + ObDASDelCtDef *ddel_ctdef_; + ObDASInsCtDef *dins_ctdef_; + DASDelCtDefArray related_del_ctdefs_; + DASInsCtDefArray related_ins_ctdefs_; + // for insert up end + common::ObIAllocator &alloc_; +}; + +struct ObTableUpdRtDef : ObTableDmlBaseRtDef +{ +public: + ObTableUpdRtDef() + : ObTableDmlBaseRtDef(), + das_rtdef_(), + related_rtdefs_(), + ddel_rtdef_(nullptr), + dins_rtdef_(nullptr), + related_del_rtdefs_(), + related_ins_rtdefs_(), + found_rows_(0) + { + } + TO_STRING_KV(K_(das_rtdef), + K_(related_rtdefs)); + ObDASUpdRtDef das_rtdef_; + DASUpdRtDefArray related_rtdefs_; + // for insert up begin + ObDASDelRtDef *ddel_rtdef_; + ObDASInsRtDef *dins_rtdef_; + DASDelRtDefArray related_del_rtdefs_; + DASInsRtDefArray related_ins_rtdefs_; + int64_t found_rows_; + // for insert up end +}; + +struct ObTableDelCtDef : ObTableDmlBaseCtDef +{ +public: + ObTableDelCtDef(common::ObIAllocator &alloc) + : ObTableDmlBaseCtDef(alloc), + das_ctdef_(alloc), + related_ctdefs_(alloc), + alloc_(alloc) + { + } + TO_STRING_KV(K_(das_ctdef), + K_(related_ctdefs)); + ObDASDelCtDef das_ctdef_; + DASDelCtDefArray related_ctdefs_; + common::ObIAllocator &alloc_; +}; + +struct ObTableDelRtDef : ObTableDmlBaseRtDef +{ +public: + ObTableDelRtDef() + : ObTableDmlBaseRtDef(), + das_rtdef_(), + related_rtdefs_() + { + } + TO_STRING_KV(K_(das_rtdef), + K_(related_rtdefs)); + ObDASDelRtDef das_rtdef_; + DASDelRtDefArray related_rtdefs_; +}; + +struct ObTableReplaceCtDef +{ +public: + ObTableReplaceCtDef(common::ObIAllocator &alloc) + : ins_ctdef_(alloc), + del_ctdef_(alloc), + alloc_(alloc) + { + } + TO_STRING_KV(K_(ins_ctdef), + K_(del_ctdef)); + ObTableInsCtDef ins_ctdef_; + ObTableDelCtDef del_ctdef_; + common::ObIAllocator &alloc_; +}; + +struct ObTableReplaceRtDef +{ +public: + ObTableReplaceRtDef() + : ins_rtdef_(), + del_rtdef_() + { + } + TO_STRING_KV(K_(ins_rtdef), + K_(del_rtdef)) + ObTableInsRtDef ins_rtdef_; + ObTableDelRtDef del_rtdef_; +}; + +struct ObTableInsUpdCtDef +{ +public: + ObTableInsUpdCtDef(common::ObIAllocator &alloc) + : ins_ctdef_(alloc), + upd_ctdef_(alloc), + alloc_(alloc) + { + } + TO_STRING_KV(K_(ins_ctdef), + K_(upd_ctdef)); + ObTableInsCtDef ins_ctdef_; + ObTableUpdCtDef upd_ctdef_; + common::ObIAllocator &alloc_; +}; + +struct ObTableInsUpdRtDef +{ +public: + ObTableInsUpdRtDef() + : ins_rtdef_(), + upd_rtdef_() + { + } + TO_STRING_KV(K_(ins_rtdef), + K_(upd_rtdef)) + ObTableInsRtDef ins_rtdef_; + ObTableUpdRtDef upd_rtdef_; +}; + +struct ObTableLockCtDef : ObTableDmlBaseCtDef +{ +public: + ObTableLockCtDef(common::ObIAllocator &alloc) + : ObTableDmlBaseCtDef(alloc), + das_ctdef_(alloc), + alloc_(alloc) + { + } + TO_STRING_KV(K_(das_ctdef)); + ObDASLockCtDef das_ctdef_; + common::ObIAllocator &alloc_; +}; + +struct ObTableLockRtDef : ObTableDmlBaseRtDef +{ +public: + ObTableLockRtDef() + : ObTableDmlBaseRtDef(), + das_rtdef_() + { + } + TO_STRING_KV(K_(das_rtdef)); + ObDASLockRtDef das_rtdef_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_CONTEXT_H_ */ \ No newline at end of file diff --git a/src/observer/table/ob_table_delete_executor.cpp b/src/observer/table/ob_table_delete_executor.cpp new file mode 100644 index 0000000000..069ed01437 --- /dev/null +++ b/src/observer/table/ob_table_delete_executor.cpp @@ -0,0 +1,175 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_delete_executor.h" +#include "ob_table_scan_executor.h" +#include "ob_htable_utils.h" +#include "ob_htable_filter_operator.h" +#include "ob_table_cg_service.h" + +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ + +int ObTableApiDeleteExecutor::open() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_del_rtdef(del_spec_.get_ctdef(), del_rtdef_))) { + LOG_WARN("fail to generate delete rtdef"); + } + + return ret; +} + +int ObTableApiDeleteExecutor::process_single_operation(const ObTableEntity *entity) +{ + int ret = OB_SUCCESS; + common::ObIArray &key_ranges = tb_ctx_.get_key_ranges(); + + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else { + ObRowkey rowkey = entity->get_rowkey(); + ObNewRange range; + // init key_ranges_ + key_ranges.reset(); + if (OB_FAIL(range.build_range(tb_ctx_.get_ref_table_id(), rowkey))) { + LOG_WARN("fail to build key range", K(ret), K_(tb_ctx), K(rowkey)); + } else if (OB_FAIL(key_ranges.push_back(range))) { + LOG_WARN("fail to push back key range", K(ret), K(range)); + } else { + clear_evaluated_flag(); + if (OB_FAIL(child_->open())) { + LOG_WARN("fail to open child executor", K(ret)); + } else if (OB_FAIL(child_->get_next_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } + } + } + return ret; +} + +int ObTableApiDeleteExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + if (cur_idx_ >= 1) { + // prevent the second time process operation if the first one successfully + ret = OB_ITER_END; + } else if (OB_FAIL(process_single_operation(entity))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to process single delete operation", K(ret)); + } + } + + return ret; +} + +int ObTableApiDeleteExecutor::del_rows_post_proc() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(submit_all_dml_task())) { + LOG_WARN("fail to execute all delete das task", K(ret)); + } else { + affected_rows_ = del_rtdef_.das_rtdef_.affected_rows_; + } + + return ret; +} + +int ObTableApiDeleteExecutor::delete_row_skip_scan() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(entity_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is NULL", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::refresh_delete_exprs_frame(tb_ctx_, + del_spec_.get_ctdef().old_row_, + static_cast(*entity_)))) { + LOG_WARN("fail to refresh delete exprs frame", K(ret)); + } else if (OB_FAIL(delete_row_to_das(del_spec_.get_ctdef(), del_rtdef_))) { + LOG_WARN("fail to delete row to das", K(ret)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(del_rows_post_proc())) { + LOG_WARN("fail to post process after delete row", K(ret)); + } + + return ret; +} + +int ObTableApiDeleteExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + + if (is_skip_scan()) { + if (OB_FAIL(delete_row_skip_scan())) { + LOG_WARN("fail to process delete", K(ret)); + } + } else { + while(OB_SUCC(ret)) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } else if (OB_FAIL(delete_row_to_das(del_spec_.get_ctdef(), del_rtdef_))) { + LOG_WARN("fail to delete row to das", K(ret)); + } + + int tmp_ret = ret; + if (OB_FAIL(child_->close())) { // 需要写到das后才close child算子，否则扫描的行已经被析构 + LOG_WARN("fail to close scan executor", K(ret)); + } + ret = OB_SUCC(tmp_ret) ? ret : tmp_ret; + cur_idx_++; + } + + if (OB_ITER_END == ret) { + if (OB_FAIL(del_rows_post_proc())) { + LOG_WARN("fail to post process after delete row", K(ret)); + } else { + ret = OB_ITER_END; + } + } + } + + return ret; +} + +int ObTableApiDeleteExecutor::close() +{ + int ret = OB_SUCCESS; + + if (!is_opened_) { + // do nothing + } else if (OB_FAIL(ObTableApiModifyExecutor::close())) { + LOG_WARN("fail to close ObTableApiModifyExecutor", K(ret)); + } + + return ret; +} + +} // namespace table +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_delete_executor.h b/src/observer/table/ob_table_delete_executor.h new file mode 100644 index 0000000000..44392f1627 --- /dev/null +++ b/src/observer/table/ob_table_delete_executor.h @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_DELETE_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_DELETE_EXECUTOR_H +#include "ob_table_modify_executor.h" +#include "ob_table_scan_executor.h" +#include "ob_table_context.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTableApiDelSpec : public ObTableApiModifySpec +{ +public: + ObTableApiDelSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + del_ctdef_(alloc) + { + } +public: + OB_INLINE const ObTableDelCtDef& get_ctdef() const { return del_ctdef_; } + OB_INLINE ObTableDelCtDef& get_ctdef() { return del_ctdef_; } +private: + ObTableDelCtDef del_ctdef_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiDelSpec); +}; + +class ObTableApiDeleteExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiDeleteExecutor(ObTableCtx &ctx, const ObTableApiDelSpec &spec) + : ObTableApiModifyExecutor(ctx), + entity_(nullptr), + is_skip_scan_(false), + del_spec_(spec), + del_rtdef_(), + cur_idx_(0) + { + } + ~ObTableApiDeleteExecutor() + { + if (OB_NOT_NULL(child_)) { + ObTableApiScanExecutor *scan_executor = static_cast(child_); + scan_executor->~ObTableApiScanExecutor(); + } + } +public: + virtual int open(); + virtual int get_next_row(); + virtual int close(); + OB_INLINE void set_entity(const ObITableEntity *entity) { entity_ = entity; } + OB_INLINE void set_skip_scan(const bool &is_skip_scan) { is_skip_scan_ = is_skip_scan; } + OB_INLINE int is_skip_scan() { return is_skip_scan_; } +private: + int get_next_row_from_child(); + int del_rows_post_proc(); + int process_single_operation(const ObTableEntity *entity); + int delete_row_skip_scan(); +private: + // for refresh expr frame + const ObITableEntity *entity_; + bool is_skip_scan_; + const ObTableApiDelSpec &del_spec_; + ObTableDelRtDef del_rtdef_; + int64_t cur_idx_; +}; + +} // namespace table +} // namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_DELETE_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_end_trans_cb.cpp b/src/observer/table/ob_table_end_trans_cb.cpp index c20bd2f4b8..082d445352 100644 --- a/src/observer/table/ob_table_end_trans_cb.cpp +++ b/src/observer/table/ob_table_end_trans_cb.cpp @@ -60,7 +60,7 @@ void ObTableExecuteEndTransCb::callback(int cb_param) if (OB_FAIL(response_sender_.response(cb_param))) { LOG_WARN("failed to send response", K(ret), K(cb_param)); } else { - LOG_DEBUG("yzfdebug async send execute response", K(cb_param)); + LOG_INFO("async send execute response", K(cb_param)); } this->destroy_cb_if_no_ref(); } diff --git a/src/observer/table/ob_table_execute_processor.cpp b/src/observer/table/ob_table_execute_processor.cpp index d1c034db8e..4d77d53374 100644 --- a/src/observer/table/ob_table_execute_processor.cpp +++ b/src/observer/table/ob_table_execute_processor.cpp @@ -18,6 +18,9 @@ #include "sql/optimizer/ob_table_location.h" // ObTableLocation #include "lib/stat/ob_session_stat.h" #include "storage/tx_storage/ob_access_service.h" +#include "ob_table_scan_executor.h" +#include "ob_table_cg_service.h" +#include "observer/ob_req_time_service.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -33,13 +36,13 @@ int ObTableRpcProcessorUtil::negate_htable_timestamp(table::ObITableEntity &enti int64_t val = 0; if (3 == entity.get_rowkey_size()) { if (OB_FAIL(entity.get_rowkey_value(2, T_val))) { - LOG_WARN("failed to get T from entity", K(ret), K(entity)); + LOG_WARN("fail to get T from entity", K(ret), K(entity)); } else if (OB_FAIL(T_val.get_int(val))) { LOG_WARN("invalid obj type for T", K(ret), K(T_val)); } else { T_val.set_int(-val); if (OB_FAIL(entity.set_rowkey_value(2, T_val))) { - LOG_WARN("failed to negate T value", K(ret)); + LOG_WARN("fail to negate T value", K(ret)); } else { LOG_DEBUG("[yzfdebug] nenative T value", K(ret), K(T_val)); } @@ -51,8 +54,8 @@ int ObTableRpcProcessorUtil::negate_htable_timestamp(table::ObITableEntity &enti //////////////////////////////////////////////////////////////// ObTableApiExecuteP::ObTableApiExecuteP(const ObGlobalContext &gctx) :ObTableRpcProcessor(gctx), - allocator_(ObModIds::TABLE_PROC), - get_ctx_(allocator_), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + tb_ctx_(allocator_), need_rollback_trans_(false), query_timeout_ts_(0) { @@ -87,14 +90,100 @@ int ObTableApiExecuteP::check_arg() int ObTableApiExecuteP::check_arg2() const { int ret = OB_SUCCESS; - if (arg_.returning_rowkey_ - || arg_.returning_affected_entity_) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("some options not supported yet", K(ret), - "returning_rowkey", arg_.returning_rowkey_, - "returning_affected_entity", arg_.returning_affected_entity_, - "operation_type", arg_.table_operation_.type()); + ObTableOperationType::Type op_type = arg_.table_operation_.type(); + + if (ObTableOperationType::Type::APPEND != op_type && + ObTableOperationType::Type::INCREMENT != op_type) { + if (arg_.returning_rowkey_ || arg_.returning_affected_entity_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("some options not supported yet", K(ret), + "returning_rowkey", arg_.returning_rowkey_, + "returning_affected_entity", arg_.returning_affected_entity_, + "operation_type", op_type); + } } + + return ret; +} + +int ObTableApiExecuteP::init_tb_ctx() +{ + int ret = OB_SUCCESS; + ObExprFrameInfo *expr_frame_info = nullptr; + ObTableOperationType::Type op_type = arg_.table_operation_.type(); + tb_ctx_.set_entity(&arg_.table_operation_.entity()); + tb_ctx_.set_operation_type(op_type); + + if (tb_ctx_.is_init()) { + LOG_INFO("tb ctx has been inited", K_(tb_ctx)); + } else if (OB_FAIL(tb_ctx_.init_common(credential_, + arg_.tablet_id_, + arg_.table_name_, + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), K(arg_.table_name_)); + } else { + switch(op_type) { + case ObTableOperationType::INSERT: { + if (OB_FAIL(tb_ctx_.init_insert())) { + LOG_WARN("fail to init insert ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::UPDATE: { + if (OB_FAIL(tb_ctx_.init_update())) { + LOG_WARN("fail to init update ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::DEL: { + if (OB_FAIL(tb_ctx_.init_delete())) { + LOG_WARN("fail to init delete ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::REPLACE: { + if (OB_FAIL(tb_ctx_.init_replace())) { + LOG_WARN("fail to init replace ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::INSERT_OR_UPDATE: { + if (OB_FAIL(tb_ctx_.init_insert_up())) { + LOG_WARN("fail to init insert up ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::APPEND: { + if (OB_FAIL(tb_ctx_.init_append(arg_.returning_affected_entity_, + arg_.returning_rowkey_))) { + LOG_WARN("fail to init append ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::INCREMENT: { + if (OB_FAIL(tb_ctx_.init_increment(arg_.returning_affected_entity_, + arg_.returning_rowkey_))) { + LOG_WARN("fail to init increment ctx", K(ret), K(tb_ctx_)); + } + break; + } + case ObTableOperationType::GET: { + if (OB_FAIL(tb_ctx_.init_get())) { + LOG_WARN("fail to init get ctx", K(ret), K(tb_ctx_)); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid operation type", K(ret), K(op_type)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tb_ctx_.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(tb_ctx_)); + } + } + return ret; } @@ -102,10 +191,6 @@ int ObTableApiExecuteP::process() { int ret = OB_SUCCESS; ret = ParentType::process(); - int tmp_ret = revert_get_ctx(); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("fail to revert get ctx", K(tmp_ret)); - } return ret; } @@ -131,7 +216,7 @@ int ObTableApiExecuteP::try_process() switch (table_operation.type()) { case ObTableOperationType::INSERT: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_INSERT; - ret = process_insert(); + ret = process_dml_op(); break; case ObTableOperationType::GET: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_GET; @@ -139,28 +224,27 @@ int ObTableApiExecuteP::try_process() break; case ObTableOperationType::DEL: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_DELETE; - ret = process_del(); + ret = process_dml_op(); break; case ObTableOperationType::UPDATE: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_UPDATE; - ret = process_update(); + ret = process_dml_op(); break; case ObTableOperationType::INSERT_OR_UPDATE: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_INSERT_OR_UPDATE; - ret = process_insert_or_update(); + ret = process_dml_op(); break; case ObTableOperationType::REPLACE: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_REPLACE; - ret = process_replace(); + ret = process_dml_op(); break; case ObTableOperationType::INCREMENT: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_INCREMENT; - ret = process_increment(); + ret = process_dml_op(); break; case ObTableOperationType::APPEND: stat_event_type_ = ObTableProccessType::TABLE_API_SINGLE_APPEND; - // for both increment and append - ret = process_increment(); + ret = process_dml_op(); break; default: ret = OB_INVALID_ARGUMENT; @@ -181,25 +265,6 @@ int ObTableApiExecuteP::try_process() return ret; } -int ObTableApiExecuteP::revert_get_ctx() -{ - int ret = OB_SUCCESS; - if (ObTableOperationType::GET == arg_.table_operation_.type()) { - if (NULL != get_ctx_.scan_result_) { - access_service_->revert_scan_iter(get_ctx_.scan_result_); - get_ctx_.scan_result_ = NULL; - } - if (query_timeout_ts_ <= 0) { - // for robust purpose - query_timeout_ts_ = ObTimeUtility::current_time() + 1000000; - } - if (OB_FAIL(end_trans(need_rollback_trans_, req_, query_timeout_ts_))) { - LOG_WARN("failed to end trans", K(ret)); - } - } - return ret; -} - void ObTableApiExecuteP::audit_on_finish() { audit_record_.consistency_level_ = ObTableConsistencyLevel::STRONG == arg_.consistency_level_ ? @@ -241,8 +306,6 @@ int ObTableApiExecuteP::response(const int retcode) void ObTableApiExecuteP::reset_ctx() { - (void)revert_get_ctx(); - get_ctx_.reset_dml(); ObTableApiProcessorBase::reset_ctx(); need_rollback_trans_ = false; need_retry_in_queue_ = false; @@ -256,9 +319,9 @@ int ObTableApiExecuteP::get_tablet_id(uint64_t table_id, const ObRowkey &rowkey, ObSEArray rowkeys; ObSEArray tablet_ids; if (OB_FAIL(rowkeys.push_back(rowkey))) { - LOG_WARN("failed to push back", K(ret)); + LOG_WARN("fail to push back", K(ret)); } else if (OB_FAIL(get_tablet_by_rowkey(table_id, rowkeys, tablet_ids))) { - LOG_WARN("failed to get partition", K(ret), K(table_id), K(rowkeys)); + LOG_WARN("fail to get partition", K(ret), K(table_id), K(rowkeys)); } else if (1 != tablet_ids.count()) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("should have one tablet", K(ret)); @@ -271,43 +334,85 @@ int ObTableApiExecuteP::get_tablet_id(uint64_t table_id, const ObRowkey &rowkey, return ret; } -//////////////////////////////////////////////////////////////// -// get +template +int ObTableApiExecuteP::process_dml_op() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_arg2())) { + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_tb_ctx())) { + LOG_WARN("fail to init tb ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_INSERT, + arg_.consistency_level_, + tb_ctx_.get_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("fail to start trans", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret)); + } else if (OB_FAIL(ObTableOpWrapper::process_op(tb_ctx_, result_))) { + LOG_WARN("fail to process op", K(ret)); + } + + result_.set_errno(ret); + ObTableRpcProcessorUtil::replace_ret_code(ret); + int tmp_ret = ret; + if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { + LOG_WARN("fail to end trans", K(ret)); + } + + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + return ret; +} + int ObTableApiExecuteP::process_get() { int ret = OB_SUCCESS; - need_rollback_trans_ = false; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - const bool is_readonly = true; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; + ObNewRow *row = nullptr; if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_SELECT, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start readonly transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_get(get_ctx_, arg_.table_operation_, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute get", K(ret), K(table_id)); + LOG_WARN("fail to check arg", K(ret)); + } else if (OB_FAIL(init_tb_ctx())) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(init_read_trans(arg_.consistency_level_, + tb_ctx_.get_ls_id(), + tb_ctx_.get_timeout_ts()))) { + LOG_WARN("fail to init wead read trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(ObTableOpWrapper::process_get(tb_ctx_, row))) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get row", K(ret)); } - } else {} - // end trans in after_process() - need_rollback_trans_ = (OB_SUCCESS != ret); + } else { + // fill result entity + ObITableEntity *result_entity = nullptr; + const ObITableEntity *request_entity = tb_ctx_.get_entity(); + ObArray properties; + const ObTableSchema *table_schema = tb_ctx_.get_table_schema(); + if (OB_FAIL(request_entity->get_properties_names(properties))) { + LOG_WARN("fail to get entity properties", K(ret)); + } else if (OB_FAIL(result_.get_entity(result_entity))) { + LOG_WARN("fail to get result entity", K(ret)); + } else if (OB_FAIL(ObTableApiUtil::construct_entity_from_row(row, + table_schema, + properties, + result_entity))) { + LOG_WARN("fail to fill result entity", K(ret)); + } + } + + release_read_trans(); + result_.set_errno(ret); + ObTableRpcProcessorUtil::replace_ret_code(ret); + result_.set_type(arg_.table_operation_.type()); + return ret; } + //////////////////////////////////////////////////////////////// // insert_or_update ObTableAPITransCb *ObTableApiExecuteP::new_callback(rpc::ObRequest *req) @@ -317,7 +422,7 @@ ObTableAPITransCb *ObTableApiExecuteP::new_callback(rpc::ObRequest *req) // @todo optimize to avoid this copy int ret = OB_SUCCESS; if (OB_FAIL(cb->assign_execute_result(result_))) { - LOG_WARN("failed to assign result", K(ret)); + LOG_WARN("fail to assign result", K(ret)); cb->~ObTableExecuteEndTransCb(); cb = NULL; } else { @@ -326,232 +431,3 @@ ObTableAPITransCb *ObTableApiExecuteP::new_callback(rpc::ObRequest *req) } return cb; } - -int ObTableApiExecuteP::process_insert_or_update() -{ - int ret = OB_SUCCESS; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_INSERT, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_insert_or_update(get_ctx_, arg_.table_operation_, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert_or_update", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret)); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - -int ObTableApiExecuteP::process_del() -{ - int ret = OB_SUCCESS; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_DELETE, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_delete(get_ctx_, arg_.table_operation_, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to delete", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret)); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - -int ObTableApiExecuteP::process_replace() -{ - int ret = OB_SUCCESS; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_REPLACE, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(table_service_->execute_replace(get_ctx_, arg_.table_operation_, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to replace", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret)); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - -int ObTableApiExecuteP::process_insert() -{ - int ret = OB_SUCCESS; - ObNewRowIterator *duplicate_row_iter = nullptr; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_INSERT, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_insert(get_ctx_, - arg_.table_operation_, result_, duplicate_row_iter))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - const bool did_rollback = (OB_SUCCESS != ret || OB_SUCCESS != result_.get_errno()); - if (OB_FAIL(end_trans(did_rollback, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret)); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - - -int ObTableApiExecuteP::process_update() -{ - int ret = OB_SUCCESS; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_); - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; - if (OB_FAIL(check_arg2())) { - } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_UPDATE, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_update(get_ctx_, arg_.table_operation_, nullptr, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to update", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret)); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} - -int ObTableApiExecuteP::process_increment() -{ - int ret = OB_SUCCESS; - uint64_t &table_id = get_ctx_.param_table_id(); - get_ctx_.init_param(get_timeout_ts(), this, &allocator_, - arg_.returning_affected_rows_, - arg_.entity_type_, - arg_.binlog_row_image_type_, - arg_.returning_affected_entity_, - arg_.returning_rowkey_); - const bool is_readonly = false; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObRowkey rowkey = const_cast(arg_.table_operation_.entity()).get_rowkey(); - ObSEArray tablet_ids; - if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret), K(table_id)); - } else if (OB_FAIL(get_tablet_id(table_id, rowkey, get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(get_ls_id(get_ctx_.param_tablet_id(), get_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(tablet_ids.push_back(get_ctx_.param_tablet_id()))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_UPDATE, consistency_level, - table_id, get_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_increment(get_ctx_, arg_.table_operation_, result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to update", K(ret), K(table_id)); - } - } - int tmp_ret = ret; - if (OB_FAIL(end_trans(OB_SUCCESS != ret, req_, get_timeout_ts()))) { - LOG_WARN("failed to end trans", K(ret)); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - return ret; -} diff --git a/src/observer/table/ob_table_execute_processor.h b/src/observer/table/ob_table_execute_processor.h index 47f54221c1..f3a93999d5 100644 --- a/src/observer/table/ob_table_execute_processor.h +++ b/src/observer/table/ob_table_execute_processor.h @@ -17,6 +17,12 @@ #include "share/table/ob_table_rpc_proxy.h" #include "ob_table_rpc_processor.h" #include "ob_table_service.h" +#include "ob_table_context.h" +#include "ob_table_executor.h" +#include "ob_table_cache.h" +#include "sql/plan_cache/ob_cache_object_factory.h" +#include "sql/plan_cache/ob_plan_cache.h" +#include "ob_table_op_wrapper.h" namespace oceanbase { @@ -42,23 +48,19 @@ protected: virtual uint64_t get_request_checksum() override; private: + int init_tb_ctx(); int check_arg2() const; - int revert_get_ctx(); int get_tablet_id(uint64_t table_id, const ObRowkey &rowkey, common::ObTabletID &tablet_id); + template + int process_dml_op(); int process_get(); - int process_insert(); - int process_del(); - int process_update(); - int process_insert_or_update(); - int process_replace(); - int process_increment(); private: table::ObTableEntity request_entity_; table::ObTableEntity result_entity_; common::ObArenaAllocator allocator_; + table::ObTableCtx tb_ctx_; + table::ObTableApiCacheGuard cache_guard_; table::ObTableEntityFactory default_entity_factory_; - // the life of scan_ctx_ should be longer than process() - ObTableServiceGetCtx get_ctx_; bool need_rollback_trans_; int64_t query_timeout_ts_; }; diff --git a/src/observer/table/ob_table_executor.cpp b/src/observer/table/ob_table_executor.cpp new file mode 100644 index 0000000000..5f711d42c7 --- /dev/null +++ b/src/observer/table/ob_table_executor.cpp @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_executor.h" +#include "ob_table_executor_factory.h" +#include "ob_table_context.h" + +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ + +// NOTE: In fact, the table api executor/spec tree is just a doubly linked list +int ObTableApiSpec::create_executor(ObTableCtx &ctx, ObTableApiExecutor *&executor) +{ + int ret = OB_SUCCESS; + ObTableApiSpec *cur_spec = this; + ObTableApiExecutor *pre_executor = nullptr; + ObTableApiExecutor *cur_executor = nullptr; + ObTableApiExecutor *root_executor = nullptr; + while(cur_spec != nullptr && OB_SUCC(ret)) { + if (OB_FAIL(ObTableExecutorFactory::alloc_executor(ctx.get_allocator(), + ctx, + *cur_spec, + cur_executor))) { + LOG_WARN("fail to alloc executor", K(ret)); + } else { + if (root_executor == nullptr) { + root_executor = cur_executor; + } + if (pre_executor != nullptr) { + pre_executor->set_child(cur_executor); + cur_executor->set_parent(pre_executor); + } + pre_executor = cur_executor; + cur_spec = cur_spec->child_; + } + } + + if (OB_SUCC(ret)) { + executor = root_executor; + } + + return ret; +} + +void ObTableApiExecutor::set_parent(ObTableApiExecutor *parent) +{ + parent_ = parent; +} + +void ObTableApiExecutor::set_child(ObTableApiExecutor *child) +{ + child_ = child; +} + + void ObTableApiExecutor::clear_evaluated_flag() + { + if (tb_ctx_.get_table_schema()->has_generated_column() || tb_ctx_.is_inc_or_append()) { + ObExprFrameInfo* expr_info = const_cast(tb_ctx_.get_expr_frame_info()); + for (int64_t i = 0; i < expr_info->rt_exprs_.count(); i++) { + expr_info->rt_exprs_.at(i).clear_evaluated_flag(eval_ctx_); + } + } + } + +} // namespace table +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_executor.h b/src/observer/table/ob_table_executor.h new file mode 100644 index 0000000000..4d8694bbf3 --- /dev/null +++ b/src/observer/table/ob_table_executor.h @@ -0,0 +1,108 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_H +#include "sql/engine/ob_exec_context.h" +#include "ob_table_context.h" // for ObTableCtx +namespace oceanbase +{ +namespace table +{ + +class ObTableApiExecutor +{ +public: + ObTableApiExecutor(ObTableCtx &ctx) + : tb_ctx_(ctx), + exec_ctx_(ctx.get_exec_ctx()), + eval_ctx_(exec_ctx_), + parent_(nullptr), + child_(nullptr), + is_opened_(false) + { + } + virtual ~ObTableApiExecutor() {} +public: + virtual int open() = 0; + virtual int get_next_row() = 0; + virtual int close() = 0; +public: + void set_parent(ObTableApiExecutor *parent); + void set_child(ObTableApiExecutor *child); + const ObTableApiExecutor* get_parent() const { return parent_; } + const ObTableApiExecutor* get_child() const { return child_; } + OB_INLINE const ObTableCtx& get_table_ctx() const { return tb_ctx_; } + OB_INLINE sql::ObEvalCtx &get_eval_ctx() { return eval_ctx_; } + void clear_evaluated_flag(); + +protected: + ObTableCtx &tb_ctx_; + sql::ObExecContext &exec_ctx_; + sql::ObEvalCtx eval_ctx_; + ObTableApiExecutor *parent_; + ObTableApiExecutor *child_; + bool is_opened_; +}; + +class ObTableApiSpec +{ +public: + ObTableApiSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : alloc_(alloc), + type_(type), + parent_(nullptr), + child_(nullptr), + expr_frame_info_(nullptr) + { + } + virtual ~ObTableApiSpec() {}; +public: + // getter + OB_INLINE const ObTableApiSpec* get_parent() const { return parent_; } + OB_INLINE const ObTableApiSpec* get_child() const { return child_; } + OB_INLINE ObTableExecutorType get_type() const { return type_; } + OB_INLINE sql::ObExprFrameInfo* get_expr_frame_info() const { return expr_frame_info_; } + // setter + OB_INLINE void set_parent(ObTableApiSpec *parent) { parent_ = parent; } + OB_INLINE void set_child(ObTableApiSpec *child) { child_ = child; } + OB_INLINE void set_expr_frame_info(sql::ObExprFrameInfo *info) { expr_frame_info_ = info; } +public: + int create_executor(ObTableCtx &ctx, ObTableApiExecutor *&executor); + template + void destroy_executor(T *executor) + { + if (OB_NOT_NULL(executor)) { + executor->~T(); + } + } +protected: + common::ObIAllocator &alloc_; + ObTableExecutorType type_; + ObTableApiSpec *parent_; + ObTableApiSpec *child_; + sql::ObExprFrameInfo *expr_frame_info_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSpec); +}; + +struct ObTableApiDmlBaseCtDef +{ +public: + sql::ExprFixedArray old_row_; + sql::ExprFixedArray new_row_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_executor_factory.cpp b/src/observer/table/ob_table_executor_factory.cpp new file mode 100644 index 0000000000..3c36638bc3 --- /dev/null +++ b/src/observer/table/ob_table_executor_factory.cpp @@ -0,0 +1,161 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_executor_factory.h" +#include "share/datum/ob_datum_util.h" + +namespace oceanbase +{ +namespace table +{ + +template + struct AllocTableExecutorHelper; + +template + struct GenTableSpecHelper; + +int report_not_registered(int type) +{ + int ret = OB_ERR_UNEXPECTED; + LOG_WARN("executor not registered", K(ret), K(type)); + return ret; +} + +template <> +struct AllocTableExecutorHelper<0> +{ + static int alloc(ObIAllocator &alloc, + ObTableCtx &ctx, + const ObTableApiSpec &spec, + ObTableApiExecutor *&executor) + { return report_not_registered(0); } +}; + +template <> +struct GenTableSpecHelper<0> +{ + static int generate(ObIAllocator &alloc, + const ObTableExecutorType &type, + ObTableCtx &ctx, + ObTableApiSpec *&spec) + { return report_not_registered(0); } +}; + +template +struct AllocTableExecutorHelper +{ + static int alloc(ObIAllocator &alloc, + ObTableCtx &ctx, + const ObTableApiSpec &spec, + ObTableApiExecutor *&executor) + { + int ret = OB_SUCCESS; + typedef typename ObTableApiExecutorTypeTraits::Executor ExecutorType; + typedef typename ObTableApiExecutorTypeTraits::Spec SpecType; + void *buf = alloc.alloc(sizeof(ExecutorType)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc table api executor", K(ret)); + } else { + executor = new(buf) ExecutorType(ctx, static_cast(spec)); + } + return ret; + } +}; + +template +struct GenTableSpecHelper +{ + static int generate(ObIAllocator &alloc, + const ObTableExecutorType &type, + ObTableCtx &ctx, + ObTableApiSpec *&spec) + { + int ret = OB_SUCCESS; + typedef typename ObTableApiExecutorTypeTraits::Spec SpecType; + void *buf = alloc.alloc(sizeof(SpecType)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc table api spec", K(ret)); + } else { + SpecType *spec_ptr = new(buf) SpecType(alloc, type); + if (OB_FAIL(ObTableSpecCgService::generate_spec(alloc, ctx, *spec_ptr))) { + LOG_WARN("fail to generate table api spec", K(ret)); + spec_ptr->~SpecType(); + alloc.free(buf); + } else { + spec_ptr->set_expr_frame_info(ctx.get_expr_frame_info()); + spec = spec_ptr; + } + } + return ret; + } +}; + +static ObTableExecutorFactory::AllocFunc G_TABLE_API_ALLOC_FUNCS_ARRAY[TABLE_API_EXEC_MAX]; +static_assert(TABLE_API_EXEC_MAX == ARRAYSIZEOF(G_TABLE_API_ALLOC_FUNCS_ARRAY), + "alloc function array is too small"); + +ObTableExecutorFactory::AllocFunc *ObTableExecutorFactory::G_TABLE_API_ALLOC_FUNCS_ = G_TABLE_API_ALLOC_FUNCS_ARRAY; + +template +struct TableApiInitAllocFunc +{ + static void init_array() + { + static constexpr int registered =ObTableApiExecutorTypeTraits::registered_; + G_TABLE_API_ALLOC_FUNCS_ARRAY[N] = ObTableExecutorFactory::AllocFunc { + (&AllocTableExecutorHelper::alloc), + (&GenTableSpecHelper::generate) + }; + } +}; + +bool G_TABLE_API_ALLOC_FUNC_SET = common::ObArrayConstIniter::init(); + +int ObTableExecutorFactory::alloc_executor(ObIAllocator &alloc, + ObTableCtx &ctx, + const ObTableApiSpec &spec, + ObTableApiExecutor *&executor) +{ + int ret = OB_SUCCESS; + ObTableExecutorType exec_type = spec.get_type(); + if (!is_registered(exec_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("table api executor is not supported", K(ret), K(exec_type)); + } else if (OB_FAIL(G_TABLE_API_ALLOC_FUNCS_[exec_type].exec_func_(alloc, ctx, spec, executor))) { + LOG_WARN("fail to alloc table api executor", K(ret), K(exec_type)); + } else {} + + return ret; +} + +int ObTableExecutorFactory::generate_spec(ObIAllocator &alloc, + const ObTableExecutorType &type, + ObTableCtx &ctx, + ObTableApiSpec *&spec) +{ + int ret = OB_SUCCESS; + if (!is_registered(type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("table api executor is not supported", K(ret), K(type)); + } else if (OB_FAIL(G_TABLE_API_ALLOC_FUNCS_[type].gen_spec_func_(alloc, type, ctx, spec))) { + LOG_WARN("fail to alloc table api executor", K(ret), K(type)); + } else {} + + return ret; +} + +} // namespace table +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_executor_factory.h b/src/observer/table/ob_table_executor_factory.h new file mode 100644 index 0000000000..ba4683948f --- /dev/null +++ b/src/observer/table/ob_table_executor_factory.h @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_FACTORY_H +#define OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_FACTORY_H +#include "ob_table_executor_reg.h" +#include "ob_table_cg_service.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTableExecutorFactory +{ +public: + static int alloc_executor(common::ObIAllocator &alloc, + ObTableCtx &ctx, + const ObTableApiSpec &spec, + ObTableApiExecutor *&executor); + + static int generate_spec(common::ObIAllocator &alloc, + const ObTableExecutorType &type, + ObTableCtx &ctx, + ObTableApiSpec *&spec); + + static inline bool is_registered(const ObTableExecutorType exec_type) + { + return exec_type >= TABLE_API_EXEC_INVALID + && exec_type < TABLE_API_EXEC_MAX + && NULL != G_TABLE_API_ALLOC_FUNCS_[exec_type].exec_func_; + } + + struct AllocFunc + { + // use typeof instead + typeof(&ObTableExecutorFactory::alloc_executor) exec_func_; + typeof(&ObTableExecutorFactory::generate_spec) gen_spec_func_; + }; + +private: + static AllocFunc *G_TABLE_API_ALLOC_FUNCS_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif // OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_FACTORY_H \ No newline at end of file diff --git a/src/observer/table/ob_table_executor_reg.h b/src/observer/table/ob_table_executor_reg.h new file mode 100644 index 0000000000..da2de12cb4 --- /dev/null +++ b/src/observer/table/ob_table_executor_reg.h @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_REG_H +#define OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_REG_H +#include "ob_table_scan_executor.h" +#include "ob_table_insert_executor.h" +#include "ob_table_delete_executor.h" +#include "ob_table_update_executor.h" +#include "ob_table_insert_up_executor.h" +#include "ob_table_replace_executor.h" +#include "ob_table_lock_executor.h" + +namespace oceanbase +{ +namespace table +{ + +template +struct ObTableApiExecutorTypeTraits +{ + constexpr static bool registered_ = false; + typedef char Spec; + typedef char Executor; +}; + +template +struct ObTableApiExecutorTraits +{ + constexpr static int type_ = 0; +}; + +#define REGISTER_TABLE_API_EXECUTOR(type, spec, executor) \ + template <> struct ObTableApiExecutorTypeTraits { \ + constexpr static bool registered_ = true; \ + typedef spec Spec; \ + typedef executor Executor; \ + }; \ + template <> struct ObTableApiExecutorTraits { \ + constexpr static int type_ = type; \ + }; \ + template <> struct ObTableApiExecutorTraits { \ + constexpr static int type_ = type; \ + }; + +// REGISTER_TABLE_API_EXECUTOR(executor_type, spec, executor) + +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_SCAN, ObTableApiScanSpec, ObTableApiScanExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_INSERT, ObTableApiInsertSpec, ObTableApiInsertExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_DELETE, ObTableApiDelSpec, ObTableApiDeleteExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_UPDATE, ObTableApiUpdateSpec, ObTableApiUpdateExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_INSERT_UP, ObTableApiInsertUpSpec, ObTableApiInsertUpExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_REPLACE, ObTableApiReplaceSpec, ObTableApiReplaceExecutor); +REGISTER_TABLE_API_EXECUTOR(TABLE_API_EXEC_LOCK, ObTableApiLockSpec, ObTableApiLockExecutor); + +#undef REGISTER_TABLE_API_EXECUTOR + +} // end namespace table +} // end namespace oceanbase + +#endif // OCEANBASE_OBSERVER_OB_TABLE_EXECUTOR_REG_H diff --git a/src/observer/table/ob_table_insert_executor.cpp b/src/observer/table/ob_table_insert_executor.cpp new file mode 100644 index 0000000000..090cd3c81c --- /dev/null +++ b/src/observer/table/ob_table_insert_executor.cpp @@ -0,0 +1,129 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_insert_executor.h" +#include "ob_table_cg_service.h" +#include "sql/engine/dml/ob_dml_service.h" + +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ +int ObTableApiInsertExecutor::open() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_ins_rtdef(ins_spec_.get_ctdef(), ins_rtdef_))) { + LOG_WARN("fail to generate insert rtdef", K(ret)); + } + + return ret; +} + +int ObTableApiInsertExecutor::insert_row_to_das() +{ + return ObTableApiModifyExecutor::insert_row_to_das(ins_spec_.get_ctdef(), ins_rtdef_); +} + +int ObTableApiInsertExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + + while(OB_SUCC(ret)) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } else if (OB_FAIL(insert_row_to_das())) { + LOG_WARN("fail to insert row to das", K(ret)); + } else { + cur_idx_++; + } + } + + if (OB_ITER_END == ret) { + if (OB_FAIL(ins_rows_post_proc())) { + LOG_WARN("fail to post process after insert row", K(ret)); + } else { + ret = OB_ITER_END; + } + } + + return ret; +} + +int ObTableApiInsertExecutor::process_single_operation(const ObTableEntity *entity) +{ + int ret = OB_SUCCESS; + + clear_evaluated_flag(); + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::refresh_insert_exprs_frame(tb_ctx_, + ins_spec_.get_ctdef().new_row_, + *entity))) { + LOG_WARN("fail to refresh insert exprs frame", K(ret), K(*entity)); + } + + return ret; +} + +int ObTableApiInsertExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + if (cur_idx_ >= 1) { + ret = OB_ITER_END; + } else if (OB_FAIL(process_single_operation(entity))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to process single insert operation", K(ret)); + } + } + + return ret; +} + +int ObTableApiInsertExecutor::ins_rows_post_proc() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(submit_all_dml_task())) { + LOG_WARN("fail to execute all insert das task", K(ret)); + } else { + affected_rows_ = 1; + } + + return ret; +} + +int ObTableApiInsertExecutor::close() +{ + int ret = OB_SUCCESS; + + if (!is_opened_) { + // do nothing + } else if (OB_FAIL(ObTableApiModifyExecutor::close())) { + LOG_WARN("fail to close ObTableApiModifyExecutor", K(ret)); + } + + return ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_insert_executor.h b/src/observer/table/ob_table_insert_executor.h new file mode 100644 index 0000000000..9c1f16dd71 --- /dev/null +++ b/src/observer/table/ob_table_insert_executor.h @@ -0,0 +1,69 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_INSERT_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_INSERT_EXECUTOR_H +#include "ob_table_modify_executor.h" +#include "ob_table_context.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTableApiInsertSpec : public ObTableApiModifySpec +{ +public: + ObTableApiInsertSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + ins_ctdef_(alloc) + { + } +public: + OB_INLINE const ObTableInsCtDef& get_ctdef() const { return ins_ctdef_; } + OB_INLINE ObTableInsCtDef& get_ctdef() { return ins_ctdef_; } +private: + ObTableInsCtDef ins_ctdef_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiInsertSpec); +}; + +class ObTableApiInsertExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiInsertExecutor(ObTableCtx &ctx, const ObTableApiInsertSpec &spec) + : ObTableApiModifyExecutor(ctx), + ins_spec_(spec), + cur_idx_(0) + { + } + +public: + virtual int open() override; + virtual int get_next_row() override; + virtual int close() override; +private: + int process_single_operation(const ObTableEntity *entity); + int get_next_row_from_child(); + int ins_rows_post_proc(); + int insert_row_to_das(); + +private: + const ObTableApiInsertSpec &ins_spec_; + ObTableInsRtDef ins_rtdef_; + int64_t cur_idx_; +}; + +} // namespace table +} // namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_INSERT_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_insert_up_executor.cpp b/src/observer/table/ob_table_insert_up_executor.cpp new file mode 100644 index 0000000000..c5ad4cf27e --- /dev/null +++ b/src/observer/table/ob_table_insert_up_executor.cpp @@ -0,0 +1,651 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_insert_up_executor.h" +#include "ob_table_cg_service.h" +#include "lib/utility/ob_tracepoint.h" +#include "sql/das/ob_das_insert_op.h" +#include "sql/engine/dml/ob_dml_service.h" +#include "ob_htable_utils.h" +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ +int ObTableApiInsertUpExecutor::generate_insert_up_rtdef(const ObTableInsUpdCtDef &ctdef, + ObTableInsUpdRtDef &rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_ins_rtdef(ctdef.ins_ctdef_, rtdef.ins_rtdef_))) { + LOG_WARN("fail to generate insert rtdef", K(ret)); + } else if (OB_FAIL(generate_upd_rtdef(ctdef.upd_ctdef_, + rtdef.upd_rtdef_))) { + LOG_WARN("fail to generate update rtdef", K(ret)); + } else { + rtdef.ins_rtdef_.das_rtdef_.table_loc_->is_writing_ = true; + } + + return ret; +} + +int ObTableApiInsertUpExecutor::modify_htable_timestamp() +{ + int ret = OB_SUCCESS; + int64_t now_ms = -ObHTableUtils::current_time_millis(); + const ObITableEntity *entity = static_cast(tb_ctx_.get_entity()); + if (entity->get_rowkey_size() != 3) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("htable should be with 3 rowkey columns", K(ret), K(entity)); + } else { + ObRowkey rowkey = entity->get_rowkey(); + ObObj &t_obj = const_cast(rowkey.get_obj_ptr()[ObHTableConstants::COL_IDX_T]); // column T + ObHTableCellEntity3 htable_cell(entity); + bool row_is_null = htable_cell.last_get_is_null(); + int64_t timestamp = htable_cell.get_timestamp(); + bool timestamp_is_null = htable_cell.last_get_is_null(); + if (row_is_null || timestamp_is_null) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument for htable put", K(ret), K(row_is_null), K(timestamp_is_null)); + } else if (ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // update timestamp iff LATEST_TIMESTAMP + t_obj.set_int(now_ms); + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::open() +{ + int ret = OB_SUCCESS; + + const ObSQLSessionInfo &session = tb_ctx_.get_session_info(); + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_insert_up_rtdef(insert_up_spec_.get_ctdef(), insert_up_rtdef_))) { + LOG_WARN("fail to init insert up rtdef", K(ret)); + } else { + ObDASTabletLoc *tablet_loc = nullptr; + ObDASTableLoc *table_loc = insert_up_rtdef_.ins_rtdef_.das_rtdef_.table_loc_; + if (OB_ISNULL(table_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table location is invalid", K(ret)); + } else if (OB_FAIL(conflict_checker_.init_conflict_checker(insert_up_spec_.get_expr_frame_info(), + table_loc))) { + LOG_WARN("fail to init conflict_checker", K(ret)); + } else if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + conflict_checker_.set_local_tablet_loc(tablet_loc); + // init update das_ref + ObMemAttr mem_attr; + mem_attr.tenant_id_ = session.get_effective_tenant_id(); + mem_attr.label_ = "TableApiInsUpd"; + upd_rtctx_.das_ref_.set_expr_frame_info(insert_up_spec_.get_expr_frame_info()); + upd_rtctx_.das_ref_.set_mem_attr(mem_attr); + upd_rtctx_.das_ref_.set_execute_directly(true); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (tb_ctx_.is_htable() && OB_FAIL(modify_htable_timestamp())) { + LOG_WARN("fail to modify htable timestamp", K(ret)); + } + + return ret; +} + +void ObTableApiInsertUpExecutor::set_need_fetch_conflict() +{ + insert_up_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = true; + dml_rtctx_.set_non_sub_full_task(); + upd_rtctx_.set_pick_del_task_first(); + upd_rtctx_.set_non_sub_full_task(); +} + +int ObTableApiInsertUpExecutor::refresh_exprs_frame(const ObTableEntity *entity) +{ + int ret = OB_SUCCESS; + const ObTableInsCtDef &ins_ctdef = insert_up_spec_.get_ctdef().ins_ctdef_; + const ObTableUpdCtDef &upd_ctdef = insert_up_spec_.get_ctdef().upd_ctdef_; + + clear_evaluated_flag(); + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::refresh_insert_up_exprs_frame(tb_ctx_, + ins_ctdef.new_row_, + upd_ctdef.delta_exprs_, + *entity))) { + LOG_WARN("fail to refresh insert up exprs frame", K(ret), K(*entity)); + } + + return ret; +} + +int ObTableApiInsertUpExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + if (cur_idx_ >= 1) { + ret = OB_ITER_END; + } else if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } + + return ret; +} + +int ObTableApiInsertUpExecutor::insert_constraint_row_to_das(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value) +{ + int ret = OB_SUCCESS; + const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); + + if (OB_FAIL(insert_row_to_das(insert_up_ctdef.ins_ctdef_, insert_up_rtdef_.ins_rtdef_))) { + LOG_WARN("fail to insert row to das", K(ret)); + } + + return ret; +} + +int ObTableApiInsertUpExecutor::load_insert_up_rows(bool &is_iter_end, + int64_t &insert_rows) +{ + int ret = OB_SUCCESS; + is_iter_end = false; + int64_t row_cnt = 0; + const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); + int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_INSERT_UP_BATCH_ROW_COUNT); + int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? + simulate_batch_row_cnt : DEFAULT_INSERT_UP_BATCH_ROW_COUNT; + + while (OB_SUCC(ret) && ++row_cnt <= default_row_batch_cnt) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to load next row from child", K(ret)); + } + } else if (OB_FAIL(insert_row_to_das(insert_up_ctdef.ins_ctdef_, insert_up_rtdef_.ins_rtdef_))) { + LOG_WARN("fail to insert row to das", K(ret)); + } else { + cur_idx_++; + insert_rows++; + } + } + + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + is_iter_end = true; + } + + return ret; +} + +int ObTableApiInsertUpExecutor::post_das_task(ObDMLRtCtx &dml_rtctx, + bool del_task_ahead) +{ + int ret = OB_SUCCESS; + + if (dml_rtctx.das_ref_.has_task()) { + if (del_task_ahead) { + if (OB_FAIL(dml_rtctx.das_ref_.pick_del_task_to_first())) { + LOG_WARN("fail to remove delete das task first", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(dml_rtctx.das_ref_.execute_all_task())) { + LOG_WARN("fail to execute all das task", K(ret)); + } + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::fetch_conflict_rowkey() +{ + int ret = OB_SUCCESS; + bool got_row = false; + DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); + + while (OB_SUCC(ret) && !task_iter.is_end()) { + if (OB_FAIL(get_next_conflict_rowkey(task_iter))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next conflict rowkey from das_result", K(ret)); + } + } else if (OB_FAIL(conflict_checker_.build_primary_table_lookup_das_task())) { + LOG_WARN("fail to build lookup_das_task", K(ret)); + } + } + + ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); + return ret; +} + +// todo:linjing 和replace executor有相同函数，可以抽成公共函数 +int ObTableApiInsertUpExecutor::get_next_conflict_rowkey(DASTaskIter &task_iter) +{ + int ret = OB_SUCCESS; + bool got_row = false; + + while (OB_SUCC(ret) && !got_row) { + ObNewRow *dup_row = nullptr; + ObChunkDatumStore::StoredRow *stored_row = nullptr; + ObDASWriteBuffer::DmlShadowRow ssr; + ObDASInsertOp *ins_op = static_cast(*task_iter); + ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); + if (OB_ISNULL(conflict_result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("duplicted key result is null", K(ret)); + } else if (OB_FAIL(conflict_result->get_next_row(dup_row))) { + if (OB_ITER_END == ret) { + ++task_iter; + if (!task_iter.is_end()) { + ret = OB_SUCCESS; + } + } else { + LOG_WARN("fail to get next row from das result", K(ret)); + } + } else if (OB_FAIL(ssr.init(dml_rtctx_.get_das_alloc(), ins_ctdef->table_rowkey_types_, false))) { + LOG_WARN("fail to init shadow stored row", K(ret), K(ins_ctdef->table_rowkey_types_)); + } else if (OB_FAIL(ssr.shadow_copy(*dup_row))) { + LOG_WARN("fail to shadow copy ob new row", K(ret)); + } else if (OB_ISNULL(stored_row = ssr.get_store_row())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("stored row is null", K(ret)); + } else if (OB_FAIL(stored_row->to_expr_skip_const( + conflict_checker_.checker_ctdef_.data_table_rowkey_expr_, + conflict_checker_.eval_ctx_))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row from result iterator", K(ret)); + } + } else { + got_row = true; + } + } + + return ret; +} + +// todo:linjing 和replace executor有相同函数，可以抽成公共函数 +int ObTableApiInsertUpExecutor::reset_das_env() +{ + int ret = OB_SUCCESS; + // 释放第一次try insert的das task + if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { + LOG_WARN("fail to close all das task", K(ret)); + } else { + dml_rtctx_.das_ref_.reuse(); + } + + // 因为第二次插入不需要fetch conflict result了，如果有conflict + // 就直接报错 + insert_up_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = false; + insert_up_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_ = false; + + return ret; +} + +int ObTableApiInsertUpExecutor::do_insert_up_cache() +{ + int ret = OB_SUCCESS; + ObSEArray constraint_values; + bool is_skipped = false; + ObChunkDatumStore::StoredRow *insert_row = NULL; + ObTableUpdRtDef &upd_rtdef = insert_up_rtdef_.upd_rtdef_; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + const ObExprPtrIArray &new_row_exprs = get_primary_table_insert_row(); + + if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } else if (OB_FAIL(ObChunkDatumStore::StoredRow::build(insert_row, new_row_exprs, eval_ctx_, allocator_))) { + LOG_WARN("failed to build stored row", K(ret)); + } else { + constraint_values.reuse(); + if (OB_ISNULL(insert_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert row is NULL", K(ret)); + } else if (OB_FAIL(conflict_checker_.check_duplicate_rowkey(insert_row, + constraint_values, + true))) { + LOG_WARN("fail to check duplicated key", K(ret), KPC(insert_row)); + } else { + // do update + const ObChunkDatumStore::StoredRow *upd_new_row = insert_row; + const ObChunkDatumStore::StoredRow *upd_old_row = constraint_values.at(0).current_datum_row_; + clear_evaluated_flag(); + if (FALSE_IT(upd_rtdef.found_rows_++)) { + // do nothing + } else if (OB_FAIL(conflict_checker_.update_row(upd_new_row, upd_old_row))) { + LOG_WARN("fail to update row in conflict_checker", K(ret), KPC(upd_new_row), KPC(upd_old_row)); + } else { + upd_changed_rows_++; + } + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::prepare_final_insert_up_task() +{ + int ret = OB_SUCCESS; + ObConflictRowMap *primary_map = NULL; + OZ(conflict_checker_.get_primary_table_map(primary_map)); + CK(OB_NOT_NULL(primary_map)); + ObConflictRowMap::iterator start_row_iter = primary_map->begin(); + ObConflictRowMap::iterator end_row_iter = primary_map->end(); + for (; OB_SUCC(ret) && start_row_iter != end_row_iter; ++start_row_iter) { + const ObRowkey &constraint_rowkey = start_row_iter->first; + const ObConflictValue &constraint_value = start_row_iter->second; + if (constraint_value.new_row_source_ == ObNewRowSource::FROM_UPDATE) { + OZ(do_update(constraint_rowkey, constraint_value)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected row source", K(ret), K(constraint_value.new_row_source_)); + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::delete_upd_old_row_to_das(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value) +{ + int ret = OB_SUCCESS; + ObDASTabletLoc *tablet_loc = nullptr; + + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); + const ObTableUpdCtDef &upd_ctdef = insert_up_ctdef.upd_ctdef_; + ObTableUpdRtDef &upd_rtdef = insert_up_rtdef_.upd_rtdef_; + if (OB_ISNULL(upd_ctdef.ddel_ctdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ddel_ctdef can't be null", K(ret)); + } else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) { + if (OB_FAIL(ObDASTaskFactory::alloc_das_rtdef(DAS_OP_TABLE_DELETE, + tb_ctx_.get_allocator(), + upd_rtdef.ddel_rtdef_))) { + LOG_WARN("fail to create das delete rtdef", K(ret)); + } else if (OB_FAIL(generate_del_rtdef_for_update(upd_ctdef, upd_rtdef))) { + LOG_WARN("fail to generate del rtdef for update", K(ret), K(upd_ctdef), K(upd_rtdef)); + } + } + + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_ISNULL(upd_rtdef.ddel_rtdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ddel_rtdef is null", K(ret)); + } else if (OB_FAIL(ObDMLService::delete_row(*upd_ctdef.ddel_ctdef_, + *upd_rtdef.ddel_rtdef_, + tablet_loc, + upd_rtctx_, + upd_ctdef.old_row_))) { + LOG_WARN("fail to delete row with das", K(ret)); + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(*upd_ctdef.ddel_ctdef_, + *upd_rtdef.ddel_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_del_ctdefs_, + upd_rtdef.related_del_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + upd_rtdef.ddel_rtdef_->related_ctdefs_ = &upd_ctdef.related_del_ctdefs_; + upd_rtdef.ddel_rtdef_->related_rtdefs_ = &upd_rtdef.related_del_rtdefs_; + } + + return ret; +} + +int ObTableApiInsertUpExecutor::generate_ins_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(*upd_ctdef.dins_ctdef_, + *upd_rtdef.dins_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_ins_ctdefs_, + upd_rtdef.related_ins_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + upd_rtdef.dins_rtdef_->related_ctdefs_ = &upd_ctdef.related_ins_ctdefs_; + upd_rtdef.dins_rtdef_->related_rtdefs_ = &upd_rtdef.related_ins_rtdefs_; + } + + return ret; +} + +int ObTableApiInsertUpExecutor::insert_upd_new_row_to_das() +{ + int ret = OB_SUCCESS; + const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); + ObDASTabletLoc *tablet_loc = nullptr; + + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + const ObTableUpdCtDef &upd_ctdef = insert_up_ctdef.upd_ctdef_; + ObTableUpdRtDef &upd_rtdef = insert_up_rtdef_.upd_rtdef_; + if (OB_ISNULL(upd_ctdef.dins_ctdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dins_ctdef_ can't be null", K(ret)); + } else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) { + if (OB_FAIL(ObDASTaskFactory::alloc_das_rtdef(DAS_OP_TABLE_INSERT, + tb_ctx_.get_allocator(), + upd_rtdef.dins_rtdef_))) { + LOG_WARN("fail to create das insert rtdef", K(ret)); + } else if (OB_FAIL(generate_ins_rtdef_for_update(upd_ctdef, upd_rtdef))) { + LOG_WARN("fail to generate del rtdef for update", K(ret), K(upd_ctdef), K(upd_rtdef)); + } + } + + clear_evaluated_flag(); + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_ISNULL(upd_rtdef.dins_rtdef_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dins_rtdef_ is null", K(ret)); + } else if (OB_FAIL(ObDMLService::insert_row(*upd_ctdef.dins_ctdef_, + *upd_rtdef.dins_rtdef_, + tablet_loc, + upd_rtctx_, + upd_ctdef.new_row_))) { + LOG_WARN("fail to insert row with das", K(ret)); + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, + const ObRowkey &constraint_rowkey) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = tb_ctx_.get_table_schema(); + const ObTableInsUpdCtDef &insert_up_ctdef = insert_up_spec_.get_ctdef(); + const ObTableUpdCtDef &upd_ctdef = insert_up_ctdef.upd_ctdef_; + const ObIArray &new_row = upd_ctdef.new_row_; + if (OB_UNLIKELY(store_row.cnt_ != new_row.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("datum count mismatch", K(ret), K(store_row.cnt_), K(new_row.count())); + } else { + // 1. refresh rowkey expr datum + const int64_t rowkey_col_cnt = tb_ctx_.get_table_schema()->get_rowkey_column_num(); + for (uint64_t i = 0; OB_SUCC(ret) && i < rowkey_col_cnt; ++i) { + const ObExpr *expr = new_row.at(i); + expr->locate_expr_datum(eval_ctx_) = store_row.cells()[i]; + expr->get_eval_info(eval_ctx_).evaluated_ = true; + expr->get_eval_info(eval_ctx_).projected_ = true; + } + + // 2. refresh assign column expr datum + const ObTableCtx::ObAssignIds &assign_ids = tb_ctx_.get_assign_ids(); + const int64_t N = assign_ids.count(); + for (uint64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + uint64_t assign_id = assign_ids.at(i).idx_; + const ObColumnSchemaV2 *col_schema = nullptr; + if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(assign_id))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(assign_id), K(*table_schema)); + } else if (assign_id >= store_row.cnt_) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(store_row.cnt_)); + } else if (assign_id >= new_row.count()) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(new_row.count())); + } else if (col_schema->is_virtual_generated_column()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should not have virtual generated expr", K(ret)); + } else if (col_schema->is_stored_generated_column()) { + ObTableCtx &ctx = const_cast(tb_ctx_); + if (OB_FAIL(ObTableExprCgService::refresh_generated_column_related_frame(ctx, + upd_ctdef.old_row_, + upd_ctdef.full_assign_row_, + assign_ids, + *col_schema))) { + LOG_WARN("fail to refresh generated column related frame", K(ret), K(ctx), K(*col_schema)); + } + } else { + const ObExpr *expr = new_row.at(assign_id); + expr->locate_expr_datum(eval_ctx_) = store_row.cells()[assign_id]; + expr->get_eval_info(eval_ctx_).evaluated_ = true; + expr->get_eval_info(eval_ctx_).projected_ = true; + } + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::do_update(const ObRowkey &constraint_rowkey, + const ObConflictValue &constraint_value) +{ + int ret = OB_SUCCESS; + + if (constraint_value.new_row_source_ == ObNewRowSource::FROM_UPDATE) { + // current_datum_row_ 是update的new_row + if (NULL != constraint_value.baseline_datum_row_ && + NULL != constraint_value.current_datum_row_) { + // base_line 和 curr_row 都存在 + OZ(constraint_value.baseline_datum_row_->to_expr(get_primary_table_upd_old_row(), + eval_ctx_)); + OZ(delete_upd_old_row_to_das(constraint_rowkey, constraint_value)); + OZ(to_expr_skip_old(*constraint_value.current_datum_row_, constraint_rowkey)); + clear_evaluated_flag(); + OZ(insert_upd_new_row_to_das()); + } else if (NULL == constraint_value.baseline_datum_row_ && + NULL != constraint_value.current_datum_row_) { + OZ(to_expr_skip_old(*constraint_value.current_datum_row_, constraint_rowkey)); + OZ(insert_upd_new_row_to_das()); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected constraint_value", K(ret)); + } + } + + return ret; +} + +int ObTableApiInsertUpExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + bool is_iter_end = false; + + while (OB_SUCC(ret) && !is_iter_end) { + int64_t insert_rows = 0; + int64_t savepoint_no = 0; + // must set conflict_row fetch flag + set_need_fetch_conflict(); + if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(exec_ctx_, savepoint_no))) { + LOG_WARN("fail to create save_point", K(ret)); + } else if (OB_FAIL(load_insert_up_rows(is_iter_end, insert_rows))) { + LOG_WARN("fail to load all row", K(ret)); + } else if (OB_FAIL(post_das_task(dml_rtctx_, false))) { + LOG_WARN("fail to post all das task", K(ret)); + } else if (!is_duplicated()) { + insert_rows_ += insert_rows; + LOG_TRACE("try insert is not duplicated", K(ret), K(insert_rows_)); + } else if (OB_FAIL(fetch_conflict_rowkey())) { + LOG_WARN("fail to fetch conflict row", K(ret)); + } else if (OB_FAIL(reset_das_env())) { + // 这里需要reuse das 相关信息 + LOG_WARN("fail to reset das env", K(ret)); + } else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(exec_ctx_, savepoint_no))) { + // 本次插入存在冲突, 回滚到save_point + LOG_WARN("fail to rollback to save_point", K(ret)); + } else if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(1))) { + LOG_WARN("fail to build conflict map", K(ret)); + } else if (OB_FAIL(do_insert_up_cache())) { + LOG_WARN("fail to do insert_up in cache", K(ret)); + } else if (OB_FAIL(prepare_final_insert_up_task())) { + LOG_WARN("fail to prepare final das task", K(ret)); + } else if (OB_FAIL(post_das_task(upd_rtctx_, true))) { + LOG_WARN("fail to post upd_rtctx_ das task", K(ret)); + } else if (OB_FAIL(post_das_task(dml_rtctx_, false))) { + LOG_WARN("fail to post dml_rtctx_ das task", K(ret)); + } + } + + if (OB_SUCC(ret)) { + affected_rows_ += insert_rows_ + insert_up_rtdef_.upd_rtdef_.found_rows_; + } + + return ret; +} + +int ObTableApiInsertUpExecutor::close() +{ + int ret = OB_SUCCESS; + int close_ret = OB_SUCCESS; + + if (is_opened_) { + if (OB_FAIL(conflict_checker_.close())) { + LOG_WARN("fail to close conflict_checker", K(ret)); + } + + if (upd_rtctx_.das_ref_.has_task()) { + close_ret = (upd_rtctx_.das_ref_.close_all_task()); + if (OB_SUCCESS == close_ret) { + upd_rtctx_.das_ref_.reset(); + } + } + ret = OB_SUCCESS == ret ? close_ret : ret; + // close dml das tasks + close_ret = ObTableApiModifyExecutor::close(); + } + + return (OB_SUCCESS == ret) ? close_ret : ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_insert_up_executor.h b/src/observer/table/ob_table_insert_up_executor.h new file mode 100644 index 0000000000..59b95dab84 --- /dev/null +++ b/src/observer/table/ob_table_insert_up_executor.h @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_INSERT_UP_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_INSERT_UP_EXECUTOR_H +#include "ob_table_modify_executor.h" +#include "ob_table_context.h" +#include "sql/engine/dml/ob_conflict_checker.h" + +namespace oceanbase +{ +namespace table +{ +class ObTableApiInsertUpSpec : public ObTableApiModifySpec +{ +public: + ObTableApiInsertUpSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + insert_up_ctdef_(alloc), + conflict_checker_ctdef_(alloc), + all_saved_exprs_(alloc) + { + } +public: + OB_INLINE const ObTableInsUpdCtDef& get_ctdef() const { return insert_up_ctdef_; } + OB_INLINE ObTableInsUpdCtDef& get_ctdef() { return insert_up_ctdef_; } + OB_INLINE const sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() const { return conflict_checker_ctdef_; } + OB_INLINE sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() { return conflict_checker_ctdef_; } + OB_INLINE const common::ObIArray& get_all_saved_exprs() const { return all_saved_exprs_; } + OB_INLINE common::ObIArray& get_all_saved_exprs() { return all_saved_exprs_; } +private: + ObTableInsUpdCtDef insert_up_ctdef_; + sql::ObConflictCheckerCtdef conflict_checker_ctdef_; + sql::ExprFixedArray all_saved_exprs_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiInsertUpSpec); +}; + +class ObTableApiInsertUpExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiInsertUpExecutor(ObTableCtx &ctx, const ObTableApiInsertUpSpec &spec) + : ObTableApiModifyExecutor(ctx), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + insert_up_spec_(spec), + insert_up_rtdef_(), + insert_rows_(0), + upd_changed_rows_(0), + upd_rtctx_(eval_ctx_, exec_ctx_, get_fake_modify_op()), + conflict_checker_(allocator_, eval_ctx_, spec.get_conflict_checker_ctdef()), + is_duplicated_(false), + cur_idx_(0) + { + } +public: + virtual int open(); + virtual int get_next_row(); + virtual int close(); +public: + OB_INLINE bool is_insert_duplicated() + { + return is_duplicated_; + } +private: + static const int64_t DEFAULT_INSERT_UP_BATCH_ROW_COUNT = 1000L; + OB_INLINE bool is_duplicated() + { + is_duplicated_ = insert_up_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_; + return is_duplicated_; + } + OB_INLINE const common::ObIArray& get_primary_table_new_row() + { + return insert_up_spec_.get_ctdef().ins_ctdef_.new_row_; + } + OB_INLINE const common::ObIArray &get_primary_table_upd_new_row() + { + return insert_up_spec_.get_ctdef().upd_ctdef_.new_row_; + } + OB_INLINE const common::ObIArray &get_primary_table_upd_old_row() + { + return insert_up_spec_.get_ctdef().upd_ctdef_.old_row_; + } + OB_INLINE const common::ObIArray &get_primary_table_insert_row() + { + return insert_up_spec_.get_ctdef().ins_ctdef_.new_row_; + } + int generate_insert_up_rtdef(const ObTableInsUpdCtDef &ctdef, + ObTableInsUpdRtDef &rtdef); + void set_need_fetch_conflict(); + int refresh_exprs_frame(const ObTableEntity *entity); + int get_next_row_from_child(); + int insert_constraint_row_to_das(const ObRowkey &constraint_rowkey, + const ObConflictValue &constraint_value); + int load_insert_up_rows(bool &is_iter_end, int64_t &insert_rows); + int post_das_task(sql::ObDMLRtCtx &dml_rtctx, bool del_task_ahead); + int fetch_conflict_rowkey(); + int get_next_conflict_rowkey(sql::DASTaskIter &task_iter); + int reset_das_env(); + int do_insert_up_cache(); + int prepare_final_insert_up_task(); + int do_update(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value); + int do_insert(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value); + int delete_upd_old_row_to_das(const ObRowkey &constraint_rowkey, + const sql::ObConflictValue &constraint_value); + int insert_upd_new_row_to_das(); + int generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef); + int generate_ins_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef); + int to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, + const ObRowkey &constraint_rowkey); + // for htable + int modify_htable_timestamp(); + +private: + common::ObArenaAllocator allocator_; + const ObTableApiInsertUpSpec &insert_up_spec_; + ObTableInsUpdRtDef insert_up_rtdef_; + int64_t insert_rows_; + int64_t upd_changed_rows_; + sql::ObDMLRtCtx upd_rtctx_; + sql::ObConflictChecker conflict_checker_; + bool is_duplicated_; + int64_t cur_idx_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_INSERT_UP_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_lock_executor.cpp b/src/observer/table/ob_table_lock_executor.cpp new file mode 100644 index 0000000000..55e7a13cc4 --- /dev/null +++ b/src/observer/table/ob_table_lock_executor.cpp @@ -0,0 +1,164 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_lock_executor.h" +#include "sql/engine/dml/ob_dml_service.h" + +namespace oceanbase +{ +namespace table +{ +int ObTableApiLockExecutor::generate_lock_rtdef() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(lock_spec_.get_ctdef().das_ctdef_, + lock_rtdef_.das_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } + + return ret; +} + +int ObTableApiLockExecutor::open() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_lock_rtdef())) { + LOG_WARN("fail to generate lock rtdef"); + } + + return ret; +} + +int ObTableApiLockExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + + if (cur_idx_ >= 1) { + ret = OB_ITER_END; + } else { + common::ObIArray &key_ranges = tb_ctx_.get_key_ranges(); + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else { + ObRowkey rowkey = entity->get_rowkey(); + ObNewRange range; + // init key_ranges_ + key_ranges.reset(); + if (OB_FAIL(range.build_range(tb_ctx_.get_ref_table_id(), rowkey))) { + LOG_WARN("fail to build key range", K(ret), K(tb_ctx_), K(rowkey)); + } else if (OB_FAIL(key_ranges.push_back(range))) { + LOG_WARN("fail to push back key range", K(ret), K(range)); + } else { + clear_evaluated_flag(); + if (OB_FAIL(child_->open())) { + LOG_WARN("fail to open child executor", K(ret)); + } else if (OB_FAIL(child_->get_next_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } + int tmp_ret = ret; + if (OB_FAIL(child_->close())) { + LOG_WARN("fail to close child executor", K(ret)); + } else { + ret = tmp_ret; + } + } + } + } + + return ret; +} + +int ObTableApiLockExecutor::lock_row_to_das() +{ + int ret = OB_SUCCESS; + ObDASTabletLoc *tablet_loc = nullptr; + + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail tp calc tablet location", K(ret)); + } else if (OB_FAIL(ObDMLService::lock_row(lock_spec_.get_ctdef().das_ctdef_, + lock_rtdef_.das_rtdef_, + tablet_loc, + dml_rtctx_, + lock_spec_.get_ctdef().old_row_))) { + if (OB_TRY_LOCK_ROW_CONFLICT != ret && + OB_TRANSACTION_SET_VIOLATION != ret && + OB_ERR_EXCLUSIVE_LOCK_CONFLICT != ret) { + LOG_WARN("fail to lock row with das", K(ret)); + } + } + + return ret; +} + +int ObTableApiLockExecutor::lock_rows_post_proc() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(submit_all_dml_task())) { + if (OB_TRY_LOCK_ROW_CONFLICT != ret && + OB_TRANSACTION_SET_VIOLATION != ret && + OB_ERR_EXCLUSIVE_LOCK_CONFLICT != ret) { + LOG_WARN("fail to lock row with das", K(ret)); + } + } + + return ret; +} + +int ObTableApiLockExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + + while(OB_SUCC(ret)) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } else if (OB_FAIL(lock_row_to_das())) { + LOG_WARN("fail to lock row to das", K(ret)); + } + } + + if (OB_ITER_END == ret) { + if (OB_FAIL(lock_rows_post_proc())) { + LOG_WARN("fail to post process after lock row", K(ret)); + } else { + ret = OB_ITER_END; + } + } + + return ret; +} + +int ObTableApiLockExecutor::close() +{ + int ret = OB_SUCCESS; + if (!is_opened_) { + // do nothing + } else { + ret = ObTableApiModifyExecutor::close(); + } + return ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_lock_executor.h b/src/observer/table/ob_table_lock_executor.h new file mode 100644 index 0000000000..7cf0f84127 --- /dev/null +++ b/src/observer/table/ob_table_lock_executor.h @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_LOCK_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_LOCK_EXECUTOR_H +#include "ob_table_modify_executor.h" +#include "ob_table_scan_executor.h" +#include "ob_table_context.h" +namespace oceanbase +{ +namespace table +{ + +class ObTableApiLockSpec : public ObTableApiModifySpec +{ +public: + ObTableApiLockSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + lock_ctdef_(alloc) + { + } +public: + OB_INLINE const ObTableLockCtDef& get_ctdef() const { return lock_ctdef_; } + OB_INLINE ObTableLockCtDef& get_ctdef() { return lock_ctdef_; } +private: + ObTableLockCtDef lock_ctdef_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiLockSpec); +}; + +class ObTableApiLockExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiLockExecutor(ObTableCtx &ctx, const ObTableApiLockSpec &lock_spec) + : ObTableApiModifyExecutor(ctx), + lock_spec_(lock_spec), + lock_rtdef_(), + cur_idx_(0) + { + } + ~ObTableApiLockExecutor() + { + if (OB_NOT_NULL(child_)) { + ObTableApiScanExecutor *scan_executor = static_cast(child_); + scan_executor->~ObTableApiScanExecutor(); + } + } +public: + virtual int open(); + virtual int get_next_row(); + virtual int close(); +private: + int generate_lock_rtdef(); + int get_next_row_from_child(); + int lock_row_to_das(); + int lock_rows_post_proc(); +private: + const ObTableApiLockSpec &lock_spec_; + ObTableLockRtDef lock_rtdef_; + int64_t cur_idx_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_LOCK_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_modify_executor.cpp b/src/observer/table/ob_table_modify_executor.cpp new file mode 100644 index 0000000000..f5a3ffcf14 --- /dev/null +++ b/src/observer/table/ob_table_modify_executor.cpp @@ -0,0 +1,271 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_modify_executor.h" +#include "sql/engine/dml/ob_dml_service.h" + +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ +int ObTableApiModifyExecutor::open() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(init_das_ref())) { + LOG_WARN("fail to init das dml ctx", K(ret)); + } else { + is_opened_ = true; + } + return ret; +} + +OB_INLINE int ObTableApiModifyExecutor::init_das_ref() +{ + int ret = OB_SUCCESS; + ObDASRef &das_ref = dml_rtctx_.das_ref_; + ObSQLSessionInfo *session = GET_MY_SESSION(exec_ctx_); + + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null session", K(ret)); + } else { + // todo@dazhi: use different labels for each dml type + const char *label = "DmlDASCtx"; + const bool use_dist_das = false; + ObMemAttr memattr(session->get_effective_tenant_id(), label, ObCtxIds::EXECUTE_CTX_ID); + // das_ref.set_expr_frame_info(expr_frame_info_); + das_ref.set_mem_attr(memattr); + das_ref.set_execute_directly(!use_dist_das); + } + + return OB_SUCCESS; +} + +int ObTableApiModifyExecutor::submit_all_dml_task() +{ + int ret = OB_SUCCESS; + ObDASRef &das_ref = dml_rtctx_.das_ref_; + + if (das_ref.has_task()) { + if (OB_FAIL(das_ref.execute_all_task())) { + LOG_WARN("fail to execute all dml das tasks", K(ret)); + } else if (OB_FAIL(das_ref.close_all_task())) { + LOG_WARN("fail to close all dml tasks", K(ret)); + } else { + das_ref.reuse(); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::close() +{ + int ret = OB_SUCCESS; + ObDASRef &das_ref = dml_rtctx_.das_ref_; + + if (!is_opened_) { + // do nothing + } else if (das_ref.has_task()) { + if (OB_FAIL(das_ref.close_all_task())) { + LOG_WARN("fail to close all insert das task", K(ret)); + } else { + das_ref.reset(); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::init_das_dml_rtdef(const ObDASDMLBaseCtDef &das_ctdef, + ObDASDMLBaseRtDef &das_rtdef, + const ObDASTableLocMeta *loc_meta) +{ + int ret = OB_SUCCESS; + + ObDASRef &das_ref = dml_rtctx_.das_ref_; + ObSQLSessionInfo *my_session = GET_MY_SESSION(das_ref.get_exec_ctx()); + ObDASCtx &das_ctx = das_ref.get_exec_ctx().get_das_ctx(); + uint64_t table_loc_id = das_ctdef.table_id_; + uint64_t ref_table_id = das_ctdef.index_tid_; + das_rtdef.timeout_ts_ = tb_ctx_.get_timeout_ts(); + das_rtdef.prelock_ = my_session->get_prelock(); + das_rtdef.tenant_schema_version_ = tb_ctx_.get_tenant_schema_version(); + das_rtdef.sql_mode_ = my_session->get_sql_mode(); + das_rtdef.table_loc_ = das_ctx.get_table_loc_by_id(table_loc_id, ref_table_id); + if (OB_ISNULL(das_rtdef.table_loc_)) { + if (OB_ISNULL(loc_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("loc meta is null", K(ret), K(table_loc_id), + K(ref_table_id), K(das_ctx.get_table_loc_list())); + } else if (OB_FAIL(das_ctx.extended_table_loc(*loc_meta, das_rtdef.table_loc_))) { + LOG_WARN("extended table location failed", K(ret), KPC(loc_meta)); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::init_related_das_rtdef(const DASDMLCtDefArray &das_ctdefs, + DASDMLRtDefArray &das_rtdefs) +{ + int ret = OB_SUCCESS; + ObDASRef &das_ref = dml_rtctx_.das_ref_; + int64_t ct_count = das_ctdefs.count(); + + if (!das_ctdefs.empty()) { + ObIAllocator &allocator = das_ref.get_exec_ctx().get_allocator(); + if (OB_FAIL(das_rtdefs.allocate_array(allocator, ct_count))) { + SQL_DAS_LOG(WARN, "fail to create das insert rtdef array", K(ret), K(ct_count)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < ct_count; ++i) { + ObDASTaskFactory &das_factory = das_ref.get_exec_ctx().get_das_ctx().get_das_factory(); + ObDASBaseRtDef *das_rtdef = nullptr; + if (OB_FAIL(das_factory.create_das_rtdef(das_ctdefs.at(i)->op_type_, das_rtdef))) { + SQL_DAS_LOG(WARN, "fail to create das insert rtdef", K(ret)); + } else if (OB_FAIL(init_das_dml_rtdef(*das_ctdefs.at(i), + static_cast(*das_rtdef), + nullptr))) { + SQL_DAS_LOG(WARN, "fail to init das dml rtdef", K(ret)); + } else { + das_rtdefs.at(i) = static_cast(das_rtdef); + } + } + + return ret; +} + +int ObTableApiModifyExecutor::calc_tablet_loc(ObDASTabletLoc *&tablet_loc) +{ + int ret = OB_SUCCESS; + ObTableID table_loc_id = get_table_ctx().get_ref_table_id(); + ObTableID ref_table_id = get_table_ctx().get_ref_table_id();; + ObDASCtx &das_ctx = exec_ctx_.get_das_ctx(); + ObDASTableLoc *table_loc = nullptr; + + if (OB_ISNULL(table_loc = das_ctx.get_table_loc_by_id(table_loc_id, ref_table_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get table location by table id failed", K(ret), + K(table_loc_id), K(ref_table_id), K(das_ctx.get_table_loc_list())); + } else { + tablet_loc = table_loc->get_first_tablet_loc(); + } + + return ret; +} + +int ObTableApiModifyExecutor::generate_ins_rtdef(const ObTableInsCtDef &ins_ctdef, + ObTableInsRtDef &ins_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(ins_ctdef.das_ctdef_, + ins_rtdef.das_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(ins_ctdef.related_ctdefs_, + ins_rtdef.related_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + ins_rtdef.das_rtdef_.related_ctdefs_ = &ins_ctdef.related_ctdefs_; + ins_rtdef.das_rtdef_.related_rtdefs_ = &ins_rtdef.related_rtdefs_; + } + + return ret; +} + +int ObTableApiModifyExecutor::generate_del_rtdef(const ObTableDelCtDef &del_ctdef, + ObTableDelRtDef &del_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(del_ctdef.das_ctdef_, + del_rtdef.das_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(del_ctdef.related_ctdefs_, + del_rtdef.related_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + del_rtdef.das_rtdef_.related_ctdefs_ = &del_ctdef.related_ctdefs_; + del_rtdef.das_rtdef_.related_rtdefs_ = &del_rtdef.related_rtdefs_; + } + + return ret; +} + +int ObTableApiModifyExecutor::generate_upd_rtdef(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(init_das_dml_rtdef(upd_ctdef.das_ctdef_, + upd_rtdef.das_rtdef_, + nullptr))) { + LOG_WARN("fail to init das dml rtdef", K(ret)); + } else if (OB_FAIL(init_related_das_rtdef(upd_ctdef.related_ctdefs_, + upd_rtdef.related_rtdefs_))) { + LOG_WARN("fail to init related das ctdef", K(ret)); + } else { + upd_rtdef.das_rtdef_.related_ctdefs_ = &upd_ctdef.related_ctdefs_; + upd_rtdef.das_rtdef_.related_rtdefs_ = &upd_rtdef.related_rtdefs_; + dml_rtctx_.get_exec_ctx().set_update_columns(&upd_ctdef.assign_columns_); + } + + return ret; +} + +int ObTableApiModifyExecutor::insert_row_to_das(const ObTableInsCtDef &ins_ctdef, + ObTableInsRtDef &ins_rtdef) +{ + int ret = OB_SUCCESS; + ObDASTabletLoc *tablet_loc = nullptr; + + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc partition key", K(ret)); + } else if (OB_FAIL(ObDMLService::insert_row(ins_ctdef.das_ctdef_, + ins_rtdef.das_rtdef_, + tablet_loc, + dml_rtctx_, + ins_ctdef.new_row_))) { + LOG_WARN("fail to insert row by dml service", K(ret)); + } + + return ret; +} + +int ObTableApiModifyExecutor::delete_row_to_das(const ObTableDelCtDef &del_ctdef, + ObTableDelRtDef &del_rtdef) +{ + int ret = OB_SUCCESS; + ObDASTabletLoc *tablet_loc = nullptr; + + // todo:linjing check rowkey null and skip + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail tp calc tablet location", K(ret)); + } else if (OB_FAIL(ObDMLService::delete_row(del_ctdef.das_ctdef_, + del_rtdef.das_rtdef_, + tablet_loc, + dml_rtctx_, + del_ctdef.old_row_))) { + LOG_WARN("fail to delete row to das op", K(ret), K(del_ctdef), K(del_rtdef)); + } + + return ret; +} + +} // namespace table +} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_modify_executor.h b/src/observer/table/ob_table_modify_executor.h new file mode 100644 index 0000000000..cba8c9a09a --- /dev/null +++ b/src/observer/table/ob_table_modify_executor.h @@ -0,0 +1,100 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_MODIFY_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_MODIFY_EXECUTOR_H +#include "ob_table_executor.h" +#include "sql/engine/dml/ob_dml_ctx_define.h" +#include "sql/engine/dml/ob_table_insert_op.h" + +namespace oceanbase +{ +namespace table +{ +// todo@dazhi: 编译需要，后续修改 dml_rtctx 中的 modify op 引用为指针之后会移除 +static sql::ObTableModifyOp& get_fake_modify_op() +{ + static common::ObArenaAllocator alloc; + static sql::ObPhyOperatorType op_type; + static sql::ObTableInsertSpec op_spec(alloc, op_type); + static sql::ObExecContext exec_ctx(alloc); + static sql::ObSQLSessionInfo session; + exec_ctx.set_my_session(&session); + static sql::ObTableInsertOpInput input(exec_ctx, op_spec); // ObDMLService::init_das_dml_rtdef 需要 + static sql::ObPhysicalPlan phy_plan; // ObDMLService::init_das_dml_rtdef 需要 + phy_plan.set_plan_type(OB_PHY_PLAN_LOCAL); + op_spec.plan_ = &phy_plan; + static sql::ObTableInsertOp ins_op(exec_ctx, op_spec, &input); + return ins_op; +} + +class ObTableApiModifySpec : public ObTableApiSpec +{ +public: + ObTableApiModifySpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiSpec(alloc, type), + expr_frame_info_(nullptr) + { + } + virtual ~ObTableApiModifySpec() + { + } + sql::ObExprFrameInfo *expr_frame_info_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiModifySpec); +}; + +class ObTableApiModifyExecutor : public ObTableApiExecutor +{ +public: + ObTableApiModifyExecutor(ObTableCtx &ctx) + : ObTableApiExecutor(ctx), + dml_rtctx_(eval_ctx_, exec_ctx_, get_fake_modify_op()), + affected_rows_(0) + { + } + virtual ~ObTableApiModifyExecutor() + { + } +public: + virtual int open() override; + virtual int close() override; + int init_das_ref(); + int submit_all_dml_task(); + int init_das_dml_rtdef(const sql::ObDASDMLBaseCtDef &das_ctdef, + sql::ObDASDMLBaseRtDef &das_rtdef, + const sql::ObDASTableLocMeta *loc_meta); + int init_related_das_rtdef(const sql::DASDMLCtDefArray &das_ctdefs, + sql::DASDMLRtDefArray &das_rtdefs); + int calc_tablet_loc(sql::ObDASTabletLoc *&tablet_loc); + OB_INLINE int64_t get_affected_rows() const { return affected_rows_; } + int get_affected_entity(ObITableEntity *&entity); +protected: + int generate_ins_rtdef(const ObTableInsCtDef &ins_ctdef, + ObTableInsRtDef &ins_rtdef); + int generate_del_rtdef(const ObTableDelCtDef &del_ctdef, + ObTableDelRtDef &del_rtdef); + int generate_upd_rtdef(const ObTableUpdCtDef &upd_ctdef, + ObTableUpdRtDef &upd_rtdef); + int insert_row_to_das(const ObTableInsCtDef &ins_ctdef, + ObTableInsRtDef &ins_rtdef); + int delete_row_to_das(const ObTableDelCtDef &del_ctdef, + ObTableDelRtDef &del_rtdef); +protected: + sql::ObDMLRtCtx dml_rtctx_; + int64_t affected_rows_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_MODIFY_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_op_wrapper.cpp b/src/observer/table/ob_table_op_wrapper.cpp new file mode 100644 index 0000000000..ed0379acb2 --- /dev/null +++ b/src/observer/table/ob_table_op_wrapper.cpp @@ -0,0 +1,448 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_op_wrapper.h" +#include "ob_table_modify_executor.h" +#include "share/table/ob_table.h" +#include "ob_table_scan_executor.h" +#include "ob_htable_utils.h" +#include "ob_htable_filter_operator.h" +#include "ob_table_insert_up_executor.h" +#include "ob_table_cg_service.h" + +namespace oceanbase +{ +namespace table +{ +template +int ObTableOpWrapper::process_op(ObTableCtx &tb_ctx, ObTableOperationResult &op_result) +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + ObTableApiCacheGuard cache_guard; + if (OB_FAIL(get_or_create_spec(tb_ctx, cache_guard, spec))) { + LOG_WARN("fail to get or create spec", K(ret), K(TYPE)); + }else if (OB_FAIL(process_op_with_spec(tb_ctx, spec, op_result))) { + LOG_WARN("fail to process op with spec", K(ret), K(TYPE)); + } else { + tb_ctx.set_expr_info(nullptr); + } + + return ret; +} + +// 生成/匹配计划 +template +int ObTableOpWrapper::get_or_create_spec(ObTableCtx &tb_ctx, + ObTableApiCacheGuard &cache_guard, + ObTableApiSpec *&spec) +{ + int ret = OB_SUCCESS; + ObExprFrameInfo *expr_frame_info; + if (OB_FAIL(cache_guard.init(&tb_ctx))) { + LOG_WARN("fail to init cache guard", K(ret)); + } else if (OB_FAIL(cache_guard.get_expr_info(&tb_ctx, expr_frame_info))) { + LOG_WARN("fail to get expr frame info", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::alloc_exprs_memory(tb_ctx, *expr_frame_info))) { + LOG_WARN("fail to alloc exprs memory", K(ret)); + } else if (FALSE_IT(tb_ctx.set_expr_info(expr_frame_info))) { + } else if (FALSE_IT(tb_ctx.set_init_flag(true))) { + } else if (OB_FAIL(cache_guard.get_spec(&tb_ctx, spec))) { + LOG_WARN("fail to get spec from cache", K(ret), K(TYPE)); + } + return ret; +} + +// 根据执行计划驱动executor执行 +int ObTableOpWrapper::process_op_with_spec(ObTableCtx &tb_ctx, + ObTableApiSpec *spec, + ObTableOperationResult &op_result) +{ + int ret = OB_SUCCESS; + ObTableApiExecutor *executor = nullptr; + if (OB_ISNULL(spec)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ObTableApiSpec is NULL", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx, executor))) { + LOG_WARN("fail to create executor", K(ret)); + } else if (OB_FAIL(executor->open())) { + LOG_WARN("fail to open", K(ret)); + } else if (OB_FAIL(executor->get_next_row())) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to execute", K(ret)); + } + } + + if (OB_SUCC(ret)) { + ObTableApiModifyExecutor *modify_executor = static_cast(executor); + op_result.set_affected_rows(modify_executor->get_affected_rows()); + if (tb_ctx.return_affected_entity() + && OB_FAIL(process_affected_entity(tb_ctx, *spec, *executor, op_result))) { + LOG_WARN("fail to process affected entity", K(ret), K(tb_ctx)); + } + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = executor->close())) { + LOG_WARN("fail to close executor", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + op_result.set_errno(ret); + op_result.set_type(tb_ctx.get_opertion_type()); + spec->destroy_executor(executor); + return ret; +} + +int ObTableOpWrapper::process_affected_entity(ObTableCtx &tb_ctx, + const ObTableApiSpec &spec, + ObTableApiExecutor &executor, + ObTableOperationResult &op_result) +{ + int ret = OB_SUCCESS; + ObITableEntity *result_entity = nullptr; + if (TABLE_API_EXEC_INSERT_UP != spec.get_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid spec type", K(ret), K(spec.get_type())); + } else if (OB_FAIL(op_result.get_entity(result_entity))) { + LOG_WARN("fail to get result entity", K(ret), K(result_entity)); + } else { + const ObTableApiInsertUpSpec &ins_up_spec = static_cast(spec); + const ObIArray &full_assign_exprs = ins_up_spec.get_ctdef().upd_ctdef_.full_assign_row_; + const ObIArray &ins_exprs = ins_up_spec.get_ctdef().ins_ctdef_.new_row_; + const ObTableCtx::ObAssignIds &assign_ids = tb_ctx.get_assign_ids(); + const int64_t N = assign_ids.count(); + ObObj* obj_array = static_cast(tb_ctx.get_allocator().alloc(sizeof(ObObj) * N)); + if (OB_ISNULL(obj_array)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("faild to alloc memory for objs", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + uint64_t idx = assign_ids.at(i).idx_; + uint64_t column_id = assign_ids.at(i).column_id_; + const ObColumnSchemaV2 *column_schema = nullptr; + if (OB_ISNULL(column_schema = tb_ctx.get_table_schema()->get_column_schema(column_id))) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("column not exist", K(ret), K(column_id)); + } else { + bool is_duplicated = static_cast(executor).is_insert_duplicated(); + ObExpr *rt_expr = is_duplicated ? full_assign_exprs.at(idx) : ins_exprs.at(idx); + ObDatum *datum = nullptr; + const ObString &column_name = column_schema->get_column_name_str(); + if (OB_FAIL(rt_expr->eval(executor.get_eval_ctx(), datum))) { + LOG_WARN("fail to eval datum", K(ret), K(*rt_expr)); + } else if (OB_FAIL(datum->to_obj(obj_array[i], column_schema->get_meta_type()))){ + LOG_WARN("fail to datum to obj", K(ret), K(*datum), K(column_schema->get_meta_type())); + } else if (OB_FAIL(result_entity->set_property(column_name, obj_array[i]))) { + LOG_WARN("fail to set property", K(ret), K(column_name), K(obj_array[i])); + } + } + } + + if (OB_SUCC(ret) && tb_ctx.return_rowkey()) { + if (OB_FAIL(result_entity->deep_copy_rowkey(tb_ctx.get_allocator(), *tb_ctx.get_entity()))) { + LOG_WARN("fail to copy entity rowkey", K(ret)); + } + } + } + + return ret; +} + +// get操作单独处理 +int ObTableOpWrapper::process_get(ObTableCtx &tb_ctx, ObNewRow *&row) +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + ObTableApiCacheGuard cache_guard; + if (OB_FAIL(get_or_create_spec(tb_ctx, cache_guard, spec))) { + LOG_WARN("fail to get or create scan spec", K(ret)); + } else if (OB_FAIL(process_get_with_spec(tb_ctx, spec, row))) { + LOG_WARN("fail to process get with spec", K(ret)); + } + return ret; +} + +int ObTableOpWrapper::process_get_with_spec(ObTableCtx &tb_ctx, + ObTableApiSpec *spec, + ObNewRow *&row) +{ + int ret = OB_SUCCESS; + ObTableApiExecutor *executor = nullptr; + ObTableApiScanRowIterator row_iter; + // fill key range + const ObITableEntity *entity = tb_ctx.get_entity(); + ObRowkey rowkey = entity->get_rowkey(); + ObNewRange range; + tb_ctx.get_key_ranges().reset(); + if (OB_FAIL(range.build_range(tb_ctx.get_ref_table_id(), rowkey))) { + LOG_WARN("fail to build key range", K(ret), K(rowkey)); + } else if (OB_FAIL(tb_ctx.get_key_ranges().push_back(range))) { + LOG_WARN("fail to push back key range", K(ret), K(range)); + } else if (OB_ISNULL(spec)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to spec is NULL", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx, executor))) { + LOG_WARN("fail to create scan executor", K(ret)); + } else if (OB_FAIL(row_iter.open(static_cast(executor)))) { + LOG_WARN("fail to open scan row iterator", K(ret)); + } else if (OB_FAIL(row_iter.get_next_row(row))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next row", K(ret)); + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = row_iter.close())) { + LOG_WARN("fail to close row iterator", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + spec->destroy_executor(executor); + return ret; +} + +int ObTableApiUtil::construct_entity_from_row(ObNewRow *row, + const ObTableSchema *table_schema, + const ObIArray &cnames, + ObITableEntity *entity) +{ + int ret = OB_SUCCESS; + const int64_t N = cnames.count(); + const ObColumnSchemaV2 *column_schema = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + const ObString &name = cnames.at(i); + if (OB_ISNULL(column_schema = table_schema->get_column_schema(name))) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("column not exist", K(ret), K(name)); + } else { + int64_t column_idx = table_schema->get_column_idx(column_schema->get_column_id()); + const ObObj &cell = row->get_cell(column_idx); + if (OB_FAIL(entity->set_property(name, cell))) { + LOG_WARN("fail to set property", K(ret), K(name), K(cell)); + } + } + } // end for + return ret; +} + +int ObHTableDeleteExecutor::open() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(executor_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("executor is NULL", K(ret)); + } else if (OB_FAIL(executor_->open())) { + LOG_WARN("fail to open delete executor", K(ret)); + } + return ret; +} + +int ObHTableDeleteExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + const ObTableBatchOperation *batch_op = tb_ctx_.get_batch_operation(); + ObTableQuery query; + if (OB_ISNULL(batch_op)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("batch operation is null", K(ret)); + } else if (OB_FAIL(build_range(query))) { + LOG_WARN("fail to build query range", K(ret)); + } else { + ObHTableFilter &filter = query.htable_filter(); + for (int64_t i = 0; OB_SUCC(ret) && i < batch_op->count(); i++) { + const ObTableOperation &op = batch_op->at(i); + if (OB_FAIL(generate_filter(op.entity(), filter))) { + LOG_WARN("fail to generate htable filter", K(ret), K(op.entity())); + } else if (OB_FAIL(query_and_delete(query))) { + LOG_WARN("fail to query and delete", K(ret), K(query)); + } + } + } + return ret; +} + +int ObHTableDeleteExecutor::close() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(executor_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("executor is NULL", K(ret)); + } else if (OB_FAIL(executor_->close())) { + LOG_WARN("fail to close delete executor", K(ret)); + } + return ret; +} + +int ObHTableDeleteExecutor::build_range(ObTableQuery &query) +{ + int ret = OB_SUCCESS; + common::ObIArray &key_ranges = tb_ctx_.get_key_ranges(); + const ObTableBatchOperation *batch_op = tb_ctx_.get_batch_operation(); + const ObTableOperation &del_op = batch_op->at(0); + const ObITableEntity &entity = del_op.entity(); + ObHTableCellEntity3 htable_cell(&entity); + ObString row = htable_cell.get_rowkey(); + // generate scan range + if (OB_FAIL(query.add_select_column(ObHTableConstants::ROWKEY_CNAME_STR))) { + LOG_WARN("fail to add K", K(ret)); + } else if (OB_FAIL(query.add_select_column(ObHTableConstants::CQ_CNAME_STR))) { + LOG_WARN("fail to add Q", K(ret)); + } else if (OB_FAIL(query.add_select_column(ObHTableConstants::VERSION_CNAME_STR))) { + LOG_WARN("fail to add T", K(ret)); + } else if (OB_FAIL(query.add_select_column(ObHTableConstants::VALUE_CNAME_STR))) { + LOG_WARN("fail to add V", K(ret)); + } else { + query.set_batch(1); // mutate for each row + query.set_max_result_size(-1); + } + ObNewRange range; + pk_objs_start_[0].set_varbinary(row); + pk_objs_start_[1].set_min_value(); + pk_objs_start_[2].set_min_value(); + pk_objs_end_[0].set_varbinary(row); + pk_objs_end_[1].set_max_value(); + pk_objs_end_[2].set_max_value(); + range.start_key_.assign(pk_objs_start_, 3); + range.end_key_.assign(pk_objs_end_, 3); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + + if (OB_FAIL(key_ranges.push_back(range))) { + LOG_WARN("fail to push back hdelete scan range", K(ret), K(key_ranges)); + } + + return ret; +} + +int ObHTableDeleteExecutor::query_and_delete(const ObTableQuery &query) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator; + ObTableApiScanRowIterator row_iter; + ObTableQueryResultIterator *result_iter = nullptr; + ObHTableFilterOperator *htable_result_iter = nullptr; + ObTableQueryResult tmp_result; + ObTableQueryResult *one_result = nullptr; + ObTableApiExecutor *child = nullptr; + if (OB_ISNULL(htable_result_iter = OB_NEWx(ObHTableFilterOperator, (&allocator), query, tmp_result))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to alloc htable query result iterator", K(ret)); + } else if (OB_FAIL(htable_result_iter->parse_filter_string(&allocator))) { + LOG_WARN("failed to parse htable filter string", K(ret)); + } else if (OB_ISNULL(child = const_cast(executor_->get_child()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("scan executor is null", K(ret)); + } else if (OB_FAIL(row_iter.open(static_cast(child)))) { + LOG_WARN("fail to open scan row iterator", K(ret)); + } else { + result_iter = htable_result_iter; + htable_result_iter->set_scan_result(&row_iter); + ObHColumnDescriptor desc; + const ObString &comment = tb_ctx_.get_table_schema()->get_comment_str(); + if (OB_FAIL(desc.from_string(comment))) { + LOG_WARN("fail to parse hcolumn_desc from comment string", K(ret), K(comment)); + } else if (desc.get_time_to_live() > 0) { + htable_result_iter->set_ttl(desc.get_time_to_live()); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(result_iter->get_next_result(one_result))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next result", K(ret)); + } + } else if (OB_ISNULL(one_result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("one_result is NULL", K(ret)); + } else if (FALSE_IT(one_result->rewind())) { + // do nothing + } else if (OB_FAIL(delete_rows(*one_result))) { + LOG_WARN("fail to delete rows", K(ret)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = row_iter.close())) { + LOG_WARN("fail to close row iterator", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + + return ret; +} + +int ObHTableDeleteExecutor::delete_rows(ObTableQueryResult &result) +{ + int ret = OB_SUCCESS; + const ObITableEntity *entity = nullptr; + executor_->set_skip_scan(true); + while (OB_SUCC(result.get_next_entity(entity))) { + executor_->set_entity(entity); + if (OB_FAIL(executor_->get_next_row())) { + LOG_WARN("fail to call delete executor get next row", K(ret), K(entity)); + } else { + affected_rows_++; + } + } + if (OB_FAIL(ret)) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next entity to delete", K(ret)); + } else { + ret = OB_SUCCESS; + } + } + return ret; +} + +int ObHTableDeleteExecutor::generate_filter(const ObITableEntity &entity, + ObHTableFilter &filter) +{ + int ret = OB_SUCCESS; + ObHTableCellEntity3 htable_cell(&entity); + ObString row = htable_cell.get_rowkey(); + ObString qualifier; + filter.set_valid(true); + filter.clear_columns(); + if (htable_cell.last_get_is_null()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("K is null", K(ret), K(entity)); + } else if (FALSE_IT(qualifier = htable_cell.get_qualifier())) { + // do nothing + } else if (htable_cell.last_get_is_null()) { + // delete column family, so we need to scan all qualifier + // wildcard scan + } else if (OB_FAIL(filter.add_column(qualifier))) { + LOG_WARN("failed to add column", K(ret)); + } + int64_t timestamp = -htable_cell.get_timestamp(); // negative to get the original value + if (-ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // INT64_MAX + // delete the most recently added cell + filter.set_max_versions(1); + filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); + } else if (timestamp > 0) { + // delete the specific version + filter.set_max_versions(1); + filter.set_timestamp(timestamp); + } else if (ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // -INT64_MAX + // delete all version + filter.set_max_versions(INT32_MAX); + filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); + } else { + // delete all versions less than or equal to the timestamp + filter.set_max_versions(INT32_MAX); + filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, (-timestamp) + 1); + } + + return ret; +} +} // end namespace table +} // end namespace oceanbase \ No newline at end of file diff --git a/src/observer/table/ob_table_op_wrapper.h b/src/observer/table/ob_table_op_wrapper.h new file mode 100644 index 0000000000..56414843c9 --- /dev/null +++ b/src/observer/table/ob_table_op_wrapper.h @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_OP_WARPPER_H_ +#define OCEANBASE_OBSERVER_OB_TABLE_OP_WARPPER_H_ +#include "ob_table_context.h" +#include "ob_table_executor.h" +#include "ob_table_delete_executor.h" +#include "ob_table_cache.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTableOpWrapper +{ +public: + // dml操作模板函数 + template + static int process_op(ObTableCtx &tb_ctx, ObTableOperationResult &op_result); + // 生成/匹配计划 + template + static int get_or_create_spec(ObTableCtx &tb_ctx, ObTableApiCacheGuard &cache_guard, ObTableApiSpec *&spec); + // 根据执行计划驱动executor执行 + static int process_op_with_spec(ObTableCtx &tb_ctx, ObTableApiSpec *spec, ObTableOperationResult &op_result); + // get特有的逻辑，单独处理 + static int process_get(ObTableCtx &tb_ctx, ObNewRow *&row); + static int process_get_with_spec(ObTableCtx &tb_ctx, ObTableApiSpec *spec, ObNewRow *&row); +private: + static int process_affected_entity(ObTableCtx &tb_ctx, + const ObTableApiSpec &spec, + ObTableApiExecutor &executor, + ObTableOperationResult &op_result); +}; + +class ObTableApiUtil +{ +public: + // schema序的ObNewRow组装成ObTableEntity + static int construct_entity_from_row(ObNewRow *row, + const ObTableSchema *table_schema, + const ObIArray &cnames, + ObITableEntity *entity); +}; + +class ObHTableDeleteExecutor +{ +public: + ObHTableDeleteExecutor(ObTableCtx& tb_ctx, ObTableApiDeleteExecutor *executor) + : tb_ctx_(tb_ctx), + executor_(executor), + affected_rows_(0) {} + virtual ~ObHTableDeleteExecutor() {} + +public: + int open(); + int get_next_row(); + int close(); + int64_t get_affected_rows() { return affected_rows_; } + +private: + int build_range(ObTableQuery &query); + int query_and_delete(const ObTableQuery &query); + int delete_rows(ObTableQueryResult &result); + int generate_filter(const ObITableEntity &entity, + ObHTableFilter &filter); + +private: + ObTableCtx &tb_ctx_; + ObTableApiDeleteExecutor *executor_; + int64_t affected_rows_; + ObObj pk_objs_start_[3]; + ObObj pk_objs_end_[3]; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_OP_WRAPPER_H_ */ \ No newline at end of file diff --git a/src/observer/table/ob_table_query_and_mutate_processor.cpp b/src/observer/table/ob_table_query_and_mutate_processor.cpp index 767444605b..2e480b8e6a 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.cpp +++ b/src/observer/table/ob_table_query_and_mutate_processor.cpp @@ -20,6 +20,10 @@ #include "lib/stat/ob_diagnose_info.h" #include "lib/stat/ob_session_stat.h" #include "ob_htable_utils.h" +#include "ob_table_cg_service.h" +#include "ob_htable_filter_operator.h" +#include "ob_table_op_wrapper.h" + using namespace oceanbase::observer; using namespace oceanbase::common; using namespace oceanbase::table; @@ -28,8 +32,8 @@ using namespace oceanbase::sql; ObTableQueryAndMutateP::ObTableQueryAndMutateP(const ObGlobalContext &gctx) :ObTableRpcProcessor(gctx), - allocator_(ObModIds::TABLE_PROC), - query_ctx_(allocator_) + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + tb_ctx_(allocator_) { } @@ -105,7 +109,6 @@ uint64_t ObTableQueryAndMutateP::get_request_checksum() void ObTableQueryAndMutateP::reset_ctx() { - query_ctx_.reset_query_ctx(access_service_); need_retry_in_queue_ = false; one_result_.reset(); ObTableApiProcessorBase::reset_ctx(); @@ -144,211 +147,653 @@ int ObTableQueryAndMutateP::get_tablet_ids(uint64_t table_id, ObIArray &key_ranges = query.get_scan_ranges(); - if (key_ranges.count() != 1) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("the count of key range of increment query must be 1", K(ret)); - } else { - const ObIArray &columns = htable_filter.get_columns(); - if (columns.count() < 1 && ObTableOperationType::DEL != mutaion.type()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("must specified at least one column qualifier except delete", K(ret)); - } else if (columns.count() == 1) { // from tableapi java client's view, all ops are based on cq - const ObObj *start_key_ptr = key_ranges.at(0).start_key_.get_obj_ptr(); - int64_t start_key_cnt = key_ranges.at(0).start_key_.length(); - const ObObj *end_key_ptr = key_ranges.at(0).end_key_.get_obj_ptr(); - int64_t end_key_cnt = key_ranges.at(0).end_key_.length(); - if (start_key_cnt < 2 || end_key_cnt < 2) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("the rowkey must be longer than 2", K(ret), K(start_key_cnt), K(end_key_cnt)); - } else { - ObObj htable_filter_cq; - htable_filter_cq.set_varbinary(columns.at(0)); - ObObj &start_key_cq = const_cast(start_key_ptr[1]); - ObObj &end_key_cq = const_cast(end_key_ptr[1]); - if (OB_FAIL(ob_write_obj(allocator_, htable_filter_cq, start_key_cq))) { - LOG_WARN("fail to deep copy obobj", K(ret)); - } else if (OB_FAIL(ob_write_obj(allocator_, htable_filter_cq, end_key_cq))) { - LOG_WARN("fail to deep copy obobj", K(ret)); - } else { - if (ObTableOperationType::DEL != mutaion.type()) { // checkAnddelete delete all version - query.set_limit(1); // only lock one row - } - } - } - } else {} // we have to scan additional rows to get result with multi-column - } + ObExprFrameInfo *expr_frame_info = nullptr; + ctx.set_entity(&entity); + ctx.set_entity_type(arg_.entity_type_); + ctx.set_operation_type(op_type); + ctx.set_batch_operation(&arg_.query_and_mutate_.get_mutations()); + + if (!ctx.is_init()) { + if (OB_FAIL(ctx.init_common(credential_, + arg_.tablet_id_, + arg_.table_name_, + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), K(arg_.table_name_)); } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("htable query and mutate must have a valid htable filter", K(ret)); + switch (op_type) { + case ObTableOperationType::GET: { + if (OB_FAIL(ctx.init_get())) { + LOG_WARN("fail to init get ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::INSERT: { + if (OB_FAIL(ctx.init_insert())) { + LOG_WARN("fail to init insert ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::DEL: { + if (OB_FAIL(ctx.init_delete())) { + LOG_WARN("fail to init delete ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::UPDATE: { + if (OB_FAIL(ctx.init_update())) { + LOG_WARN("fail to init update ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::APPEND: + case ObTableOperationType::INCREMENT: + case ObTableOperationType::INSERT_OR_UPDATE: { + if (OB_FAIL(ctx.init_insert_up())) { + LOG_WARN("fail to init insert up ctx", K(ret), K(ctx)); + } + break; + } + case ObTableOperationType::REPLACE: { + if (OB_FAIL(ctx.init_replace())) { + LOG_WARN("fail to init replace ctx", K(ret), K(ctx)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected operation type", "type", op_type); + break; + } + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(ctx.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(ctx)); } } + + return ret; +} + +int ObTableQueryAndMutateP::init_scan_tb_ctx() +{ + int ret = OB_SUCCESS; + ObExprFrameInfo *expr_frame_info = nullptr; + const ObTableQuery &query = arg_.query_and_mutate_.get_query(); + bool is_weak_read = false; + tb_ctx_.set_scan(true); + + if (tb_ctx_.is_init()) { + LOG_INFO("tb ctx has been inited", K_(tb_ctx)); + } else if (OB_FAIL(tb_ctx_.init_common(credential_, + arg_.tablet_id_, + arg_.table_name_, + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), K(arg_.table_name_)); + } else if (OB_FAIL(tb_ctx_.init_scan(query, is_weak_read))) { + LOG_WARN("fail to init table ctx scan part", K(ret), K(arg_.table_name_)); + } else if (OB_FAIL(cache_guard_.init(&tb_ctx_))) { + LOG_WARN("fail to init cache guard", K(ret)); + } else if (OB_FAIL(cache_guard_.get_expr_info(&tb_ctx_, expr_frame_info))) { + LOG_WARN("fail to get expr frame info from cache", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::alloc_exprs_memory(tb_ctx_, *expr_frame_info))) { + LOG_WARN("fail to alloc expr memory", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(tb_ctx_)); + } else { + tb_ctx_.set_init_flag(true); + tb_ctx_.set_expr_info(expr_frame_info); + } + + return ret; +} + +int ObTableQueryAndMutateP::generate_query_result(ObTableApiScanRowIterator &row_iter, + ObTableQueryResultIterator *&result_iter) +{ + int ret = OB_SUCCESS; + const ObTableQuery &query = arg_.query_and_mutate_.get_query(); + ObHTableFilterOperator *htable_result_iter = nullptr; + + if (OB_FAIL(ObTableService::check_htable_query_args(query))) { + LOG_WARN("fail to check htable query args", K(ret)); + } else if (OB_ISNULL(htable_result_iter = OB_NEWx(table::ObHTableFilterOperator, + (&allocator_), + query, + one_result_))) { + LOG_WARN("fail to alloc htable query result iterator", K(ret)); + } else if (htable_result_iter->parse_filter_string(&allocator_)) { + LOG_WARN("failed to parse htable filter string", K(ret)); + } else { + result_iter = htable_result_iter; + htable_result_iter->set_scan_result(&row_iter); + ObHColumnDescriptor desc; + const ObString &comment = tb_ctx_.get_table_schema()->get_comment_str(); + if (OB_FAIL(desc.from_string(comment))) { + LOG_WARN("fail to parse hcolumn_desc from comment string", K(ret), K(comment)); + } else if (desc.get_time_to_live() > 0) { + htable_result_iter->set_ttl(desc.get_time_to_live()); + } + } + + return ret; +} + +int ObTableQueryAndMutateP::execute_htable_delete() +{ + int ret = OB_SUCCESS; + + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableApiCacheGuard cache_guard; + ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); + const ObTableOperation &op = mutations.at(0); + + if (OB_FAIL(init_tb_ctx(tb_ctx, + ObTableOperationType::Type::DEL, + op.entity()))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(ObTableOpWrapper::get_or_create_spec(tb_ctx, cache_guard, spec))) { + LOG_WARN("fail to get spec from cache", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx, executor))) { + LOG_WARN("fail to generate executor", K(ret), K(tb_ctx)); + } else if (OB_FAIL(tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else { + ObHTableDeleteExecutor delete_executor(tb_ctx, static_cast(executor)); + if (OB_FAIL(delete_executor.open())) { + LOG_WARN("fail to open htable delete executor", K(ret)); + } else if (OB_FAIL(delete_executor.get_next_row())) { + LOG_WARN("fail to call htable delete get_next_row", K(ret)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = delete_executor.close())) { + LOG_WARN("fail to close htable delete executor", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + + if (OB_NOT_NULL(spec)) { + spec->destroy_executor(executor); + tb_ctx.set_expr_info(nullptr); + } + } + + return ret; +} + +int ObTableQueryAndMutateP::execute_htable_put() +{ + int ret = OB_SUCCESS; + ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); + int64_t N = mutations.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + const ObTableOperation &op = mutations.at(i); + if (OB_FAIL(execute_htable_put(op.entity()))) { + LOG_WARN("fail to execute hatable insert", K(ret)); + } + } + + return ret; +} + +int ObTableQueryAndMutateP::execute_htable_put(const ObITableEntity &new_entity) +{ + int ret = OB_SUCCESS; + + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + ObTableOperationResult op_result; + if (OB_FAIL(init_tb_ctx(tb_ctx, + ObTableOperationType::Type::INSERT_OR_UPDATE, + new_entity))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else if (OB_FAIL(ObTableOpWrapper::process_op(tb_ctx, op_result))) { + LOG_WARN("fail to process insert up op", K(ret)); + } + } + + return ret; +} + +class ObTableQueryAndMutateP::ColumnIdxComparator +{ +public: + bool operator()(const ColumnIdx &a, const ColumnIdx &b) const + { + return a.first.compare(b.first) < 0; + } +}; + +int ObTableQueryAndMutateP::sort_qualifier(common::ObIArray &columns, + const table::ObTableBatchOperation &increment) +{ + int ret = OB_SUCCESS; + const int64_t N = increment.count(); + if (N <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("empty increment", K(ret)); + } + ObString htable_row; + for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) + { + const ObTableOperation &mutation = increment.at(i); + const ObITableEntity &entity = mutation.entity(); + if (entity.get_rowkey_size() != 3) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("htable should be with 3 rowkey columns", K(ret), K(entity)); + } else { + ObHTableCellEntity3 htable_cell(&entity); + ObString row = htable_cell.get_rowkey(); + bool row_is_null = htable_cell.last_get_is_null(); + ObString qualifier = htable_cell.get_qualifier(); + bool qualifier_is_null = htable_cell.last_get_is_null(); + (void)htable_cell.get_timestamp(); + bool timestamp_is_null = htable_cell.last_get_is_null(); + if (row_is_null || timestamp_is_null || qualifier_is_null) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument for htable put", K(ret), + K(row_is_null), K(timestamp_is_null), K(qualifier_is_null)); + } else { + if (0 == i) { + htable_row = row; // shallow copy + } else if (htable_row != row) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("rowkey not the same", K(ret), K(row), K(htable_row)); + break; + } + if (OB_FAIL(columns.push_back(std::make_pair(qualifier, i)))) { + LOG_WARN("failed to push back", K(ret)); + break; + } + } + } + } // end for + if (OB_SUCC(ret)) { + // sort qualifiers + ColumnIdx *end = &columns.at(columns.count()-1); + ++end; + std::sort(&columns.at(0), end, ColumnIdxComparator()); + } + if (OB_SUCC(ret)) { + // check duplicated qualifiers + for (int64_t i = 0; OB_SUCCESS == ret && i < N-1; ++i) + { + if (columns.at(i).first == columns.at(i+1).first) { + ret = OB_ERR_PARAM_DUPLICATE; + LOG_WARN("duplicated qualifiers", K(ret), "cq", columns.at(i).first, K(i)); + } + } // end for + } + return ret; +} + +int ObTableQueryAndMutateP::get_old_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableApiScanRowIterator row_iter; + + if (OB_FAIL(cache_guard_.get_spec(&tb_ctx_, spec))) { + LOG_WARN("fail to get spec from cache", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx_, executor))) { + LOG_WARN("fail to generate executor", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(row_iter.open(static_cast(executor)))) { + LOG_WARN("fail to open scan row iterator", K(ret)); + } else if (OB_FAIL(row_iter.get_next_row(row))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } else { + ret = OB_SUCCESS; // not exist + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = row_iter.close())) { + LOG_WARN("fail to close row iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + + if (OB_NOT_NULL(spec)) { + spec->destroy_executor(executor); + } + + return ret; +} + +int ObTableQueryAndMutateP::refresh_query_range(const ObObj &new_q_obj) +{ + int ret = OB_SUCCESS; + const ObTableQuery &query = arg_.query_and_mutate_.get_query(); + const ObIArray &ranges = query.get_scan_ranges(); + if (ranges.count() != 1) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid range count", K(ret), K(ranges.count())); + } else { + ObIArray &ctx_ranges = tb_ctx_.get_key_ranges(); + ctx_ranges.reset(); + ObNewRange new_range = ranges.at(0); // shawdow copy + tb_ctx_.set_limit(1); // 设置limit 1，只获取最新版本的数据 + if (FALSE_IT(new_range.start_key_.get_obj_ptr()[ObHTableConstants::COL_IDX_Q] = new_q_obj)) { + // 将mutate的qualifier作为新的扫描范围 + } else if (FALSE_IT(new_range.end_key_.get_obj_ptr()[ObHTableConstants::COL_IDX_Q] = new_q_obj)) { + // 将mutate的qualifier作为新的扫描范围 + } else if (OB_FAIL(ctx_ranges.push_back(new_range))) { + LOG_WARN("fail to push back new range", K(ret)); + } + } + + return ret; +} + +int ObTableQueryAndMutateP::add_to_results(const ObObj &rk, + const ObObj &cq, + const ObObj &ts, + const ObObj &value) +{ + int ret = OB_SUCCESS; + table::ObTableQueryResult &results = result_.affected_entity_; + + if (results.get_property_count() <= 0) { + if (OB_FAIL(results.add_property_name(ObHTableConstants::ROWKEY_CNAME_STR))) { + LOG_WARN("failed to copy name", K(ret)); + } else if (OB_FAIL(results.add_property_name(ObHTableConstants::CQ_CNAME_STR))) { + LOG_WARN("failed to copy name", K(ret)); + } else if (OB_FAIL(results.add_property_name(ObHTableConstants::VERSION_CNAME_STR))) { + LOG_WARN("failed to copy name", K(ret)); + } else if (OB_FAIL(results.add_property_name(ObHTableConstants::VALUE_CNAME_STR))) { + LOG_WARN("failed to copy name", K(ret)); + } + } + if (OB_SUCC(ret)) { + ObObj objs[4]; + objs[0] = rk; + objs[1] = cq; + objs[2] = ts; + int64_t timestamp = 0; + objs[2].get_int(timestamp); + objs[2].set_int(-timestamp); // negate_htable_timestamp + objs[3] = value; + common::ObNewRow row(objs, 4); + if (OB_FAIL(results.add_row(row))) { // deep copy + LOG_WARN("failed to add row to results", K(ret), K(row)); + } + } + return ret; +} + +int ObTableQueryAndMutateP::generate_new_value(const ObNewRow *old_row, + const ObITableEntity &src_entity, + bool is_increment, + ObTableEntity &new_entity) +{ + int ret = OB_SUCCESS; + int64_t now_ms = -ObHTableUtils::current_time_millis(); + ObHTableCellEntity3 htable_cell(&src_entity); + bool row_is_null = htable_cell.last_get_is_null(); + const ObString delta_str = htable_cell.get_value(); + int64_t delta_int = 0; + bool v_is_null = htable_cell.last_get_is_null(); + if (row_is_null || v_is_null) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("row is null or value is null", K(ret), K(row_is_null), K(v_is_null)); + } else if (is_increment && OB_FAIL(ObHTableUtils::java_bytes_to_int64(delta_str, delta_int))) { + LOG_WARN("fail to convert bytes to integer", K(ret), K(delta_str)); + } else { + ObString orig_str; + int64_t new_ts = htable_cell.get_timestamp(); // default insert timestamp + if (ObHTableConstants::LATEST_TIMESTAMP == new_ts) { + new_ts = now_ms; + } + if (OB_NOT_NULL(old_row)) { // 旧行存在，构造新值（base + delta） + if (ObHTableConstants::COL_IDX_V > old_row->count_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid cell count", K(ret), K(old_row->count_)); + } else { + const ObObj &base_obj_t = old_row->get_cell(ObHTableConstants::COL_IDX_T); + const ObObj &base_obj_v = old_row->get_cell(ObHTableConstants::COL_IDX_V); + orig_str = base_obj_v.get_varbinary(); + int64_t orig_ts = base_obj_t.get_int(); + new_ts = min(orig_ts - 1, now_ms); // adapt hbase + if (is_increment) { + int64_t orig_int = 0; + if (OB_FAIL(ObHTableUtils::java_bytes_to_int64(orig_str, orig_int))) { + LOG_WARN("fail to convert bytes to integer", K(ret), K(orig_str)); + } else { + delta_int += orig_int; + } + } + } + } + if (OB_SUCC(ret)) { + ObObj new_k, new_q, new_t, new_v; + // K + new_k.set_varbinary(htable_cell.get_rowkey()); + // Q + new_q.set_varbinary(htable_cell.get_qualifier()); + // T + new_t.set_int(new_ts); + // V + if (is_increment) { + char *bytes = static_cast(allocator_.alloc(sizeof(int64_t))); + if (OB_ISNULL(bytes)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret), KP(bytes)); + } else if (OB_FAIL(ObHTableUtils::int64_to_java_bytes(delta_int, bytes))) { + LOG_WARN("fail to convert bytes", K(ret), K(delta_int)); + } else { + ObString v(sizeof(int64_t), bytes); + new_v.set_varbinary(v); + } + } else { + if (orig_str.empty()) { + new_v.set_varbinary(delta_str); + } else { + int32_t total_len = orig_str.length() + delta_str.length(); + char *bytes = static_cast(allocator_.alloc(total_len)); + if (OB_ISNULL(bytes)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret), KP(bytes)); + } else { + MEMCPY(bytes, orig_str.ptr(), orig_str.length()); + MEMCPY(bytes + orig_str.length(), delta_str.ptr(), delta_str.length()); + ObString new_str(total_len, bytes); + new_v.set_varbinary(new_str); + } + } + } + if (OB_SUCC(ret)) { // generate new entity + if (OB_FAIL(new_entity.add_rowkey_value(new_k))) { + LOG_WARN("fail to add rowkey value", K(ret), K(new_k)); + } else if (OB_FAIL(new_entity.add_rowkey_value(new_q))) { + LOG_WARN("fail to add rowkey value", K(ret), K(new_q)); + } else if (OB_FAIL(new_entity.add_rowkey_value(new_t))) { + LOG_WARN("fail to add rowkey value", K(ret), K(new_t)); + } else if (OB_FAIL(new_entity.set_property(ObHTableConstants::VALUE_CNAME_STR, new_v))) { + LOG_WARN("fail to set property", K(ret), K(new_v)); + } + } + if (OB_SUCC(ret) && arg_.query_and_mutate_.return_affected_entity()) { // set return accected entity + if (OB_FAIL(add_to_results(new_k, new_q, new_t, new_v))) { + LOG_WARN("fail to add to results", K(ret), K(new_k), K(new_q), K(new_t), K(new_v)); + } + } + } + } + + return ret; +} + +// 1. 执行query获取到最新版本旧行的V +// 2. 基于旧V执行mutate +int ObTableQueryAndMutateP::execute_htable_increment() +{ + int ret = OB_SUCCESS; + ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); + int64_t N = mutations.count(); + ObSEArray columns; + if (OB_FAIL(sort_qualifier(columns, mutations))) { + LOG_WARN("fail to sort qualifier", K(ret), K(mutations)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { + ObNewRow *old_row = nullptr; + ObTableEntity new_entity; + const ObTableOperation &op = mutations.at(columns.at(i).second); + bool is_increment = ObTableOperationType::INCREMENT == op.type(); + const ObRowkey &rowkey = op.entity().get_rowkey(); + const ObObj &q_obj = rowkey.get_obj_ptr()[ObHTableConstants::COL_IDX_Q]; // column Q + if (OB_FAIL(refresh_query_range(q_obj))) { + LOG_WARN("fail to refresh query range", K(ret), K(q_obj)); + } else if (OB_FAIL(get_old_row(old_row))) { // 获取旧行，存在/不存在 + LOG_WARN("fail to get old row", K(ret)); + } else if (OB_FAIL(generate_new_value(old_row, op.entity(), is_increment, new_entity))) { + LOG_WARN("fail to generate new value", K(ret), K(op.entity()), KP(old_row)); + } else if (OB_FAIL(execute_htable_insert(new_entity))) { + LOG_WARN("fail to execute hatable insert", K(ret)); + } + } + + return ret; +} + +int ObTableQueryAndMutateP::execute_htable_insert(const ObITableEntity &new_entity) +{ + int ret = OB_SUCCESS; + + SMART_VAR(ObTableCtx, tb_ctx, allocator_) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableOperationResult op_result; + if (OB_FAIL(init_tb_ctx(tb_ctx, + ObTableOperationType::Type::INSERT, + new_entity))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else if (OB_FAIL(ObTableOpWrapper::process_op(tb_ctx, op_result))) { + LOG_WARN("fail to process insert op", K(ret)); + } + } + return ret; } int ObTableQueryAndMutateP::try_process() { int ret = OB_SUCCESS; - const ObTableQuery &query = arg_.query_and_mutate_.get_query(); - ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); - int64_t rpc_timeout = 0; - if (NULL != rpc_pkt_) { - rpc_timeout = rpc_pkt_->get_timeout(); - } - uint64_t &table_id = query_ctx_.param_table_id(); - query_ctx_.init_param(get_timeout_ts(), this, &allocator_, - false/*ignored*/, - arg_.entity_type_, - table::ObBinlogRowImageType::MINIMAL/*ignored*/); - ObSEArray tablet_ids; - const bool is_readonly = false; // query_and_mutate request arg does not contain consisteny_level_ // @see ObTableQueryAndMutateRequest const ObTableConsistencyLevel consistency_level = ObTableConsistencyLevel::STRONG; - ObTableQueryResultIterator *result_iterator = nullptr; - int32_t result_count = 0; - int64_t affected_rows = 0; + ObTableBatchOperation &mutations = arg_.query_and_mutate_.get_mutations(); const ObTableOperation &mutation = mutations.at(0); - bool is_index_supported = true; - if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(check_table_index_supported(table_id, is_index_supported))) { - LOG_WARN("fail to check index supported", K(ret), K(table_id)); - } else if (OB_UNLIKELY(!is_index_supported)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("index type is not supported by table api", K(ret)); - } else if (OB_FAIL(rewrite_htable_query_if_need(mutation, const_cast(query)))) { - LOG_WARN("fail to rewrite query", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(query_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), query_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_UPDATE, consistency_level, table_id, query_ctx_.param_ls_id(), get_timeout_ts()))) { - LOG_WARN("failed to start readonly transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_query(query_ctx_, query, - one_result_, result_iterator, - true /* for update */))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute query", K(ret), K(table_id), K(arg_.entity_type_)); + int affected_rows = 0; + + if (OB_FAIL(init_scan_tb_ctx())) { + LOG_WARN("fail to init scan table ctx", K(ret)); + } else if (OB_FAIL(start_trans(false, /* is_readonly */ + sql::stmt::T_UPDATE, + consistency_level, + tb_ctx_.get_ref_table_id(), + tb_ctx_.get_ls_id(), + get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (ObTableOperationType::INCREMENT == mutation.type()) { + stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_INCREMENT; + if (OB_FAIL(execute_htable_increment())) { + LOG_WARN("fail to execute hatable increment", K(ret)); + } else { + affected_rows = 1; + } + } else if (ObTableOperationType::APPEND == mutation.type()) { + stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_APPEND; + if (OB_FAIL(execute_htable_increment())) { + LOG_WARN("fail to execute hatable increment", K(ret)); + } else { + affected_rows = 1; } } else { - // one_result references to result_ - ObTableQueryResult *one_result = nullptr; - // htable queryAndXXX only check one row - ret = result_iterator->get_next_result(one_result); - if (OB_ITER_END == ret || OB_SUCC(ret)) { - ret = OB_SUCCESS; - one_result = &one_result_; // empty result is OK for APPEND and INCREMENT - switch(mutation.type()) { - case ObTableOperationType::DEL: // checkAndDelete - stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_DELETE; - if (one_result->get_row_count() > 0) { // not empty result means check passed - affected_rows = 1; - int64_t deleted_cells = 0; - SMART_VAR(ObHTableDeleteExecutor, delete_executor, allocator_, - table_id, - tablet_ids.at(0), - query_ctx_.param_ls_id(), - get_timeout_ts(), - this, - table_service_, - access_service_) { - ret = delete_executor.htable_delete(mutations, deleted_cells); - } - } - break; - case ObTableOperationType::INSERT_OR_UPDATE: // checkAndPut - stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_PUT; - if (one_result->get_row_count() > 0) { // not empty result means check passed - affected_rows = 1; - int64_t put_rows = 0; - SMART_VAR(ObHTablePutExecutor, put_executor, allocator_, - table_id, - tablet_ids.at(0), - query_ctx_.param_ls_id(), - get_timeout_ts(), - this, - table_service_) { - ret = put_executor.htable_put(mutations, put_rows); - } - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("put with empty check result is not supported currently", K(ret)); - } - break; - case ObTableOperationType::INCREMENT: // Increment - stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_INCREMENT; - if (one_result->get_row_count() > 0) { // not empty result means check passed - affected_rows = 1; - SMART_VAR(ObHTableIncrementExecutor, inc_executor, ObTableOperationType::INCREMENT, - allocator_, - table_id, - tablet_ids.at(0), - query_ctx_.param_ls_id(), - get_timeout_ts(), - this, - table_service_) { - int64_t put_cells = 0; - table::ObTableQueryResult *results = NULL; - if (arg_.query_and_mutate_.return_affected_entity()) { - results = &result_.affected_entity_; - } - ret = inc_executor.htable_increment(*one_result, mutations, - put_cells, results); - } - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("increment with empty check result is not supported currently", K(ret)); - } - break; - case ObTableOperationType::APPEND: // Append - stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_APPEND; - if (one_result->get_row_count() > 0) { // not empty result means check passed - affected_rows = 1; - SMART_VAR(ObHTableIncrementExecutor, apd_executor, ObTableOperationType::APPEND, - allocator_, - table_id, - tablet_ids.at(0), - query_ctx_.param_ls_id(), - get_timeout_ts(), - this, - table_service_) { - int64_t put_cells = 0; - table::ObTableQueryResult *results = NULL; - if (arg_.query_and_mutate_.return_affected_entity()) { - results = &result_.affected_entity_; - } - ret = apd_executor.htable_increment(*one_result, mutations, - put_cells, results); - } - } else { - ret = OB_NOT_SUPPORTED; - LOG_WARN("append with empty check result is not supported currently", K(ret)); - } - break; - default: - ret = OB_NOT_SUPPORTED; - LOG_WARN("not supported mutation type", K(ret), "type", mutation.type()); - break; - } + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableQueryResultIterator *result_iterator = nullptr; + ObTableApiScanRowIterator row_iter; + if (OB_FAIL(cache_guard_.get_spec(&tb_ctx_, spec))) { + LOG_WARN("fail to get spec from cache", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx_, executor))) { + LOG_WARN("fail to generate executor", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(row_iter.open(static_cast(executor)))) { + LOG_WARN("fail to open scan row iterator", K(ret)); + } else if (OB_FAIL(generate_query_result(row_iter, result_iterator))) { + LOG_WARN("fail to generate query result iterator", K(ret)); } else { - LOG_WARN("failed to get one row", K(ret)); + ObTableQueryResult *one_result = nullptr; + // htable queryAndXXX only check one row + ret = result_iterator->get_next_result(one_result); + if (OB_FAIL(ret) && OB_ITER_END != ret) { + LOG_WARN("fail to get one result", K(ret)); + } else { + ret = OB_SUCCESS; + one_result = &one_result_; // empty result is OK for APPEND and INCREMENT + switch(mutation.type()) { + case ObTableOperationType::DEL: { // checkAndDelete + stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_DELETE; + if (one_result->get_row_count() > 0) { // not empty result means check passed + if (OB_FAIL(execute_htable_delete())) { + LOG_WARN("fail to execute hatable delete", K(ret)); + } else { + affected_rows = 1; + } + } + break; + } + case ObTableOperationType::INSERT_OR_UPDATE: { // checkAndPut + stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_CHECK_AND_PUT; + if (one_result->get_row_count() > 0) { // not empty result means check passed + if (OB_FAIL(execute_htable_put())) { + LOG_WARN("fail to execute hatable put", K(ret)); + } else { + affected_rows = 1; + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("insert or update with empty check result is not supported currently", K(ret)); + } + break; + } + default: { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported mutation type", K(ret), "type", mutation.type()); + break; + } + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = row_iter.close())) { + LOG_WARN("fail to close row iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } } - NG_TRACE_EXT(tag1, OB_ID(found_rows), result_count, - OB_ID(affected_rows), affected_rows); + if (OB_NOT_NULL(spec)) { + spec->destroy_executor(executor); + tb_ctx_.set_expr_info(nullptr); + } } - query_ctx_.destroy_result_iterator(access_service_); + bool need_rollback_trans = (OB_SUCCESS != ret); int tmp_ret = ret; const bool use_sync = true; @@ -356,24 +801,25 @@ int ObTableQueryAndMutateP::try_process() LOG_WARN("failed to end trans", K(ret), "rollback", need_rollback_trans); } ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; - if (OB_SUCC(ret)) { - result_.affected_rows_ = affected_rows; - } else { - result_.affected_rows_ = 0; - } + + result_.affected_rows_ = affected_rows; // record events audit_row_count_ = 1; + int64_t rpc_timeout = 0; + if (NULL != rpc_pkt_) { + rpc_timeout = rpc_pkt_->get_timeout(); + } #ifndef NDEBUG // debug mode LOG_INFO("[TABLE] execute query_and_mutate", K(ret), K_(arg), K(rpc_timeout), - K_(retry_count), K(result_count)); + K_(retry_count)); #else // release mode LOG_TRACE("[TABLE] execute query_and_mutate", K(ret), K_(arg), K(rpc_timeout), K_(retry_count), - "receive_ts", get_receive_timestamp(), K(result_count)); + "receive_ts", get_receive_timestamp()); #endif return ret; } diff --git a/src/observer/table/ob_table_query_and_mutate_processor.h b/src/observer/table/ob_table_query_and_mutate_processor.h index bc144d5e97..863bb8bf6c 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.h +++ b/src/observer/table/ob_table_query_and_mutate_processor.h @@ -17,6 +17,13 @@ #include "share/table/ob_table_rpc_proxy.h" #include "ob_table_rpc_processor.h" #include "ob_table_service.h" +#include "ob_table_context.h" +#include "ob_table_scan_executor.h" +#include "ob_table_update_executor.h" +#include "ob_table_insert_executor.h" +#include "ob_table_cache.h" +#include "ob_table_op_wrapper.h" + namespace oceanbase { @@ -39,18 +46,45 @@ protected: virtual uint64_t get_request_checksum() override; private: + typedef std::pair ColumnIdx; + class ColumnIdxComparator; + int sort_qualifier(common::ObIArray &columns, + const table::ObTableBatchOperation &increment); + int init_scan_tb_ctx(); + int init_tb_ctx(table::ObTableCtx &ctx, + table::ObTableOperationType::Type op_type, + const table::ObITableEntity &entity); + int refresh_query_range(const ObObj &new_q_obj); + int generate_new_value(const ObNewRow *old_row, + const table::ObITableEntity &src_entity, + bool is_increment, + table::ObTableEntity &new_entity); + int add_to_results(const ObObj &rk, + const ObObj &cq, + const ObObj &ts, + const ObObj &value); + int get_old_row(ObNewRow *&row); + int execute_htable_delete(); + int execute_htable_put(); + int execute_htable_increment(); + int execute_htable_insert(const table::ObITableEntity &new_entity); + int execute_htable_put(const table::ObITableEntity &new_entity); + int generate_query_result(table::ObTableApiScanRowIterator &row_iter, + table::ObTableQueryResultIterator *&result_iter); int get_tablet_ids(uint64_t table_id, ObIArray &tablet_ids); int check_rowkey_and_generate_mutations( ObTableQueryResult &one_row, ObTableBatchOperation *&mutations); - // rewrite htable query to avoid lock too much rows for update + // rewrite htable query to avoid lock too much rows for update int rewrite_htable_query_if_need(const ObTableOperation &mutaion, ObTableQuery &query); DISALLOW_COPY_AND_ASSIGN(ObTableQueryAndMutateP); private: common::ObArenaAllocator allocator_; + table::ObTableCtx tb_ctx_; table::ObTableEntityFactory default_entity_factory_; - ObTableServiceQueryCtx query_ctx_; + table::ObTableQueryResult one_result_; + table::ObTableApiCacheGuard cache_guard_; }; } // end namespace observer } // end namespace oceanbase diff --git a/src/observer/table/ob_table_query_processor.cpp b/src/observer/table/ob_table_query_processor.cpp index b50986f026..cd09cf26c3 100644 --- a/src/observer/table/ob_table_query_processor.cpp +++ b/src/observer/table/ob_table_query_processor.cpp @@ -18,6 +18,9 @@ #include "sql/optimizer/ob_table_location.h" // ObTableLocation #include "lib/stat/ob_diagnose_info.h" #include "lib/stat/ob_session_stat.h" +#include "ob_table_scan_executor.h" +#include "ob_table_cg_service.h" +#include "ob_htable_filter_operator.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -26,10 +29,10 @@ using namespace oceanbase::share; using namespace oceanbase::sql; ObTableQueryP::ObTableQueryP(const ObGlobalContext &gctx) - :ObTableRpcProcessor(gctx), - allocator_(ObModIds::TABLE_PROC), - table_service_ctx_(allocator_), - result_row_count_(0) + : ObTableRpcProcessor(gctx), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + tb_ctx_(allocator_), + result_row_count_(0) { // the streaming interface may return multi packet. The memory may be freed after the first packet has been sended. // the deserialization of arg_ is shallow copy, so we need deep copy data to processor @@ -79,7 +82,6 @@ uint64_t ObTableQueryP::get_request_checksum() void ObTableQueryP::reset_ctx() { - table_service_ctx_.reset_query_ctx(access_service_); need_retry_in_queue_ = false; result_row_count_ = 0; ObTableApiProcessorBase::reset_ctx(); @@ -118,54 +120,108 @@ int ObTableQueryP::get_tablet_ids(uint64_t table_id, ObIArray &table return ret; } -int ObTableQueryP::try_process() +int ObTableQueryP::init_tb_ctx() { int ret = OB_SUCCESS; - int64_t rpc_timeout = 0; - if (NULL != rpc_pkt_) { - rpc_timeout = rpc_pkt_->get_timeout(); + ObExprFrameInfo *expr_frame_info = nullptr; + bool is_weak_read = arg_.consistency_level_ == ObTableConsistencyLevel::EVENTUAL; + tb_ctx_.set_scan(true); + + if (tb_ctx_.is_init()) { + LOG_INFO("tb ctx has been inited", K_(tb_ctx)); + } else if (OB_FAIL(tb_ctx_.init_common(credential_, + arg_.tablet_id_, + arg_.table_name_, + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), K(arg_.table_name_)); + } else if (OB_FAIL(tb_ctx_.init_scan(arg_.query_, is_weak_read))) { + LOG_WARN("fail to init table ctx scan part", K(ret), K(arg_.table_name_)); + } else if (OB_FAIL(cache_guard_.init(&tb_ctx_))) { + LOG_WARN("fail to init cache guard", K(ret)); + } else if (OB_FAIL(cache_guard_.get_expr_info(&tb_ctx_, expr_frame_info))) { + LOG_WARN("fail to get expr frame info from cache", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::alloc_exprs_memory(tb_ctx_, *expr_frame_info))) { + LOG_WARN("fail to alloc expr memory", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(tb_ctx_)); + } else { + tb_ctx_.set_init_flag(true); + tb_ctx_.set_expr_info(expr_frame_info); } - const int64_t timeout_ts = get_timeout_ts(); - uint64_t &table_id = table_service_ctx_.param_table_id(); - table_service_ctx_.init_param(timeout_ts, this, &allocator_, - false/*ignored*/, - arg_.entity_type_, - table::ObBinlogRowImageType::MINIMAL/*ignored*/); - ObSEArray tablet_ids; - const bool is_readonly = true; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; - ObTableQueryResultIterator *result_iterator = nullptr; + + return ret; +} + +int ObTableQueryP::query_and_result(ObTableApiScanExecutor *executor) +{ + int ret = OB_SUCCESS; + ObTableQueryResultIterator *result_iter = nullptr; + ObHTableFilterOperator *htable_result_iter = nullptr; + ObNormalTableQueryResultIterator *normal_result_iter = nullptr; + bool is_htable = arg_.query_.get_htable_filter().is_valid(); int32_t result_count = 0; - if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_.param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_.param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_SELECT, consistency_level, table_id, table_service_ctx_.param_ls_id(), timeout_ts))) { - LOG_WARN("failed to start readonly transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_query(table_service_ctx_, arg_.query_, - result_, result_iterator))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute query", K(ret), K(table_id)); + const int64_t timeout_ts = get_timeout_ts(); + ObTableApiScanRowIterator row_iter; + + // 1. create result iterator + if (is_htable) { + if (OB_FAIL(ObTableService::check_htable_query_args(arg_.query_))) { + LOG_WARN("fail to check htable query args", K(ret)); + } else { + htable_result_iter = OB_NEWx(ObHTableFilterOperator, (&allocator_), arg_.query_, result_); + if (OB_ISNULL(htable_result_iter)) { + LOG_WARN("fail to alloc htable query result iterator", K(ret)); + } else if (htable_result_iter->parse_filter_string(&allocator_)) { + LOG_WARN("fail to parse htable filter string", K(ret)); + } else { + result_iter = htable_result_iter; + } } } else { - if (arg_.query_.get_htable_filter().is_valid()) { - // hbase model, compress the result packet + normal_result_iter = OB_NEWx(ObNormalTableQueryResultIterator, (&allocator_), arg_.query_, result_); + if (OB_ISNULL(normal_result_iter)) { + LOG_WARN("fail to alloc normal query result iterator", K(ret)); + } else { + result_iter = normal_result_iter; + } + } + + // 2. set scan row iter to result iterator + if (OB_SUCC(ret)) { + if (OB_ISNULL(executor)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("executor is null", K(ret)); + } else if (OB_FAIL(row_iter.open(executor))) { + LOG_WARN("fail to open scan row iterator", K(ret)); + } else { + if (is_htable) { + htable_result_iter->set_scan_result(&row_iter); + ObHColumnDescriptor desc; + const ObString &comment = executor->get_table_ctx().get_table_schema()->get_comment_str(); + if (OB_FAIL(desc.from_string(comment))) { + LOG_WARN("fail to parse hcolumn_desc from comment string", K(ret), K(comment)); + } else if (desc.get_time_to_live() > 0) { + htable_result_iter->set_ttl(desc.get_time_to_live()); + } + } else { + normal_result_iter->set_scan_result(&row_iter); + } + } + } + + // 3. loop get row and serialize row + if (OB_SUCC(ret)) { + // hbase model, compress the result packet + if (is_htable) { ObCompressorType compressor_type = INVALID_COMPRESSOR; if (OB_FAIL(ObCompressorPool::get_instance().get_compressor_type( - GCONF.tableapi_transport_compress_func, compressor_type))) { + GCONF.tableapi_transport_compress_func, compressor_type))) { compressor_type = INVALID_COMPRESSOR; } else if (NONE_COMPRESSOR == compressor_type) { compressor_type = INVALID_COMPRESSOR; } this->set_result_compress_type(compressor_type); ret = OB_SUCCESS; // reset ret - LOG_DEBUG("[yzfdebug] use compressor", K(compressor_type)); } // one_result references to result_ ObTableQueryResult *one_result = nullptr; @@ -175,19 +231,19 @@ int ObTableQueryP::try_process() if (ObTimeUtility::current_time() > timeout_ts) { ret = OB_TRANS_TIMEOUT; LOG_WARN("exceed operatiton timeout", K(ret)); - } else if (OB_FAIL(result_iterator->get_next_result(one_result))) { + } else if (OB_FAIL(result_iter->get_next_result(one_result))) { if (OB_ITER_END != ret) { LOG_WARN("fail to get next result", K(ret)); } - } else if (result_iterator->has_more_result()) { + } else if (result_iter->has_more_result()) { if (OB_FAIL(this->flush())) { if (OB_ITER_END != ret) { - LOG_WARN("failed to flush result packet", K(ret)); + LOG_WARN("fail to flush result packet", K(ret)); } else { LOG_TRACE("user abort the stream rpc", K(ret)); } } else { - LOG_DEBUG("[yzfdebug] flush one result", K(ret), "row_count", result_.get_row_count()); + LOG_DEBUG("flush one result", K(ret), "row_count", result_.get_row_count()); result_row_count_ += result_.get_row_count(); result_.reset_except_property(); } @@ -197,22 +253,22 @@ int ObTableQueryP::try_process() break; } } - if (OB_ITER_END == ret) { + + if (OB_LIKELY(ret == OB_ITER_END)) { ret = OB_SUCCESS; } - LOG_DEBUG("[yzfdebug] last result", K(ret), "row_count", result_.get_row_count()); + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = row_iter.close())) { + LOG_WARN("fail to close row iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + + LOG_DEBUG("last result", K(ret), "row_count", result_.get_row_count()); NG_TRACE_EXT(tag1, OB_ID(return_rows), result_count, OB_ID(arg2), result_row_count_); } - table_service_ctx_.destroy_result_iterator(access_service_); - bool need_rollback_trans = (OB_SUCCESS != ret); - int tmp_ret = ret; - if (OB_FAIL(end_trans(need_rollback_trans, req_, timeout_ts))) { - LOG_WARN("failed to end trans", K(ret), "rollback", need_rollback_trans); - } - ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; // record events - if (arg_.query_.get_htable_filter().is_valid()) { + if (is_htable) { stat_event_type_ = ObTableProccessType::TABLE_API_HBASE_QUERY; // hbase query } else { stat_event_type_ = ObTableProccessType::TABLE_API_TABLE_QUERY;// table query @@ -221,12 +277,53 @@ int ObTableQueryP::try_process() #ifndef NDEBUG // debug mode - LOG_INFO("[TABLE] execute query", K(ret), K_(arg), K(rpc_timeout), + LOG_INFO("[TABLE] execute query", K(ret), K_(arg), K(timeout_ts), K_(retry_count), K(result_count), K_(result_row_count)); #else // release mode - LOG_TRACE("[TABLE] execute query", K(ret), K_(arg), K(rpc_timeout), K_(retry_count), + LOG_TRACE("[TABLE] execute query", K(ret), K_(arg), K(timeout_ts), K_(retry_count), "receive_ts", get_receive_timestamp(), K(result_count), K_(result_row_count)); #endif + return ret; } + +int ObTableQueryP::try_process() +{ + int ret = OB_SUCCESS; + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + + if (OB_FAIL(init_tb_ctx())) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(cache_guard_.get_spec(&tb_ctx_, spec))) { + LOG_WARN("fail to get spec from cache", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx_, executor))) { + LOG_WARN("fail to generate executor", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(start_trans(true, /* is_readonly */ + sql::stmt::T_SELECT, + arg_.consistency_level_, + tb_ctx_.get_ref_table_id(), + tb_ctx_.get_ls_id(), + tb_ctx_.get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret)); + } else if (OB_FAIL(tb_ctx_.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx_)); + } else if (OB_FAIL(query_and_result(static_cast(executor)))) { + LOG_WARN("fail to query and result", K(ret)); + } + + if (OB_NOT_NULL(spec)) { + spec->destroy_executor(executor); + tb_ctx_.set_expr_info(nullptr); + } + + int tmp_ret = ret; + bool need_rollback_trans = (OB_SUCCESS != ret); + if (OB_FAIL(end_trans(need_rollback_trans, req_, tb_ctx_.get_timeout_ts()))) { + LOG_WARN("fail to end trans", K(ret), K(need_rollback_trans)); + } + ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; + + return ret; +} \ No newline at end of file diff --git a/src/observer/table/ob_table_query_processor.h b/src/observer/table/ob_table_query_processor.h index 048eb688dd..40828345bd 100644 --- a/src/observer/table/ob_table_query_processor.h +++ b/src/observer/table/ob_table_query_processor.h @@ -17,6 +17,9 @@ #include "share/table/ob_table_rpc_proxy.h" #include "ob_table_rpc_processor.h" #include "ob_table_service.h" +#include "ob_table_context.h" +#include "ob_table_executor.h" +#include "ob_table_cache.h" namespace oceanbase { @@ -39,12 +42,17 @@ protected: virtual uint64_t get_request_checksum() override; private: + int init_tb_ctx(); + int query_and_result(table::ObTableApiScanExecutor *executor); int get_tablet_ids(uint64_t table_id, ObIArray &tablet_ids); - DISALLOW_COPY_AND_ASSIGN(ObTableQueryP); + private: common::ObArenaAllocator allocator_; - ObTableServiceQueryCtx table_service_ctx_; + table::ObTableCtx tb_ctx_; int64_t result_row_count_; + table::ObTableApiCacheGuard cache_guard_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableQueryP); }; } // end namespace observer diff --git a/src/observer/table/ob_table_query_sync_processor.cpp b/src/observer/table/ob_table_query_sync_processor.cpp index 67c51a081a..935dec213d 100644 --- a/src/observer/table/ob_table_query_sync_processor.cpp +++ b/src/observer/table/ob_table_query_sync_processor.cpp @@ -22,6 +22,8 @@ #include "lib/string/ob_strings.h" #include "lib/rc/ob_rc.h" #include "storage/tx/ob_trans_service.h" +#include "ob_table_cg_service.h" +#include "ob_htable_filter_operator.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -32,23 +34,6 @@ using namespace oceanbase::sql; /** * ---------------------------------------- ObTableQuerySyncSession ---------------------------------------- */ -int ObTableQuerySyncSession::deep_copy_select_columns(const ObTableQuery &query) -{ - int ret = OB_SUCCESS; - const ObIArray &select_columns = query.get_select_columns(); - const int64_t N = select_columns.count(); - ObString tmp_str; - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - if (OB_FAIL(ob_write_string(*get_allocator(), select_columns.at(i), tmp_str))) { - LOG_WARN("failed to copy column name", K(ret)); - break; - } else if (OB_FAIL(query_.add_select_column(tmp_str))) { - LOG_WARN("failed to add column name", K(ret)); - } - } // end for - return ret; -} - void ObTableQuerySyncSession::set_result_iterator(ObNormalTableQueryResultIterator *query_result) { result_iterator_ = query_result; @@ -75,8 +60,6 @@ int ObTableQuerySyncSession::init() ret = OB_ERR_UNEXPECTED; LOG_WARN("iterator_mementity_ should be NULL", K(ret)); } else { - table_service_ctx_.scan_param_.scan_allocator_ = &mem_context->get_arena_allocator(); - table_service_ctx_.scan_param_.allocator_ = &mem_context->get_arena_allocator(); iterator_mementity_ = mem_context; } return ret; @@ -143,6 +126,7 @@ int ObQuerySyncMgr::init() } if (OB_FAIL(query_session_map_.create(QUERY_SESSION_MAX_SIZE, ObModIds::TABLE_PROC, ObModIds::TABLE_PROC))) { LOG_WARN("fail to create query session map", K(ret)); + } else if (FALSE_IT(timer_.set_run_wrapper(MTL_CTX()))) { // 设置当前租户上下文 } else if (OB_FAIL(timer_.init())) { LOG_WARN("fail to init timer_", K(ret)); } else if (OB_FAIL(timer_.schedule(query_session_recycle_, QUERY_SESSION_CLEAN_DELAY, true))) { @@ -217,14 +201,7 @@ void ObQuerySyncMgr::clean_timeout_query_session() if (OB_FAIL(rollback_trans(*query_session))) { LOG_WARN("failed to rollback trans for query session", K(ret), K(sess_id)); } - access_service = MTL(ObAccessService *); - ObTableServiceQueryCtx *table_service_ctx = query_session->get_table_service_ctx(); - if (OB_ISNULL(table_service_ctx)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("free query session fail, table service context null", K(ret)); - } else { - table_service_ctx->destroy_result_iterator(access_service); - } + query_session->destory_query_ctx(); (void)query_session_map_.erase_refactored(sess_id); OB_DELETE(ObTableQuerySyncSession, ObModIds::TABLE_PROC, query_session); // connection loses or bug exists @@ -298,10 +275,9 @@ int ObQuerySyncMgr::ObGetAllSessionIdOp::operator()(QuerySessionPair &entry) { */ ObTableQuerySyncP::ObTableQuerySyncP(const ObGlobalContext &gctx) : ObTableRpcProcessor(gctx), - table_service_ctx_(nullptr), result_row_count_(0), query_session_id_(0), - allocator_(ObModIds::TABLE_PROC), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), query_session_(nullptr) {} @@ -351,7 +327,6 @@ void ObTableQuerySyncP::reset_ctx() { result_row_count_ = 0; query_session_ = nullptr; - table_service_ctx_ = nullptr; ObTableApiProcessorBase::reset_ctx(); } @@ -484,7 +459,6 @@ int ObTableQuerySyncP::query_scan_with_new_context( } } else if (result_iterator->has_more_result()){ result_.is_end_ = false; - query_session->deep_copy_select_columns(arg_.query_); query_session->set_result_iterator(dynamic_cast(result_iterator)); query_session->set_trans_desc(trans_desc_); // save processor's trans_desc_ to query session } else { @@ -493,42 +467,156 @@ int ObTableQuerySyncP::query_scan_with_new_context( return ret; } +int ObTableQuerySyncP::init_tb_ctx(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + ObExprFrameInfo *expr_frame_info = nullptr; + bool is_weak_read = arg_.consistency_level_ == ObTableConsistencyLevel::EVENTUAL; + ctx.set_scan(true); + + if (ctx.is_init()) { + LOG_INFO("tb ctx has been inited", K(ctx)); + } else if (OB_FAIL(ctx.init_common(credential_, + arg_.tablet_id_, + arg_.table_name_, + get_timeout_ts()))) { + LOG_WARN("fail to init table ctx common part", K(ret), K(arg_.table_name_)); + } else if (OB_FAIL(ctx.init_scan(query_session_->get_query(), is_weak_read))) { + LOG_WARN("fail to init table ctx scan part", K(ret), K(arg_.table_name_)); + } else if (OB_FAIL(cache_guard_.init(&ctx))) { + LOG_WARN("fail to init cache guard", K(ret)); + } else if (OB_FAIL(cache_guard_.get_expr_info(&ctx, expr_frame_info))) { + LOG_WARN("fail to get expr frame info from cache", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::alloc_exprs_memory(ctx, *expr_frame_info))) { + LOG_WARN("fail to alloc expr memory", K(ret)); + } else if (OB_FAIL(ctx.init_exec_ctx())) { + LOG_WARN("fail to init exec ctx", K(ret), K(ctx)); + } else { + ctx.set_init_flag(true); + ctx.set_expr_info(expr_frame_info); + } + + return ret; +} + +int ObTableQuerySyncP::execute_query(ObTableQuerySyncSession &query_session) +{ + int ret = OB_SUCCESS; + ObArenaAllocator *allocator = query_session.get_allocator(); + ObTableQuerySyncCtx &query_ctx = query_session.get_query_ctx(); + ObTableQuery &query = query_session_->get_query(); + ObTableCtx &tb_ctx = query_ctx.tb_ctx_; + ObTableApiScanRowIterator &row_iter = query_ctx.row_iter_; + ObHTableFilterOperator *htable_result_iter = nullptr; + ObNormalTableQueryResultIterator *normal_result_iter = nullptr; + ObTableQueryResultIterator *result_iter = nullptr; + bool is_htable = query.get_htable_filter().is_valid(); + + // 1. create result iterator + if (is_htable) { + if (OB_FAIL(ObTableService::check_htable_query_args(query))) { + LOG_WARN("fail to check htable query args", K(ret)); + } else { + htable_result_iter = OB_NEWx(ObHTableFilterOperator, (allocator), query, result_); + if (OB_ISNULL(htable_result_iter)) { + LOG_WARN("fail to alloc htable query result iterator", K(ret)); + } else if (htable_result_iter->parse_filter_string(allocator)) { + LOG_WARN("fail to parse htable filter string", K(ret)); + } else { + result_iter = htable_result_iter; + } + } + } else { + normal_result_iter = OB_NEWx(ObNormalTableQueryResultIterator, (allocator), query, result_); + if (OB_ISNULL(normal_result_iter)) { + LOG_WARN("fail to alloc normal query result iterator", K(ret)); + } else { + result_iter = normal_result_iter; + } + } + + // 2. create scan executor + if (OB_SUCC(ret)) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + if (OB_FAIL(cache_guard_.get_spec(&tb_ctx, spec))) { + LOG_WARN("fail to get spec from cache", K(ret)); + } else if (OB_FAIL(spec->create_executor(tb_ctx, executor))) { + LOG_WARN("fail to generate executor", K(ret), K(tb_ctx)); + } else { + query_ctx.executor_ = static_cast(executor); + } + } + + // 3. set scan row iter to result iterator + if (OB_SUCC(ret)) { + if (OB_FAIL(row_iter.open(query_ctx.executor_))) { + LOG_WARN("fail to open scan row iterator", K(ret)); + } else { + if (is_htable) { + htable_result_iter->set_scan_result(&row_iter); + ObHColumnDescriptor desc; + const ObString &comment = query_ctx.executor_->get_table_ctx().get_table_schema()->get_comment_str(); + if (OB_FAIL(desc.from_string(comment))) { + LOG_WARN("fail to parse hcolumn_desc from comment string", K(ret), K(comment)); + } else if (desc.get_time_to_live() > 0) { + htable_result_iter->set_ttl(desc.get_time_to_live()); + } + } else { + normal_result_iter->set_scan_result(&row_iter); + } + } + } + + // 4. do scan and save result iter + if (OB_SUCC(ret)) { + ObTableQueryResult *one_result = nullptr; + query_session.set_result_iterator(dynamic_cast(result_iter)); + if (ObTimeUtility::current_time() > timeout_ts_) { + ret = OB_TRANS_TIMEOUT; + LOG_WARN("exceed operatiton timeout", K(ret)); + } else if (OB_FAIL(result_iter->get_next_result(one_result))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next result", K(ret)); + } else { + ret = OB_SUCCESS; + result_.is_end_ = true; + } + } else if (result_iter->has_more_result()) { + result_.is_end_ = false; + query_session.set_trans_desc(trans_desc_); // save processor's trans_desc_ to query session + } else { + // no more result + result_.is_end_ = true; + } + } + + return ret; +} + int ObTableQuerySyncP::query_scan_with_init() { int ret = OB_SUCCESS; - table_service_ctx_ = query_session_->get_table_service_ctx(); - table_service_ctx_->scan_param_.is_thread_scope_ = false; - uint64_t &table_id = table_service_ctx_->param_table_id(); - table_service_ctx_->init_param( - timeout_ts_, - this, - query_session_->get_allocator(), - false /*ignored*/, - arg_.entity_type_, - table::ObBinlogRowImageType::MINIMAL /*ignored*/); - ObSEArray tablet_ids; - table::ObTableQueryResultIterator *result_iterator = nullptr; - const bool is_readonly = true; - const ObTableConsistencyLevel consistency_level = arg_.consistency_level_; + ObArenaAllocator *allocator = query_session_->get_allocator(); + ObTableQuerySyncCtx &query_ctx = query_session_->get_query_ctx(); + ObTableCtx &tb_ctx = query_ctx.tb_ctx_; + ObTableQuery &query = query_session_->get_query(); - if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(get_tablet_ids(table_id, tablet_ids))) { - LOG_WARN("failed to get part id", K(ret)); - } else if (1 != tablet_ids.count()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("should have one tablet", K(ret), K(tablet_ids)); - } else if (FALSE_IT(table_service_ctx_->param_tablet_id() = tablet_ids.at(0))) { - } else if (OB_FAIL(get_ls_id(tablet_ids.at(0), table_service_ctx_->param_ls_id()))) { - LOG_WARN("failed to get ls id", K(ret)); - } else if (OB_FAIL(start_trans(is_readonly, sql::stmt::T_SELECT, consistency_level, table_id, table_service_ctx_->param_ls_id(), timeout_ts_))) { - LOG_WARN("failed to start readonly transaction", K(ret)); - } else if (OB_FAIL(table_service_->execute_query(*table_service_ctx_, arg_.query_, result_, result_iterator))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute query", K(ret), K(table_id)); - } - } else if (OB_FAIL(query_scan_with_new_context(query_session_, result_iterator, timeout_ts_))) { - LOG_WARN("fail to query, need rollback", K(ret)); + if (OB_FAIL(arg_.query_.deep_copy(*allocator, query))) { // 存储的key range是引用，所以这里需要深拷贝 + LOG_WARN("fail to deep copy query", K(ret), K(arg_.query_)); + } else if (OB_FAIL(init_tb_ctx(tb_ctx))) { + LOG_WARN("fail to init table ctx", K(ret)); + } else if (OB_FAIL(start_trans(true, /* is_readonly */ + sql::stmt::T_SELECT, + arg_.consistency_level_, + tb_ctx.get_ref_table_id(), + tb_ctx.get_ls_id(), + tb_ctx.get_timeout_ts()))) { + LOG_WARN("fail to start readonly transaction", K(ret), K(tb_ctx)); + } else if (OB_FAIL(tb_ctx.init_trans(get_trans_desc(), get_tx_snapshot()))) { + LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); + } else if (OB_FAIL(execute_query(*query_session_))) { + LOG_WARN("fail to execute query", K(ret)); } else { audit_row_count_ = result_.get_row_count(); result_.query_session_id_ = query_session_id_; @@ -540,15 +628,31 @@ int ObTableQuerySyncP::query_scan_with_init() int ObTableQuerySyncP::query_scan_without_init() { int ret = OB_SUCCESS; - if (OB_ISNULL(query_session_->get_result_iterator()) || OB_ISNULL(query_session_->get_table_service_ctx())) { + ObTableQueryResultIterator *result_iter = query_session_->get_result_iterator(); + + if (OB_ISNULL(result_iter)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("unexpected null result iterator or table service context", K(ret)); - } else if (OB_FAIL(query_scan_with_old_context(timeout_ts_))) { - LOG_WARN("fail to query scan with old context", K(ret)); + LOG_WARN("unexpected null result iterator", K(ret)); } else { - audit_row_count_ = result_.get_row_count(); - result_.query_session_id_ = query_session_id_; + ObTableQueryResult *query_result = nullptr; + result_iter->set_one_result(&result_); // set result_ as container + if (ObTimeUtility::current_time() > timeout_ts_) { + ret = OB_TRANS_TIMEOUT; + LOG_WARN("exceed operatiton timeout", K(ret)); + } else if (OB_FAIL(result_iter->get_next_result(query_result))) { + if (OB_ITER_END == ret) { + result_.is_end_ = true; // set scan end + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to scan result", K(ret)); + } + } else { + result_.is_end_ = !result_iter->has_more_result(); + result_.query_session_id_ = query_session_id_; + audit_row_count_ = result_.get_row_count(); + } } + return ret; } @@ -583,7 +687,6 @@ int ObTableQuerySyncP::try_process() LOG_WARN("fail to get query session id", K(ret), K(arg_.query_session_id_)); } else if (OB_FAIL(get_query_session(query_session_id_, query_session_))) { LOG_WARN("fail to get query session", K(ret), K(query_session_id_)); - } else if (FALSE_IT(table_service_ctx_ = query_session_->get_table_service_ctx())) { } else if (FALSE_IT(timeout_ts_ = get_timeout_ts())) { } else { if (ObQueryOperationType::QUERY_START == arg_.query_type_) { @@ -599,7 +702,11 @@ int ObTableQuerySyncP::try_process() } ret = tmp_ret; } else if (result_.is_end_) { - if (OB_FAIL(destory_query_session(false))) { + // destroy session后session中的allocator_析构，导致session.query_的select columns内存被回收 + // 因此需要深拷出来 + if (OB_FAIL(deep_copy_result_property_names())) { + LOG_WARN("fail to deep copy result property names", K(ret)); + } else if (OB_FAIL(destory_query_session(false))) { LOG_WARN("fail to destory query session", K(ret), K(query_session_id_)); } } else { @@ -616,19 +723,19 @@ int ObTableQuerySyncP::try_process() int ObTableQuerySyncP::destory_query_session(bool need_rollback_trans) { int ret = OB_SUCCESS; + query_session_->destory_query_ctx(); if (OB_FAIL(end_trans(need_rollback_trans, req_, timeout_ts_))) { LOG_WARN("failed to end trans", K(ret), K(need_rollback_trans)); } int tmp_ret = ret; ObQuerySyncMgr::get_instance().get_locker(query_session_id_).lock(); - if (OB_ISNULL(query_session_) || OB_ISNULL(table_service_ctx_)) { + if (OB_ISNULL(query_session_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected null value", K(ret), KP_(query_session), KP_(table_service_ctx)); + LOG_WARN("Unexpected null value", K(ret), KP_(query_session)); } else if (OB_FAIL(ObQuerySyncMgr::get_instance().get_query_session_map()->erase_refactored(query_session_id_))) { LOG_WARN("fail to erase query session from query sync mgr", K(ret)); } else { - table_service_ctx_->destroy_result_iterator(access_service_); OB_DELETE(ObTableQuerySyncSession, ObModIds::TABLE_PROC, query_session_); LOG_DEBUG("destory query session success", K(ret), K(query_session_id_)); } @@ -648,3 +755,26 @@ int ObTableQuerySyncP::check_query_type() } return ret; } + +int ObTableQuerySyncP::deep_copy_result_property_names() +{ + int ret = OB_SUCCESS; + ObTableQuery &query = query_session_->get_query(); + ObNormalTableQueryResultIterator *result_iterator = query_session_->get_result_iterator(); + if ((OB_NOT_NULL(result_iterator))) { + const ObIArray &select_columns = query.get_select_columns(); + ObTableQueryResult *one_result = result_iterator->get_one_result(); + if (OB_NOT_NULL(one_result)) { + one_result->reset_property_names(); + for (int64_t i = 0; OB_SUCC(ret) && i < select_columns.count(); i++) { + ObString select_column; + if (OB_FAIL(ob_write_string(allocator_, select_columns.at(i), select_column))) { + LOG_WARN("Fail to deep copy select column", K(ret), K(select_columns.at(i))); + } else if (OB_FAIL(one_result->add_property_name(select_column))) { + LOG_WARN("fail to add property name", K(ret), K(select_column)); + } + } + } + } + return ret; +} \ No newline at end of file diff --git a/src/observer/table/ob_table_query_sync_processor.h b/src/observer/table/ob_table_query_sync_processor.h index ff373cceda..d6046ae34b 100644 --- a/src/observer/table/ob_table_query_sync_processor.h +++ b/src/observer/table/ob_table_query_sync_processor.h @@ -17,6 +17,12 @@ #include "share/table/ob_table_rpc_proxy.h" #include "ob_table_rpc_processor.h" #include "ob_table_service.h" +#include "ob_table_context.h" +#include "ob_table_scan_executor.h" +#include "ob_table_cache.h" +#include "sql/plan_cache/ob_cache_object_factory.h" +#include "sql/plan_cache/ob_plan_cache.h" + namespace oceanbase { @@ -28,6 +34,26 @@ class ObPartitionService; namespace observer { +/** + * ---------------------------------------- ObTableQuerySyncCtx ---------------------------------------- + */ +struct ObTableQuerySyncCtx +{ + explicit ObTableQuerySyncCtx(common::ObIAllocator &allocator) + : tb_ctx_(allocator) + {} + table::ObTableCtx tb_ctx_; + table::ObTableApiScanExecutor *executor_; + table::ObTableApiScanRowIterator row_iter_; +public: + void destory() + { + row_iter_.close(); + tb_ctx_.~ObTableCtx(); + executor_->~ObTableApiScanExecutor(); + } +}; + /** * ---------------------------------------- ObTableQuerySyncSession ---------------------------------------- */ @@ -42,37 +68,36 @@ public: tenant_id_(MTL_ID()), query_(), result_iterator_(nullptr), - allocator_(ObModIds::TABLE_PROC), - table_service_ctx_(allocator_), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + query_ctx_(allocator_), iterator_mementity_(nullptr) {} ~ObTableQuerySyncSession(); void set_result_iterator(ObNormalTableQueryResultIterator* iter); - int deep_copy_select_columns(const ObTableQuery &query); void set_in_use(bool in_use) {in_use_ = in_use;} bool is_in_use() {return in_use_;} int init(); void set_timout_ts(uint64_t timeout_ts) { timeout_ts_ = timeout_ts; } - ObTableServiceQueryCtx *get_table_service_ctx() {return &table_service_ctx_;} ObNormalTableQueryResultIterator *get_result_iterator() { return result_iterator_; } ObArenaAllocator *get_allocator() {return &allocator_;} common::ObObjectID get_tenant_id() { return tenant_id_; } - + table::ObTableQuery &get_query() { return query_; } + ObTableQuerySyncCtx &get_query_ctx() { return query_ctx_; } + void destory_query_ctx() { return query_ctx_.destory(); } public: sql::TransState* get_trans_state() {return &trans_state_;} transaction::ObTxDesc* get_trans_desc() {return trans_desc_;} void set_trans_desc(transaction::ObTxDesc *trans_desc) { trans_desc_ = trans_desc; } - private: bool in_use_; uint64_t timeout_ts_; common::ObObjectID tenant_id_; - ObTableQuery query_; // only select_columns is correct + ObTableQuery query_; // deep copy from arg_.query_ ObNormalTableQueryResultIterator *result_iterator_; ObArenaAllocator allocator_; - ObTableServiceQueryCtx table_service_ctx_; + ObTableQuerySyncCtx query_ctx_; lib::MemoryContext iterator_mementity_; private: @@ -191,14 +216,17 @@ private: private: void set_trans_from_session(ObTableQuerySyncSession *query_session); int check_query_type(); + int init_tb_ctx(table::ObTableCtx &ctx); + int execute_query(ObTableQuerySyncSession &query_session); + int deep_copy_result_property_names(); private: - ObTableServiceQueryCtx *table_service_ctx_; int64_t result_row_count_; uint64_t query_session_id_; ObArenaAllocator allocator_; ObTableQuerySyncSession *query_session_; int64_t timeout_ts_; + table::ObTableApiCacheGuard cache_guard_; }; } // end namespace observer diff --git a/src/observer/table/ob_table_replace_executor.cpp b/src/observer/table/ob_table_replace_executor.cpp new file mode 100644 index 0000000000..99e8f6accb --- /dev/null +++ b/src/observer/table/ob_table_replace_executor.cpp @@ -0,0 +1,406 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_replace_executor.h" +#include "lib/utility/ob_tracepoint.h" +#include "sql/das/ob_das_insert_op.h" +#include "ob_table_cg_service.h" + +namespace oceanbase +{ +namespace table +{ + +int ObTableApiReplaceExecutor::generate_replace_rtdef(const ObTableReplaceCtDef &ctdef, + ObTableReplaceRtDef &rtdef) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_ins_rtdef(ctdef.ins_ctdef_, + rtdef.ins_rtdef_))) { + LOG_WARN("fail to generate insert rtdef", K(ret)); + } else if (OB_FAIL(generate_del_rtdef(ctdef.del_ctdef_, + rtdef.del_rtdef_))) { + LOG_WARN("fail to generate delete rtdef", K(ret)); + } else { + rtdef.ins_rtdef_.das_rtdef_.table_loc_->is_writing_ = true; //todo:linjing其他的executor还没有设置is_writting + } + + return ret; +} + +int ObTableApiReplaceExecutor::open() +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_replace_rtdef(replace_spec_.get_ctdef(), replace_rtdef_))) { + LOG_WARN("fail to init replace rtdef", K(ret)); + } else { + ObDASTabletLoc *tablet_loc = nullptr; + ObDASTableLoc *table_loc = replace_rtdef_.ins_rtdef_.das_rtdef_.table_loc_; + if (OB_ISNULL(table_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table location is invalid", K(ret)); + } else if (OB_FAIL(conflict_checker_.init_conflict_checker(replace_spec_.get_expr_frame_info(), + table_loc))) { + LOG_WARN("fail to init conflict_checker", K(ret)); + } else if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("fail to calc tablet location", K(ret)); + } else { + conflict_checker_.set_local_tablet_loc(tablet_loc); + } + } + + return ret; +} + +void ObTableApiReplaceExecutor::set_need_fetch_conflict() +{ + replace_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = true; + dml_rtctx_.set_pick_del_task_first(); + dml_rtctx_.set_non_sub_full_task(); +} + +int ObTableApiReplaceExecutor::refresh_exprs_frame(const ObTableEntity *entity) +{ + int ret = OB_SUCCESS; + const ObTableReplaceCtDef &ctdef = replace_spec_.get_ctdef(); + + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else if (OB_FAIL(ObTableExprCgService::refresh_replace_exprs_frame(tb_ctx_, + ctdef.ins_ctdef_.new_row_, + *entity))) { + LOG_WARN("fail to refresh replace exprs frame", K(ret), K(*entity)); + } + + return ret; +} + +int ObTableApiReplaceExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + if (cur_idx_ >= 1) { + ret = OB_ITER_END; + } else if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } + + return ret; +} + +int ObTableApiReplaceExecutor::load_replace_rows(bool &is_iter_end) +{ + int ret = OB_SUCCESS; + int64_t row_cnt = 0; + const ObTableReplaceCtDef &ctdef = replace_spec_.get_ctdef(); + int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_REPLACE_BATCH_ROW_COUNT); + int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? + simulate_batch_row_cnt : DEFAULT_REPLACE_BATCH_ROW_COUNT; + + while (OB_SUCC(ret) && ++row_cnt < default_row_batch_cnt) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to load next row from child", K(ret)); + } + } else if (OB_FAIL(insert_row_to_das(ctdef.ins_ctdef_, replace_rtdef_.ins_rtdef_))) { + LOG_WARN("fail to insert row to das", K(ret)); + } else { + replace_rtdef_.ins_rtdef_.cur_row_num_ = 1; + cur_idx_++; + } + } + + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + is_iter_end = true; + } + + return ret; +} + +int ObTableApiReplaceExecutor::post_das_task() +{ + int ret = OB_SUCCESS; + + if (dml_rtctx_.das_ref_.has_task()) { + if (OB_FAIL(dml_rtctx_.das_ref_.execute_all_task())) { + LOG_WARN("fail to execute all das task", K(ret)); + } + } + + return ret; +} + +int ObTableApiReplaceExecutor::fetch_conflict_rowkey() +{ + int ret = OB_SUCCESS; + bool got_row = false; + DASTaskIter task_iter = dml_rtctx_.das_ref_.begin_task_iter(); + + while (OB_SUCC(ret) && !task_iter.is_end()) { + if (OB_FAIL(get_next_conflict_rowkey(task_iter))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next conflict rowkey from das_result", K(ret)); + } + } else if (OB_FAIL(conflict_checker_.build_primary_table_lookup_das_task())) { + LOG_WARN("fail to build lookup_das_task", K(ret)); + } + } + + ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); + return ret; +} + +int ObTableApiReplaceExecutor::reset_das_env() +{ + int ret = OB_SUCCESS; + + // 释放第一次try insert的das task + if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { + LOG_WARN("close all das task failed", K(ret)); + } else { + dml_rtctx_.das_ref_.reuse(); + } + + // 因为第二次插入不需要fetch conflict result了，如果有conflict + // 就说明replace into的某些逻辑处理有问题 + replace_rtdef_.ins_rtdef_.das_rtdef_.need_fetch_conflict_ = false; + replace_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_ = false; + + return ret; +} + +int ObTableApiReplaceExecutor::check_values(bool &is_equal, + const ObChunkDatumStore::StoredRow *replace_row, + const ObChunkDatumStore::StoredRow *delete_row) +{ + int ret = OB_SUCCESS; + is_equal = true; + const ObIArray &new_row = get_primary_table_new_row(); + const ObIArray &old_row = get_primary_table_old_row(); + CK(OB_NOT_NULL(delete_row)); + CK(OB_NOT_NULL(replace_row)); + CK(replace_row->cnt_ == new_row.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < new_row.count(); ++i) { + const UIntFixedArray &column_ids = replace_spec_.get_ctdef().ins_ctdef_.column_ids_; + CK(new_row.at(i)->basic_funcs_->null_first_cmp_ == old_row.at(i)->basic_funcs_->null_first_cmp_); + if (OB_SUCC(ret)) { + if (share::schema::ObColumnSchemaV2::is_hidden_pk_column_id(column_ids[i])) { + //隐藏主键列不处理 + } else { + const ObDatum &insert_datum = replace_row->cells()[i]; + const ObDatum &del_datum = delete_row->cells()[i]; + if (0 != new_row.at(i)->basic_funcs_->null_first_cmp_(insert_datum, del_datum)) { + is_equal = false; + } + } + } + } + return ret; +} + + +int ObTableApiReplaceExecutor::get_next_conflict_rowkey(DASTaskIter &task_iter) +{ + int ret = OB_SUCCESS; + bool got_row = false; + + while (OB_SUCC(ret) && !got_row) { + ObNewRow *dup_row = nullptr; + ObChunkDatumStore::StoredRow *stored_row = nullptr; + ObDASWriteBuffer::DmlShadowRow ssr; + ObDASInsertOp *ins_op = static_cast(*task_iter); + ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); + if (OB_ISNULL(conflict_result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("duplicted key result is null", K(ret)); + } else if (OB_FAIL(conflict_result->get_next_row(dup_row))) { + if (OB_ITER_END == ret) { + ++task_iter; + if (!task_iter.is_end()) { + ret = OB_SUCCESS; + } + } else { + LOG_WARN("fail to get next row from das result", K(ret)); + } + } else if (OB_FAIL(ssr.init(dml_rtctx_.get_das_alloc(), ins_ctdef->table_rowkey_types_, false))) { + LOG_WARN("fail to init shadow stored row", K(ret), K(ins_ctdef->table_rowkey_types_)); + } else if (OB_FAIL(ssr.shadow_copy(*dup_row))) { + LOG_WARN("fail to shadow copy ob new row", K(ret)); + } else if (OB_ISNULL(stored_row = ssr.get_store_row())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("stored row is null", K(ret)); + } else if (OB_FAIL(stored_row->to_expr_skip_const( + conflict_checker_.checker_ctdef_.data_table_rowkey_expr_, + conflict_checker_.eval_ctx_))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row from result iterator", K(ret)); + } + } else { + got_row = true; + } + } + + return ret; +} + +int ObTableApiReplaceExecutor::do_delete(ObConflictRowMap *primary_map) +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = nullptr; + const ObTableReplaceCtDef &ctdef = replace_spec_.get_ctdef(); + ObConflictRowMap::iterator start_row_iter = primary_map->begin(); + ObConflictRowMap::iterator end_row_iter = primary_map->end(); + + for (; OB_SUCC(ret) && start_row_iter != end_row_iter; ++start_row_iter) { + const ObRowkey &constraint_rowkey = start_row_iter->first; + ObConflictValue &constraint_value = start_row_iter->second; + if (NULL != constraint_value.baseline_datum_row_) { + //baseline row is not empty, delete it + if (OB_FAIL(constraint_value.baseline_datum_row_->to_expr(get_primary_table_old_row(), + eval_ctx_))) { + LOG_WARN("fail to stored row to expr", K(ret)); + } else if (OB_FAIL(delete_row_to_das(ctdef.del_ctdef_, replace_rtdef_.del_rtdef_))) { + LOG_WARN("fail to shuffle delete row", K(ret), K(constraint_value)); + } else { + replace_rtdef_.del_rtdef_.cur_row_num_ = 1; + } + } + } + + return ret; +} + +int ObTableApiReplaceExecutor::do_insert() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + const ObTableReplaceCtDef &ctdef = replace_spec_.get_ctdef(); + + if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh expr frame", K(ret)); + } else if (OB_FAIL(insert_row_to_das(ctdef.ins_ctdef_, replace_rtdef_.ins_rtdef_))) { + LOG_WARN("shuffle insert row failed", K(ret)); + } else { + replace_rtdef_.ins_rtdef_.cur_row_num_ = 1; + } + + return ret; +} + +int ObTableApiReplaceExecutor::prepare_final_replace_task() +{ + int ret = OB_SUCCESS; + ObConflictRowMap *primary_map = NULL; + + OZ(conflict_checker_.get_primary_table_map(primary_map)); + CK(OB_NOT_NULL(primary_map)); + OZ(do_delete(primary_map)); + OZ(do_insert()); + + return ret; +} + +int ObTableApiReplaceExecutor::reuse() +{ + int ret = OB_SUCCESS; + + if (dml_rtctx_.das_ref_.has_task()) { + if (OB_FAIL(dml_rtctx_.das_ref_.close_all_task())) { + LOG_WARN("fail to close all insert das task", K(ret)); + } else { + dml_rtctx_.das_ref_.reuse(); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(conflict_checker_.reuse())) { + LOG_WARN("fail to reuse conflict checker", K(ret)); + } + } + + return ret; +} + +int ObTableApiReplaceExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + bool is_iter_end = false; + + while (OB_SUCC(ret) && !is_iter_end) { + int64_t savepoint_no = 0; + set_need_fetch_conflict(); + if (OB_FAIL(ObSqlTransControl::create_anonymous_savepoint(exec_ctx_, savepoint_no))) { + LOG_WARN("fail to create save_point", K(ret)); + } else if (OB_FAIL(load_replace_rows(is_iter_end))) { + LOG_WARN("fail to load all row", K(ret)); + } else if (OB_FAIL(post_das_task())) { + LOG_WARN("fail to post all das task", K(ret)); + } else if (!is_duplicated()) { + LOG_DEBUG("try insert is not duplicated", K(ret)); + } else if (OB_FAIL(fetch_conflict_rowkey())) { + LOG_WARN("fail to fetch conflict row", K(ret)); + } else if (OB_FAIL(reset_das_env())) { + // 这里需要reuse das 相关信息 + LOG_WARN("fail to reset das env", K(ret)); + } else if (OB_FAIL(ObSqlTransControl::rollback_savepoint(exec_ctx_, savepoint_no))) { + // 本次插入存在冲突, 回滚到save_point + LOG_WARN("fail to rollback to save_point", K(ret)); + } else if (OB_FAIL(conflict_checker_.do_lookup_and_build_base_map(1))) { + LOG_WARN("fail to do table lookup", K(ret)); + } else if (OB_FAIL(prepare_final_replace_task())) { + LOG_WARN("fail to prepare final das task", K(ret)); + } else if (OB_FAIL(post_das_task())) { + LOG_WARN("do insert rows post process failed", K(ret)); + } + + if (OB_SUCC(ret) && !is_iter_end) { + // 只有还有下一个batch时才需要做reuse，如果没有下一个batch，close和destroy中会释放内存 + // 前边逻辑执行成功，这一批batch成功完成replace, reuse环境, 准备下一个batch + if (OB_FAIL(reuse())) { + LOG_WARN("fail to reuse", K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + affected_rows_ = replace_rtdef_.ins_rtdef_.cur_row_num_ + replace_rtdef_.del_rtdef_.cur_row_num_; + } + + return ret; +} + +int ObTableApiReplaceExecutor::close() +{ + int ret = OB_SUCCESS; + int close_ret = OB_SUCCESS; + + if (is_opened_) { + if (OB_FAIL(conflict_checker_.close())) { + LOG_WARN("fail to close conflict_checker", K(ret)); + } + // close dml das tasks + close_ret = ObTableApiModifyExecutor::close(); + } + + return (OB_SUCCESS == ret) ? close_ret : ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_replace_executor.h b/src/observer/table/ob_table_replace_executor.h new file mode 100644 index 0000000000..d0128b1f7d --- /dev/null +++ b/src/observer/table/ob_table_replace_executor.h @@ -0,0 +1,106 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_TABLE_REPLACE_EXECUTOR_H +#define OCEANBASE_OBSERVER_TABLE_REPLACE_EXECUTOR_H +#include "ob_table_modify_executor.h" +#include "ob_table_context.h" +#include "sql/engine/dml/ob_conflict_checker.h" + +namespace oceanbase +{ +namespace table +{ + +class ObTableApiReplaceSpec : public ObTableApiModifySpec +{ +public: + ObTableApiReplaceSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + replace_ctdef_(alloc), + conflict_checker_ctdef_(alloc) + { + } +public: + OB_INLINE const ObTableReplaceCtDef& get_ctdef() const { return replace_ctdef_; } + OB_INLINE ObTableReplaceCtDef& get_ctdef() { return replace_ctdef_; } + OB_INLINE const sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() const { return conflict_checker_ctdef_; } + OB_INLINE sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() { return conflict_checker_ctdef_; } +private: + ObTableReplaceCtDef replace_ctdef_; + sql::ObConflictCheckerCtdef conflict_checker_ctdef_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiReplaceSpec); +}; + +class ObTableApiReplaceExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiReplaceExecutor(ObTableCtx &ctx, const ObTableApiReplaceSpec &replace_spec) + : ObTableApiModifyExecutor(ctx), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + replace_spec_(replace_spec), + insert_rows_(0), + delete_rows_(0), + conflict_checker_(allocator_, eval_ctx_, replace_spec_.get_conflict_checker_ctdef()), + cur_idx_(0) + { + } +public: + virtual int open(); + virtual int get_next_row(); + virtual int close(); +private: + const static int64_t DEFAULT_REPLACE_BATCH_ROW_COUNT = 1000L; + OB_INLINE const common::ObIArray& get_primary_table_new_row() + { + return replace_spec_.get_ctdef().ins_ctdef_.new_row_; + } + OB_INLINE const common::ObIArray& get_primary_table_old_row() + { + return replace_spec_.get_ctdef().del_ctdef_.old_row_; + } + OB_INLINE bool is_duplicated() + { + return replace_rtdef_.ins_rtdef_.das_rtdef_.is_duplicated_; + } + int generate_replace_rtdef(const ObTableReplaceCtDef &ctdef, + ObTableReplaceRtDef &rtdef); + int refresh_exprs_frame(const ObTableEntity *entity); + void set_need_fetch_conflict(); + int load_replace_rows(bool &is_iter_end); + int get_next_row_from_child(); + int post_das_task(); + int fetch_conflict_rowkey(); + int get_next_conflict_rowkey(sql::DASTaskIter &task_iter); + int reset_das_env(); + int check_values(bool &is_equal, + const ObChunkDatumStore::StoredRow *replace_row, + const ObChunkDatumStore::StoredRow *delete_row); + int prepare_final_replace_task(); + int do_delete(ObConflictRowMap *primary_map); + int do_insert(); + int reuse(); +private: + common::ObArenaAllocator allocator_; + const ObTableApiReplaceSpec &replace_spec_; + ObTableReplaceRtDef replace_rtdef_; + int64_t insert_rows_; + int64_t delete_rows_; + sql::ObConflictChecker conflict_checker_; + int64_t cur_idx_; +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_TABLE_REPLACE_EXECUTOR_H */ \ No newline at end of file diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index da7e66c1c8..c24e73554c 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -27,6 +27,7 @@ #include "observer/mysql/ob_mysql_request_manager.h" #include "share/ob_define.h" #include "storage/tx/ob_trans_service.h" +#include "ob_table_session_pool.h" #include "storage/tx/wrs/ob_weak_read_util.h" using namespace oceanbase::observer; @@ -79,6 +80,8 @@ int ObTableLoginP::process() } else if (OB_FAIL(generate_credential(result_.tenant_id_, result_.user_id_, result_.database_id_, login.ttl_us_, user_token, result_.credential_))) { LOG_WARN("failed to generate credential", K(ret), K(login)); + } else if (OB_FAIL(GCTX.table_service_->get_sess_mgr().update_sess(credential_))) { + LOG_WARN("failed to update session pool", K(ret), K_(credential)); } else { result_.reserved1_ = 0; result_.reserved2_ = 0; @@ -179,58 +182,26 @@ int ObTableLoginP::verify_password(const ObString &tenant, const ObString &user, return ret; } - -OB_SERIALIZE_MEMBER(ObTableApiCredential, - cluster_id_, - tenant_id_, - user_id_, - database_id_, - expire_ts_, - hash_val_); - -ObTableApiCredential::ObTableApiCredential() - :cluster_id_(0), - tenant_id_(0), - user_id_(0), - database_id_(0), - expire_ts_(0), - hash_val_(0) -{ - -} - -ObTableApiCredential::~ObTableApiCredential() -{ - -} - -uint64_t ObTableApiCredential::hash(uint64_t seed /*= 0*/) const -{ - uint64_t hash_val = murmurhash(&cluster_id_, sizeof(cluster_id_), seed); - hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); - hash_val = murmurhash(&user_id_, sizeof(user_id_), hash_val); - hash_val = murmurhash(&database_id_, sizeof(database_id_), hash_val); - hash_val = murmurhash(&expire_ts_, sizeof(expire_ts_), hash_val); - return hash_val; -} - -int ObTableLoginP::generate_credential(uint64_t tenant_id, uint64_t user_id, uint64_t database, - int64_t ttl_us, uint64_t user_token, ObString &credential_str) +int ObTableLoginP::generate_credential(uint64_t tenant_id, + uint64_t user_id, + uint64_t database, + int64_t ttl_us, + uint64_t user_token, + ObString &credential_str) { int ret = OB_SUCCESS; - ObTableApiCredential credential; - credential.cluster_id_ = GCONF.cluster_id; - credential.tenant_id_ = tenant_id; - credential.user_id_ = user_id; - credential.database_id_ = database; + credential_.cluster_id_ = GCONF.cluster_id; + credential_.tenant_id_ = tenant_id; + credential_.user_id_ = user_id; + credential_.database_id_ = database; if (ttl_us > 0) { - credential.expire_ts_ = ObTimeUtility::current_time() + ttl_us; + credential_.expire_ts_ = ObTimeUtility::current_time() + ttl_us; } else { - credential.expire_ts_ = 0; + credential_.expire_ts_ = 0; } - credential.hash_val_ = credential.hash(user_token); + credential_.hash_val_ = credential_.hash(user_token); int64_t pos = 0; - if (OB_FAIL(serialization::encode(credential_buf_, CREDENTIAL_BUF_SIZE, pos, credential))) { + if (OB_FAIL(serialization::encode(credential_buf_, CREDENTIAL_BUF_SIZE, pos, credential_))) { LOG_WARN("failed to serialize credential", K(ret), K(pos)); } else { credential_str.assign_ptr(credential_buf_, static_cast(pos)); @@ -280,36 +251,24 @@ int ObTableApiProcessorBase::check_user_access(const ObString &credential_str) { int ret = OB_SUCCESS; int64_t pos = 0; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObUserInfo *user_info = NULL; - if (OB_ISNULL(gctx_.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid schema service", K(ret)); - } else if (OB_FAIL(serialization::decode(credential_str.ptr(), credential_str.length(), pos, credential_))) { + ObTableApiSessGuard guard; + const ObTableApiCredential *sess_credetial = nullptr; + if (OB_FAIL(serialization::decode(credential_str.ptr(), credential_str.length(), pos, credential_))) { LOG_WARN("failed to serialize credential", K(ret), K(pos)); - } else if (OB_FAIL(gctx_.schema_service_->get_tenant_schema_guard(credential_.tenant_id_, schema_guard))) { - LOG_WARN("fail to get schema guard", K(ret), "tenant_id", credential_.tenant_id_); - } else if (OB_FAIL(schema_guard.get_user_info(credential_.tenant_id_, credential_.user_id_, user_info))) { - LOG_WARN("fail to get user info", K(ret), K(credential_)); - } else if (OB_ISNULL(user_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("user info is null", K(ret), K(credential_)); + } else if (OB_FAIL(GCTX.table_service_->get_sess_mgr().get_sess_info(credential_.tenant_id_, + credential_.user_id_, + guard))) { + LOG_WARN("fail to get session info", K(ret), K_(credential)); + } else if (OB_FAIL(guard.get_credential(sess_credetial))) { + LOG_WARN("fail to get credential", K(ret)); + } else if (sess_credetial->hash_val_ != credential_.hash_val_) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("invalid credential", K(ret), K_(credential), K(*sess_credetial)); + } else if (sess_credetial->cluster_id_ != credential_.cluster_id_) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("invalid credential cluster id", K(ret), K_(credential), K(*sess_credetial)); } else { - const uint64_t user_token = user_info->get_passwd_str().hash(); - uint64_t hash_val = credential_.hash(user_token); - uint64_t my_cluster_id = GCONF.cluster_id; - if (hash_val != credential_.hash_val_) { - ret = OB_ERR_NO_PRIVILEGE; - LOG_WARN("invalid credential", K(ret), K_(credential), K(hash_val)); - } else if (my_cluster_id != credential_.cluster_id_) { - ret = OB_ERR_NO_PRIVILEGE; - LOG_WARN("invalid credential cluster id", K(ret), K_(credential), K(my_cluster_id)); - } else if (user_info->get_is_locked()) { // check whether user is locked. - ret = OB_ERR_USER_IS_LOCKED; - LOG_WARN("user is locked", K(ret), K(credential_)); - } else { - LOG_DEBUG("user can access", K(credential_)); - } + LOG_DEBUG("user can access", K(credential_)); } return ret; } @@ -396,6 +355,34 @@ int ObTableApiProcessorBase::get_tablet_by_rowkey(uint64_t table_id, const ObIAr return ret; } +int ObTableApiProcessorBase::init_read_trans(const ObTableConsistencyLevel consistency_level, + const ObLSID &ls_id, + int64_t timeout_ts) +{ + int ret = OB_SUCCESS; + bool strong_read = ObTableConsistencyLevel::STRONG == consistency_level; + transaction::ObTransService *txs = MTL(transaction::ObTransService*); + + if (OB_FAIL(txs->acquire_tx(trans_desc_, session().get_sessid()))) { + LOG_WARN("failed to acquire tx desc", K(ret)); + } else if (OB_FAIL(setup_tx_snapshot_(*trans_desc_, strong_read, ls_id, timeout_ts))) { + LOG_WARN("setup txn snapshot fail", K(ret), KPC_(trans_desc), K(strong_read), K(ls_id), K(timeout_ts)); + txs->release_tx(*trans_desc_); + trans_desc_ = NULL; + } + + return ret; +} + +void ObTableApiProcessorBase::release_read_trans() +{ + if (OB_NOT_NULL(trans_desc_)) { + transaction::ObTransService *txs = MTL(transaction::ObTransService*); + txs->release_tx(*trans_desc_); + trans_desc_ = NULL; + } +} + int ObTableApiProcessorBase::setup_tx_snapshot_(transaction::ObTxDesc &trans_desc, const bool strong_read, const share::ObLSID &ls_id, @@ -956,594 +943,6 @@ void ObTableRpcProcessor::generate_sql_id() snprintf(audit_record_.sql_id_, (int32_t)sizeof(audit_record_.sql_id_), "TABLEAPI0x%04Xvv%016lX", RpcProcessor::PCODE, checksum); } - -//////////////////////////////////////////////////////////////// -ObHTableDeleteExecutor::ObHTableDeleteExecutor(common::ObArenaAllocator &alloc, - uint64_t table_id, - ObTabletID tablet_id, - ObLSID ls_id, - int64_t timeout_ts, - ObTableApiProcessorBase *processor, - ObTableService *table_service, - ObAccessService *access_service) - :table_service_(table_service), - access_service_(access_service), - query_ctx_(alloc), - mutate_ctx_(alloc) -{ - query_ctx_.param_table_id() = table_id; - query_ctx_.param_tablet_id() = tablet_id; - query_ctx_.param_ls_id() = ls_id; - query_ctx_.init_param(timeout_ts, processor, &alloc, - false/*ignored*/, table::ObTableEntityType::ET_HKV, - table::ObBinlogRowImageType::MINIMAL/*ignored*/); - mutate_ctx_.param_table_id() = table_id; - mutate_ctx_.param_tablet_id() = tablet_id; - mutate_ctx_.param_ls_id() = ls_id; - mutate_ctx_.init_param(timeout_ts, processor, &alloc, - false/*no affected rows*/, table::ObTableEntityType::ET_HKV, - table::ObBinlogRowImageType::MINIMAL/*hbase cell can use put*/); - mutations_result_.set_entity_factory(&entity_factory_); -} - -// @see https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Delete.html -int ObHTableDeleteExecutor::htable_delete(const ObTableBatchOperation &batch_operation, int64_t &affected_rows) -{ - int ret = OB_SUCCESS; - affected_rows = 0; - ObHTableFilter &htable_filter = query_.htable_filter(); - htable_filter.set_valid(true); - if (OB_FAIL(query_.add_select_column(ObHTableConstants::ROWKEY_CNAME_STR))) { - LOG_WARN("failed to add K", K(ret)); - } else if (OB_FAIL(query_.add_select_column(ObHTableConstants::CQ_CNAME_STR))) { - LOG_WARN("failed to add Q", K(ret)); - } else if (OB_FAIL(query_.add_select_column(ObHTableConstants::VERSION_CNAME_STR))) { - LOG_WARN("failed to add T", K(ret)); - } else if (OB_FAIL(query_.add_select_column(ObHTableConstants::VALUE_CNAME_STR))) { - LOG_WARN("failed to add V", K(ret)); - } else { - query_.set_batch(1); // mutate for each row - query_.set_max_result_size(-1); - } - ObObj pk_objs_start[3]; - ObObj pk_objs_end[3]; - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - - const int64_t N = batch_operation.count(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) // for each delete - { - const ObTableOperation &del_op = batch_operation.at(i); - const ObITableEntity &entity = del_op.entity(); - ObHTableCellEntity3 htable_cell(&entity); - ObString row = htable_cell.get_rowkey(); - if (htable_cell.last_get_is_null()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("K is null", K(ret), K(entity)); - break; - } - if (0 == i) { - // generate scan range by K - pk_objs_start[0].set_varbinary(row); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - pk_objs_end[0].set_varbinary(row); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - if (OB_FAIL(query_.add_scan_range(range))) { - LOG_WARN("failed to add range", K(ret)); - break; - } - } - htable_filter.clear_columns(); - ObString qualifier = htable_cell.get_qualifier(); - if (htable_cell.last_get_is_null()) { - // delete column family, so we need to scan all qualifier - // wildcard scan - } else if (OB_FAIL(htable_filter.add_column(qualifier))) { - LOG_WARN("failed to add column", K(ret)); - break; - } - int64_t timestamp = -htable_cell.get_timestamp(); // negative to get the original value - if (-ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // INT64_MAX - // delete the most recently added cell - htable_filter.set_max_versions(1); - htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); - } else if (timestamp > 0) { - // delete the specific version - htable_filter.set_max_versions(1); - htable_filter.set_timestamp(timestamp); - } else if (ObHTableConstants::LATEST_TIMESTAMP == timestamp) { // -INT64_MAX - // delete all version - htable_filter.set_max_versions(INT32_MAX); - htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); - } else { - // delete all versions less than or equal to the timestamp - htable_filter.set_max_versions(INT32_MAX); - htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, (-timestamp)+1); - } - // execute the query - ObTableQueryResultIterator *result_iterator = nullptr; - ObTableQueryResult *one_result = nullptr; - if (OB_FAIL(execute_query(query_, result_iterator))) { - } else { - ret = result_iterator->get_next_result(one_result); - if (OB_ITER_END == ret) { - // empty - ret = OB_SUCCESS; - } else if (OB_SUCCESS != ret) { - LOG_WARN("failed to query", K(ret)); - } else if (OB_ISNULL(one_result)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("one_result is NULL", K(ret)); - } else { - if (OB_FAIL(generate_delete_cells(*one_result, entity_factory_, mutations_))) { - LOG_WARN("failed to delete cells", K(ret)); - } else if (OB_FAIL(execute_mutation(mutations_, mutations_result_))) { - LOG_WARN("failed to execute mutations", K(ret)); - } else { - const int64_t result_num = mutations_result_.count(); - affected_rows += result_num; - } - } // end else - } - query_ctx_.reset_query_ctx(access_service_); - mutate_ctx_.reset_get_ctx(); - } // end for each delete op - return ret; -} - -int ObHTableDeleteExecutor::execute_query(const table::ObTableQuery &query, - ObTableQueryResultIterator *&result_iterator) -{ - int ret = OB_SUCCESS; - one_result_.reset(); - if (OB_FAIL(table_service_->execute_query(query_ctx_, query, - one_result_, result_iterator))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to execute query", K(ret)); - } - } - return ret; -} - -int ObHTableDeleteExecutor::generate_delete_cells( - ObTableQueryResult &one_row, - table::ObTableEntityFactory &entity_factory, - ObTableBatchOperation &mutations_out) -{ - int ret = OB_SUCCESS; - mutations_out.reset(); - entity_factory.free_and_reuse(); - one_row.rewind(); - ObObj rk, cq, ts; - ObObj key1, key2, key3; - // delete all the selected key-values - const ObITableEntity *key_value = nullptr; - while (OB_SUCC(ret) && OB_SUCC(one_row.get_next_entity(key_value))) { - // for each cell of the row - ObHTableCellEntity2 cell(key_value); - key1.set_varbinary(cell.get_rowkey()); // K - key2.set_varbinary(cell.get_qualifier()); // Q - key3.set_int(-cell.get_timestamp()); // T - ObITableEntity* new_entity = entity_factory.alloc(); - if (NULL == new_entity) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no memory", K(ret)); - } else if (OB_FAIL(new_entity->add_rowkey_value(key1))) { - } else if (OB_FAIL(new_entity->add_rowkey_value(key2))) { - } else if (OB_FAIL(new_entity->add_rowkey_value(key3))) { - } else if (OB_FAIL(mutations_out.del(*new_entity))) { - LOG_WARN("failed to add delete operation", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] delete cell", K(ret), "htable_cell", *new_entity, "kv", *key_value); - } - } // end while - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - return ret; -} - -int ObHTableDeleteExecutor::execute_mutation(const ObTableBatchOperation &mutations, - ObTableBatchOperationResult &mutations_result) -{ - int ret = OB_SUCCESS; - mutations_result.reset(); - if (OB_FAIL(table_service_->multi_delete(mutate_ctx_, mutations, mutations_result))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_delete", K(ret)); - } - } - return ret; -} - -//////////////////////////////////////////////////////////////// -ObHTablePutExecutor::ObHTablePutExecutor(common::ObArenaAllocator &alloc, - uint64_t table_id, - common::ObTabletID tablet_id, - share::ObLSID ls_id, - int64_t timeout_ts, - ObTableApiProcessorBase *processor, - ObTableService *table_service) - :table_service_(table_service), - mutate_ctx_(alloc) -{ - mutate_ctx_.param_table_id() = table_id; - mutate_ctx_.param_tablet_id() = tablet_id; - mutate_ctx_.param_ls_id() = ls_id; - mutate_ctx_.init_param(timeout_ts, processor, &alloc, - false/*no affected rows*/, table::ObTableEntityType::ET_HKV, - table::ObBinlogRowImageType::MINIMAL/*hbase cell can use put*/); - - mutations_result_.set_entity_factory(&entity_factory_); -} - -int ObHTablePutExecutor::htable_put(const ObTableBatchOperation &mutations, int64_t &affected_rows, int64_t now_ms/*=0*/) -{ - int ret = OB_SUCCESS; - if (0 == now_ms) { - now_ms = -ObHTableUtils::current_time_millis(); - } - // store not set timestamp cell for tmp - ObArray reset_timestamp_obj; - //ObString htable_row; - const int64_t N = mutations.count(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) - { - const ObTableOperation &mutation = mutations.at(i); - const ObITableEntity &entity = mutation.entity(); - if (ObTableOperationType::INSERT_OR_UPDATE != mutation.type()) { // for insert_or_update only - ret = OB_INVALID_ARGUMENT; - LOG_WARN("htable put should use INSERT_OR_UPDATE", K(ret), K(mutation)); - } else if (entity.get_rowkey_size() != 3) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("htable should be with 3 rowkey columns", K(ret), K(entity)); - } else { - ObRowkey mutate_rowkey = const_cast(entity).get_rowkey(); - ObObj &hbase_timestamp = const_cast(mutate_rowkey.get_obj_ptr()[ObHTableConstants::COL_IDX_T]); // column T - ObHTableCellEntity3 htable_cell(&entity); - bool row_is_null = htable_cell.last_get_is_null(); - int64_t timestamp = htable_cell.get_timestamp(); - bool timestamp_is_null = htable_cell.last_get_is_null(); - if (row_is_null || timestamp_is_null) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument for htable put", K(ret), K(row_is_null), K(timestamp_is_null)); - } else { - // update timestamp iff LATEST_TIMESTAMP - if (ObHTableConstants::LATEST_TIMESTAMP == timestamp) { - hbase_timestamp.set_int(now_ms); - reset_timestamp_obj.push_back(&hbase_timestamp); - } - } - } - } // end for - if (OB_SUCC(ret)) { - // do the multi_put - mutations_result_.reset(); - affected_rows = 0; - if (OB_FAIL(table_service_->multi_insert_or_update(mutate_ctx_, mutations, mutations_result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_delete", K(ret)); - } - if ( OB_TRANSACTION_SET_VIOLATION == ret ) { - // When OB_TRANSACTION_SET_VIOLATION happen, there will not refresh timestamp - // and will cover old row data in processor local retry. so here reset timestamp - // to origin LATEST_TIMESTAMP in order to retry in local - for (int64_t i = 0; i < reset_timestamp_obj.count(); i++) { - reset_timestamp_obj.at(i)->set_int(ObHTableConstants::LATEST_TIMESTAMP); - } - } - } else { - affected_rows = 1; - } - } - return ret; -} -//////////////////////////////////////////////////////////////// -ObHTableIncrementExecutor::ObHTableIncrementExecutor(table::ObTableOperationType::Type type, - common::ObArenaAllocator &alloc, - uint64_t table_id, - common::ObTabletID tablet_id, - share::ObLSID ls_id, - int64_t timeout_ts, - ObTableApiProcessorBase *processor, - ObTableService *table_service) - :type_(type), - table_service_(table_service), - mutate_ctx_(alloc) -{ - mutate_ctx_.param_table_id() = table_id; - mutate_ctx_.param_tablet_id() = tablet_id; - mutate_ctx_.param_ls_id() = ls_id; - mutate_ctx_.init_param(timeout_ts, processor, &alloc, - false/*no affected rows*/, table::ObTableEntityType::ET_HKV, - table::ObBinlogRowImageType::MINIMAL/*hbase cell can use put*/); - mutations_result_.set_entity_factory(&entity_factory_); -} - -class ObHTableIncrementExecutor::ColumnIdxComparator -{ -public: - bool operator()(const ColumnIdx &a, const ColumnIdx &b) const - { - return a.first.compare(b.first) < 0; - } -}; - - -int ObHTableIncrementExecutor::sort_qualifier(const table::ObTableBatchOperation &increment) -{ - int ret = OB_SUCCESS; - const int64_t N = increment.count(); - if (N <= 0) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("empty increment", K(ret)); - } - ObString htable_row; - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) - { - const ObTableOperation &mutation = increment.at(i); - const ObITableEntity &entity = mutation.entity(); - if (type_ != mutation.type()) { // increment or append - ret = OB_INVALID_ARGUMENT; - LOG_WARN("should use INCREMENT/APPEND", K(ret), K_(type), K(mutation)); - } else if (entity.get_rowkey_size() != 3) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("htable should be with 3 rowkey columns", K(ret), K(entity)); - } else { - ObHTableCellEntity3 htable_cell(&entity); - ObString row = htable_cell.get_rowkey(); - bool row_is_null = htable_cell.last_get_is_null(); - ObString qualifier = htable_cell.get_qualifier(); - bool qualifier_is_null = htable_cell.last_get_is_null(); - (void)htable_cell.get_timestamp(); - bool timestamp_is_null = htable_cell.last_get_is_null(); - if (row_is_null || timestamp_is_null || qualifier_is_null) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument for htable put", K(ret), - K(row_is_null), K(timestamp_is_null), K(qualifier_is_null)); - } else { - if (0 == i) { - htable_row = row; // shallow copy - } else if (htable_row != row) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("rowkey not the same", K(ret), K(row), K(htable_row)); - break; - } - if (OB_FAIL(columns_.push_back(std::make_pair(qualifier, i)))) { - LOG_WARN("failed to push back", K(ret)); - break; - } - } - } - } // end for - if (OB_SUCC(ret)) { - // sort qualifiers - ColumnIdx *end = &columns_.at(columns_.count()-1); - ++end; - std::sort(&columns_.at(0), end, ColumnIdxComparator()); - } - if (OB_SUCC(ret)) { - // check duplicated qualifiers - for (int64_t i = 0; OB_SUCCESS == ret && i < N-1; ++i) - { - if (columns_.at(i).first == columns_.at(i+1).first) { - ret = OB_ERR_PARAM_DUPLICATE; - LOG_WARN("duplicated qualifiers", K(ret), "cq", columns_.at(i).first, K(i)); - } - } // end for - } - return ret; -} - -int ObHTableIncrementExecutor::htable_increment(ObTableQueryResult &row_cells, - const table::ObTableBatchOperation &increment, - int64_t &affected_rows, - table::ObTableQueryResult *results) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(sort_qualifier(increment))) { - LOG_WARN("failed to sort qualifier", K(ret)); - } - row_cells.rewind(); - int64_t now_ms = -ObHTableUtils::current_time_millis(); - ObObj rk, cq, ts; - const ObITableEntity *get_value = nullptr; - const int64_t N = increment.count(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) - { - const ObTableOperation &mutation = increment.at(columns_.at(i).second); - const ObITableEntity &kv_entity = mutation.entity(); - ObHTableCellEntity3 kv(&kv_entity); - ObString qualifier = kv.get_qualifier(); - bool need_write = false; - int64_t delta_int = 0; - ObString delta_str = kv.get_value(); - bool value_is_null = kv.last_get_is_null(); - if (value_is_null) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("increment value is invalid", K(ret), K(kv_entity)); - break; - } - - if (type_ == ObTableOperationType::INCREMENT) { - if (OB_FAIL(ObHTableUtils::java_bytes_to_int64(delta_str, delta_int))) { - LOG_WARN("failed to convert bytes to integer", K(ret), K(delta_str)); - break; - } else { - need_write = (0 != delta_int); - } - } else { // ObTableOperationType::APPEND - need_write = true; // always apply for APPEND - } - - if (nullptr == get_value) { - if (OB_FAIL(row_cells.get_next_entity(get_value))) { - if (OB_ITER_END == ret) { - get_value = nullptr; - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get next", K(ret)); - break; - } - } - } - bool first_write = false; - int64_t orig_ts = -1; - ObString orig_str; - if (nullptr != get_value) { - ObHTableCellEntity2 cell(get_value); - ObString qualifier2 = cell.get_qualifier(); - int cmp_ret = ObHTableUtils::compare_qualifier(qualifier, qualifier2); - if (0 == cmp_ret) { - // qualifier exists - orig_str = cell.get_value(); - orig_ts = cell.get_timestamp(); - if (type_ == ObTableOperationType::INCREMENT) { - int64_t orig_int = 0; - if (OB_FAIL(ObHTableUtils::java_bytes_to_int64(orig_str, orig_int))) { - LOG_WARN("failed to convert bytes to integer", K(ret), K(orig_str)); - break; - } else { - delta_int += orig_int; - } - } else { // APPEND - // nothing - } - get_value = nullptr; // next cell - } else { - // qualifier not exist, first write - first_write = true; - } - } else { - // no more cells from get - first_write = true; - } - - rk.set_varbinary(kv.get_rowkey()); // K - cq.set_varbinary(qualifier); // Q - // generate timestamp - if (orig_ts >= 0) { - // already exists - ts.set_int(std::min(-orig_ts, now_ms)); // T - } else { - int64_t new_ts = kv.get_timestamp(); - if (ObHTableConstants::LATEST_TIMESTAMP == new_ts) { - ts.set_int(now_ms); - } else { - ts.set_int(new_ts); - } - } - - // generate V - ObObj value_obj; // V - if (type_ == ObTableOperationType::INCREMENT) { - char* bytes = static_cast(allocator_.alloc(sizeof(int64_t))); - if (NULL == bytes) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no memory", K(ret), KP(bytes)); - } else if (OB_FAIL(ObHTableUtils::int64_to_java_bytes(delta_int, bytes))) { - LOG_WARN("failed to convert bytes", K(ret), K(delta_int)); - } else { - ObString v(sizeof(int64_t), bytes); - value_obj.set_varbinary(v); - } - } else { // APPEND - if (orig_str.empty()) { - value_obj.set_varbinary(delta_str); - } else { - int32_t total_len = orig_str.length() + delta_str.length(); - char* bytes = static_cast(allocator_.alloc(total_len)); - if (NULL == bytes) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no memory", K(ret), KP(bytes)); - } else { - MEMCPY(bytes, orig_str.ptr(), orig_str.length()); - MEMCPY(bytes+orig_str.length(), delta_str.ptr(), delta_str.length()); - ObString new_str(total_len, bytes); - value_obj.set_varbinary(new_str); - } - } - } - - // generate entity - ObITableEntity* new_entity = nullptr; - if (OB_FAIL(ret)) { - } else if (nullptr == (new_entity = entity_factory_.alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no memory", K(ret), KP(new_entity)); - } else { - if (OB_FAIL(new_entity->add_rowkey_value(rk))) { - } else if (OB_FAIL(new_entity->add_rowkey_value(cq))) { - } else if (OB_FAIL(new_entity->add_rowkey_value(ts))) { - } else if (OB_FAIL(new_entity->set_property(ObHTableConstants::VALUE_CNAME_STR, value_obj))) { - } - } - - if (OB_SUCC(ret) && (need_write || first_write)) { - if (OB_FAIL(mutations_.insert_or_update(*new_entity))) { - LOG_WARN("failed to add put operation", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] put cell", K(ret), "new_cell", *new_entity, "kv", kv); - } - } // end if need_write - if (OB_SUCC(ret) && NULL != results) { - // Add to results to get returned to the Client. If null, cilent does not want results. - ret = add_to_results(*results, rk, cq, ts, value_obj); - } - } // end for - if (OB_SUCC(ret) && mutations_.count() > 0) { - // do the multi_put - mutations_result_.reset(); - affected_rows = 0; - if (OB_FAIL(table_service_->multi_insert_or_update(mutate_ctx_, mutations_, mutations_result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to multi_delete", K(ret)); - } - } else { - affected_rows = 1; - } - } - return ret; -} - -int ObHTableIncrementExecutor::add_to_results(table::ObTableQueryResult &results, - const ObObj &rk, const ObObj &cq, - const ObObj &ts, const ObObj &value) -{ - int ret = OB_SUCCESS; - if (results.get_property_count() <= 0) { - if (OB_FAIL(results.add_property_name(ObHTableConstants::ROWKEY_CNAME_STR))) { - LOG_WARN("failed to copy name", K(ret)); - } else if (OB_FAIL(results.add_property_name(ObHTableConstants::CQ_CNAME_STR))) { - LOG_WARN("failed to copy name", K(ret)); - } else if (OB_FAIL(results.add_property_name(ObHTableConstants::VERSION_CNAME_STR))) { - LOG_WARN("failed to copy name", K(ret)); - } else if (OB_FAIL(results.add_property_name(ObHTableConstants::VALUE_CNAME_STR))) { - LOG_WARN("failed to copy name", K(ret)); - } - } - if (OB_SUCC(ret)) { - ObObj objs[4]; - objs[0] = rk; - objs[1] = cq; - objs[2] = ts; - int64_t timestamp = 0; - objs[2].get_int(timestamp); - objs[2].set_int(-timestamp); // negate_htable_timestamp - objs[3] = value; - common::ObNewRow row(objs, 4); - if (OB_FAIL(results.add_row(row))) { // deep copy - LOG_WARN("failed to add row to results", K(ret), K(row)); - } - } - return ret; -} - bool oceanbase::observer::is_bad_routing_err(const int err) { // bad routing check : whether client should refresh location cache @@ -1557,4 +956,4 @@ bool oceanbase::observer::is_bad_routing_err(const int err) || is_has_no_readable_replica_err(err) || is_select_dup_follow_replic_err(err) || is_trans_stmt_need_retry_error(err)); -} +} \ No newline at end of file diff --git a/src/observer/table/ob_table_rpc_processor.h b/src/observer/table/ob_table_rpc_processor.h index 1433bd22b2..58e44bbe85 100644 --- a/src/observer/table/ob_table_rpc_processor.h +++ b/src/observer/table/ob_table_rpc_processor.h @@ -34,29 +34,6 @@ using oceanbase::table::ObTableConsistencyLevel; struct ObGlobalContext; class ObTableService; -struct ObTableApiCredential final -{ - OB_UNIS_VERSION(1); -public: - ObTableApiCredential(); - ~ObTableApiCredential(); -public: - int64_t cluster_id_; - uint64_t tenant_id_; - uint64_t user_id_; - uint64_t database_id_; - int64_t expire_ts_; - uint64_t hash_val_; -public: - uint64_t hash(uint64_t seed = 0) const; - TO_STRING_KV(K_(cluster_id), - K_(tenant_id), - K_(user_id), - K_(database_id), - K_(expire_ts), - K_(hash_val)); -}; - /// @see RPC_S(PR5 login, obrpc::OB_TABLE_API_LOGIN, (table::ObTableLoginRequest), table::ObTableLoginResult); class ObTableLoginP: public obrpc::ObRpcProcessor > { @@ -77,6 +54,7 @@ private: static const int64_t CREDENTIAL_BUF_SIZE = 256; private: const ObGlobalContext &gctx_; + table::ObTableApiCredential credential_; char credential_buf_[CREDENTIAL_BUF_SIZE]; }; @@ -128,10 +106,15 @@ public: static int init_session(); int check_user_access(const ObString &credential_str); // transaction control - int start_trans(bool is_readonly, const sql::stmt::StmtType stmt_type, + int start_trans(bool is_readonly, const sql::stmt::StmtType stmt_type, const table::ObTableConsistencyLevel consistency_level, uint64_t table_id, const share::ObLSID &ls_id, int64_t timeout_ts); int end_trans(bool is_rollback, rpc::ObRequest *req, int64_t timeout_ts, bool use_sync = false); + // for get + int init_read_trans(const table::ObTableConsistencyLevel consistency_level, + const share::ObLSID &ls_id, + int64_t timeout_ts); + void release_read_trans(); inline bool did_async_end_trans() const { return did_async_end_trans_; } inline transaction::ObTxDesc *get_trans_desc() { return trans_desc_; } int get_tablet_by_rowkey(uint64_t table_id, const ObIArray &rowkeys, @@ -165,7 +148,7 @@ protected: ObTableService *table_service_; storage::ObAccessService *access_service_; share::ObLocationService *location_service_; - ObTableApiCredential credential_; + table::ObTableApiCredential credential_; int32_t stat_event_type_; int64_t audit_row_count_; bool need_audit_; @@ -207,104 +190,6 @@ protected: virtual uint64_t get_request_checksum() = 0; }; -class ObHTableDeleteExecutor final -{ -public: - ObHTableDeleteExecutor(common::ObArenaAllocator &alloc, - uint64_t table_id, - common::ObTabletID tablet_id, - share::ObLSID ls_id, - int64_t timeout_ts, - ObTableApiProcessorBase *processor, - ObTableService *table_service, - ObAccessService *access_service); - ~ObHTableDeleteExecutor() {} - // @param affected_rows [out] deleted number of htable cells - int htable_delete(const table::ObTableBatchOperation &delete_op, int64_t &affected_rows); -private: - int execute_query(const table::ObTableQuery &query, - table::ObTableQueryResultIterator *&result_iterator); - int generate_delete_cells( - table::ObTableQueryResult &one_row, - table::ObTableEntityFactory &entity_factory, - table::ObTableBatchOperation &mutations_out); - int execute_mutation(const table::ObTableBatchOperation &mutations, - table::ObTableBatchOperationResult &mutations_result); -private: - ObTableService *table_service_; - storage::ObAccessService *access_service_; - ObTableServiceQueryCtx query_ctx_; - table::ObTableQuery query_; - table::ObTableQueryResult one_result_; - table::ObTableEntityFactory entity_factory_; - table::ObTableBatchOperation mutations_; - table::ObTableBatchOperationResult mutations_result_; - ObTableServiceGetCtx mutate_ctx_; - // disallow copy - DISALLOW_COPY_AND_ASSIGN(ObHTableDeleteExecutor); -}; - -class ObHTablePutExecutor final -{ -public: - ObHTablePutExecutor(common::ObArenaAllocator &alloc, - uint64_t table_id, - common::ObTabletID tablet_id, - share::ObLSID ls_id, - int64_t timeout_ts, - ObTableApiProcessorBase *processor, - ObTableService *table_service); - ~ObHTablePutExecutor() {} - - int htable_put(const ObTableBatchOperation &put_op, int64_t &affected_rows, int64_t now_ms = 0); -private: - ObTableService *table_service_; - table::ObTableEntityFactory entity_factory_; - table::ObTableBatchOperationResult mutations_result_; - ObTableServiceGetCtx mutate_ctx_; - // disallow copy - DISALLOW_COPY_AND_ASSIGN(ObHTablePutExecutor); -}; - -// executor of Increment and Append -class ObHTableIncrementExecutor final -{ -public: - ObHTableIncrementExecutor(table::ObTableOperationType::Type type, - common::ObArenaAllocator &alloc, - uint64_t table_id, - common::ObTabletID tablet_id, - share::ObLSID ls_id, - int64_t timeout_ts, - ObTableApiProcessorBase *processor, - ObTableService *table_service); - ~ObHTableIncrementExecutor() {} - - int htable_increment(ObTableQueryResult &row_cells, - const table::ObTableBatchOperation &increment_op, - int64_t &affected_rows, - table::ObTableQueryResult *results); -private: - typedef std::pair ColumnIdx; - class ColumnIdxComparator; - int sort_qualifier(const table::ObTableBatchOperation &increment); - int execute_mutation(const table::ObTableBatchOperation &mutations, - table::ObTableBatchOperationResult &mutations_result); - static int add_to_results(table::ObTableQueryResult &results, const ObObj &rk, const ObObj &cq, - const ObObj &ts, const ObObj &value); -private: - table::ObTableOperationType::Type type_; - ObTableService *table_service_; - table::ObTableEntityFactory entity_factory_; - table::ObTableBatchOperation mutations_; - table::ObTableBatchOperationResult mutations_result_; - ObTableServiceGetCtx mutate_ctx_; - common::ObSEArray columns_; - common::ObArenaAllocator allocator_; - // disallow copy - DISALLOW_COPY_AND_ASSIGN(ObHTableIncrementExecutor); -}; - template int64_t ObTableRpcProcessor::get_timeout_ts() const { diff --git a/src/observer/table/ob_table_rpc_processor_util.h b/src/observer/table/ob_table_rpc_processor_util.h index 0c4d7aa9b0..4717a1d9fb 100644 --- a/src/observer/table/ob_table_rpc_processor_util.h +++ b/src/observer/table/ob_table_rpc_processor_util.h @@ -257,6 +257,17 @@ public: } static int negate_htable_timestamp(table::ObITableEntity &entity); + static void replace_ret_code(int &ret) + { + if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret + || OB_BAD_NULL_ERROR == ret + || OB_OBJ_TYPE_ERROR == ret + || OB_ERR_COLLATION_MISMATCH == ret + || OB_ERR_DATA_TOO_LONG == ret + || OB_DATA_OUT_OF_RANGE == ret) { + ret = OB_SUCCESS; + } + } private: // disallow copy DISALLOW_COPY_AND_ASSIGN(ObTableRpcProcessorUtil); diff --git a/src/observer/table/ob_table_scan_executor.cpp b/src/observer/table/ob_table_scan_executor.cpp new file mode 100644 index 0000000000..6e7da2ed0a --- /dev/null +++ b/src/observer/table/ob_table_scan_executor.cpp @@ -0,0 +1,352 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_scan_executor.h" +#include "ob_table_context.h" +#include "sql/das/ob_das_utils.h" + +namespace oceanbase +{ +using namespace common; +using namespace share; +namespace table +{ + +int ObTableApiScanExecutor::init_das_scan_rtdef(const ObDASScanCtDef &das_ctdef, + ObDASScanRtDef &das_rtdef, + const ObDASTableLocMeta *loc_meta) +{ + int ret = OB_SUCCESS; + const ObTableCtx &tb_ctx = get_table_ctx(); + const ObTableApiScanCtDef &tsc_ctdef = scan_spec_.get_ctdef(); + bool is_lookup = (&das_ctdef == tsc_ctdef.lookup_ctdef_); + das_rtdef.timeout_ts_ = tb_ctx.get_timeout_ts(); + das_rtdef.scan_flag_.scan_order_ = tb_ctx.get_scan_order(); + das_rtdef.scan_flag_.index_back_ = tb_ctx.is_index_back(); + bool is_get_op = tb_ctx.get_opertion_type() == ObTableOperationType::Type::GET; + das_rtdef.scan_flag_.read_latest_ = is_get_op ? false : true; + das_rtdef.need_check_output_datum_ = false; + das_rtdef.sql_mode_ = SMO_DEFAULT; + das_rtdef.stmt_allocator_.set_alloc(&das_ref_.get_das_alloc()); + das_rtdef.scan_allocator_.set_alloc(&das_ref_.get_das_alloc()); + das_rtdef.eval_ctx_ = &get_eval_ctx(); + if (!is_lookup) { + das_rtdef.limit_param_.limit_ = tb_ctx.get_limit(); + das_rtdef.limit_param_.offset_ = tb_ctx.get_offset(); + } + if (OB_FAIL(das_rtdef.init_pd_op(exec_ctx_, das_ctdef))) { + LOG_WARN("fail to init pushdown storage filter", K(ret)); + } else { + das_rtdef.tenant_schema_version_ = tb_ctx.get_tenant_schema_version(); + ObTableID table_loc_id = tb_ctx.get_ref_table_id(); + das_rtdef.table_loc_ = exec_ctx_.get_das_ctx().get_table_loc_by_id(table_loc_id, das_ctdef.ref_table_id_); + if (OB_ISNULL(das_rtdef.table_loc_)) { + ObDASTabletLoc *tablet_loc = nullptr; + if (OB_ISNULL(loc_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get table loc by id", K(ret), K(table_loc_id), K(das_ctdef.ref_table_id_)); + } else if (OB_FAIL(exec_ctx_.get_das_ctx().extended_table_loc(*loc_meta, das_rtdef.table_loc_))) { + LOG_WARN("fail to extend table loc", K(ret), KPC(loc_meta)); + } else if (OB_FAIL(exec_ctx_.get_das_ctx().extended_tablet_loc(*das_rtdef.table_loc_, + tb_ctx.get_tablet_id(), + tablet_loc))) { + LOG_WARN("fail to extend tablet loc", K(ret), K(tb_ctx.get_tablet_id())); + } + } + } + return ret; +} + +int ObTableApiScanExecutor::init_tsc_rtdef() +{ + int ret = OB_SUCCESS; + // init das_ref_ + ObMemAttr mem_attr; + mem_attr.tenant_id_ = MTL_ID(); + mem_attr.label_ = "ScanDASCtx"; + das_ref_.set_mem_attr(mem_attr); + das_ref_.set_expr_frame_info(scan_spec_.get_expr_frame_info()); + das_ref_.set_execute_directly(true); + // init rtdef + const ObDASScanCtDef &scan_ctdef = scan_spec_.get_ctdef().scan_ctdef_; + ObDASScanRtDef &scan_rtdef = tsc_rtdef_.scan_rtdef_; + if (OB_FAIL(init_das_scan_rtdef(scan_ctdef, scan_rtdef, NULL))) { + LOG_WARN("fail to init das scan rtdef", K(ret)); + } + + if (OB_SUCC(ret)) { + if (scan_spec_.get_ctdef().lookup_ctdef_ != NULL) { + const ObDASScanCtDef &lookup_ctdef = *scan_spec_.get_ctdef().lookup_ctdef_; + ObDASBaseRtDef *das_rtdef = NULL; + ObDASTaskFactory &das_factory = exec_ctx_.get_das_ctx().get_das_factory(); + if (OB_FAIL(das_factory.create_das_rtdef(DAS_OP_TABLE_SCAN, das_rtdef))) { + LOG_WARN("fail to create das rtdef", K(ret)); + } else { + tsc_rtdef_.lookup_rtdef_ = static_cast(das_rtdef); + if (OB_FAIL(init_das_scan_rtdef(lookup_ctdef, + *tsc_rtdef_.lookup_rtdef_, + scan_spec_.get_ctdef().lookup_loc_meta_))) { + LOG_WARN("fail to init das scan rtdef", K(ret), K(lookup_ctdef)); + } + } + } + } + + return ret; +} + +int ObTableApiScanExecutor::prepare_das_task() +{ + int ret = OB_SUCCESS; + ObIDASTaskOp *task_op = nullptr; + ObDASScanOp *scan_op = nullptr; + ObDASTabletLoc *tablet_loc = tsc_rtdef_.scan_rtdef_.table_loc_->get_first_tablet_loc(); + if (OB_FAIL(das_ref_.create_das_task(tablet_loc, + DAS_OP_TABLE_SCAN, + task_op))) { + LOG_WARN("fail to prepare das task", K(ret)); + } else { + scan_op = static_cast(task_op); + scan_op->set_scan_ctdef(&scan_spec_.get_ctdef().scan_ctdef_); + scan_op->set_scan_rtdef(&tsc_rtdef_.scan_rtdef_); + scan_op->set_can_part_retry(nullptr == tsc_rtdef_.scan_rtdef_.sample_info_); + tsc_rtdef_.scan_rtdef_.table_loc_->is_reading_ = true; + if (scan_spec_.get_ctdef().lookup_ctdef_ != nullptr) { + //is local index lookup, need to set the lookup ctdef to the das scan op + ObDASTableLoc *lookup_table_loc = tsc_rtdef_.lookup_rtdef_->table_loc_; + ObDASTabletLoc *tablet_loc_list = tsc_rtdef_.lookup_rtdef_->table_loc_->get_first_tablet_loc(); + ObDASTabletLoc *lookup_tablet_loc = ObDASUtils::get_related_tablet_loc( + *tablet_loc_list, lookup_table_loc->loc_meta_->ref_table_id_); + if (OB_ISNULL(lookup_tablet_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lookup tablet loc is nullptr", K(ret), KPC(lookup_table_loc->loc_meta_)); + } else if (OB_FAIL(scan_op->set_lookup_ctdef(scan_spec_.get_ctdef().lookup_ctdef_))) { + LOG_WARN("set lookup ctdef failed", K(ret)); + } else if (OB_FAIL(scan_op->set_lookup_rtdef(tsc_rtdef_.lookup_rtdef_))) { + LOG_WARN("set lookup rtdef failed", K(ret)); + } else if (OB_FAIL(scan_op->set_lookup_tablet_id(lookup_tablet_loc->tablet_id_))) { + LOG_WARN("set lookup tablet id failed", K(ret), KPC(lookup_tablet_loc)); + } else { + lookup_table_loc->is_reading_ = true; + } + } + + if (OB_SUCC(ret)) { + // set scan range + ObIArray &scan_ranges = scan_op->get_scan_param().key_ranges_; + if (OB_FAIL(scan_ranges.assign(get_table_ctx().get_key_ranges()))) { + LOG_WARN("fail to assign scan ranges", K(ret)); + } + } + } + + return ret; +} + +int ObTableApiScanExecutor::do_table_scan() +{ + int ret = OB_SUCCESS; + if (das_ref_.has_task()) { + if (OB_FAIL(das_ref_.execute_all_task())) { + LOG_WARN("fail to execute all das scan task", K(ret)); + } + } + if (OB_SUCC(ret)) { + //prepare to output row + scan_result_ = das_ref_.begin_result_iter(); + } + return ret; +} + +int ObTableApiScanExecutor::do_init_before_get_row() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_das_task())) { + LOG_WARN("fail to prepare das task", K(ret)); + } else if (OB_FAIL(do_table_scan())) { + if (OB_TRY_LOCK_ROW_CONFLICT != ret) { + LOG_WARN("fail to do table scan", K(ret)); + } + } + + if (OB_SUCC(ret)) { + need_do_init_ = false; + } + + return ret; +} + +int ObTableApiScanExecutor::get_next_row_with_das() +{ + int ret = OB_SUCCESS; + bool got_row = false; + while (OB_SUCC(ret) && !got_row) { + clear_evaluated_flag(); + if (OB_FAIL(scan_result_.get_next_row())) { + if (OB_ITER_END == ret) { + if (OB_FAIL(scan_result_.next_result())) { + if (OB_ITER_END != ret) { + LOG_WARN("fetch next task failed", K(ret)); + } + } + } else { + LOG_WARN("get next row from das result failed", K(ret)); + } + } else { + ++input_row_cnt_; + ++output_row_cnt_; + got_row = true; + } + } + return ret; +} + +int ObTableApiScanExecutor::open() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(init_tsc_rtdef())) { + LOG_WARN("fail to init table scan rtdef", K(ret)); + } else { + is_opened_ = true; + } + return ret; +} + +int ObTableApiScanExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + if (0 == get_table_ctx().get_limit()) { + // limit 0，直接返回iter end + ret = OB_ITER_END; + } else if (need_do_init_ && OB_FAIL(do_init_before_get_row())) { + LOG_WARN("fail to do init before get row", K(ret)); + } else if (OB_FAIL(get_next_row_with_das())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row from das", K(ret)); + } + } + return ret; +} + +int ObTableApiScanExecutor::close() +{ + int ret = OB_SUCCESS; + + if (!is_opened_) { + // do nothing + } else if (das_ref_.has_task()) { + if (OB_FAIL(das_ref_.close_all_task())) { + LOG_WARN("fail to close all das task", K(ret)); + } else { + reset(); + } + } + + return ret; +} + +int ObTableApiScanRowIterator::open(ObTableApiScanExecutor *executor) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(executor)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("scan executor is null", K(ret)); + } else if (OB_FAIL(executor->open())) { + LOG_WARN("fail to open scan executor", K(ret)); + } else { + scan_executor_ = executor; + } + + return ret; +} + +int ObTableApiScanRowIterator::get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + ObNewRow *tmp_row = nullptr; + char *row_buf = nullptr; + ObObj *cells = nullptr; + const ObTableCtx &tb_ctx = scan_executor_->get_table_ctx(); + ObIAllocator &allocator = tb_ctx.get_allocator(); + const ExprFixedArray &output_exprs = scan_executor_->get_spec().get_ctdef().get_das_output_exprs(); + const int64_t cells_cnt = output_exprs.count(); + + if (OB_ISNULL(scan_executor_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("scan executor is null", K(ret)); + } else if (OB_FAIL(scan_executor_->get_next_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row by scan executor", K(ret)); + } + } else if (OB_ISNULL(row_buf = static_cast(allocator.alloc(sizeof(ObNewRow))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObNewRow buffer", K(ret)); + } else if (OB_ISNULL(cells = static_cast(allocator.alloc(sizeof(ObObj) * cells_cnt)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc cells buffer", K(ret), K(cells_cnt)); + } else { + // 循环select_exprs,eval获取datum，并将datum转ObObj，最后组成ObNewRow + tmp_row = new(row_buf)ObNewRow(cells, cells_cnt); + ObDatum *datum = nullptr; + ObEvalCtx &eval_ctx = scan_executor_->get_eval_ctx(); + const ObIArray &output_metas = tb_ctx.get_select_metas(); + if (tb_ctx.is_scan()) { // 转为用户select的顺序 + const ObIArray &select_col_ids = tb_ctx.get_select_col_ids(); + const ObIArray &query_col_ids = tb_ctx.get_query_col_ids(); + for (int64_t i = 0; OB_SUCC(ret) && i < query_col_ids.count(); i++) { + uint64_t col_id = query_col_ids.at(i); + int64_t idx = -1; + if (!has_exist_in_array(select_col_ids, col_id, &idx)) { + ret = OB_ERR_COLUMN_NOT_FOUND; + LOG_WARN("query column id not found", K(ret), K(select_col_ids), K(col_id), K(query_col_ids)); + } else if (OB_FAIL(output_exprs.at(idx)->eval(eval_ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else if (OB_FAIL(datum->to_obj(cells[i], output_metas.at(idx)))) { + LOG_WARN("fail to datum to obj", K(ret), K(output_metas.at(idx)), K(i), K(idx)); + } + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < cells_cnt; i++) { + if (OB_FAIL(output_exprs.at(i)->eval(eval_ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else if (OB_FAIL(datum->to_obj(cells[i], output_metas.at(i)))) { + LOG_WARN("fail to datum to obj", K(ret), K(output_metas.at(i))); + } + } + } + } + + if (OB_SUCC(ret)) { + row = tmp_row; + } + + return ret; +} + +int ObTableApiScanRowIterator::close() +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(scan_executor_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("scan executor is null", K(ret)); + } else if (OB_FAIL(scan_executor_->close())) { + LOG_WARN("fail to close scan executor", K(ret)); + } + + return ret; +} + +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_scan_executor.h b/src/observer/table/ob_table_scan_executor.h new file mode 100644 index 0000000000..8afd0d5e06 --- /dev/null +++ b/src/observer/table/ob_table_scan_executor.h @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_TABLE_OB_TABLE_SCAN_EXECUTOR_H_ +#define OCEANBASE_OBSERVER_TABLE_OB_TABLE_SCAN_EXECUTOR_H_ + +#include "ob_table_executor.h" // for ObTableApiExecutor + +namespace oceanbase +{ +namespace table +{ +class ObTableApiScanSpec : public ObTableApiSpec +{ +public: + ObTableApiScanSpec(common::ObIAllocator &allocator, const ObTableExecutorType type) + : ObTableApiSpec(allocator, type), + tsc_ctdef_(allocator) + { + } + virtual ~ObTableApiScanSpec() {} +public: + // getter + OB_INLINE ObTableApiScanCtDef& get_ctdef() { return tsc_ctdef_; } + OB_INLINE const ObTableApiScanCtDef& get_ctdef() const { return tsc_ctdef_; } +private: + ObTableApiScanCtDef tsc_ctdef_; +}; + +class ObTableApiScanExecutor : public ObTableApiExecutor +{ +public: + ObTableApiScanExecutor(ObTableCtx &ctx, const ObTableApiScanSpec &spec) + : ObTableApiExecutor(ctx), + scan_spec_(spec), + tsc_rtdef_(ctx.get_allocator()), + das_ref_(eval_ctx_, ctx.get_exec_ctx()) + { + reset(); + } + virtual ~ObTableApiScanExecutor() {} + int open() override; + int get_next_row() override; + int close() override; +public: + OB_INLINE const ObTableApiScanSpec& get_spec() const { return scan_spec_; } + OB_INLINE void reset() + { + input_row_cnt_ = 0; + output_row_cnt_ = 0; + need_do_init_ = true; + } +protected: + const ObTableApiScanSpec &scan_spec_; + ObTableApiScanRtDef tsc_rtdef_; + sql::DASOpResultIter scan_result_; + sql::ObDASRef das_ref_; + int64_t input_row_cnt_; + int64_t output_row_cnt_; + bool need_do_init_; +private: + int init_tsc_rtdef(); + int init_das_scan_rtdef(const sql::ObDASScanCtDef &das_ctdef, + sql::ObDASScanRtDef &das_rtdef, + const sql::ObDASTableLocMeta *loc_meta); + int do_init_before_get_row(); + int prepare_scan_range(); + int prepare_das_task(); + int do_table_scan(); + int get_next_row_with_das(); +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiScanExecutor); +}; + +class ObTableApiScanRowIterator +{ +public: + ObTableApiScanRowIterator() + : scan_executor_(nullptr) + { + } + virtual ~ObTableApiScanRowIterator() {}; +public: + int open(ObTableApiScanExecutor *executor); + int get_next_row(common::ObNewRow *&row); + int close(); +private: + ObTableApiScanExecutor *scan_executor_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiScanRowIterator); +}; + +} // table +} // oceanbase +#endif // OCEANBASE_OBSERVER_TABLE_OB_TABLE_SCAN_EXECUTOR_H_ diff --git a/src/observer/table/ob_table_service.cpp b/src/observer/table/ob_table_service.cpp index 30199d44ed..5f4bad132f 100644 --- a/src/observer/table/ob_table_service.cpp +++ b/src/observer/table/ob_table_service.cpp @@ -12,7 +12,6 @@ #define USING_LOG_PREFIX SERVER #include "ob_table_service.h" -#include "ob_table_api_row_iterator.h" #include "observer/ob_service.h" #include "ob_table_rpc_processor.h" #include "sql/engine/expr/ob_expr_res_type.h" @@ -28,2055 +27,47 @@ using namespace oceanbase::table; using namespace oceanbase::share; using namespace oceanbase::sql; -OB_INLINE void replace_ret_code(int &ret) -{ - if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret - || OB_BAD_NULL_ERROR == ret - || OB_OBJ_TYPE_ERROR == ret - || OB_ERR_COLLATION_MISMATCH == ret - || OB_ERR_DATA_TOO_LONG == ret - || OB_DATA_OUT_OF_RANGE == ret) { - ret = OB_SUCCESS; - } -} - -OB_INLINE int get_rowkey_column_ids( - const ObIArray &full_column_ids, - const int64_t rowkey_column_cnt, - ObIArray &rowkey_column_ids) -{ - int ret = OB_SUCCESS; - rowkey_column_ids.reset(); - if (OB_UNLIKELY(full_column_ids.count() <= 0 - || rowkey_column_cnt <= 0 - || rowkey_column_cnt > full_column_ids.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(full_column_ids), K(rowkey_column_cnt)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_column_cnt; ++i) { - if (OB_FAIL(rowkey_column_ids.push_back(full_column_ids.at(i)))) { - LOG_WARN("fail to push back current column id", K(ret), K(i), K(full_column_ids.at(i))); - } - } - } - return ret; -} - int ObTableService::init(ObGlobalContext &gctx) { int ret = OB_SUCCESS; schema_service_ = gctx.schema_service_; - return ret; -} - -int ObTableService::cons_rowkey_infos(const share::schema::ObTableSchema &table_schema, - ObIArray *column_ids, - ObIArray *columns_type) -{ - int ret = OB_SUCCESS; - const ObRowkeyInfo &rowkey_info = table_schema.get_rowkey_info(); - if (NULL != column_ids) { - if (OB_FAIL(rowkey_info.get_column_ids(*column_ids))) { - LOG_WARN("failed to get rowkey column ids", K(ret), K(rowkey_info)); - } - } - if (OB_SUCC(ret) && NULL != columns_type) { - const share::schema::ObColumnSchemaV2 *column_schema = NULL; - uint64_t column_id = OB_INVALID_ID; - ObExprResType column_type; - const int64_t N = rowkey_info.get_size(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) - { - if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { - LOG_WARN("failed to get column id", K(ret), K(i)); - } else if (NULL == (column_schema = table_schema.get_column_schema(column_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("rowkey column not exists", K(ret), K(column_id)); - } else if (OB_FAIL(cons_column_type(*column_schema, column_type))) { - LOG_WARN("failed to cons column type", K(ret)); - } else if (OB_FAIL(columns_type->push_back(column_type))) { - LOG_WARN("failed to push back", K(ret)); - } - } // end for + if (OB_FAIL(sess_pool_mgr_.init())) { + LOG_WARN("fail to init tableapi session pool manager", K(ret)); } return ret; } -int ObTableService::cons_properties_infos(const share::schema::ObTableSchema &table_schema, - const ObIArray &properties, - ObIArray &column_ids, - ObIArray *columns_type) +int ObTableService::check_htable_query_args(const ObTableQuery &query) { int ret = OB_SUCCESS; - const share::schema::ObColumnSchemaV2 *column_schema = NULL; - ObExprResType column_type; - const int64_t N = properties.count(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - const ObString &cname = properties.at(i); - if (NULL == (column_schema = table_schema.get_column_schema(cname))) { - ret = OB_ERR_COLUMN_NOT_FOUND; - LOG_WARN("column not exists", K(ret), K(cname)); - } else if (OB_FAIL(column_ids.push_back(column_schema->get_column_id()))) { - LOG_WARN("failed to add column id", K(ret)); - } else if (NULL != columns_type) { - if (OB_FAIL(cons_column_type(*column_schema, column_type))) { - LOG_WARN("failed to cons column type", K(ret)); - } else if (OB_FAIL(columns_type->push_back(column_type))) { - LOG_WARN("failed to push back", K(ret)); - } - } - } // end for - return ret; -} - -int ObTableService::cons_column_type(const share::schema::ObColumnSchemaV2 &column_schema, ObExprResType &column_type) -{ - int ret = OB_SUCCESS; - column_type.set_type(column_schema.get_data_type()); - column_type.set_result_flag(ObRawExprUtils::calc_column_result_flag(column_schema)); - if (ob_is_string_type(column_schema.get_data_type()) || ob_is_json(column_schema.get_data_type()) - || ob_is_geometry(column_schema.get_data_type())) { - column_type.set_collation_type(column_schema.get_collation_type()); - column_type.set_collation_level(CS_LEVEL_IMPLICIT); - } else { - column_type.set_collation_type(CS_TYPE_BINARY); - column_type.set_collation_level(CS_LEVEL_NUMERIC); - } - const ObAccuracy &accuracy = column_schema.get_accuracy(); - column_type.set_accuracy(accuracy); - const bool is_zerofill = column_type.has_result_flag(ZEROFILL_FLAG); - if (is_zerofill) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("modifing column with ZEROFILL flag is not supported", K(ret), K(column_schema)); - } - return ret; -} - -// @pre column is not zerofill -int ObTableService::check_column_type(const ObExprResType &column_type, ObObj &obj) -{ - int ret = OB_SUCCESS; - const bool is_not_nullable = column_type.is_not_null_for_read(); - const ObCollationType cs_type = column_type.get_collation_type(); - // 1. check nullable - if (is_not_nullable && obj.is_null()) { - ret = OB_BAD_NULL_ERROR; - } else if (obj.is_null()) { - // continue - } else if (column_type.get_type() != obj.get_type() - && !(ob_is_string_type(column_type.get_type()) && ob_is_string_type(obj.get_type()))) { - // 2. data type mismatch - ret = OB_OBJ_TYPE_ERROR; - LOG_WARN("object type mismatch with column type", K(ret), K(column_type), K(obj)); - } else { - // 3. check collation - if (!ob_is_string_type(obj.get_type())) { - // not string type, continue - } else { - if (cs_type == obj.get_collation_type()) { - // same collation type - } else if (cs_type == CS_TYPE_BINARY) { - // any collation type can be compatible with cs_type_binary - obj.set_collation_type(cs_type); - } else if (ObCharset::charset_type_by_coll(cs_type) == ObCharset::charset_type_by_coll(obj.get_collation_type())) { - // same charset, convert it - obj.set_collation_type(cs_type); - } else { - ret = OB_ERR_COLLATION_MISMATCH; - LOG_WARN("collation type mismatch with column", K(ret), K(column_type), K(obj)); - } - if (OB_SUCC(ret)) { - // convert obj type to the column type (char, varchar or text) - obj.set_type(column_type.get_type()); - } - } - // 4. check accuracy - if (OB_SUCC(ret)) { - if (OB_FAIL(ob_obj_accuracy_check_only(column_type.get_accuracy(), cs_type, obj))) { - LOG_WARN("accuracy check failed", K(ret), K(obj), K(column_type)); - } - } - } - return ret; -} - -int ObTableService::insert_or_update_can_use_put(ObTableEntityType entity_type, uint64_t table_id, const ObITableEntity &entity, bool &use_put) -{ - int ret = OB_SUCCESS; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTableSchema *table_schema = NULL; - const uint64_t tenant_id = MTL_ID(); - if (ObTableEntityType::ET_HKV == entity_type) { - // hbase model table does not have secondary index and always specify all the properties (column V) - use_put = true; - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("failed to get schema guard", K(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("get table schema failed", K(tenant_id), K(table_id), K(ret)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("NULL ptr", K(ret), K(table_schema)); - } else { - // 1. there is not any index - // 2. if there are some columns missing, the missing columns don't have a default value - use_put = (table_schema->get_index_tid_count() <= 0); - if (use_put) { - if (table_schema->get_column_count() - table_schema->get_rowkey_column_num() <= entity.get_properties_count()) { - // all columns are fine - } else { - // some columns are missing - share::schema::ObTableSchema::const_column_iterator iter = table_schema->column_begin(); - share::schema::ObTableSchema::const_column_iterator end = table_schema->column_end(); - ObObj obj; - ObExprResType column_type; - for (; OB_SUCC(ret) && iter != end; ++iter) { - const share::schema::ObColumnSchemaV2 *column = *iter; - if (OB_ISNULL(column)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid column schema", K(column)); - } else if (!column->is_rowkey_column()) { - const ObString &column_name = column->get_column_name_str(); - if (OB_FAIL(entity.get_property(column_name, obj))) { - ret = OB_SUCCESS; - // missing column - if (!column->get_cur_default_value().is_null()) { - use_put = false; - break; - } - } - } - } // end for - } - } - } - return ret; -} - -int ObTableService::execute_insert_or_update(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - const ObITableEntity &entity = table_operation.entity(); - bool can_use_put = true; - if (OB_FAIL(insert_or_update_can_use_put(ctx.param_.entity_type_, ctx.param_.table_id_, entity, can_use_put))) { - LOG_WARN("failed to check", K(ret)); - } else if (can_use_put - && ctx.param_.binlog_row_image_type_ != ObBinlogRowImageType::FULL) { - ret = do_put(ctx, table_operation, result); - } else { - ret = do_insert_or_update(ctx, table_operation, result); - } - return ret; -} - -// If there are some columns missing in the property, can not fill default value. -// Because if row exists, it will do update, fill default value will cover the old value. -// If row does not exist, it will do insert, fill null value to missing columns is wrong. -// So, if and only if all columns are given or missing columns have not default value and there is not any index, we can use do_put -int ObTableService::do_put(ObTableServiceCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - SMART_VAR(ObTableApiInsertRowIterator, put_row_iter) { - ObTableDMLParam table_param(*ctx.param_.allocator_); - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(put_row_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init put row iterator, ", K(ret), K(table_id)); - } else if (OB_FAIL(put_row_iter.open(table_operation))) { - LOG_WARN("Fail to open put row iterator, ", K(ret), K(table_id)); - } else if (OB_FAIL(build_table_param(table_id, put_row_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - int64_t affected_rows = 0; - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = false; // always false for put - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = put_row_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - if (OB_FAIL(access_service->put_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - put_row_iter.get_column_ids(), - &put_row_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to put row", K(ret), K(table_id)); - } - } else { - result.set_affected_rows(1); - } - } - } - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(ObTableOperationType::INSERT_OR_UPDATE); - return ret; -} - -int ObTableService::do_insert_or_update(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - ObNewRowIterator *duplicate_row_iter = nullptr; - if (OB_FAIL(execute_insert(ctx, table_operation, result, duplicate_row_iter))) { - LOG_WARN("failed to execute_insert", K(ret)); - } else { - ret = result.get_errno(); - if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { - ret = OB_SUCCESS; - ctx.reset_get_ctx(); - ObNewRow *duplicate_row = nullptr; - // update on duplicate key - if (OB_ISNULL(duplicate_row_iter)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("duplicate row iterator is null", K(ret)); - } else if (OB_FAIL(duplicate_row_iter->get_next_row(duplicate_row))) { - LOG_WARN("fail to get duplicate row", K(ret)); - } else if (OB_FAIL(execute_update(ctx, table_operation, duplicate_row, result))) { - LOG_WARN("failed to update", K(ret)); - } else { - // update succ - } - } else { - // insert succ - } - } - if (nullptr != duplicate_row_iter) { - ObAccessService *access_service = MTL(ObAccessService *); - int tmp_ret = access_service->revert_insert_iter(duplicate_row_iter); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("fail to revert duplicate_row_iter", K(tmp_ret), KP(duplicate_row_iter)); - } else { - duplicate_row_iter = nullptr; - } - } - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(ObTableOperationType::INSERT_OR_UPDATE); - return ret; -} - -OB_INLINE int ObTableService::add_one_result(ObTableBatchOperationResult &result, - ObTableOperationType::Type op_type, - int32_t error_code, - int64_t affected_rows) -{ - int ret = OB_SUCCESS; - ObTableOperationResult one_result; - ObITableEntity *result_entity = result.get_entity_factory()->alloc(); - if (NULL == result_entity) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memroy for result_entity", K(ret)); - } else { - one_result.set_entity(*result_entity); - one_result.set_type(op_type); - one_result.set_errno(error_code); - one_result.set_affected_rows(affected_rows); - if (OB_FAIL(result.push_back(one_result))) { - LOG_WARN("failed to push back", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] add result", K(ret), K(one_result)); - } - } - return ret; -} - -int ObTableService::multi_insert_or_update(ObTableServiceGetCtx &ctx, - const ObTableBatchOperation &batch_operation, - ObTableBatchOperationResult &result) -{ - NG_TRACE(insertup_start_do); - int ret = OB_SUCCESS; - const ObTableOperation &one_op = batch_operation.at(0); - bool can_use_put = true; - if (OB_FAIL(insert_or_update_can_use_put(ctx.param_.entity_type_, - ctx.param_.table_id_, one_op.entity(), can_use_put))) { - LOG_WARN("failed to check", K(ret)); - } else if (can_use_put - && ctx.param_.binlog_row_image_type_ != ObBinlogRowImageType::FULL) { - ret = multi_put(ctx, batch_operation, result); - } else { - ret = do_multi_insert_or_update(ctx, batch_operation, result); - } - NG_TRACE(insertup_end); - return ret; -} - -int ObTableService::multi_put(ObTableServiceCtx &ctx, - const ObTableBatchOperation &batch_operation, - ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - SMART_VAR(ObTableApiMultiInsertRowIterator, multi_put_iter) { - ObTableDMLParam table_param(*ctx.param_.allocator_); - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(multi_put_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init multi put iterator, ", K(ret), K(table_id)); - } else if (OB_FAIL(multi_put_iter.open(batch_operation))) { - LOG_WARN("Fail to open multi put iterator, ", K(ret), K(table_id)); - } else if (OB_FAIL(build_table_param(table_id, multi_put_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - int64_t affected_rows = 0; - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = false; // always false for put - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = multi_put_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - const int64_t N = batch_operation.count(); - NG_TRACE_EXT(insertup_calc_new_row, OB_ID(input_count), N); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - multi_put_iter.continue_iter(); - if (OB_FAIL(access_service->put_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - multi_put_iter.get_column_ids(), - &multi_put_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to put rows", K(ret), K(table_id)); - } else { - NG_TRACE(locked); - } - } else if (OB_FAIL(add_one_result(result, ObTableOperationType::INSERT_OR_UPDATE, OB_SUCCESS, affected_rows))) { - LOG_WARN("failed to add result", K(ret)); - } - } // end for - } - } - - return ret; -} - -int ObTableService::do_multi_insert_or_update(ObTableServiceGetCtx &ctx, - const ObTableBatchOperation &batch_operation, - ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - const uint64_t table_id = ctx.param_.table_id_; - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else { - int64_t N = batch_operation.count(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - const ObTableOperation &table_operation = batch_operation.at(i); - ObTableOperationResult op_result; - ObITableEntity *result_entity = result.get_entity_factory()->alloc(); - if (NULL == result_entity) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memroy for result_entity", K(ret)); - break; - } - op_result.set_entity(*result_entity); - if (OB_FAIL(do_insert_or_update(ctx, table_operation, op_result))) { - LOG_WARN("failed to do insert_or_update", K(ret)); - } else if (OB_FAIL(result.push_back(op_result))) { - LOG_WARN("failed to push back result", K(ret)); - } else { - ctx.reset_get_ctx(); - } - } // end for - } - return ret; -} - -//////////////////////////////////////////////////////////////// - -int ObTableService::fill_scan_param(ObTableServiceCtx &ctx, - const ObIArray &output_column_ids, - int64_t schema_version, - storage::ObTableScanParam &scan_param) -{ - int ret = OB_SUCCESS; - const uint64_t table_id = ctx.param_.table_id_; - const uint64_t tenant_id = MTL_ID(); - scan_param.timeout_ = ctx.param_.timeout_ts_; - ObQueryFlag query_flag(ObQueryFlag::KeepOrder, // scan_order KeepOrder! - false, // daily_merge - false, // optimize - false, // whole_macro_scan - false, // full_row - false, // index_back - false, // query_stat - ObQueryFlag::MysqlMode, // sql_mode - true // read_latest - ); - scan_param.scan_flag_.flag_ = query_flag.flag_; - scan_param.reserved_cell_count_ = output_column_ids.count() + 10; - scan_param.for_update_ = false; - scan_param.column_ids_.reset(); - scan_param.tablet_id_ = ctx.param_.tablet_id_; - // scan_param.index_tablet_id_ = ctx.param_.index_tablet_id_; todo@dazhi: tableapi index back - scan_param.ls_id_ = ctx.param_.ls_id_; - scan_param.schema_version_ = schema_version; - if (OB_FAIL(scan_param.column_ids_.assign(output_column_ids))) { - LOG_WARN("fail to assign column id", K(ret)); - } else { - SQL_ENG_LOG(DEBUG, "set scan param", K(output_column_ids)); - scan_param.limit_param_.limit_ = -1; - scan_param.limit_param_.offset_ = 0; - scan_param.trans_desc_ = ctx.param_.processor_->get_trans_desc(); - scan_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - scan_param.tx_id_ = ctx.param_.processor_->get_trans_desc()->get_tx_id(); - scan_param.index_id_ = table_id; - scan_param.sql_mode_ = SMO_DEFAULT; - CURRENT_CONTEXT->get_arena_allocator().set_tenant_id(tenant_id); - } - return ret; -} - -int ObTableService::fill_get_result( - ObTableServiceCtx &ctx, - const ObIArray &properties, - ObTableApiRowIterator *scan_result, - ObTableOperationResult &operation_result) -{ - int ret = OB_SUCCESS; - ObNewRow *row = NULL; - table::ObITableEntity *entity = NULL; - if (OB_FAIL(operation_result.get_entity(entity))) { - LOG_WARN("failed to get entity", K(ret)); - } else if (OB_FAIL(scan_result->get_next_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("failed to get next row", K(ret)); - } - } else { - const int64_t rowkey_column_cnt = scan_result->get_rowkey_column_cnt(); - const int64_t N = row->get_count(); - ObObj cell_clone; - for (int64_t i = rowkey_column_cnt; OB_SUCCESS == ret && i < N; ++i) { - const ObString &name = properties.at(i - rowkey_column_cnt); - const ObObj &cell = row->get_cell(i); - if (OB_FAIL(ob_write_obj(*ctx.param_.allocator_, cell, cell_clone))) { - LOG_WARN("failed to copy obj", K(ret)); - } else if (OB_FAIL(entity->set_property(name, cell_clone))) { - LOG_WARN("failed to set property", K(ret), K(name), K(cell)); - } else { - LOG_DEBUG("yzfdebug get cell", K(name), K(i), K(cell), "row", (*row)); - } - } // end for - } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - return ret; -} - -ObTableServiceGetCtx::ObTableServiceGetCtx(common::ObArenaAllocator &alloc) - :table_param_on_stack_(alloc), - table_param_(&table_param_on_stack_), - scan_result_(NULL) -{ - table_param_ = &table_param_on_stack_; - scan_param_.table_param_ = table_param_; -} - -int ObTableService::execute_get(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - NG_TRACE(S_table_scan_begin); - SMART_VAR(ObTableApiGetRowIterator, get_row_iter) { - ObAccessService *access_service = MTL(ObAccessService *); - ctx.scan_result_ = NULL; - if (OB_FAIL(get_row_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init get row iterator, ", K(ret)); - } else if (OB_FAIL(get_row_iter.open(table_operation))) { - LOG_WARN("Fail to open get row iterator, ", K(ret)); - } else if (OB_FAIL(fill_get_result(ctx, get_row_iter.get_properties(), &get_row_iter, result))) { - LOG_WARN("failed to send result"); - } else { - LOG_DEBUG("[yzfdebug] execute_get", "prop", get_row_iter.get_properties(), "table_param", *(ctx.table_param_), "scan_param", ctx.scan_param_, "trans_desc", *ctx.param_.processor_->get_trans_desc()); - } - } - result.set_type(ObTableOperationType::GET); - result.set_errno(ret); - replace_ret_code(ret); - NG_TRACE(S_table_scan_end); - return ret; -} - -int ObTableService::fill_multi_get_result( - ObTableServiceGetCtx &ctx, - const ObTableBatchOperation &batch_operation, - ObTableApiRowIterator *scan_result, - ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - const int64_t rowkey_size = batch_operation.at(0).entity().get_rowkey_size(); - ObNewRow *row = NULL; - const int64_t N = batch_operation.count(); - bool did_get_next_row = true; - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - // left join - const ObTableEntity &entity = static_cast(batch_operation.at(i).entity()); - ObRowkey expected_key = const_cast(entity).get_rowkey(); - ObTableOperationResult op_result; - ObITableEntity *result_entity = result.get_entity_factory()->alloc(); - if (NULL == result_entity) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memroy for result_entity", K(ret)); - break; - } - op_result.set_entity(*result_entity); - op_result.set_type(ObTableOperationType::GET); - if (did_get_next_row) { - if (OB_FAIL(scan_result->get_next_row(row))) { - if (OB_ITER_END == ret) { - // push empty entity - ret = OB_SUCCESS; - op_result.set_errno(OB_SUCCESS); - if (OB_FAIL(result.push_back(op_result))) { - LOG_WARN("failed to push back result", K(ret), K(i)); - } - continue; - } else { - LOG_WARN("failed to get next row", K(ret)); - } - } else { - LOG_DEBUG("[yzfdebug] multi get row", K(ret), K(i), K(*row)); - } - } - if (OB_SUCC(ret)) { - ObRowkey the_key(row->cells_, rowkey_size); - ObObj cell_clone; - if (expected_key.simple_equal(the_key)) { - const int64_t N = row->get_count(); - for (int64_t i = rowkey_size; OB_SUCCESS == ret && i < N; ++i) { - const ObString &name = scan_result->get_properties().at(i-rowkey_size); - ObObj &cell = row->get_cell(i); - if (OB_FAIL(ob_write_obj(*ctx.param_.allocator_, cell, cell_clone))) { - LOG_WARN("failed to copy obj", K(ret)); - } else if (OB_FAIL(result_entity->set_property(name, cell_clone))) { - LOG_WARN("failed to set property", K(ret)); - } else { - LOG_DEBUG("get cell", K(name), K(i), K(cell)); - } - } // end for - did_get_next_row = true; - } else { - did_get_next_row = false; - LOG_DEBUG("[yzfdebug] the row not exist", K(ret), K(expected_key)); - } - op_result.set_errno(OB_SUCCESS); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(result.push_back(op_result))) { - LOG_WARN("failed to push back result", K(ret), K(i)); - } - } - } // end for - return ret; -} - -int ObTableService::multi_get(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - SMART_VAR(ObTableApiMultiGetRowIterator, multi_get_iter) { - ObAccessService *access_service = MTL(ObAccessService *); - if (OB_FAIL(multi_get_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init multi get iter, ", K(ret)); - } else if (OB_FAIL(multi_get_iter.open(batch_operation))) { - LOG_WARN("Fail to open multi get iter, ", K(ret)); - } else if (OB_FAIL(fill_multi_get_result(ctx, batch_operation, &multi_get_iter, result))) { - LOG_WARN("failed to send result"); - } - } - return ret; -} - -// for update, if schema is index type, all columns in index schema -// should be added, including index defined columns, rowkey columns -// in data schema, and storing columns. -// for delete, the storing columns might be useless, but still added. -// -int ObTableService::add_index_columns_if_missing(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const uint64_t data_table_id, - const share::schema::ObTableSchema *index_schema, - ObIArray &column_ids, - ObIArray *columns_type) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_INVALID_ID; - const share::schema::ObColumnSchemaV2 *column_schema = NULL; - ObExprResType column_type; - share::schema::ObTableSchema::const_column_iterator b = index_schema->column_begin(); - share::schema::ObTableSchema::const_column_iterator e = index_schema->column_end(); - for (; OB_SUCC(ret) && b != e; ++b) { // for all columns of the index, include the storing columns - if (NULL == (*b)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to to get column schema", K(*b)); - } else { - if ((*b)->is_shadow_column()) { - continue; - } - column_id = (*b)->get_column_id(); - bool found = false; - const int64_t N = column_ids.count(); - for (int64_t i = 0; !found && i < N; ++i) { - if (column_id == column_ids.at(i)) { - found = true; - } - } // end for - if (!found) { - if (OB_FAIL(schema_guard.get_column_schema(tenant_id, data_table_id, column_id, column_schema))) { - LOG_WARN("get column schema failed", KR(ret), K(tenant_id), K(data_table_id), K(column_id)); - } else if (NULL == column_schema) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get column schema", K(data_table_id), K(column_id)); - } else { - if (OB_FAIL(column_ids.push_back(column_schema->get_column_id()))) { - LOG_WARN("failed to add column id", K(ret)); - } else if (NULL != columns_type) { - if (OB_FAIL(cons_column_type(*column_schema, column_type))) { - LOG_WARN("failed to cons column type", K(ret)); - } else if (OB_FAIL(columns_type->push_back(column_type))) { - LOG_WARN("failed to push back", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] add missing index column for update/delete", - K(column_id), K(column_type)); - } - } - } - } // end if !found - } - } // end for - return ret; -} - -int ObTableService::delete_can_use_put(table::ObTableEntityType entity_type, uint64_t table_id, bool &use_put) -{ - int ret = OB_SUCCESS; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTableSchema *table_schema = NULL; - const uint64_t tenant_id = MTL_ID(); - if (entity_type == ObTableEntityType::ET_HKV) { - // hbase model table does not have secondary index - use_put = true; - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("failed to get schema guard", K(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("get table schema failed", K(tenant_id), K(table_id), K(ret)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("NULL ptr", K(ret), K(table_schema)); - } else { - // 1. there is not any index - use_put = (table_schema->get_index_tid_count() <= 0); - } - return ret; -} -//////////////////////////////////////////////////////////////// -int ObTableService::execute_delete(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - int64_t affected_rows = 0; - SMART_VAR(ObTableApiDeleteRowIterator, delete_row_iter) { - const uint64_t table_id = ctx.param_.table_id_; - ObTableDMLParam table_param(*ctx.param_.allocator_); - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(delete_row_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init delete row iterator, ", K(ret)); - } else if (OB_FAIL(delete_row_iter.open(table_operation))) { - LOG_WARN("Fail to open delete row iterator, ", K(ret)); - } else if (OB_FAIL(build_table_param(table_id, delete_row_iter.get_delete_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = delete_row_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - if (OB_FAIL(access_service->delete_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - delete_row_iter.get_delete_column_ids(), - &delete_row_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to delete", K(ret), K(table_id)); - } - } else { - LOG_DEBUG("[yzfdebug] delete rows", K(ret), K(affected_rows)); - } - } - - - if (ctx.param_.returning_affected_rows_) { - result.set_affected_rows(affected_rows); - } else { - result.set_affected_rows(-1); // always return -1 - } - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(ObTableOperationType::DEL); - } - return ret; -} - -int ObTableService::cons_all_index_properties(share::schema::ObSchemaGetterGuard &schema_guard, - const share::schema::ObTableSchema &table_schema, - ObIArray &column_ids, - ObIArray *columns_type) -{ - int ret = OB_SUCCESS; - if (table_schema.get_index_tid_count() <= 0) { - // no index, do nothing - } else { - const share::schema::ObTableSchema *index_schema = NULL; - const uint64_t tenant_id = table_schema.get_tenant_id(); - const uint64_t table_id = table_schema.get_table_id(); - uint64_t index_tids[OB_MAX_INDEX_PER_TABLE + 1]; - int64_t index_count = ARRAYSIZEOF(index_tids); - // get all the indexes - if (OB_FAIL(schema_guard.get_can_write_index_array(tenant_id, table_id, index_tids, index_count))) { - LOG_WARN("fail to get index", K(ret), K(tenant_id), K(table_id)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < index_count; ++i) { // for each writable index - const uint64_t index_id = index_tids[i]; - // get index schema - if (OB_FAIL(schema_guard.get_table_schema(tenant_id, index_id, index_schema))) { - LOG_WARN("get index schema failed", KR(ret), K(tenant_id), K(index_id)); - } else { - if (OB_FAIL(add_index_columns_if_missing(schema_guard, tenant_id, - table_id, index_schema, - column_ids, columns_type))) { - LOG_WARN("failed to add missing index column", K(ret)); - } - } - } // end for - } - } - return ret; -} - -int ObTableService::multi_delete(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - int64_t affected_rows = 0; - SMART_VAR(ObTableApiMultiDeleteRowIterator, delete_row_iter) { - const uint64_t table_id = ctx.param_.table_id_; - ObTableDMLParam table_param(*ctx.param_.allocator_); - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(delete_row_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init delete row iterator, ", K(ret)); - } else if (OB_FAIL(delete_row_iter.open(batch_operation))) { - LOG_WARN("Fail to open delete row iterator, ", K(ret)); - } else if (OB_FAIL(build_table_param(table_id, delete_row_iter.get_delete_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = delete_row_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - int64_t result_idx = 0; - while(OB_SUCC(ret) && !delete_row_iter.has_finished()) { - affected_rows = 0; - delete_row_iter.continue_iter(); - // get one row - if (OB_FAIL(access_service->delete_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - delete_row_iter.get_delete_column_ids(), - &delete_row_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to delete", K(ret), K(table_id)); - } - } else { - // add result for not exist rows - const int64_t cur_delete_idx = delete_row_iter.get_cur_delete_idx(); - for (; OB_SUCC(ret) && result_idx < cur_delete_idx; ++result_idx) { - if (OB_FAIL(add_one_result(result, ObTableOperationType::DEL, OB_SUCCESS, 0))) { - LOG_WARN("failed to add result", K(ret), K(result_idx)); - } else { - LOG_DEBUG("Success to add delete result, ", K(result_idx), K(cur_delete_idx), K(tablet_id)); - } - } // end for - // add result for this row - if (OB_SUCC(ret) && result_idx <= cur_delete_idx) { - if (OB_FAIL(add_one_result( - result, - ObTableOperationType::DEL, - OB_SUCCESS, - ctx.param_.returning_affected_rows_ ? affected_rows : -1))) { - LOG_WARN("failed to add result", K(ret)); - } else { - result_idx++; - LOG_DEBUG("Success to add delete result, ", K(result_idx), K(cur_delete_idx), K(affected_rows), K(tablet_id)); - } - } - } - } //end while - - if (OB_SUCC(ret)) { - const int64_t N = batch_operation.count(); - for (; OB_SUCC(ret) && result_idx < N; result_idx++) { - if (OB_FAIL(add_one_result(result, ObTableOperationType::DEL, OB_SUCCESS, 0))) { - LOG_WARN("failed to add result", K(ret), K(result_idx)); - } - } - } - } - } - return ret; -} - -//////////////////////////////////////////////////////////////// -int ObTableService::execute_insert( - ObTableServiceCtx &ctx, - const ObTableOperation &table_operation, - ObTableOperationResult &result, - ObNewRowIterator *&duplicate_row_iter) -{ - int ret = OB_SUCCESS; - duplicate_row_iter = NULL; - SMART_VAR(ObTableApiInsertRowIterator, insert_iter) { - ObNewRow *row = NULL; - const uint64_t table_id = ctx.param_.table_id_; - ObTableDMLParam table_param(*ctx.param_.allocator_); - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(insert_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init insert row iterator, ", K(ret)); - } else if (OB_FAIL(insert_iter.open(table_operation))) { - LOG_WARN("Fail to open insert row iterator, ", K(ret)); - } else if (OB_FAIL(insert_iter.get_next_row(row))) { - LOG_WARN("Fail to get next row, ", K(ret)); - } else { - int64_t affected_rows = 0; - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = insert_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - if (ObTableOperationType::INSERT_OR_UPDATE == table_operation.type()) { - // use insert row with duplicate rows - ObSEArray rowkey_column_ids; - ObIArray &full_column_ids = insert_iter.get_column_ids(); - const int64_t rowkey_column_cnt = table_operation.entity().get_rowkey_size(); - if (OB_FAIL(build_table_param(table_id, full_column_ids, table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - dml_param.table_param_ = &table_param; - if (OB_FAIL(get_rowkey_column_ids(full_column_ids, rowkey_column_cnt, rowkey_column_ids))) { - LOG_WARN("failed to fill rowkey column ids, ", K(ret)); - } else if (OB_FAIL(access_service->insert_row( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - full_column_ids, - rowkey_column_ids, - *row, - storage::ObInsertFlag::INSERT_RETURN_ONE_DUP, - affected_rows, - duplicate_row_iter))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert", K(ret), K(table_id)); - } - } - } - } else { - // directly insert - ObSingleRowIteratorWrapper single_row_iter(row); - if (OB_FAIL(build_table_param(table_id, insert_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id)); - } else { - dml_param.table_param_ = &table_param; - if (OB_FAIL(access_service->insert_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - insert_iter.get_column_ids(), - &single_row_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert", K(ret), K(table_id)); - } - } - } - } - - if (OB_SUCC(ret)) { - result.set_affected_rows(1); - // for execute_increment() - if (ObTableOperationType::INSERT != table_operation.type() /*increment/append*/ - && ctx.param_.returning_affected_entity_) { - // need to return the new values to the client - ObITableEntity *new_entity = NULL; - if (OB_FAIL(result.get_entity(new_entity)) || OB_ISNULL(new_entity)) { - LOG_WARN("failed to get entity", K(ret), K(new_entity)); - } else if (OB_FAIL(fill_new_entity( - ctx.param_.returning_rowkey_, - *row, - table_operation.entity().get_rowkey_size(), - insert_iter.get_properties(), - *ctx.param_.allocator_, - new_entity))) { - LOG_WARN("failed to return new entity", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] increment return new entity", K(*new_entity)); - } - } - } - } - } - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(table_operation.type()); - return ret; -} - -int ObTableService::multi_insert(ObTableServiceCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - SMART_VAR(ObTableApiMultiInsertRowIterator, insert_iter) { - const uint64_t table_id = ctx.param_.table_id_; - ObTableDMLParam table_param(*ctx.param_.allocator_); - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(insert_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init multi insert iterator, ", K(ret)); - } else if (OB_FAIL(insert_iter.open(batch_operation))) { - LOG_WARN("Fail to open multi insert iterator, ", K(ret)); - } else if (OB_FAIL(build_table_param(table_id, insert_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - int64_t affected_rows = 0; - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = insert_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - const int64_t N = batch_operation.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { - insert_iter.continue_iter(); - if (OB_FAIL(access_service->insert_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - insert_iter.get_column_ids(), - &insert_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert", K(ret), K(table_id)); - } - } - if (OB_SUCCESS == ret || OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { - // ignore OB_ERR_PRIMARY_KEY_DUPLICATE - if (OB_FAIL(add_one_result(result, ObTableOperationType::INSERT, ret, affected_rows))) { - LOG_WARN("failed to add result", K(ret)); - } - } - } // end for - } - } - return ret; -} - -//////////////////////////////////////////////////////////////// -int ObTableService::do_replace( - ObTableServiceCtx &ctx, - storage::ObDMLBaseParam &dml_param, - ObIArray &column_ids, - ObIArray &rowkey_column_ids, - ObNewRow &row, - int64_t &affected_rows) -{ - int ret = OB_SUCCESS; - ObSingleRowIteratorWrapper single_row_iter(&row); - ObNewRowIterator *duplicated_rows = NULL; - int64_t del_rows = 0; - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (OB_FAIL(access_service->insert_row( // 1. try to insert - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - column_ids, - rowkey_column_ids, - row, - storage::ObInsertFlag::INSERT_RETURN_ALL_DUP, - affected_rows, - duplicated_rows))) { - if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { - // 2. delete the row - if (OB_FAIL(do_replace_delete( - ctx, - dml_param, - column_ids, - duplicated_rows, - del_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("Fail to delete replace duplicate row, ", K(ret), K(tablet_id), K(ls_id)); - } - } else { - // 3. insert the row again - single_row_iter.reset(); - if (OB_FAIL(access_service->insert_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - column_ids, - &single_row_iter, - affected_rows))) { - if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("we have already delete the rowkey and lock it, maybe have unique index", K(ret), K(row)); - } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert", K(ret), K(tablet_id), K(ls_id), K(row)); - } - } else { - // replace succ. - affected_rows += del_rows; - } - } - } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to insert", K(ret), K(tablet_id)); - } - } else { - // insert succ - } - - if (NULL != duplicated_rows) { - (void) access_service->revert_insert_iter(duplicated_rows); - duplicated_rows = NULL; - } - return ret; -} - -int ObTableService::do_replace_delete( - ObTableServiceCtx &ctx, - storage::ObDMLBaseParam &dml_param, - common::ObIArray &column_ids, - common::ObNewRowIterator *duplicated_rows, - int64_t &affected_rows) -{ - int ret = OB_SUCCESS; - storage::ObTableScanParam scan_param; - ObNewRow *dup_row = NULL; - common::ObNewRowIterator *scan_iter = NULL; - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - - if (NULL == duplicated_rows) { + const ObIArray &select_columns = query.get_select_columns(); + int64_t N = select_columns.count(); + if (N != 4) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("Invalid duplicate row iterator, ", K(ret)); + LOG_WARN("TableQuery with htable_filter should select 4 columns", K(ret), K(N)); } - - while (OB_SUCC(ret) && OB_SUCC(duplicated_rows->get_next_row(dup_row))) { - ObRowkey tmp_key(dup_row->cells_, dup_row->count_); - ObRowkey key; - if (OB_FAIL(tmp_key.deep_copy(key, *ctx.param_.allocator_))) { - LOG_WARN("fail to deep copy rowkey", K(ret)); - } else { - common::ObNewRange range; - if (OB_FAIL(range.build_range(table_id, key))) { - LOG_WARN("fail to build key range", K(ret), K(table_id), K(key)); - } else if (OB_FAIL(scan_param.key_ranges_.push_back(range))) { - LOG_WARN("fail to push back key range", K(ret), K(range)); - } - } - } - if (ret != OB_ITER_END) { - LOG_WARN("get next row not return ob_iter_end", K(ret)); - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - } - } else if (OB_FAIL(fill_query_scan_param( - ctx, - column_ids, - dml_param.schema_version_, - ObQueryFlag::ScanOrder::Forward, - table_id, - -1, //limit - 0, //offset - scan_param))) { - LOG_WARN("fail to fill query scan param, ", K(ret)); - } else if (OB_FAIL(access_service->table_scan(scan_param, scan_iter))) { - LOG_WARN("fail to table scan, ", K(ret)); - } else if (OB_FAIL(access_service->delete_rows(ls_id, tablet_id, *ctx.param_.processor_->get_trans_desc(), dml_param, - column_ids, scan_iter, affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to delete", K(ret), K(tablet_id)); - } - } else { - //delete success - } - - if (NULL != scan_iter) { - (void) access_service->revert_scan_iter(scan_iter); - scan_iter = NULL; - } - return ret; -} - -int ObTableService::execute_replace(ObTableServiceCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - SMART_VAR(ObTableApiInsertRowIterator, replace_iter) { - ObNewRow *row = NULL; - const uint64_t table_id = ctx.param_.table_id_; - ObTableDMLParam table_param(*ctx.param_.allocator_); - ObIArray &full_column_ids = replace_iter.get_column_ids(); - ObAccessService *access_service = MTL(ObAccessService *); - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(replace_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init replace iter, ", K(ret), K(table_id)); - } else if (OB_FAIL(replace_iter.open(table_operation))) { - LOG_WARN("Fail to open replace iter, ", K(ret), K(table_id)); - } else if (OB_FAIL(build_table_param(table_id, full_column_ids, table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(full_column_ids), K(table_param)); - } else { - int64_t affected_rows = 0; - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = replace_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - ObIArray &full_column_ids = replace_iter.get_column_ids(); - const int64_t rowkey_column_cnt = table_operation.entity().get_rowkey_size(); - ObSEArray rowkey_column_ids; - if (OB_FAIL(get_rowkey_column_ids(full_column_ids, rowkey_column_cnt, rowkey_column_ids))) { - LOG_WARN("failed to fill rowkey column ids, ", K(ret)); - } else if (OB_FAIL(replace_iter.get_next_row(row))) { - LOG_WARN("Fail to get replace row, ", K(ret), K(table_id)); - } else if (OB_FAIL(do_replace(ctx, dml_param, - full_column_ids, rowkey_column_ids, *row, affected_rows))) { - LOG_WARN("failed to do replace", K(ret)); - } else { - result.set_affected_rows(affected_rows); - } - } - } - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(ObTableOperationType::REPLACE); - return ret; -} - -int ObTableService::multi_replace(ObTableServiceCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - const uint64_t table_id = ctx.param_.table_id_; - ObAccessService *access_service = MTL(ObAccessService *); - SMART_VAR(ObTableApiMultiInsertRowIterator, replace_iter) { - ObNewRow *row = NULL; - ObTableDMLParam table_param(*ctx.param_.allocator_); - ObIArray &column_ids = replace_iter.get_column_ids(); - - if (OB_INVALID_ID == table_id) { - ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table id is invalid", K(ret), K(table_id)); - } else if (OB_FAIL(replace_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init insert iter, ", K(ret), K(table_id)); - } else if (OB_FAIL(replace_iter.open(batch_operation))) { - LOG_WARN("Fail to open batch operation, ", K(ret), K(table_id)); - } else if (OB_FAIL(build_table_param(table_id, column_ids, table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(column_ids), K(table_param)); - } else { - // init dml param - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = replace_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - // fill column ids - ObIArray &column_ids = replace_iter.get_column_ids(); - ObSEArray rowkey_column_ids; - const ObTableOperation &one_operation = batch_operation.at(0); - const ObITableEntity &one_entity = one_operation.entity(); - - for (int64_t i = 0; OB_SUCC(ret) && i < one_entity.get_rowkey_size(); ++i) { - if (OB_FAIL(rowkey_column_ids.push_back(column_ids.at(i)))) { - LOG_WARN("failed to fill rowkey column ids, ", K(ret)); - } - } - - if (OB_SUCC(ret)) { - int64_t affected_rows = 0; - const int64_t N = batch_operation.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { - replace_iter.continue_iter(); - if (OB_FAIL(replace_iter.get_next_row(row))) { - LOG_WARN("Fail to get next replace row, ", K(ret), K(i)); - } else if (OB_FAIL(do_replace(ctx, dml_param, column_ids, rowkey_column_ids, *row, affected_rows))) { - LOG_WARN("failed to do replace", K(ret)); - } else if (OB_FAIL(add_one_result(result, ObTableOperationType::REPLACE, OB_SUCCESS, affected_rows))) { - LOG_WARN("failed to add result", K(ret)); - } - } // end for - } - } - } - return ret; -} -//////////////////////////////////////////////////////////////// -// update & multi_update -//////////////////////////////////////////////////////////////// - -int ObTableService::execute_update(ObTableServiceGetCtx &ctx, - const ObTableOperation &table_operation, ObNewRow *target_row, ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - int64_t affected_rows = 0; - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - SMART_VAR(ObTableApiUpdateRowIterator, update_iter) { - ObRowkey rowkey; - ObTableDMLParam table_param(*ctx.param_.allocator_); - - if (NULL == target_row) { - rowkey = const_cast(table_operation.entity()).get_rowkey(); - } else { - const int64_t rowkey_cnt = table_operation.entity().get_rowkey_size(); - if (OB_UNLIKELY(rowkey_cnt > target_row->count_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(rowkey_cnt), K(target_row)); - } else { - rowkey.assign(target_row->cells_, rowkey_cnt); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(update_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init update row iterator, ", K(ret)); - } else if (OB_FAIL(update_iter.open(table_operation, rowkey, - NULL != target_row/*need_update_rowkey*/))) { - LOG_WARN("Fail to open update row iterator, ", K(ret)); - } else if (OB_FAIL(build_table_param(table_id, update_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = update_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - if (OB_FAIL(access_service->update_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - update_iter.get_column_ids(), - update_iter.get_update_column_ids(), - &update_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to update_rows", K(ret), K(tablet_id), K(ls_id)); - } - } else { - result.set_affected_rows(affected_rows); - } - } - } - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(ObTableOperationType::UPDATE); - return ret; -} - - -int ObTableService::fill_new_entity( - bool returning_rowkey, - const common::ObNewRow &row, - const int64_t primary_key_size, - const common::ObIArray &properties, - common::ObIAllocator &alloc, - ObITableEntity *new_entity) -{ - int ret = OB_SUCCESS; - new_entity->reset(); - ObObj cell_clone; - ObString name_clone; - if (primary_key_size > row.count_ || OB_ISNULL(new_entity)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected rowkey size", K(ret), K(primary_key_size), KP(new_entity)); - } else if (returning_rowkey) { - for (int64_t i = 0; i < primary_key_size && OB_SUCCESS == ret; ++i) { - if (OB_FAIL(ob_write_obj(alloc, row.cells_[i], cell_clone))) { - LOG_WARN("failed to copy obj", K(ret)); - } else if (OB_FAIL(new_entity->add_rowkey_value(cell_clone))) { - LOG_WARN("failed to add rowkey value", K(ret), K(cell_clone)); - } + if (OB_SUCC(ret)) { + if (ObHTableConstants::ROWKEY_CNAME_STR != select_columns.at(0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("TableQuery with htable_filter should select K as the first column", K(ret), K(select_columns)); + } else if (ObHTableConstants::CQ_CNAME_STR != select_columns.at(1)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("TableQuery with htable_filter should select Q as the second column", K(ret), K(select_columns)); + } else if (ObHTableConstants::VERSION_CNAME_STR != select_columns.at(2)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("TableQuery with htable_filter should select T as the third column", K(ret), K(select_columns)); + } else if (ObHTableConstants::VALUE_CNAME_STR != select_columns.at(3)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("TableQuery with htable_filter should select V as the fourth column", K(ret), K(select_columns)); } } if (OB_SUCC(ret)) { - const int64_t N = primary_key_size + properties.count(); - for (int64_t i = primary_key_size, j = 0; OB_SUCCESS == ret && i < N; ++i, ++j) { - // deep copy property - const ObString &name = properties.at(j); - const ObObj &cell = row.cells_[i]; - if (OB_FAIL(ob_write_string(alloc, name, name_clone))) { - LOG_WARN("failed to copy string", K(ret), K(name)); - } else if (OB_FAIL(ob_write_obj(alloc, cell, cell_clone))) { - LOG_WARN("failed to copy obj", K(ret)); - } else if (OB_FAIL(new_entity->set_property(name_clone, cell_clone))) { - LOG_WARN("failed to set property", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] affected new cell", K(name), K(i), K(cell_clone)); - } - } // end for - } - return ret; -} - -int ObTableService::execute_increment_by_update( - ObTableServiceGetCtx &ctx, - const ObTableOperation &table_operation, - ObTableOperationResult &result) -{ - int ret = OB_SUCCESS; - int64_t affected_rows = 0; - SMART_VAR(ObTableApiUpdateRowIterator, increment_row_iter) { - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - ObRowkey rowkey = const_cast(table_operation.entity()).get_rowkey(); - ObTableDMLParam table_param(*ctx.param_.allocator_); - - if (OB_FAIL(increment_row_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init increment row iter, ", K(ret)); - } else if (OB_FAIL(increment_row_iter.open(table_operation, rowkey))) { - LOG_WARN("Fail to open increment row iter, ", K(ret)); - } else if (OB_FAIL(build_table_param(table_id, increment_row_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = increment_row_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - if (OB_FAIL(access_service->update_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - increment_row_iter.get_column_ids(), - increment_row_iter.get_update_column_ids(), - &increment_row_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to update_rows", K(ret), K(tablet_id), K(ls_id)); - } - } else { - if (affected_rows > 0) { - affected_rows = 1; - if (ctx.param_.returning_affected_entity_) { - // need to return the new values to the client - ObITableEntity *new_entity = NULL; - ObNewRow *new_row = NULL; - if (NULL == (new_row = increment_row_iter.get_cur_new_row())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected error, the new row is null, ", K(ret), K(tablet_id)); - } else if (OB_FAIL(result.get_entity(new_entity)) || OB_ISNULL(new_entity)) { - LOG_WARN("failed to get entity", K(ret), K(new_entity)); - } else if (OB_FAIL(fill_new_entity( - ctx.param_.returning_rowkey_, - *new_row, - table_operation.entity().get_rowkey_size(), - increment_row_iter.get_properties(), - *ctx.param_.allocator_, - new_entity))) { - LOG_WARN("failed to return new entity", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] increment return new entity", K(*new_entity)); - } - } - } - - if (NULL == (increment_row_iter.get_cur_new_row())) { - ret = OB_EMPTY_RESULT; - } - } - } - - if (OB_EMPTY_RESULT != ret) { - result.set_affected_rows(affected_rows); - result.set_errno(ret); - replace_ret_code(ret); - result.set_type(table_operation.type()); - } - } - return ret; -} - -int ObTableService::execute_increment(ObTableServiceGetCtx &ctx, - const ObTableOperation &table_operation, - ObTableOperationResult &result) -{ - int ret = execute_increment_by_update(ctx, table_operation, result); - if (OB_EMPTY_RESULT == ret) { - // the row not exist, insert it - ctx.reset_get_ctx(); - ObNewRowIterator *duplicate_row_iter = nullptr; - ret = execute_insert(ctx, table_operation, result, duplicate_row_iter); - if (OB_SUCC(ret) && OB_ERR_PRIMARY_KEY_DUPLICATE == result.get_errno()) { - // concurrent inserted by another thread, try again in process() - ret = OB_TRY_LOCK_ROW_CONFLICT; - LOG_WARN("insert failed again after update failed", K(ret), K(result)); - } - } - - if (OB_FAIL(ret)) { - LOG_WARN("Fail to execute increment, ", K(ret)); - } else { - LOG_DEBUG("Success to execute increment, ", K(result)); - } - return ret; -} - -//////////////////////////////////////////////////////////////// -// multi update -int ObTableService::multi_update(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - SMART_VAR(ObTableApiMultiUpdateRowIterator, update_row_iter) { - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - ObAccessService *access_service = MTL(ObAccessService *); - ObLSID ls_id = ctx.param_.ls_id_; - int64_t affected_rows = 0; - int64_t result_idx = 0; - ObTableDMLParam table_param(*ctx.param_.allocator_); - - if (OB_FAIL(update_row_iter.init(*access_service, *schema_service_, ctx))) { - LOG_WARN("Fail to init multi update row iterator, ", K(ret)); - } else if (OB_FAIL(update_row_iter.open(batch_operation))) { - LOG_WARN("Fail to open multi update row iterator, ", K(ret)); - } else if (OB_FAIL(build_table_param(table_id, update_row_iter.get_column_ids(), table_param))) { - LOG_WARN("fail to convert table param", K(ret), K(table_id), K(table_param)); - } else { - storage::ObDMLBaseParam dml_param; - dml_param.timeout_ = ctx.param_.timeout_ts_; - dml_param.is_total_quantity_log_ = (ObBinlogRowImageType::FULL == ctx.param_.binlog_row_image_type_); - dml_param.tz_info_ = NULL; - dml_param.sql_mode_ = SMO_DEFAULT; - dml_param.schema_version_ = update_row_iter.get_schema_version(); - dml_param.encrypt_meta_ = &dml_param.encrypt_meta_legacy_; - dml_param.table_param_ = &table_param; - dml_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - dml_param.write_flag_.set_is_table_api(); - while(OB_SUCC(ret) && !update_row_iter.has_finished()) { - affected_rows = 0; - update_row_iter.continue_iter(); - if (OB_FAIL(access_service->update_rows( - ls_id, - tablet_id, - *ctx.param_.processor_->get_trans_desc(), - dml_param, - update_row_iter.get_column_ids(), - update_row_iter.get_update_column_ids(), - &update_row_iter, - affected_rows))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to update_rows", K(ret), K(tablet_id), K(ls_id)); - } - } else { - // add result for not exist rows - const int64_t cur_update_idx = update_row_iter.get_cur_update_idx(); - for (; OB_SUCC(ret) && result_idx < cur_update_idx; ++result_idx) { - if (OB_FAIL(add_one_result(result, ObTableOperationType::UPDATE, OB_SUCCESS, 0))) { - LOG_WARN("failed to add result", K(ret), K(result_idx)); - } else { - LOG_DEBUG("Success to add result, ", K(result_idx), K(cur_update_idx), K(tablet_id)); - } - } // end for - // add result for this row - if (OB_SUCC(ret) && result_idx <= cur_update_idx) { - if (OB_FAIL(add_one_result(result, ObTableOperationType::UPDATE, OB_SUCCESS, affected_rows))) { - LOG_WARN("failed to add result", K(ret)); - } else { - result_idx++; - LOG_DEBUG("Success to add result, ", K(result_idx), K(cur_update_idx), K(tablet_id)); - } - } - } - } // end for - - if (OB_SUCC(ret)) { - const int64_t N = batch_operation.count(); - for (; OB_SUCC(ret) && result_idx < N; result_idx++) { - if (OB_FAIL(add_one_result(result, ObTableOperationType::UPDATE, OB_SUCCESS, 0))) { - LOG_WARN("failed to add result", K(ret), K(result_idx)); - } - } - } - } - } - return ret; -} - -int ObTableService::batch_execute(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result) -{ - int ret = OB_SUCCESS; - int64_t N = batch_operation.count(); - ObNewRowIterator *duplicate_row_iter = nullptr; - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - const ObTableOperation &table_operation = batch_operation.at(i); - ObTableOperationResult op_result; - ObITableEntity *result_entity = result.get_entity_factory()->alloc(); - if (NULL == result_entity) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memroy for result_entity", K(ret)); - break; - } - op_result.set_entity(*result_entity); - switch(table_operation.type()) { - case ObTableOperationType::GET: - ret = execute_get(ctx, table_operation, op_result); - break; - case ObTableOperationType::INSERT: - ret = execute_insert(ctx, table_operation, op_result, duplicate_row_iter); - break; - case ObTableOperationType::DEL: - ret = execute_delete(ctx, table_operation, op_result); - break; - case ObTableOperationType::UPDATE: - ret = execute_update(ctx, table_operation, nullptr, op_result); - break; - case ObTableOperationType::INSERT_OR_UPDATE: - ret = execute_insert_or_update(ctx, table_operation, op_result); - break; - case ObTableOperationType::REPLACE: - ret = execute_replace(ctx, table_operation, op_result); - break; - case ObTableOperationType::APPEND: - case ObTableOperationType::INCREMENT: - ret = execute_increment(ctx, table_operation, op_result); - // no need to deep copy cells here which have already been deep copied in execute_increment() - break; - default: - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("unexpected operation type", "type", batch_operation.at(0).type()); - break; - } - if (OB_SUCC(ret)) { - if (OB_FAIL(result.push_back(op_result))) { - LOG_WARN("failed to push back result", K(ret)); - } else { - ctx.reset_get_ctx(); - } - } else { - LOG_WARN("failed to execute, ", K(ret), K(table_operation.type())); - } - } // end for - return ret; -} -//////////////////////////////////////////////////////////////// -// execute query -//////////////////////////////////////////////////////////////// -int ObTableService::cons_index_key_type(share::schema::ObSchemaGetterGuard &schema_guard, - const share::schema::ObTableSchema *index_schema, - uint64_t data_table_id, - common::ObIArray &columns_type) -{ - int ret = OB_SUCCESS; - uint64_t column_id = OB_INVALID_ID; - const share::schema::ObColumnSchemaV2 *column_schema = NULL; - ObExprResType column_type; - const ObIndexInfo &index_key_info = index_schema->get_index_info(); - const int64_t N = index_key_info.get_size(); - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { - if (OB_FAIL(index_key_info.get_column_id(i, column_id))) { - LOG_WARN("failed to get index column", K(ret), K(i)); - } else if (OB_FAIL(schema_guard.get_column_schema( - index_schema->get_tenant_id(), data_table_id, column_id, column_schema))) { - LOG_WARN("get column schema failed", K(data_table_id), K(column_id)); - } else if (NULL == column_schema) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get column schema", K(data_table_id), K(column_id)); - } else if (OB_FAIL(cons_column_type(*column_schema, column_type))) { - LOG_WARN("failed to cons column type", K(ret)); - } else if (OB_FAIL(columns_type.push_back(column_type))) { - LOG_WARN("failed to push back", K(ret)); - } else { - LOG_DEBUG("[yzfdebug] add index column for scan", - K(data_table_id), K(column_id), K(column_type)); - } - } // end for - return ret; -} - -int ObTableService::get_index_id_by_name(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, uint64_t base_table_id, - const ObString &index_name, uint64_t &index_id, - ObIArray &columns_type, - const share::schema::ObTableSchema *&index_schema) -{ - int ret = OB_SUCCESS; - uint64_t tids[OB_MAX_INDEX_PER_TABLE]; - int64_t table_index_count = OB_MAX_INDEX_PER_TABLE; - index_schema = nullptr; - if (index_name.empty() - || 0 == index_name.case_compare(ObIndexHint::PRIMARY_KEY)) { - index_id = base_table_id; - } else if (OB_FAIL(schema_guard.get_can_read_index_array( - tenant_id, base_table_id, tids, table_index_count, false))) { - LOG_WARN("failed to get can read index", K(ret), K(tenant_id)); - } else if (table_index_count > OB_MAX_INDEX_PER_TABLE) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Table index count is bigger than OB_MAX_INDEX_PER_TABLE", K(ret), K(table_index_count)); - } else { - LOG_DEBUG("get readable index", K(table_index_count)); - bool found = false; - for (int64_t i = 0; OB_SUCC(ret) && !found && i < table_index_count; ++i) { - uint64_t this_index_id = tids[i]; - ObString this_index; - if (OB_FAIL(schema_guard.get_table_schema(tenant_id, this_index_id, index_schema)) - || OB_ISNULL(index_schema)) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get table schema", K(tenant_id), K(this_index_id), K(ret)); - } else if (OB_FAIL(index_schema->get_index_name(this_index))) { - LOG_WARN("fail to get index name", K(tenant_id), K(this_index), K(ret), K(this_index_id)); - } else if (0 != index_name.case_compare(this_index)) { - //do nothing, just continue - continue; - } else { - found = true; - index_id = this_index_id; - //record the column type in index to check the object type in scan range - ret = cons_index_key_type(schema_guard, index_schema, base_table_id, columns_type); - } - } // end for - if (!found) { - ret = OB_ERR_INDEX_UNKNOWN; - } - } - return ret; -} - -int ObTableService::fill_query_table_param(const uint64_t tenant_id, - const uint64_t table_id, - const ObIArray &properties, - const ObString &index_name, - share::schema::ObTableParam &table_param, - ObIArray &output_column_ids, - common::ObIArray &rowkey_columns_type, - int64_t &schema_version, - uint64_t &index_id, - int64_t &padding_num, - table::ObHColumnDescriptor *hcolumn_desc) -{ - int ret = OB_SUCCESS; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTableSchema *table_schema = NULL; - const share::schema::ObTableSchema *index_schema = NULL; - if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("failed to get schema guard", K(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("get table schema failed", K(tenant_id), K(table_id), K(ret)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("NULL ptr", K(ret), KP(table_schema)); - } else if (OB_FAIL(get_index_id_by_name(schema_guard, tenant_id, table_id, index_name, index_id, - rowkey_columns_type, index_schema))) { - LOG_WARN("failed to get index id", K(ret), K(tenant_id), K(index_name), K(table_id)); - } else { - NG_TRACE_EXT(idx, OB_ID(idx), index_id, OB_ID(tag1), index_name); - schema_version = table_schema->get_schema_version(); - const int64_t key_column_cnt = rowkey_columns_type.count(); - padding_num = (NULL == index_schema) ? -1 : (index_schema->get_rowkey_column_num() - key_column_cnt); - LOG_DEBUG("[xilin debug]padding", K(padding_num), K(key_column_cnt), K(index_name)); - - //@ATTENSION there is no index back in 4.0, need refact - const bool index_back = (index_id != table_id); - bool is_index_supported = true; - if (index_back && OB_FAIL(check_index_supported(schema_guard, table_schema, index_id, is_index_supported))) { - LOG_WARN("fail to check index supported", K(ret), K(index_id)); - } else if (OB_UNLIKELY(!is_index_supported)) { + if (0 != query.get_offset()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("htable scan should not set Offset and Limit", K(ret), K(query)); + } else if (ObQueryFlag::Forward != query.get_scan_order() && ObQueryFlag::Reverse != query.get_scan_order()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("index type is not supported by table api", K(ret), K(table_id), K(index_id)); - } else if (OB_FAIL(cons_rowkey_infos(*table_schema, NULL, index_back ? NULL : &rowkey_columns_type))) { - } else if (OB_FAIL(cons_properties_infos(*table_schema, properties, output_column_ids, NULL))) { - } else if (OB_FAIL(table_param.convert(*table_schema, output_column_ids))) { - LOG_WARN("failed to convert table param", K(ret)); - } else if (!table_schema->get_comment_str().empty() - && NULL != hcolumn_desc) { - if (OB_FAIL(hcolumn_desc->from_string(table_schema->get_comment_str()))) { - LOG_WARN("failed to parse hcolumn_desc from comment string", K(ret), - "comment", table_schema->get_comment_str()); - } else { - LOG_DEBUG("[yzfdebug] get ttl", K(table_id), - "comment", table_schema->get_comment_str(), - "ttl", hcolumn_desc->get_time_to_live()); - } - } - } - return ret; -} - -int ObTableService::fill_query_scan_ranges(ObTableServiceCtx &ctx, - const ObTableQuery &query, - int64_t padding_num, - storage::ObTableScanParam &scan_param) -{ - int ret = OB_SUCCESS; - scan_param.key_ranges_.reset(); - const ObIArray &scan_ranges = query.get_scan_ranges(); - int64_t N = scan_ranges.count(); - // check obj type in ranges - for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { // foreach range - const ObNewRange &range = scan_ranges.at(i); - // check column type - for (int64_t j = 0; OB_SUCCESS == ret && j < 2; ++j) { - const ObRowkey *p_key = nullptr; - if (0 == j) { - p_key = &range.get_start_key(); - } else { - p_key = &range.get_end_key(); - } - if (p_key->is_min_row() || p_key->is_max_row()) { - continue; - } else { - if (p_key->get_obj_cnt() != ctx.columns_type_.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("wrong rowkey size", K(ret), K(i), K(j), K(*p_key), K_(ctx.columns_type)); - } else { - const int64_t M = p_key->get_obj_cnt(); - for (int64_t k = 0; OB_SUCCESS == ret && k < M; ++k) { - ObObj &obj = const_cast(p_key->get_obj_ptr()[k]); - if (obj.is_min_value() || obj.is_max_value()) { - continue; - } else if (OB_FAIL(ObTableService::check_column_type(ctx.columns_type_.at(k), obj))) { - } - } - } - } // end else - } // end for - if (OB_UNLIKELY(padding_num > 0)) { - // index scan need fill primary key object - ObNewRange index_key_range = range; - for (int64_t j = 0; OB_SUCCESS == ret && j < 2; ++j) { - const ObRowkey *p_key = nullptr; - if (0 == j) { - p_key = &range.get_start_key(); - } else { - p_key = &range.get_end_key(); - } - if (p_key->is_min_row() || p_key->is_max_row()) { - continue; - } else { - const int64_t old_objs_num = p_key->get_obj_cnt(); - const int64_t new_objs_num = old_objs_num + padding_num; - ObObj *new_objs = static_cast(ctx.param_.allocator_->alloc(sizeof(ObObj)*new_objs_num)); - if (nullptr == new_objs) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no memory", K(ret)); - } else { - const ObObj *old_objs = p_key->get_obj_ptr(); - for (int64_t k = 0; k < old_objs_num; ++k) { - new_objs[k] = old_objs[k]; // shallow copy - } // end for - if (0 == j) { // padding for startkey - for (int64_t k = 0; k < padding_num; ++k) { - new_objs[k+old_objs_num] = ObObj::make_min_obj(); - } - index_key_range.start_key_.assign(new_objs, new_objs_num); - } else { // padding for endkey - for (int64_t k = 0; k < padding_num; ++k) { - new_objs[k+old_objs_num] = ObObj::make_max_obj(); - } - index_key_range.end_key_.assign(new_objs, new_objs_num); - } - } - } - } // end for - if (OB_SUCC(ret)) { - if (OB_FAIL(scan_param.key_ranges_.push_back(index_key_range))) { - LOG_WARN("fail to push back key range", K(ret), K(index_key_range)); - } else { - LOG_DEBUG("[yzfdebug] add key range for index scan", K(ret), K(index_key_range)); - } - } - } else { - if (OB_SUCC(ret)) { - if (OB_FAIL(scan_param.key_ranges_.push_back(range))) { - LOG_WARN("fail to push back key range", K(ret), K(range)); - } - } - } - } // end for - return ret; -} - -int ObTableService::fill_query_scan_param(ObTableServiceCtx &ctx, - const ObIArray &output_column_ids, - int64_t schema_version, - ObQueryFlag::ScanOrder scan_order, - uint64_t index_id, - int32_t limit, - int32_t offset, - storage::ObTableScanParam &scan_param, - bool for_update /* false */) -{ - int ret = OB_SUCCESS; - const uint64_t table_id = ctx.param_.table_id_; - const ObTabletID tablet_id = ctx.param_.tablet_id_; - const uint64_t tenant_id = MTL_ID(); - const bool index_back = (index_id != table_id); - ObTabletID index_tablet_id; - - if (index_back) { - share::schema::ObSchemaGetterGuard schema_guard; - const ObTableSchema *index_schema = NULL; - if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("fail to get schema guard", K(ret), K(tenant_id), KP(schema_service_)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, index_id, index_schema))) { - LOG_WARN("get index schema failed", KR(ret), K(tenant_id), K(index_id)); - } else if (OB_ISNULL(index_schema)) { - ret = OB_TABLE_NOT_EXIST; - LOG_WARN("index not exist", KR(ret), K(tenant_id), K(index_id)); - } else { - index_tablet_id = index_schema->get_tablet_id(); - } - } - if (OB_SUCC(ret)) { - scan_param.timeout_ = ctx.param_.timeout_ts_; - ObQueryFlag query_flag(scan_order, - false, // daily_merge - false, // optimize - false, // whole_macro_scan - false, // full_row - index_back, // index_back - false, // query_stat - ObQueryFlag::MysqlMode, // sql_mode - true // read_latest - ); - scan_param.scan_flag_.flag_ = query_flag.flag_; - scan_param.reserved_cell_count_ = output_column_ids.count() + 10; - scan_param.for_update_ = for_update; - scan_param.column_ids_.reset(); - scan_param.schema_version_ = schema_version; - if (OB_FAIL(scan_param.column_ids_.assign(output_column_ids))) { - LOG_WARN("fail to assign column id", K(ret)); - } else { - SQL_ENG_LOG(DEBUG, "set scan param", K(output_column_ids)); - scan_param.ls_id_ = ctx.param_.ls_id_; - scan_param.tablet_id_ = tablet_id; - // scan_param.index_tablet_id_ = index_tablet_id; todo@dazhi: tableapi index back - scan_param.limit_param_.limit_ = limit; - scan_param.limit_param_.offset_ = offset; - scan_param.trans_desc_ = ctx.param_.processor_->get_trans_desc(); - scan_param.snapshot_ = ctx.param_.processor_->get_tx_snapshot(); - scan_param.tx_id_ = ctx.param_.processor_->get_trans_desc()->get_tx_id(); - scan_param.index_id_ = index_id; - scan_param.sql_mode_ = SMO_DEFAULT; - CURRENT_CONTEXT->get_arena_allocator().set_tenant_id(tenant_id); - LOG_DEBUG("[yzfdebug] scan param", K(scan_param)); + LOG_WARN("TableQuery with htable_filter only support forward and reverse scan yet", K(ret)); } } return ret; @@ -2151,204 +142,4 @@ int ObNormalTableQueryResultIterator::get_next_result(table::ObTableQueryResult bool ObNormalTableQueryResultIterator::has_more_result() const { return has_more_rows_; -} - -ObNormalTableQueryResultIterator *ObTableServiceQueryCtx::get_normal_result_iterator( - const ObTableQuery &query, table::ObTableQueryResult &one_result) -{ - if (NULL == normal_result_iterator_) { - normal_result_iterator_ = OB_NEWx(ObNormalTableQueryResultIterator, param_.allocator_, query, one_result); - if (NULL == normal_result_iterator_) { - LOG_WARN("failed to allocate result iterator"); - } - } - return normal_result_iterator_; -} - -ObHTableFilterOperator *ObTableServiceQueryCtx::get_htable_result_iterator( - const ObTableQuery &query, table::ObTableQueryResult &one_result) -{ - if (NULL == htable_result_iterator_) { - htable_result_iterator_ = OB_NEWx(ObHTableFilterOperator, param_.allocator_, query, one_result); - if (NULL == htable_result_iterator_) { - LOG_WARN("failed to allocate htable filter"); - } - } - return htable_result_iterator_; -} - -void ObTableServiceQueryCtx::destroy_result_iterator(storage::ObAccessService *access_service) -{ - if (NULL != normal_result_iterator_) { - normal_result_iterator_->~ObNormalTableQueryResultIterator(); - normal_result_iterator_ = NULL; - } - if (NULL != htable_result_iterator_) { - htable_result_iterator_->~ObHTableFilterOperator(); - htable_result_iterator_ = NULL; - } - if (NULL != scan_result_) { - if (NULL == access_service) { - LOG_ERROR("access service is NULL, memory leak"); - } else { - access_service->revert_scan_iter(scan_result_); - scan_result_ = NULL; - } - } -} - -int ObTableService::check_htable_query_args(const ObTableQuery &query) -{ - int ret = OB_SUCCESS; - const ObIArray &select_columns = query.get_select_columns(); - int64_t N = select_columns.count(); - if (N != 4) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("TableQuery with htable_filter should select 4 columns", K(ret), K(N)); - } - if (OB_SUCC(ret)) { - if (ObHTableConstants::ROWKEY_CNAME_STR != select_columns.at(0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("TableQuery with htable_filter should select K as the first column", K(ret), K(select_columns)); - } else if (ObHTableConstants::CQ_CNAME_STR != select_columns.at(1)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("TableQuery with htable_filter should select Q as the second column", K(ret), K(select_columns)); - } else if (ObHTableConstants::VERSION_CNAME_STR != select_columns.at(2)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("TableQuery with htable_filter should select T as the third column", K(ret), K(select_columns)); - } else if (ObHTableConstants::VALUE_CNAME_STR != select_columns.at(3)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("TableQuery with htable_filter should select V as the fourth column", K(ret), K(select_columns)); - } - } - if (OB_SUCC(ret)) { - if (0 != query.get_offset()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("htable scan should not set Offset and Limit", K(ret), K(query)); - } else if (ObQueryFlag::Forward != query.get_scan_order() && ObQueryFlag::Reverse != query.get_scan_order()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("TableQuery with htable_filter only support forward and reverse scan yet", K(ret)); - } - } - return ret; -} - -int ObTableService::execute_query(ObTableServiceQueryCtx &ctx, const ObTableQuery &query, - table::ObTableQueryResult &one_result, - table::ObTableQueryResultIterator *&query_result, - bool for_update /* false */) -{ - int ret = OB_SUCCESS; - ObSEArray output_column_ids; - int64_t schema_version = 0; - ctx.scan_result_ = NULL; - const uint64_t table_id = ctx.param_.table_id_; - uint64_t index_id = OB_INVALID_ID; - int64_t padding_num = 0; - ObHColumnDescriptor hcolumn_desc; - ObHColumnDescriptor *p_hcolumn_desc = NULL; - ObAccessService *access_service = MTL(ObAccessService *); - const uint64_t tenant_id = MTL_ID(); - if (query.get_htable_filter().is_valid()) { - if (OB_FAIL(check_htable_query_args(query))) { - LOG_WARN("invalid query request", K(ret)); - } else if (NULL == (query_result = ctx.get_htable_result_iterator(query, one_result))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate htable result iterator", K(ret)); - } else if (OB_FAIL(ctx.htable_result_iterator_->parse_filter_string(ctx.param_.allocator_))) { - LOG_WARN("failed to parse htable filter string", K(ret)); - } else { - p_hcolumn_desc = &hcolumn_desc; - } - } else { - if (NULL == (query_result = ctx.get_normal_result_iterator(query, one_result))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate result iterator", K(ret)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(fill_query_table_param(tenant_id, table_id, - query.get_select_columns(), - query.get_index_name(), - *(ctx.table_param_), output_column_ids, - ctx.columns_type_, schema_version, - index_id, padding_num, - p_hcolumn_desc))) { // @todo optimize, table_param_ can be cached - LOG_WARN("failed to fill param", K(ret)); - } else if (OB_FAIL(fill_query_scan_ranges(ctx, query, - (table_id != index_id) ? padding_num : -1, - ctx.scan_param_))) { - LOG_WARN("failed to fill range", K(ret)); - } else if (OB_FAIL(fill_query_scan_param(ctx, output_column_ids, schema_version, - query.get_scan_order(), index_id, query.get_limit(), - query.get_offset(), ctx.scan_param_, for_update))) { - LOG_WARN("failed to fill param", K(ret)); - } else if (OB_FAIL(access_service->table_scan(ctx.scan_param_, ctx.scan_result_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("fail to scan table", K(ret)); - } - } else { - if (query.get_htable_filter().is_valid()) { - ctx.htable_result_iterator_->set_scan_result(ctx.scan_result_); - if (p_hcolumn_desc->get_time_to_live() > 0) { - ctx.htable_result_iterator_->set_ttl(p_hcolumn_desc->get_time_to_live()); - } - } else { - ctx.normal_result_iterator_->set_scan_result(ctx.scan_result_); - } - } - return ret; -} - -int ObTableService::build_table_param( - const uint64_t table_id, - const common::ObIArray &column_ids, - ObTableDMLParam &table_param) -{ - int ret = OB_SUCCESS; - const share::schema::ObTableSchema *table_schema = NULL; - share::schema::ObSchemaGetterGuard schema_guard; - int64_t schema_version = OB_INVALID_VERSION; - const uint64_t tenant_id = MTL_ID(); - if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("fail to get schema guard", K(ret), K(tenant_id), KP(schema_service_)); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { - LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(table_id), KP(table_schema)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret), KP(table_schema)); - } else if (OB_INVALID_VERSION == (schema_version = table_schema->get_schema_version())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema version is invalid", K(ret), K(schema_version)); - } else if (OB_FAIL(table_param.convert(table_schema, schema_version, column_ids))) { - LOG_WARN("fail to convert table param", K(ret), K(table_schema)); - } - return ret; -} - -// check whether index is supported in given table schema by table api -// global index is not supported by table api. specially, global index in non-partitioned -// table was optimized to local index, which we can support. -int ObTableService::check_index_supported(schema::ObSchemaGetterGuard &schema_guard, - const schema::ObSimpleTableSchemaV2 *table_schema, uint64_t index_id, bool &is_supported) -{ - int ret = OB_SUCCESS; - is_supported = true; - const schema::ObSimpleTableSchemaV2 *index_schema = NULL; - const uint64_t tenant_id = MTL_ID(); - - if (OB_ISNULL(table_schema)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("null table schema", K(ret)); - } else if (table_schema->is_partitioned_table()) { - if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, index_id, index_schema))) { - LOG_WARN("fail to get table schmea", K(ret), K(index_id)); - } else if (OB_ISNULL(index_schema)) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("get null index schema", K(ret), K(index_id)); - } else if (index_schema->is_global_index_table()) { - is_supported = false; - } - } - return ret; -} +} \ No newline at end of file diff --git a/src/observer/table/ob_table_service.h b/src/observer/table/ob_table_service.h index d5dd3a5d5e..f3cb964e2e 100644 --- a/src/observer/table/ob_table_service.h +++ b/src/observer/table/ob_table_service.h @@ -18,6 +18,8 @@ #include "share/table/ob_table_rpc_struct.h" #include "share/schema/ob_table_param.h" #include "storage/access/ob_dml_param.h" +#include "ob_table_scan_executor.h" +#include "ob_table_session_pool.h" namespace oceanbase { namespace table @@ -26,8 +28,8 @@ class ObHTableFilterOperator; class ObHColumnDescriptor; } // end namespace table -namespace storage -{ +namespace storage +{ class ObAccessService; } // end namespace storage @@ -43,7 +45,6 @@ using table::ObTableQueryResult; using table::ObTableQuerySyncResult; class ObTableApiProcessorBase; class ObTableService; -class ObTableApiRowIterator; class ObTableServiceCtx { @@ -52,14 +53,13 @@ public: common::ObSEArray columns_type_; protected: friend class ObTableService; - friend class ObTableApiRowIterator; struct Param { uint64_t table_id_; uint64_t partition_id_; common::ObTabletID tablet_id_; common::ObTabletID index_tablet_id_; - share::ObLSID ls_id_; + share::ObLSID ls_id_; int64_t timeout_ts_; ObTableApiProcessorBase *processor_; common::ObArenaAllocator *allocator_; @@ -114,22 +114,6 @@ public: share::ObLSID ¶m_ls_id() { return param_.ls_id_; } }; -class ObTableServiceGetCtx: public ObTableServiceCtx -{ -public: - share::schema::ObTableParam table_param_on_stack_; - share::schema::ObTableParam *table_param_; - storage::ObTableScanParam scan_param_; - common::ObNewRowIterator *scan_result_; -public: - ObTableServiceGetCtx(common::ObArenaAllocator &alloc); - void reset_get_ctx() - { - ObTableServiceCtx::reset_dml(); - table_param_->reset(); - } -}; - class ObNormalTableQueryResultIterator: public table::ObTableQueryResultIterator { public: @@ -149,8 +133,9 @@ public: virtual ~ObNormalTableQueryResultIterator() {} virtual int get_next_result(table::ObTableQueryResult *&one_result) override; virtual bool has_more_result() const override; - void set_scan_result(common::ObNewRowIterator *scan_result) { scan_result_ = scan_result; } + void set_scan_result(table::ObTableApiScanRowIterator *scan_result) { scan_result_ = scan_result; } virtual void set_one_result(ObTableQueryResult *result) {one_result_ = result;} + table::ObTableQueryResult *get_one_result() { return one_result_; } void set_query(const ObTableQuery *query) {query_ = query;} void set_query_sync() { is_query_sync_ = true ; } private: @@ -159,35 +144,12 @@ private: common::ObNewRow *last_row_; int32_t batch_size_; int64_t max_result_size_; - common::ObNewRowIterator *scan_result_; + table::ObTableApiScanRowIterator *scan_result_; bool is_first_result_; bool has_more_rows_; bool is_query_sync_; }; -struct ObTableServiceQueryCtx: public ObTableServiceGetCtx -{ -public: - ObNormalTableQueryResultIterator *normal_result_iterator_; - table::ObHTableFilterOperator *htable_result_iterator_; -public: - ObTableServiceQueryCtx(common::ObArenaAllocator &alloc) - :ObTableServiceGetCtx(alloc), - normal_result_iterator_(NULL), - htable_result_iterator_(NULL) - {} - void reset_query_ctx(storage::ObAccessService *access_service) - { - destroy_result_iterator(access_service); - ObTableServiceGetCtx::reset_get_ctx(); - } - ObNormalTableQueryResultIterator *get_normal_result_iterator(const ObTableQuery &query, - table::ObTableQueryResult &one_result); - table::ObHTableFilterOperator *get_htable_result_iterator(const ObTableQuery &query, - table::ObTableQueryResult &one_result); - void destroy_result_iterator(storage::ObAccessService *access_service); -}; - /// table service class ObTableService { @@ -199,154 +161,14 @@ public: {} virtual ~ObTableService() = default; int init(ObGlobalContext &gctx); - - int execute_get(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - int execute_insert_or_update(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - int execute_delete(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - int execute_insert(ObTableServiceCtx &ctx, const ObTableOperation &table_operation, - ObTableOperationResult &result, ObNewRowIterator *&duplicate_row_iter); - int execute_replace(ObTableServiceCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - int execute_update(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, - ObNewRow *target_row, ObTableOperationResult &result); - int execute_increment(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - - int multi_get(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int multi_insert_or_update(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int multi_delete(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int multi_insert(ObTableServiceCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int multi_replace(ObTableServiceCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int multi_update(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - - int batch_execute(ObTableServiceGetCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int execute_query(ObTableServiceQueryCtx &ctx, const ObTableQuery &query, - table::ObTableQueryResult &one_result, table::ObTableQueryResultIterator *&query_result, - bool for_update = false); -private: - static int cons_rowkey_infos(const share::schema::ObTableSchema &table_schema, - common::ObIArray *column_ids, - common::ObIArray *columns_type); - static int cons_properties_infos(const share::schema::ObTableSchema &table_schema, - const common::ObIArray &properties, - common::ObIArray &column_ids, - common::ObIArray *columns_type); - static int cons_column_type(const share::schema::ObColumnSchemaV2 &column_schema, sql::ObExprResType &column_type); - static int check_column_type(const sql::ObExprResType &column_type, common::ObObj &obj); - static int add_index_columns_if_missing(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const uint64_t data_table_id, - const share::schema::ObTableSchema *index_schema, - common::ObIArray &column_ids, - common::ObIArray *columns_type); - - int insert_or_update_can_use_put(table::ObTableEntityType entity_type, uint64_t table_id, const table::ObITableEntity &entity, bool &use_put); - int add_one_result(ObTableBatchOperationResult &result, - table::ObTableOperationType::Type op_type, - int32_t error_code, - int64_t affected_rows); - int do_put(ObTableServiceCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - int do_insert_or_update(ObTableServiceGetCtx &ctx, const ObTableOperation &table_operation, ObTableOperationResult &result); - int multi_put(ObTableServiceCtx &ctx, const ObTableBatchOperation &batch_operation, ObTableBatchOperationResult &result); - int do_multi_insert_or_update(ObTableServiceGetCtx &ctx, - const ObTableBatchOperation &batch_operation, - ObTableBatchOperationResult &result); - // for get - int fill_scan_param(ObTableServiceCtx &ctx, - const ObIArray &output_column_ids, - int64_t schema_version, - storage::ObTableScanParam &scan_param); - int fill_get_result( - ObTableServiceCtx &ctx, - const ObIArray &properties, - ObTableApiRowIterator *scan_result, - ObTableOperationResult &operation_result); - // for multi-get - int fill_multi_get_result( - ObTableServiceGetCtx &ctx, - const ObTableBatchOperation &batch_operation, - ObTableApiRowIterator *scan_result, - ObTableBatchOperationResult &result); - int delete_can_use_put(table::ObTableEntityType entity_type, uint64_t table_id, bool &use_put); - static int cons_all_index_properties(share::schema::ObSchemaGetterGuard &schema_guard, - const share::schema::ObTableSchema &table_schema, - common::ObIArray &column_ids, - common::ObIArray *columns_type); - - // for replace - int do_replace( - ObTableServiceCtx &ctx, - storage::ObDMLBaseParam &dml_param, - common::ObIArray &column_ids, - common::ObIArray &rowkey_column_ids, - common::ObNewRow &row, - int64_t &affected_rows); - // for replace delete duplicate row - int do_replace_delete( - ObTableServiceCtx &ctx, - storage::ObDMLBaseParam &dml_param, - common::ObIArray &column_ids, - common::ObNewRowIterator *duplicated_rows, - int64_t &affected_rows); - // for execute_query - int cons_index_key_type(share::schema::ObSchemaGetterGuard &schema_guard, - const share::schema::ObTableSchema *index_schema, - uint64_t data_table_id, - common::ObIArray &columns_type); - int get_index_id_by_name(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, uint64_t base_table_id, - const ObString &index_name, uint64_t &index_id, - common::ObIArray &columns_type, - const share::schema::ObTableSchema *&index_schema); - int fill_query_table_param(const uint64_t tenant_id, - const uint64_t table_id, - const common::ObIArray &properties, - const ObString &index_name, - share::schema::ObTableParam &table_param, - common::ObIArray &output_column_ids, - common::ObIArray &rowkey_columns_type, - int64_t &schema_version, - uint64_t &index_id, - int64_t &padding_num, - table::ObHColumnDescriptor *hcolumn_desc); - int fill_query_scan_ranges(ObTableServiceCtx &ctx, - const ObTableQuery &query, - int64_t padding_num, - storage::ObTableScanParam &scan_param); - int fill_query_scan_param(ObTableServiceCtx &ctx, - const common::ObIArray &output_column_ids, - int64_t schema_version, - ObQueryFlag::ScanOrder scan_order, - uint64_t index_id, - int32_t limit, - int32_t offset, - storage::ObTableScanParam &scan_param, - bool for_update = false); - int check_htable_query_args(const ObTableQuery &query); - int check_index_supported(share::schema::ObSchemaGetterGuard &schema_guard, - const share::schema::ObSimpleTableSchemaV2 *table_schema, - uint64_t index_id, - bool &is_supported); - -private: - int fill_new_entity( - bool returning_rowkey, - const common::ObNewRow &row, - const int64_t primary_key_size, - const common::ObIArray &properties, - common::ObIAllocator &alloc, - table::ObITableEntity *new_entity); - int execute_increment_by_update(ObTableServiceGetCtx &ctx, - const ObTableOperation &table_operation, - ObTableOperationResult &result); - int build_table_param( - const uint64_t table_id, - const common::ObIArray &column_ids, - ObTableDMLParam &table_param); + table::ObTableApiSessPoolMgr& get_sess_mgr() { return sess_pool_mgr_; } + static int check_htable_query_args(const ObTableQuery &query); private: // disallow copy DISALLOW_COPY_AND_ASSIGN(ObTableService); private: - static const int64_t COMMON_COLUMN_NUM = 16; share::schema::ObMultiVersionSchemaService *schema_service_; + table::ObTableApiSessPoolMgr sess_pool_mgr_; }; } // end namespace observer diff --git a/src/observer/table/ob_table_session_pool.cpp b/src/observer/table/ob_table_session_pool.cpp new file mode 100644 index 0000000000..00cd634a3d --- /dev/null +++ b/src/observer/table/ob_table_session_pool.cpp @@ -0,0 +1,732 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX SERVER + +#include "ob_table_session_pool.h" +namespace oceanbase +{ +namespace table +{ + +int ObTableApiSessPoolMgr::init() +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + ObMemAttr bucket_attr, node_attr; + bucket_attr.label_ = "HashBucApiSessM"; + node_attr.label_ = "HasNodApiSessP"; + if (OB_FAIL(sess_pool_map_.create(hash::cal_next_prime(16), + bucket_attr, + node_attr))) { + LOG_WARN("failed to init tableapi session pool", K(ret)); + } else { + elimination_task_.sess_pool_mgr_ = this; + timer_.set_run_wrapper(MTL_CTX()); + if (OB_FAIL(timer_.init())) { + LOG_WARN("fail to init timer", K(ret)); + } else if (OB_FAIL(timer_.schedule(elimination_task_, ELIMINATE_SESSION_DELAY, true))) { + LOG_WARN("fail to schedule session pool elimination task. ", K(ret)); + } else if (OB_FAIL(timer_.start())) { + LOG_WARN("fail to start session pool elimination task timer.", K(ret)); + } else { + is_inited_ = true; + } + } + } + + return ret; +} + +void ObTableApiSessPoolMgr::destroy() +{ + if (is_inited_) { + is_inited_ = false; + timer_.destroy(); + for (SessPoolMap::iterator it = sess_pool_map_.begin(); + it != sess_pool_map_.end(); + it++) { + if (OB_ISNULL(it->second)) { + BACKTRACE(ERROR, true, "session pool is null"); + } else { + it->second->~ObTableApiSessPool(); + ob_free(it->second); + } + } + } +} + +int ObTableApiSessPoolMgr::get_sess_info(uint64_t tenant_id, + uint64_t user_id, + ObTableApiSessGuard &guard) +{ + int ret = OB_SUCCESS; + ObTableApiSessPoolGuard &pool_guard = guard.get_sess_pool_guard(); + if (OB_FAIL(get_session_pool(tenant_id, pool_guard))) { + LOG_WARN("fail to get session pool", K(ret)); + } else if (OB_FAIL(pool_guard.get_sess_pool()->get_sess_info(user_id, guard))) { + LOG_WARN("fail to get sess info", K(ret), K(tenant_id), K(user_id)); + } + + return ret; +} + +int ObTableApiSessPoolMgr::update_sess(ObTableApiCredential &credential) +{ + int ret = OB_SUCCESS; + ObTableApiSessPoolGuard guard; + const uint64_t tenant_id = credential.tenant_id_; + if (OB_FAIL(get_session_pool(tenant_id, guard))) { + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(extend_sess_pool(tenant_id, guard))) { + LOG_WARN("fail to extend sess pool", K(ret), K(tenant_id)); + } + } else { + LOG_WARN("fait to get session pool", K(ret), K(tenant_id)); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(guard.get_sess_pool()->update_sess(credential))) { + LOG_WARN("fail to update sess pool", K(ret), K(tenant_id), K(credential)); + } + + return ret; +} + +int ObTableApiSessPoolMgr::extend_sess_pool(uint64_t tenant_id, + ObTableApiSessPoolGuard &guard) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + ObTableApiSessPool *tmp_pool = nullptr; + + if (OB_ISNULL(buf = ob_malloc(sizeof(ObTableApiSessPool), "ApiSessPool"))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem for ObTableApiSessPool", K(ret)); + } else if (FALSE_IT(tmp_pool = new (buf) ObTableApiSessPool(tenant_id))) { + } else { + if (OB_FAIL(tmp_pool->init())) { + LOG_WARN("fail to init sess pool", K(ret), K(tenant_id)); + } else if (OB_FAIL(sess_pool_map_.set_refactored(tenant_id, tmp_pool))) { + if (OB_HASH_EXIST != ret) { + LOG_WARN("fail to add sess pool to hash map", K(ret), K(*tmp_pool)); + } else { // this pool has been set by other thread, free it + tmp_pool->~ObTableApiSessPool(); + ob_free(tmp_pool); + tmp_pool = nullptr; + // get sess pool + ObTableApiSessPoolMgrAtomic op; + if (OB_FAIL(sess_pool_map_.read_atomic(tenant_id, op))) { + LOG_WARN("fail to get sess pool", K(ret), K(tenant_id)); + } else { + ObTableApiSessPool *pool = op.get_session_pool(); + pool->inc_ref_count(); + guard.set_sess_pool(pool); + } + } + } else { + tmp_pool->inc_ref_count(); + guard.set_sess_pool(tmp_pool); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(tmp_pool)) { + tmp_pool->~ObTableApiSessPool(); + ob_free(tmp_pool); + tmp_pool = nullptr; + } + } + + return ret; +} + +int ObTableApiSessPoolMgr::get_session_pool(uint64_t tenant_id, ObTableApiSessPoolGuard &guard) +{ + int ret = OB_SUCCESS; + + if (is_inited_) { + ObTableApiSessPoolMgrAtomic op; + ret = sess_pool_map_.read_atomic(tenant_id, op); + if (OB_SUCC(ret)) { // exist + ObTableApiSessPool *pool = op.get_session_pool(); + pool->inc_ref_count(); + guard.set_sess_pool(pool); + } else if (OB_HASH_NOT_EXIST == ret) { + // do nothing + } else { + LOG_WARN("fait to atomic get session pool", K(ret), K(tenant_id)); + } + } + + return ret; +} + +// 1. 淘汰长期未被使用的session +// 2. 回收淘汰的session +void ObTableApiSessPoolMgr::ObTableApiSessEliminationTask::runTimerTask() +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(sess_pool_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sess_pool_mgr_ not inited", K(ret)); + } else if (OB_FAIL(run_retire_sess_task())) { + LOG_WARN("fail to run retire sess task", K(ret)); + } else if (OB_FAIL(run_recycle_retired_sess_task())) { + LOG_WARN("fail to run recycle retired sess task", K(ret)); + } +} + +int ObTableApiSessPoolMgr::ObTableApiSessEliminationTask::run_retire_sess_task() +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(sess_pool_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sess_pool_mgr_ not inited", K(ret)); + } else { + ObTableApiSessPoolForeachOp op; + if (OB_FAIL(sess_pool_mgr_->sess_pool_map_.foreach_refactored(op))) { + LOG_WARN("fail to foreach sess pool hash map", K(ret)); + } else { + const ObTableApiSessPoolForeachOp::TelantIdArray &arr = op.get_telant_id_array(); + const int64_t N = arr.count(); + for (int64_t i = 0; i < N && OB_SUCC(ret); i++) { + uint64_t tenant_id = arr.at(i); + ObTableApiSessPoolGuard pool_guard; + if (OB_FAIL(sess_pool_mgr_->get_session_pool(tenant_id, pool_guard))) { + LOG_WARN("fail to get sess pool", K(ret), K(tenant_id)); + } else if (OB_FAIL(pool_guard.get_sess_pool()->move_retired_sess())) { + LOG_WARN("fail to move retired session", K(ret)); + } + } + } + } + + return ret; +} + +int ObTableApiSessPoolMgr::ObTableApiSessEliminationTask::run_recycle_retired_sess_task() +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(sess_pool_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sess_pool_mgr_ not inited", K(ret)); + } else { + ObTableApiSessPoolForeachOp op; + if (OB_FAIL(sess_pool_mgr_->sess_pool_map_.foreach_refactored(op))) { + LOG_WARN("fail to foreach sess pool hash map", K(ret)); + } else { + const ObTableApiSessPoolForeachOp::TelantIdArray &arr = op.get_telant_id_array(); + const int64_t N = arr.count(); + for (int64_t i = 0; i < N && OB_SUCC(ret); i++) { + uint64_t tenant_id = arr.at(i); + ObTableApiSessPoolGuard pool_guard; + ObTableApiSessPool *pool = nullptr; + if (OB_FAIL(sess_pool_mgr_->get_session_pool(tenant_id, pool_guard))) { + LOG_WARN("fail to get sess pool", K(ret), K(tenant_id)); + } else if (FALSE_IT(pool = pool_guard.get_sess_pool())) { + } else if (OB_FAIL(pool->evict_retired_sess())) { + LOG_WARN("fail to evict retired sess", K(ret)); + } else if (pool->is_empty()) { + // 1. 标记delete，由最后一个持有者释放内存 + // 2. 从hash表中摘出，避免新的请求获取到这个pool + pool->set_deleted(); + ObTableApiSessPool *del_pool = nullptr; + if (OB_FAIL(sess_pool_mgr_->sess_pool_map_.erase_refactored(tenant_id, &del_pool))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("fail to erase sess pool from sess pool hash map", K(ret), K(tenant_id)); + } + } + } + } + } + } + + return ret; +} + +int ObTableApiSessPoolMgrAtomic::operator() (MapKV &entry) +{ + int ret = common::OB_SUCCESS; + + if (OB_ISNULL(entry.second)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else { + sess_pool_ = entry.second; + } + + return ret; +} + +int ObTableApiSessPool::init(int64_t hash_bucket/* = SESS_POOL_DEFAULT_BUCKET_NUM */) +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + if (OB_FAIL(key_node_map_.create(hash::cal_next_prime(hash_bucket), + "HashBucApiSessP", + "HasNodApiSess", + tenant_id_))) { + LOG_WARN("fail to init sess pool", K(ret), K(hash_bucket), K_(tenant_id)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +void ObTableApiSessPool::destroy() +{ + if (is_inited_) { + if (OB_SUCCESS != evict_all_session()) { + LOG_WARN("fail to evict all seesion"); + } + is_inited_ = false; + } +} + +// 将过期的node从hash map中摘掉，加入retired_nodes_链表中 +int ObTableApiSessPool::move_retired_sess() +{ + int ret = OB_SUCCESS; + int64_t cur_time = ObTimeUtility::current_time(); + ObTableApiSessForeachOp op; + + if (OB_FAIL(key_node_map_.foreach_refactored(op))) { + LOG_WARN("fail to foreach sess key node map", K(ret)); + } else { + const ObTableApiSessForeachOp::SessKvArray &arr = op.get_key_value_array(); + const int64_t N = arr.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + const ObTableApiSessForeachOp::ObTableApiSessKV &kv = arr.at(i); + if (cur_time - kv.node_->get_last_active_ts() >= SESS_RETIRE_TIME) { + if (OB_FAIL(move_retired_sess(kv.key_))) { + LOG_WARN("fail to move retired session", K(ret), K(kv.key_)); + } + } + } + } + + return ret; +} + +int ObTableApiSessPool::move_retired_sess(uint64_t key) +{ + int ret = OB_SUCCESS; + ObTableApiSessNode *del_node = nullptr; + + if (OB_FAIL(key_node_map_.erase_refactored(key, &del_node))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("fail to erase sess from sess hash map", K(ret), K(key)); + } + } else { + ObLockGuard guard(lock_); + if (false == (retired_nodes_.add_last(del_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add retired sess node to retired list", K(ret), K(*del_node)); + } + } + + return ret; +} + +// 清理retired_nodes_中没有引用的node +int ObTableApiSessPool::evict_retired_sess() +{ + int ret = OB_SUCCESS; + + DLIST_FOREACH(node, retired_nodes_) { + if (OB_FAIL(node->remove_unused_sess())) { + LOG_WARN("fail to remove unused sess", K(ret), K(*node)); + } else { + ObLockGuard guard(lock_); + if (node->is_empty()) { + ObTableApiSessNode *rm_node = retired_nodes_.remove(node); + if (OB_NOT_NULL(rm_node)) { + rm_node->~ObTableApiSessNode(); + allocator_.free(rm_node); + rm_node = nullptr; + } + } + } + } + + return ret; +} + +int ObTableApiSessPool::evict_all_session() +{ + int ret = OB_SUCCESS; + ObTableApiSessForeachOp op; + if (OB_FAIL(key_node_map_.foreach_refactored(op))) { + LOG_WARN("fail to foreach sess key node map", K(ret)); + } else { + const ObTableApiSessForeachOp::SessKvArray &arr = op.get_key_value_array(); + const int64_t N = arr.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + const ObTableApiSessForeachOp::ObTableApiSessKV &kv = arr.at(i); + ObTableApiSessNode *del_node = nullptr; + if (OB_FAIL(key_node_map_.erase_refactored(kv.key_, &del_node))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("fail to erase sess from sess hash map", K(ret), K(kv)); + } + } else { + ObLockGuard guard(lock_); + del_node->~ObTableApiSessNode(); + allocator_.free(del_node); + } + } + } + return ret; +} + +int ObTableApiSessPool::get_sess_node(uint64_t key, + ObTableApiSessNode *&node) +{ + ObTableApiSessNodeAtomicOp op; + int ret = key_node_map_.read_atomic(key, op); + + switch (ret) { + case OB_SUCCESS: { + //get node and lock + if (OB_FAIL(op.get_value(node))) { + LOG_WARN("fail to lock and get sess node", K(ret), K(key)); + } + break; + } + case OB_HASH_NOT_EXIST: { + break; + } + default: { + LOG_WARN("fail to get sess node from hash map", K(ret), K(key)); + break; + } + } + + return ret; +} + +int ObTableApiSessPool::get_sess_info(uint64_t key, ObTableApiSessGuard &guard) +{ + int ret = OB_SUCCESS; + ObTableApiSessNode *sess_node = nullptr; + ObTableApiSessNodeVal *sess_val = nullptr; + bool need_extend = false; + + if (OB_FAIL(get_sess_node(key, sess_node))) { + LOG_WARN("fail to get sess node", K(ret), K(key)); + } else { + if (OB_FAIL(sess_node->get_sess_node_val(sess_val))) { // exist + LOG_WARN("fail to get sess node value", K(ret), K(*sess_node)); + } else if (OB_ISNULL(sess_val)) { + need_extend = true; + } else { + guard.sess_node_val_ = sess_val; + } + } + + if (need_extend) { + if (OB_FAIL(sess_node->extend_sess_val(guard))) { + LOG_WARN("fail to extend sess val", K(ret), K(*sess_node), K(key)); + } + } + + if (OB_SUCC(ret) && OB_NOT_NULL(sess_node)) { + int64_t cur_time = ObTimeUtility::current_time(); + ATOMIC_STORE(&sess_node->last_active_ts_, cur_time); + } + + return ret; +} + +int ObTableApiSessPool::create_node(ObTableApiCredential &credential, ObTableApiSessNode *&node) +{ + int ret = OB_SUCCESS; + ObLockGuard guard(lock_); + ObTableApiSessNode *tmp_node = nullptr; + void *buf = nullptr; + + if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObTableApiSessNode)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem for ObTableApiSessNode", K(ret), K(sizeof(ObTableApiSessNode))); + } else { + tmp_node = new (buf) ObTableApiSessNode(credential); + tmp_node->last_active_ts_ = ObTimeUtility::current_time(); + node = tmp_node; + } + + return ret; +} + +int ObTableApiSessPool::create_and_add_node(ObTableApiCredential &credential) +{ + int ret = OB_SUCCESS; + + ObTableApiSessNode *node = nullptr; + if (OB_FAIL(create_node(credential, node))) { + LOG_WARN("fail to create node", K(ret), K(credential)); + } else if (OB_FAIL(key_node_map_.set_refactored(credential.user_id_, node))) { + if (OB_HASH_EXIST != ret) { + LOG_WARN("fail to add sess node to hash map", K(ret), K(credential.user_id_), K(*node)); + } else { // this node has been set by other thread, free it + ObLockGuard guard(lock_); + node->~ObTableApiSessNode(); + allocator_.free(node); + node = nullptr; + } + } + + return ret; +} + +// 1. login时调用 +// 2. 不存在，创建; 存在，旧的移动到淘汰链表, 添加新的node +int ObTableApiSessPool::update_sess(ObTableApiCredential &credential) +{ + int ret = OB_SUCCESS; + + ObTableApiSessNode *node = nullptr; + const uint64_t key = credential.user_id_; + if (OB_FAIL(get_sess_node(key, node))) { + if (OB_HASH_NOT_EXIST == ret) { // not exist, create + if (OB_FAIL(create_and_add_node(credential))) { + LOG_WARN("fail to create and add node", K(ret), K(credential)); + } + } else { + LOG_WARN("fail to get session node", K(ret), K(key)); + } + } else { // exist, 摘掉原来的，添加新的 + if (OB_FAIL(move_retired_sess(key))) { + LOG_WARN("fail to move retired session", K(ret), K(key)); + } else if (OB_FAIL(create_and_add_node(credential))) { + LOG_WARN("fail to create and add new node", K(ret), K(credential)); + } + } + + return ret; +} + +int64_t ObTableApiSessPool::inc_ref_count() +{ + return ATOMIC_AAF((uint64_t *)&ref_count_, 1); +} + +void ObTableApiSessPool::dec_ref_count() +{ + (void)ATOMIC_SAF((uint64_t *)&ref_count_, 1); + if (is_deleted() && 0 == ref_count_) { + this->~ObTableApiSessPool(); + ob_free(this); + } +} + +int ObTableApiSessNodeVal::init_sess_info() +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + share::schema::ObSchemaGetterGuard schema_guard; + const ObTenantSchema *tenant_schema = nullptr; + if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret), K_(tenant_id)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K_(tenant_id)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("tenant schema is null", K(ret)); + } else if (OB_FAIL(ObTableApiSessUtil::init_sess_info(tenant_id_, + tenant_schema->get_tenant_name_str(), + &allocator_, + schema_guard, + sess_info_))) { + LOG_WARN("fail to init sess info", K(ret), K(tenant_id_)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +void ObTableApiSessNodeVal::give_back_to_free_list() +{ + if (OB_NOT_NULL(owner_node_)) { + ObDList &free_list = owner_node_->sess_lists_.free_list_; + ObDList &used_list = owner_node_->sess_lists_.used_list_; + ObLockGuard guard(owner_node_->sess_lists_.lock_); + ObTableApiSessNodeVal *rm_sess = nullptr; + if (OB_ISNULL(rm_sess = (used_list.remove(this)))) { + LOG_WARN("fail to remove sess from used list", K(*rm_sess)); + } else if (false == (free_list.add_last(rm_sess))) { + LOG_WARN("fail to add sess val to free list", K(*rm_sess)); + } + } +} + +void ObTableApiSessNode::destroy() +{ + int ret = OB_SUCCESS; + ObDList &free_list = sess_lists_.free_list_; + ObDList &used_list = sess_lists_.used_list_; + + DLIST_FOREACH_REMOVESAFE(sess, free_list) { + ObTableApiSessNodeVal *rm_sess = free_list.remove(sess); + if (OB_NOT_NULL(rm_sess)) { + rm_sess->~ObTableApiSessNodeVal(); + allocator_.free(rm_sess); + } + } + DLIST_FOREACH_REMOVESAFE(sess, used_list) { + ObTableApiSessNodeVal *rm_sess = used_list.remove(sess); + if (OB_NOT_NULL(rm_sess)) { + rm_sess->~ObTableApiSessNodeVal(); + allocator_.free(rm_sess); + } + } +} + +int ObTableApiSessNode::remove_unused_sess() +{ + int ret = OB_SUCCESS; + + ObDList &free_list = sess_lists_.free_list_; + if (free_list.is_empty()) { + // do nothing + } else { + ObLockGuard guard(sess_lists_.lock_); + DLIST_FOREACH_REMOVESAFE(sess, free_list) { + ObTableApiSessNodeVal *rm_sess = free_list.remove(sess); + if (OB_NOT_NULL(rm_sess)) { + rm_sess->~ObTableApiSessNodeVal(); + allocator_.free(rm_sess); + rm_sess = nullptr; + } + } + } + + return ret; +} + +// 从free list中取出，添加到used list中 +int ObTableApiSessNode::get_sess_node_val(ObTableApiSessNodeVal *&val) +{ + int ret = OB_SUCCESS; + ObTableApiSessNodeVal *tmp_val = nullptr; + ObDList &free_list = sess_lists_.free_list_; + ObDList &used_list = sess_lists_.used_list_; + + ObLockGuard guard(sess_lists_.lock_); + if (!free_list.is_empty()) { + tmp_val = free_list.remove_first(); + // move to used list + if (false == (used_list.add_last(tmp_val))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add sess val to used list", K(ret), K(*tmp_val)); + } else { + val = tmp_val; + } + } + + return ret; +} + +int ObTableApiSessNode::extend_sess_val(ObTableApiSessGuard &guard) +{ + int ret = OB_SUCCESS; + + void *buf = nullptr; + if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObTableApiSessNodeVal)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem for ObTableApiSessNodeVal", K(ret), K(sizeof(ObTableApiSessNodeVal))); + } else { + ObTableApiSessNodeVal *val = new (buf) ObTableApiSessNodeVal(tenant_id_, this); + if (OB_FAIL(val->init_sess_info())) { + LOG_WARN("fail to init sess info", K(ret), K(*val)); + } else { + ObLockGuard lock_guard(sess_lists_.lock_); + if (false == (sess_lists_.used_list_.add_last(val))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to add sess val to list", K(ret), K(*val)); + } else { + guard.sess_node_val_ = val; + } + } + } + + return ret; +} + +int ObTableApiSessNodeAtomicOp::get_value(ObTableApiSessNode *&node) +{ + int ret = OB_SUCCESS; + + node = nullptr; + if (OB_ISNULL(sess_node_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sess node is not init", K(ret)); + } else { + node = sess_node_; + } + + return ret; +} + +int ObTableApiSessForeachOp::operator()(MapKV &entry) +{ + int ret = common::OB_SUCCESS; + + if (OB_FAIL(key_value_array_.push_back(ObTableApiSessKV(entry.first, entry.second)))) { + LOG_WARN("fail to push back key value", K(ret), K(entry.first)); + } + + return ret; +} + +int ObTableApiSessPoolForeachOp::operator()(MapKV &entry) +{ + int ret = common::OB_SUCCESS; + + if (OB_FAIL(telant_ids_.push_back(entry.first))) { + LOG_WARN("fail to push back key", K(ret), K(entry.first)); + } + + return ret; +} + +int ObTableApiSessUtil::init_sess_info(uint64_t tenant_id, + const common::ObString &tenant_name, + ObIAllocator *allocator, + ObSchemaGetterGuard &schema_guard, + sql::ObSQLSessionInfo &sess_info) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(allocator)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator is null", K(ret)); + } else if (OB_FAIL(sess_info.init(0, 0, allocator))) { + LOG_WARN("fail to init session into", K(ret)); + } else if (OB_FAIL(sess_info.init_tenant(tenant_name, tenant_id))) { + LOG_WARN("fail to init session tenant", K(ret), K(tenant_id)); + } else if (OB_FAIL(sess_info.load_all_sys_vars(schema_guard))) { + LOG_WARN("fail to load session system variable", K(ret)); + } + + return ret; +} +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_session_pool.h b/src/observer/table/ob_table_session_pool.h new file mode 100644 index 0000000000..9a82686ae0 --- /dev/null +++ b/src/observer/table/ob_table_session_pool.h @@ -0,0 +1,372 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_SESSION_POOL_H_ +#define OCEANBASE_OBSERVER_OB_TABLE_SESSION_POOL_H_ +#include "lib/hash/ob_hashmap.h" +#include "sql/session/ob_sql_session_info.h" +#include "share/table/ob_table.h" // for ObTableApiCredential + +namespace oceanbase +{ +namespace table +{ +class ObTableApiSessPool; +class ObTableApiSessNode; +class ObTableApiSessGuard; +class ObTableApiSessNodeVal; +class ObTableApiSessNodeAtomicOp; +class ObTableApiSessPoolGuard; + +class ObTableApiSessPoolMgr final +{ +public: + // key is tenant_id + typedef common::hash::ObHashMap SessPoolMap; +public: + ObTableApiSessPoolMgr() + : is_inited_(false) + {} + virtual ~ObTableApiSessPoolMgr() { destroy(); } + TO_STRING_KV(K_(is_inited), + K_(elimination_task)); +public: + class ObTableApiSessEliminationTask : public common::ObTimerTask + { + public: + ObTableApiSessEliminationTask() + : sess_pool_mgr_(nullptr), + run_task_counter_(0) + { + } + TO_STRING_KV(K_(run_task_counter)); + void runTimerTask(void); + private: + // 回收已经淘汰的session + int run_recycle_retired_sess_task(); + // 淘汰长期未被使用的session + int run_retire_sess_task(); + public: + ObTableApiSessPoolMgr *sess_pool_mgr_; + int64_t run_task_counter_; + }; +public: + int init(); + void destroy(); + int get_session_pool(uint64_t tenant_id, ObTableApiSessPoolGuard &guard); + int get_sess_info(uint64_t tenant_id, uint64_t user_id, ObTableApiSessGuard &guard); + int update_sess(ObTableApiCredential &credential); +private: + int extend_sess_pool(uint64_t tenant_id, ObTableApiSessPoolGuard &guard); +private: + static const int64_t ELIMINATE_SESSION_DELAY = 60 * 1000 * 1000; // 60s + bool is_inited_; + SessPoolMap sess_pool_map_; + ObTableApiSessEliminationTask elimination_task_; + common::ObTimer timer_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessPoolMgr); +}; + +class ObTableApiSessPoolMgrAtomic +{ +public: + typedef common::hash::HashMapPair MapKV; +public: + ObTableApiSessPoolMgrAtomic() + : sess_pool_(nullptr) + {} + int operator() (MapKV &entry); + ObTableApiSessPool *get_session_pool() { return sess_pool_; } +private: + ObTableApiSessPool *sess_pool_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessPoolMgrAtomic); +}; + +class ObTableApiSessPool final +{ +public: + // key is user_id + typedef common::hash::ObHashMap CacheKeyNodeMap; + static const int64_t SESS_POOL_DEFAULT_BUCKET_NUM = 10; // 取决于客户端登录的用户数量 + static const int64_t SESS_RETIRE_TIME = 300 * 1000000; // 超过300s未被访问的session会被标记淘汰 +public: + explicit ObTableApiSessPool(uint64_t tenant_id) + : allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id), + is_inited_(false), + tenant_id_(tenant_id), + ref_count_(0), + is_deleted_(false) + {} + ~ObTableApiSessPool() { destroy(); }; + TO_STRING_KV(K_(is_inited), + K_(tenant_id), + K_(ref_count), + K_(is_deleted)); + int init(int64_t hash_bucket = SESS_POOL_DEFAULT_BUCKET_NUM); + void destroy(); + int64_t inc_ref_count(); + void dec_ref_count(); + void set_deleted() { ATOMIC_SET(&is_deleted_, true); } + bool is_deleted() { return ATOMIC_LOAD(&is_deleted_); } + bool is_empty() const { return key_node_map_.empty(); } + int get_sess_info(uint64_t key, ObTableApiSessGuard &guard); + int update_sess(ObTableApiCredential &credential); + // 将过期的node移动到retired_nodes_ + int move_retired_sess(); + int evict_retired_sess(); +private: + int create_node(ObTableApiCredential &credential, ObTableApiSessNode *&node); + int create_and_add_node(ObTableApiCredential &credential); + int get_sess_node(uint64_t key, ObTableApiSessNode *&node); + int evict_all_session(); + int move_retired_sess(uint64_t key); +private: + common::ObArenaAllocator allocator_; + bool is_inited_; + uint64_t tenant_id_; + CacheKeyNodeMap key_node_map_; + volatile int64_t ref_count_; + volatile bool is_deleted_; + // 已经淘汰的node，等待被后台删除 + // 前台login时、后台淘汰时都会操作retired_nodes_，因此需要加锁 + common::ObDList retired_nodes_; + ObSpinLock lock_;; // for lock retired_nodes_/allocator_ +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessPool); +}; + +class ObTableApiSessPoolGuard final +{ +public: + ObTableApiSessPoolGuard() + : pool_(nullptr) + {} + ~ObTableApiSessPoolGuard() + { + if (OB_NOT_NULL(pool_)) { + pool_->dec_ref_count(); + } + } + ObTableApiSessPool *get_sess_pool() + { + return pool_; + } + void set_sess_pool(ObTableApiSessPool *pool) { pool_ = pool; } +private: + ObTableApiSessPool *pool_; +}; + +class ObTableApiSessNodeVal : public common::ObDLinkBase +{ +friend class ObTableApiSessNode; +friend class ObTableApiSessGuard; +public: + explicit ObTableApiSessNodeVal(uint64_t tenant_id, ObTableApiSessNode *owner) + : allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id), + is_inited_(false), + tenant_id_(tenant_id), + owner_node_(owner) + {} + TO_STRING_KV(K_(is_inited), + K_(tenant_id), + K_(sess_info)); +public: + sql::ObSQLSessionInfo& get_sess_info() { return sess_info_; } + const sql::ObSQLSessionInfo& get_sess_info() const { return sess_info_; } + int init_sess_info(); + void reset_tx_desc() { sess_info_.get_tx_desc() = nullptr; } // 防止异步提交场景在 session 析构的时候 rollback 事务 + void give_back_to_free_list(); +private: + common::ObArenaAllocator allocator_; + bool is_inited_; + uint64_t tenant_id_; + sql::ObSQLSessionInfo sess_info_; + ObTableApiSessNode *owner_node_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessNodeVal); +}; + +class ObTableApiSessNode : public common::ObDLinkBase +{ +friend class ObTableApiSessPool; +friend class ObTableApiSessNodeVal; +public: + explicit ObTableApiSessNode(ObTableApiCredential &credential) + : allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, credential.tenant_id_), + tenant_id_(credential.tenant_id_), + sess_lists_(), + last_active_ts_(0), + credential_(credential) + { + } + ~ObTableApiSessNode() { destroy(); } + TO_STRING_KV(K_(tenant_id), + K_(sess_lists), + K_(last_active_ts), + K_(credential)); + class SessList + { + public: + SessList() {} + TO_STRING_KV(K_(free_list), + K_(used_list)); + bool is_empty() const { return used_list_.is_empty() && free_list_.is_empty(); } + public: + common::ObSpinLock lock_; + common::ObDList free_list_; // 空闲session链表 + common::ObDList used_list_; // 正在被使用的session链表 + private: + DISALLOW_COPY_AND_ASSIGN(SessList); + }; +public: + void destroy(); + bool is_empty() const { return sess_lists_.is_empty(); } + int get_sess_node_val(ObTableApiSessNodeVal *&val); + OB_INLINE const ObTableApiCredential& get_credential() const { return credential_; } + OB_INLINE int64_t get_last_active_ts() const { return last_active_ts_; } + int remove_unused_sess(); +private: + int extend_sess_val(ObTableApiSessGuard &guard); +private: + common::ObArenaAllocator allocator_; + uint64_t tenant_id_; + SessList sess_lists_; + int64_t last_active_ts_; + ObTableApiCredential credential_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessNode); +}; + +class ObTableApiSessGuard final +{ +friend class ObTableApiSessPool; +friend class ObTableApiSessNode; +public: + ObTableApiSessGuard() + : sess_node_val_(nullptr) + {} + // 析构需要做的两件事： + // 1. reset事务描述符，避免session析构时，回滚事务 + // 2. 将session从used list移到free list + ~ObTableApiSessGuard() + { + if (OB_NOT_NULL(sess_node_val_)) { + sess_node_val_->reset_tx_desc(); + sess_node_val_->give_back_to_free_list(); + sess_node_val_ = nullptr; + } + } +public: + ObTableApiSessNodeVal* get_sess_node_val() const { return sess_node_val_; } + sql::ObSQLSessionInfo& get_sess_info() { return sess_node_val_->get_sess_info(); } + const sql::ObSQLSessionInfo& get_sess_info() const { return sess_node_val_->get_sess_info(); } + ObTableApiSessPoolGuard &get_sess_pool_guard() { return pool_guard_; } + int get_credential(const ObTableApiCredential *&credential) const + { + int ret = OB_SUCCESS; + if (OB_ISNULL(sess_node_val_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_ISNULL(sess_node_val_->owner_node_)) { + ret = OB_ERR_UNEXPECTED; + } else { + credential = &sess_node_val_->owner_node_->get_credential(); + } + return ret; + } +private: + ObTableApiSessNodeVal *sess_node_val_; + ObTableApiSessPoolGuard pool_guard_; +}; + +class ObTableApiSessNodeAtomicOp +{ +protected: + typedef common::hash::HashMapPair MapKV; +public: + ObTableApiSessNodeAtomicOp() + : sess_node_(nullptr) + {} + virtual ~ObTableApiSessNodeAtomicOp() {} + virtual int get_value(ObTableApiSessNode *&node); + void operator()(MapKV &entry) + { + if (nullptr != entry.second) { + sess_node_ = entry.second; + } + } +protected: + ObTableApiSessNode *sess_node_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessNodeAtomicOp); +}; + +class ObTableApiSessForeachOp +{ +public: + struct ObTableApiSessKV + { + ObTableApiSessKV() : node_(nullptr) {} + ObTableApiSessKV(uint64_t key, ObTableApiSessNode *node) + : key_(key), + node_(node) {} + TO_STRING_KV(K(key_), KP(node_)); + uint64_t key_; + ObTableApiSessNode *node_; + }; +public: + typedef common::hash::HashMapPair MapKV; + typedef common::ObSEArray SessKvArray; + ObTableApiSessForeachOp() + {} + int operator()(MapKV &entry); + const SessKvArray &get_key_value_array() const { return key_value_array_; } + void reset() { key_value_array_.reset(); } +private: + SessKvArray key_value_array_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessForeachOp); +}; + +class ObTableApiSessPoolForeachOp +{ +public: + typedef common::hash::HashMapPair MapKV; + typedef common::ObSEArray TelantIdArray; + ObTableApiSessPoolForeachOp() + {} + int operator()(MapKV &entry); + const TelantIdArray &get_telant_id_array() const { return telant_ids_; } + void reset() { telant_ids_.reset(); } +private: + TelantIdArray telant_ids_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessPoolForeachOp); +}; + +class ObTableApiSessUtil final +{ +public: + static int init_sess_info(uint64_t tenant_id, + const common::ObString &tenant_name, + common::ObIAllocator *allocator, + share::schema::ObSchemaGetterGuard &schema_guard, + sql::ObSQLSessionInfo &sess_info); +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiSessUtil); +}; + +} // end namespace table +} // end namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_SESSION_POOL_H_ */ \ No newline at end of file diff --git a/src/observer/table/ob_table_update_executor.cpp b/src/observer/table/ob_table_update_executor.cpp new file mode 100644 index 0000000000..fff876ed46 --- /dev/null +++ b/src/observer/table/ob_table_update_executor.cpp @@ -0,0 +1,171 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_update_executor.h" +#include "ob_table_cg_service.h" +#include "sql/engine/dml/ob_dml_service.h" + +using namespace oceanbase::sql; + +namespace oceanbase +{ +namespace table +{ +int ObTableApiUpdateExecutor::process_single_operation(const ObTableEntity *entity, + const ObTableCtx::ObAssignIds &assign_ids) +{ + int ret = OB_SUCCESS; + common::ObIArray &key_ranges = tb_ctx_.get_key_ranges(); + + if (OB_ISNULL(entity)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entity is null", K(ret)); + } else { + ObRowkey rowkey = entity->get_rowkey(); + ObNewRange range; + // init key_ranges_ + key_ranges.reset(); + if (OB_FAIL(range.build_range(tb_ctx_.get_ref_table_id(), rowkey))) { + LOG_WARN("fail to build key range", K(ret), K(tb_ctx_), K(rowkey)); + } else if (OB_FAIL(key_ranges.push_back(range))) { + LOG_WARN("fail to push back key range", K(ret), K(range)); + } else { + clear_evaluated_flag(); + const ObTableUpdCtDef *upd_ctdef = &upd_spec_.get_ctdef(); + if (OB_FAIL(child_->open())) { + LOG_WARN("fail to open child executor", K(ret)); + } else if (OB_FAIL(child_->get_next_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } else if (OB_FAIL(ObTableExprCgService::refresh_update_exprs_frame(tb_ctx_, + upd_ctdef->old_row_, + upd_ctdef->new_row_, + upd_ctdef->full_assign_row_, + assign_ids, + *entity))) { + LOG_WARN("fail to refresh update exprs frame", K(ret), K(*entity), K(cur_idx_)); + } + } + } + + return ret; +} + +int ObTableApiUpdateExecutor::get_next_row_from_child() +{ + int ret = OB_SUCCESS; + const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); + + if (cur_idx_ >= 1) { + ret = OB_ITER_END; + } else if (OB_FAIL(process_single_operation(entity, tb_ctx_.get_assign_ids()))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to process single update operation", K(ret)); + } + } + + return ret; +} + +int ObTableApiUpdateExecutor::update_row_to_das() +{ + int ret = OB_SUCCESS; + const ObTableUpdCtDef &upd_ctdef = upd_spec_.get_ctdef(); + ObDASTabletLoc *tablet_loc = nullptr; + if (OB_FAIL(calc_tablet_loc(tablet_loc))) { + LOG_WARN("calc partition key failed", K(ret)); + } else { + clear_evaluated_flag(); + if (OB_FAIL(ObDMLService::update_row(upd_ctdef.das_ctdef_, + upd_rtdef_.das_rtdef_, + tablet_loc, + dml_rtctx_, + upd_ctdef.full_row_))) { + LOG_WARN("fail to update row to das op", K(ret), K(upd_ctdef), K(upd_rtdef_)); + } + } + + return ret; +} + +int ObTableApiUpdateExecutor::open() +{ + int ret = OB_SUCCESS; + const ObTableUpdCtDef &ctdef = upd_spec_.get_ctdef(); + + if (OB_FAIL(ObTableApiModifyExecutor::open())) { + LOG_WARN("fail to oepn ObTableApiModifyExecutor", K(ret)); + } else if (OB_FAIL(generate_upd_rtdef(ctdef, upd_rtdef_))) { + LOG_WARN("fail to generate update rtdef", K(ret)); + } + + return ret; +} + +int ObTableApiUpdateExecutor::upd_rows_post_proc() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(submit_all_dml_task())) { + LOG_WARN("fail to execute all update das task", K(ret)); + } else { + affected_rows_ += upd_rtdef_.das_rtdef_.affected_rows_; + } + return ret; +} + +int ObTableApiUpdateExecutor::get_next_row() +{ + int ret = OB_SUCCESS; + + while (OB_SUCC(ret)) { + if (OB_FAIL(get_next_row_from_child())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } + } else if (OB_FAIL(update_row_to_das())) { + LOG_WARN("fail tp update row to das", K(ret)); + } + int tmp_ret = ret; + if (OB_FAIL(child_->close())) { // 需要写到das后才close child算子，否则扫描的行已经被析构 + LOG_WARN("fail to close scan executor", K(ret)); + } + ret = OB_SUCC(tmp_ret) ? ret : tmp_ret; + cur_idx_++; + } + + if (OB_ITER_END == ret) { + if (OB_FAIL(upd_rows_post_proc())) { + LOG_WARN("fail to do update rows post process", K(ret)); + } else { + ret = OB_ITER_END; + } + } + + return ret; +} + +int ObTableApiUpdateExecutor::close() +{ + int ret = OB_SUCCESS; + + if (!is_opened_) { + // do nothing + } else if (OB_FAIL(ObTableApiModifyExecutor::close())) { + LOG_WARN("fail to close ObTableApiModifyExecutor", K(ret)); + } + + return ret; +} +} // namespace table +} // namespace oceanbase diff --git a/src/observer/table/ob_table_update_executor.h b/src/observer/table/ob_table_update_executor.h new file mode 100644 index 0000000000..47a0784b4b --- /dev/null +++ b/src/observer/table/ob_table_update_executor.h @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OBSERVER_OB_TABLE_UPDATE_EXECUTOR_H +#define OCEANBASE_OBSERVER_OB_TABLE_UPDATE_EXECUTOR_H +#include "ob_table_modify_executor.h" +#include "ob_table_context.h" + +namespace oceanbase +{ +namespace table +{ +class ObTableApiUpdateSpec : public ObTableApiModifySpec +{ +public: + ObTableApiUpdateSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) + : ObTableApiModifySpec(alloc, type), + udp_ctdef_(alloc) + { + } +public: + OB_INLINE const ObTableUpdCtDef& get_ctdef() const { return udp_ctdef_; } + OB_INLINE ObTableUpdCtDef& get_ctdef() { return udp_ctdef_; } +private: + ObTableUpdCtDef udp_ctdef_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableApiUpdateSpec); +}; + +class ObTableApiUpdateExecutor : public ObTableApiModifyExecutor +{ +public: + ObTableApiUpdateExecutor(ObTableCtx &ctx, const ObTableApiUpdateSpec &spec) + : ObTableApiModifyExecutor(ctx), + upd_spec_(spec), + cur_idx_(0) + { + } +public: + virtual int open() override; + virtual int get_next_row() override; + virtual int close() override; +private: + int get_next_row_from_child(); + int update_row_to_das(); + int upd_rows_post_proc(); + int process_single_operation(const ObTableEntity *entity, + const ObTableCtx::ObAssignIds &assign_ids); +private: + const ObTableApiUpdateSpec &upd_spec_; + ObTableUpdRtDef upd_rtdef_; + int64_t cur_idx_; +}; + +} // namespace table +} // namespace oceanbase + +#endif /* OCEANBASE_OBSERVER_OB_TABLE_UPDATE_EXECUTOR_H */ \ No newline at end of file diff --git a/src/share/table/ob_table.cpp b/src/share/table/ob_table.cpp index f8ead30923..2102366194 100644 --- a/src/share/table/ob_table.cpp +++ b/src/share/table/ob_table.cpp @@ -221,17 +221,11 @@ ObTableEntity::ObTableEntity() ObTableEntity::~ObTableEntity() {} -inline int ObTableEntity::try_init() +void ObTableEntity::reset() { - int ret = OB_SUCCESS; - static const int64_t PROPERTY_MAP_BUCKET_SIZE = 107; - if (!properties_.created()) { - if (OB_FAIL(properties_.create(PROPERTY_MAP_BUCKET_SIZE, ObModIds::OB_HASH_BUCKET_SQL_COLUMN_MAP, - ObModIds::OB_HASH_NODE_SQL_COLUMN_MAP))) { - LOG_WARN("failed to init properties", K(ret)); - } - } - return ret; + rowkey_.reset(); + properties_names_.reset(); + properties_values_.reset(); } int ObTableEntity::set_rowkey_value(int64_t idx, const ObObj &value) @@ -316,10 +310,16 @@ int64_t ObTableEntity::hash_rowkey() const int ObTableEntity::get_property(const ObString &prop_name, ObObj &prop_value) const { int ret = OB_SUCCESS; - if (OB_FAIL(const_cast(this)->try_init())) { - LOG_WARN("failed to init hash map", K(ret)); - } else if (OB_FAIL(properties_.get_refactored(prop_name, prop_value))) { - LOG_DEBUG("failed to get property", K(ret), K(prop_name)); + if (prop_name.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("property name should not be empty string", K(ret), K(prop_name)); + } else { + int64_t idx = -1; + if (has_exist_in_array(properties_names_, prop_name, &idx)) { + prop_value = properties_values_.at(idx); + } else { + ret = OB_SEARCH_NOT_FOUND; + } } return ret; } @@ -330,115 +330,63 @@ int ObTableEntity::set_property(const ObString &prop_name, const ObObj &prop_val if (prop_name.empty()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("property name should not be empty string", K(ret), K(prop_name)); - } else if (OB_FAIL(try_init())) { - LOG_WARN("failed to init hash map", K(ret)); - } else if (OB_FAIL(properties_.set_refactored(prop_name, prop_value, 1))) { - LOG_WARN("failed to set property", K(ret), K(prop_name)); + } else { + int64_t idx = -1; + if (has_exist_in_array(properties_names_, prop_name, &idx)) { + properties_values_.at(idx) = prop_value; + } else { + if (OB_FAIL(properties_names_.push_back(prop_name))) { + LOG_WARN("failed to add prop name", K(ret), K(prop_name)); + } else if (OB_FAIL(properties_values_.push_back(prop_value))) { + LOG_WARN("failed to add prop value", K(ret), K(prop_value)); + } + } } return ret; } -class ObTableEntity::GetPropertyFn -{ -public: - GetPropertyFn(ObIArray > &properties) - :properties_(properties) - {} - int operator()(const hash::HashMapPair &kv) - { - int ret = OB_SUCCESS; - if (OB_FAIL(properties_.push_back(std::make_pair(kv.first, kv.second)))) { - LOG_WARN("failed to push back", K(ret)); - } - return ret; - } -private: - ObIArray > &properties_; -}; - int ObTableEntity::get_properties(ObIArray > &properties) const { int ret = OB_SUCCESS; - GetPropertyFn func(properties); - if (OB_FAIL(const_cast(this)->try_init())) { - LOG_WARN("failed to init hash map", K(ret)); - } else if (OB_FAIL(const_cast(properties_).foreach_refactored(func))) { - LOG_WARN("for each properties fail", K(ret)); + for (int64_t i = 0; OB_SUCC(ret) && i < properties_names_.count(); i++) { + if (OB_FAIL(properties.push_back(std::make_pair( + properties_names_.at(i), + properties_values_.at(i))))) { + LOG_WARN("failed to add name-value pair", K(ret), K(i)); + } } return ret; } -class ObTableEntity::GetPropertyNameFn -{ -public: - GetPropertyNameFn(ObIArray &names) - :names_(names) - {} - int operator()(const hash::HashMapPair &kv) - { - int ret = OB_SUCCESS; - if (OB_FAIL(names_.push_back(kv.first))) { - LOG_WARN("failed to push back", K(ret)); - } - return ret; - } -private: - ObIArray &names_; -}; - int ObTableEntity::get_properties_names(ObIArray &properties_names) const { int ret = OB_SUCCESS; - GetPropertyNameFn func(properties_names); - if (OB_FAIL(const_cast(this)->try_init())) { - LOG_WARN("failed to init hash map", K(ret)); - } else if (OB_FAIL(const_cast(properties_).foreach_refactored(func))) { - LOG_WARN("for each properties fail", K(ret)); + if (OB_FAIL(properties_names.assign(properties_names_))) { + LOG_WARN("fail to assign properties name array", K(ret)); } return ret; } -class ObTableEntity::GetPropertyValueFn -{ -public: - GetPropertyValueFn(ObIArray &values) - :values_(values) - {} - int operator()(const hash::HashMapPair &kv) - { - int ret = OB_SUCCESS; - if (OB_FAIL(values_.push_back(kv.second))) { - LOG_WARN("failed to push back", K(ret)); - } - return ret; - } -private: - ObIArray &values_; -}; - int ObTableEntity::get_properties_values(ObIArray &properties_values) const { int ret = OB_SUCCESS; - GetPropertyValueFn func(properties_values); - if (OB_FAIL(const_cast(this)->try_init())) { - LOG_WARN("failed to init hash map", K(ret)); - } else if (OB_FAIL(const_cast(properties_).foreach_refactored(func))) { - LOG_WARN("for each properties fail", K(ret)); + if (OB_FAIL(properties_values.assign(properties_values_))) { + LOG_WARN("failed to assign properties values array", K(ret)); } return ret; } int64_t ObTableEntity::get_properties_count() const { - return properties_.size(); + return properties_names_.count(); } -ObRowkey ObTableEntity::get_rowkey() +ObRowkey ObTableEntity::get_rowkey() const { ObRowkey rowkey; int64_t obj_cnt = rowkey_.count(); if (obj_cnt > 0) { - rowkey.assign(&rowkey_.at(0), obj_cnt); + rowkey.assign(const_cast(&rowkey_.at(0)), obj_cnt); } return rowkey; } @@ -619,7 +567,7 @@ int ObTableBatchOperation::add(const ObTableOperation &table_operation) { const ObString &name = prev_columns.at(i); if (OB_FAIL(curr.entity().get_property(name, value))) { - if (OB_HASH_NOT_EXIST == ret) { + if (OB_SEARCH_NOT_FOUND == ret) { is_same_properties_names_ = false; } } @@ -775,6 +723,14 @@ ObTableOperationResult::ObTableOperationResult() affected_rows_(0) {} +void ObTableOperationResult::reset() +{ + ObTableResult::reset(); + operation_type_ = ObTableOperationType::GET; + entity_->reset(); + affected_rows_ = 0; +} + int ObTableOperationResult::get_entity(const ObITableEntity *&entity) const { int ret = OB_SUCCESS; @@ -1029,6 +985,48 @@ uint64_t ObTableQuery::get_checksum() const return checksum; } +int ObTableQuery::deep_copy(ObIAllocator &allocator, ObTableQuery &dst) const +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; OB_SUCC(ret) && i < key_ranges_.count(); i++) { + const ObNewRange &src_range = key_ranges_.at(i); + ObNewRange dst_range; + if (OB_FAIL(deep_copy_range(allocator, src_range, dst_range))) { + LOG_WARN("fail tp deep copy range", K(ret)); + } else if (OB_FAIL(dst.key_ranges_.push_back(dst_range))) { + LOG_WARN("fail to push back new range", K(ret)); + } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < select_columns_.count(); i++) { + ObString select_column; + if (OB_FAIL(ob_write_string(allocator, select_columns_.at(i), select_column))) { + LOG_WARN("Fail to deep copy select column", K(ret), K(select_columns_.at(i))); + } else if (OB_FAIL(dst.select_columns_.push_back(select_column))) { + LOG_WARN("fail to push back select column", K(ret), K(select_column)); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(ob_write_string(allocator, filter_string_, dst.filter_string_))) { + LOG_WARN("fail to deep copy filter string", K(ret), K_(filter_string)); + } else if (OB_FAIL(ob_write_string(allocator, index_name_, dst.index_name_))) { + LOG_WARN("fail to deep copy index name", K(ret), K_(index_name)); + } else if (OB_FAIL(htable_filter_.deep_copy(allocator, dst.htable_filter_))) { + LOG_WARN("fail to deep copy htable filter", K(ret), K_(htable_filter)); + } else { + dst.deserialize_allocator_ = deserialize_allocator_; + dst.limit_ = limit_; + dst.offset_ = offset_; + dst.scan_order_ = scan_order_; + dst.batch_size_ = batch_size_; + dst.max_result_size_ = max_result_size_; + } + return ret; +} + OB_UNIS_DEF_SERIALIZE(ObTableQuery, key_ranges_, select_columns_, @@ -1228,6 +1226,35 @@ uint64_t ObHTableFilter::get_checksum() const return checksum; } +int ObHTableFilter::deep_copy(ObIAllocator &allocator, ObHTableFilter &dst) const +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; OB_SUCC(ret) && i < select_column_qualifier_.count(); i++) { + ObString select_column; + if (OB_FAIL(ob_write_string(allocator, select_column_qualifier_.at(i), select_column))) { + LOG_WARN("Fail to deep copy select column qualifier", K(ret), K(select_column_qualifier_.at(i))); + } else if (OB_FAIL(dst.select_column_qualifier_.push_back(select_column))) { + LOG_WARN("fail to push back select column qualifier", K(ret), K(select_column)); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(ob_write_string(allocator, filter_string_, dst.filter_string_))) { + LOG_WARN("fail to write filter string", K(ret), K_(filter_string)); + } else { + dst.is_valid_ = is_valid_; + dst.min_stamp_ = min_stamp_; + dst.max_stamp_ = max_stamp_; + dst.max_versions_ = max_versions_; + dst.limit_per_row_per_cf_ = limit_per_row_per_cf_; + dst.offset_per_row_per_cf_ = offset_per_row_per_cf_; + } + + return ret; +} + // If valid_ is true, serialize the members. Otherwise, nothing/dummy is serialized. OB_SERIALIZE_MEMBER_IF(ObHTableFilter, (true == is_valid_), @@ -1544,7 +1571,41 @@ OB_SERIALIZE_MEMBER(ObTableQueryAndMutateResult, affected_rows_, affected_entity_); -OB_SERIALIZE_MEMBER((ObTableQuerySyncResult, ObTableQueryResult), +OB_SERIALIZE_MEMBER((ObTableQuerySyncResult, ObTableQueryResult), is_end_, query_session_id_ -); +); + +OB_SERIALIZE_MEMBER(ObTableApiCredential, + cluster_id_, + tenant_id_, + user_id_, + database_id_, + expire_ts_, + hash_val_); + +ObTableApiCredential::ObTableApiCredential() + :cluster_id_(0), + tenant_id_(0), + user_id_(0), + database_id_(0), + expire_ts_(0), + hash_val_(0) +{ + +} + +ObTableApiCredential::~ObTableApiCredential() +{ + +} + +uint64_t ObTableApiCredential::hash(uint64_t seed /*= 0*/) const +{ + uint64_t hash_val = murmurhash(&cluster_id_, sizeof(cluster_id_), seed); + hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); + hash_val = murmurhash(&user_id_, sizeof(user_id_), hash_val); + hash_val = murmurhash(&database_id_, sizeof(database_id_), hash_val); + hash_val = murmurhash(&expire_ts_, sizeof(expire_ts_), hash_val); + return hash_val; +} diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index f33efc6b10..ffdde13edb 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -60,7 +60,7 @@ public: virtual int add_rowkey_value(const ObObj &value) = 0; virtual int64_t get_rowkey_size() const = 0; virtual int get_rowkey_value(int64_t idx, ObObj &value) const = 0; - virtual ObRowkey get_rowkey() = 0; + virtual ObRowkey get_rowkey() const = 0; virtual int64_t hash_rowkey() const = 0; //@} //@{ property is a key-value pair. @@ -114,18 +114,16 @@ public: virtual int get_properties_names(ObIArray &properties_names) const override; virtual int get_properties_values(ObIArray &properties_values) const override; virtual int64_t get_properties_count() const override; - virtual void reset() override { rowkey_.reset(); properties_.clear(); } - virtual ObRowkey get_rowkey() override; + virtual void reset() override; + virtual ObRowkey get_rowkey() const override; + const ObIArray &get_properties_names() const { return properties_names_; } + const ObIArray &get_properties_values() const { return properties_values_; } + const ObSEArray &get_rowkey_objs() const { return rowkey_; }; DECLARE_TO_STRING; -private: - int try_init(); - class GetPropertyFn; - class GetPropertyNameFn; - class GetPropertyValueFn; - typedef common::hash::ObHashMap PropertiesMap; private: ObSEArray rowkey_; - PropertiesMap properties_; + ObSEArray properties_names_; + ObSEArray properties_values_; }; enum class ObTableEntityType @@ -214,8 +212,8 @@ void ObTableEntityFactory::free_all() } } -enum class ObQueryOperationType : int { - QUERY_START = 0, +enum class ObQueryOperationType : int { + QUERY_START = 0, QUERY_NEXT = 1, QUERY_MAX }; @@ -232,7 +230,9 @@ struct ObTableOperationType INSERT_OR_UPDATE = 4, REPLACE = 5, INCREMENT = 6, - APPEND = 7 + APPEND = 7, + SCAN = 8, + INVALID = 15 }; }; @@ -341,6 +341,12 @@ public: void set_errno(int err) { errno_ = err; } int get_errno() const { return errno_; } int assign(const ObTableResult &other); + void reset() + { + errno_ = common::OB_ERR_UNEXPECTED; + sqlstate_[0] = '\0'; + msg_[0] = '\0'; + } TO_STRING_KV(K_(errno)); private: static const int64_t MAX_MSG_SIZE = common::OB_MAX_ERROR_MSG_LEN; @@ -357,7 +363,7 @@ class ObTableOperationResult final: public ObTableResult public: ObTableOperationResult(); ~ObTableOperationResult() = default; - + void reset(); ObTableOperationType::Type type() const { return operation_type_; } int get_entity(const ObITableEntity *&entity) const; int get_entity(ObITableEntity *&entity); @@ -378,10 +384,10 @@ private: class ObIRetryPolicy { public: - virtual bool need_retry(int32_t curr_retry_count, int last_errno, int64_t &retry_interval) + virtual bool need_retry(int32_t curr_retry_count, int last_errno, int64_t &retry_interval) { UNUSEDx(curr_retry_count, last_errno, retry_interval); - return false; + return false; } }; @@ -592,6 +598,7 @@ public: const ObString &get_filter() const { return filter_string_; } void clear_columns() { select_column_qualifier_.reset(); } uint64_t get_checksum() const; + int deep_copy(ObIAllocator &allocator, ObHTableFilter &dst) const; TO_STRING_KV(K_(is_valid), "column_qualifier", select_column_qualifier_, @@ -676,6 +683,7 @@ public: void clear_scan_range() { key_ranges_.reset(); } void set_deserialize_allocator(common::ObIAllocator *allocator) { deserialize_allocator_ = allocator; } + int deep_copy(ObIAllocator &allocator, ObTableQuery &dst) const; TO_STRING_KV(K_(key_ranges), K_(select_columns), K_(filter_string), @@ -764,6 +772,7 @@ public: void rewind(); virtual int get_next_entity(const ObITableEntity *&entity) override; int add_property_name(const ObString &name); + void reset_property_names() { properties_names_.reset(); } int add_row(const common::ObNewRow &row); int add_all_property(const ObTableQueryResult &other); int add_all_row(const ObTableQueryResult &other); @@ -807,7 +816,7 @@ class ObTableQuerySyncResult: public ObTableQueryResult public: ObTableQuerySyncResult() : is_end_(false), - query_session_id_(0) + query_session_id_(0) {} virtual ~ObTableQuerySyncResult() {} public: @@ -815,6 +824,28 @@ public: uint64_t query_session_id_; // from server gen }; +struct ObTableApiCredential final +{ + OB_UNIS_VERSION(1); +public: + ObTableApiCredential(); + ~ObTableApiCredential(); +public: + int64_t cluster_id_; + uint64_t tenant_id_; + uint64_t user_id_; + uint64_t database_id_; + int64_t expire_ts_; + uint64_t hash_val_; +public: + uint64_t hash(uint64_t seed = 0) const; + TO_STRING_KV(K_(cluster_id), + K_(tenant_id), + K_(user_id), + K_(database_id), + K_(expire_ts), + K_(hash_val)); +}; } // end namespace table } // end namespace oceanbase diff --git a/src/sql/engine/dml/ob_dml_service.cpp b/src/sql/engine/dml/ob_dml_service.cpp index 047a41a0fa..003a7545ff 100644 --- a/src/sql/engine/dml/ob_dml_service.cpp +++ b/src/sql/engine/dml/ob_dml_service.cpp @@ -943,6 +943,19 @@ int ObDMLService::update_row(const ObUpdCtDef &upd_ctdef, return ret; } +int ObDMLService::update_row(const ObDASUpdCtDef &ctdef, + ObDASUpdRtDef &rtdef, + const ObDASTabletLoc *tablet_loc, + ObDMLRtCtx &dml_rtctx, + const ExprFixedArray &full_row) +{ + return write_row_to_das_op(ctdef, + rtdef, + tablet_loc, + dml_rtctx, + full_row); +} + int ObDMLService::delete_row(const ObDASDelCtDef &das_del_ctdef, ObDASDelRtDef &das_del_rtdef, const ObDASTabletLoc *tablet_loc, diff --git a/src/sql/engine/dml/ob_dml_service.h b/src/sql/engine/dml/ob_dml_service.h index 403a067fc7..0f86eaf37b 100644 --- a/src/sql/engine/dml/ob_dml_service.h +++ b/src/sql/engine/dml/ob_dml_service.h @@ -96,6 +96,11 @@ public: ObDelRtDef &del_rtdef, const ObDASTabletLoc *tablet_loc, ObDMLRtCtx &dml_rtctx); + static int update_row(const ObDASUpdCtDef &ctdef, + ObDASUpdRtDef &rtdef, + const ObDASTabletLoc *tablet_loc, + ObDMLRtCtx &dml_rtctx, + const ExprFixedArray &full_row); static int update_row(const ObUpdCtDef &upd_ctdef, ObUpdRtDef &upd_rtdef, const ObDASTabletLoc *old_tablet_loc, diff --git a/src/sql/plan_cache/ob_lib_cache_register.cpp b/src/sql/plan_cache/ob_lib_cache_register.cpp index 0c3885c3be..6aa4fe260c 100644 --- a/src/sql/plan_cache/ob_lib_cache_register.cpp +++ b/src/sql/plan_cache/ob_lib_cache_register.cpp @@ -17,6 +17,7 @@ #include "sql/plan_cache/ob_pcv_set.h" #include "pl/ob_pl.h" #include "pl/ob_pl_package.h" +#include "observer/table/ob_table_cache.h" #define USING_LOG_PREFIX SQL_PC diff --git a/src/sql/plan_cache/ob_lib_cache_register.h b/src/sql/plan_cache/ob_lib_cache_register.h index ed2e9cab59..5a74e41a06 100644 --- a/src/sql/plan_cache/ob_lib_cache_register.h +++ b/src/sql/plan_cache/ob_lib_cache_register.h @@ -17,6 +17,8 @@ LIB_CACHE_OBJ_DEF(NS_SFC, "SFC", pl::ObPLObjectKey, pl::ObPLObjectSet, pl::ObPLF LIB_CACHE_OBJ_DEF(NS_ANON, "ANON", pl::ObPLObjectKey, pl::ObPLObjectSet, pl::ObPLFunction, ObNewModIds::OB_SQL_PHY_PL_OBJ) // anonymous cache LIB_CACHE_OBJ_DEF(NS_TRGR, "TRGR", pl::ObPLObjectKey, pl::ObPLObjectSet, pl::ObPLPackage, ObNewModIds::OB_SQL_PHY_PL_OBJ) // trigger cache LIB_CACHE_OBJ_DEF(NS_PKG, "PKG", pl::ObPLObjectKey, pl::ObPLObjectSet, pl::ObPLPackage, ObNewModIds::OB_SQL_PHY_PL_OBJ) // package cache +LIB_CACHE_OBJ_DEF(NS_TABLEAPI, "TABLEAPI", table::ObTableApiCacheKey, table::ObTableApiCacheNode, table::ObTableApiCacheObj, "OB_TABLEAPI_OBJ") // tableapi cache + #endif /*LIB_CACHE_OBJ_DEF*/ #ifndef OCEANBASE_SQL_PLAN_CACHE_OB_LIB_CACHE_REGISTER_ diff --git a/src/sql/plan_cache/ob_pc_ref_handle.cpp b/src/sql/plan_cache/ob_pc_ref_handle.cpp index fef1876d13..2d1804bb21 100644 --- a/src/sql/plan_cache/ob_pc_ref_handle.cpp +++ b/src/sql/plan_cache/ob_pc_ref_handle.cpp @@ -63,6 +63,7 @@ const char* ObCacheRefHandleMgr::handle_name(const CacheRefHandleID handle_id) "lc_node_wr_handle", "lc_ref_cache_obj_stat_handle", "plan_baseline_handle", + "tableapi_node_handle" }; static_assert(sizeof(handle_names)/sizeof(const char*) == MAX_HANDLE, "invalid handle name array"); if (handle_id < MAX_HANDLE) { diff --git a/src/sql/plan_cache/ob_pc_ref_handle.h b/src/sql/plan_cache/ob_pc_ref_handle.h index 522de9484b..d5ac8b91a4 100644 --- a/src/sql/plan_cache/ob_pc_ref_handle.h +++ b/src/sql/plan_cache/ob_pc_ref_handle.h @@ -127,6 +127,7 @@ enum CacheRefHandleID LC_NODE_WR_HANDLE, LC_REF_CACHE_OBJ_STAT_HANDLE, PLAN_BASELINE_HANDLE, + TABLEAPI_NODE_HANDLE, MAX_HANDLE }; diff --git a/unittest/observer/CMakeLists.txt b/unittest/observer/CMakeLists.txt index bfa119fdde..1074cfb965 100644 --- a/unittest/observer/CMakeLists.txt +++ b/unittest/observer/CMakeLists.txt @@ -1,6 +1,8 @@ #ob_unittest(test_manage_tenant omt/test_manage_tenant.cpp) storage_unittest(test_worker_pool omt/test_worker_pool.cpp) -storage_unittest(test_hfilter_parser) +storage_unittest(test_hfilter_parser table/test_hfilter_parser.cpp) storage_unittest(test_query_response_time mysql/test_query_response_time.cpp) +storage_unittest(test_create_executor table/test_create_executor.cpp) +storage_unittest(test_table_sess_pool table/test_table_sess_pool.cpp) add_subdirectory(rpc EXCLUDE_FROM_ALL) diff --git a/unittest/observer/hfilter_parser.result b/unittest/observer/table/hfilter_parser.result similarity index 100% rename from unittest/observer/hfilter_parser.result rename to unittest/observer/table/hfilter_parser.result diff --git a/unittest/observer/hfilter_parser.test b/unittest/observer/table/hfilter_parser.test similarity index 100% rename from unittest/observer/hfilter_parser.test rename to unittest/observer/table/hfilter_parser.test diff --git a/unittest/observer/table/test_create_executor.cpp b/unittest/observer/table/test_create_executor.cpp new file mode 100644 index 0000000000..e55e27cd02 --- /dev/null +++ b/unittest/observer/table/test_create_executor.cpp @@ -0,0 +1,546 @@ +#include +#define private public // 获取private成员 +#define protected public // 获取protect成员 +#include "observer/table/ob_table_cg_service.h" +#include "observer/table/ob_table_cache.h" +#include "../share/schema/mock_schema_service.h" +#include "sql/code_generator/ob_static_engine_cg.h" +#include "sql/plan_cache/ob_plan_cache_manager.h" +#include "lib/net/ob_addr.h" +#include "sql/plan_cache/ob_lib_cache_register.h" +#include "observer/ob_req_time_service.h" + +using namespace oceanbase::common; +using namespace oceanbase::table; +using namespace oceanbase::sql; +using namespace oceanbase::share::schema; +using namespace oceanbase::observer; + +// copy from test_table_schema.cpp +void fill_table_schema(ObTableSchema &table) +{ + table.set_tenant_id(1); + table.set_database_id(1); + table.set_tablegroup_id(1); + table.set_table_id(3001); + table.set_max_used_column_id(16); + table.set_rowkey_column_num(0); + table.set_index_column_num(0); + table.set_rowkey_split_pos(11); + table.set_progressive_merge_num(11); + table.set_compress_func_name(ObString::make_string("snappy_1.0")); + table.set_autoinc_column_id(0); + table.set_auto_increment(1); + table.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table.set_def_type(TABLE_DEF_TYPE_USER); + table.set_part_level(PARTITION_LEVEL_TWO); + table.set_charset_type(CHARSET_UTF8MB4); + table.set_collation_type(CS_TYPE_UTF8MB4_BIN); + table.set_table_type(USER_TABLE); + table.set_index_type(INDEX_TYPE_IS_NOT); + table.set_index_status(INDEX_STATUS_AVAILABLE); + table.set_data_table_id(0); + table.set_is_use_bloomfilter(false); + table.set_block_size(2097152); + table.set_tablegroup_name("table group name 1"); + table.set_comment("This is a table"); + table.set_table_name("table_xxx"); + table.set_expire_info("expire: modify_time > 3000s"); + table.set_schema_version(1); + table.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_HASH); + table.get_part_option().set_part_expr (ObString::make_string("rand() mod 111")); + table.get_part_option().set_part_num(100); + table.get_sub_part_option().set_part_func_type(PARTITION_FUNC_TYPE_HASH); + table.get_sub_part_option().set_part_expr (ObString::make_string("rand() mod 111")); + table.get_sub_part_option().set_part_num(666); +} +// 填充一个column_schema，类型：ObIntType +// rowkey_pos： >0时为rowkey列，表示rowkey的顺序 +// index_key_pos：>0时为索引列，表示索引列顺序 +// part_key_pos：>0时为分区键，表示分区间顺序 +void fill_column_schema(ObColumnSchemaV2 &column, uint64_t id, const char *name, + uint64_t rowkey_pos = 1, uint64_t index_key_pos = 1, + uint64_t part_key_pos = 1, ObOrderType rowkey_order = ObOrderType::ASC) +{ + ObObj value; + column.set_column_id(id); + column.set_column_name(ObString::make_string(name)); + column.set_rowkey_position(rowkey_pos); + column.set_order_in_rowkey(rowkey_order); + column.set_tbl_part_key_pos(part_key_pos); + column.set_index_position(index_key_pos); + column.set_data_length(100); + column.set_data_precision(1100); + column.set_data_scale(88); + column.set_nullable(false); + column.set_charset_type(CHARSET_UTF8MB4); + column.set_collation_type(CS_TYPE_UTF8MB4_BIN); + column.set_data_type(ObIntType); + value.set_int(100); + column.set_orig_default_value(value); + value.set_int(101); + column.set_cur_default_value(value); + column.set_comment("black gives me black eyes"); +} + +// create table test (C1, C2, C3) +void create_table_schema(ObTableSchema *table_schema, ObColumnSchemaV2 *columns) +{ + ASSERT_NE(nullptr, table_schema); + fill_table_schema(*table_schema); + fill_column_schema(columns[0], 1, "C1", 1, 1, 1); + fill_column_schema(columns[1], 2, "C2", 0, 0, 0); + fill_column_schema(columns[2], 3, "C3", 0, 0, 0); + ASSERT_EQ(OB_SUCCESS, table_schema->add_column(columns[0])); + ASSERT_EQ(OB_SUCCESS, table_schema->add_column(columns[1])); + ASSERT_EQ(OB_SUCCESS, table_schema->add_column(columns[2])); + ASSERT_EQ(1, table_schema->get_tenant_id()); + ASSERT_EQ(1, table_schema->get_database_id()); + ASSERT_EQ(1, table_schema->get_tablegroup_id()); + ASSERT_EQ(3001, table_schema->get_table_id()); + ASSERT_EQ(3, table_schema->get_column_count()); + ASSERT_EQ(1, table_schema->get_index_column_num()); + ASSERT_EQ(1, table_schema->get_rowkey_column_num()); + ASSERT_EQ(1, table_schema->get_partition_key_column_num()); +} + +ObTableApiSessNodeVal g_sess_node_val(1, NULL); +void fake_ctx_init_common(ObTableCtx &fake_ctx, ObTableSchema *table_schema) +{ + fake_ctx.table_schema_ = table_schema; + fake_ctx.tenant_id_ = table_schema->get_tenant_id(); + fake_ctx.database_id_ = table_schema->get_database_id(); + fake_ctx.table_name_ = table_schema->get_table_name(); + fake_ctx.ref_table_id_ = table_schema->get_table_id(); + fake_ctx.index_tablet_id_ = table_schema->get_table_id(); + fake_ctx.sess_guard_.sess_node_val_ = &g_sess_node_val; + g_sess_node_val.is_inited_ = true; + g_sess_node_val.sess_info_.test_init(0, 0, 0, NULL); +} + +class TestCreateExecutor: public ::testing::Test +{ +public: + TestCreateExecutor(); + virtual ~TestCreateExecutor() {} + virtual void SetUp(); + virtual void TearDown(); +public: + ObArenaAllocator allocator_; + MockSchemaService schema_service_; + ObSchemaGetterGuard schema_guard_; + ObTableSchema table_schema_; + ObColumnSchemaV2 columns_[3]; +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestCreateExecutor); +}; + +TestCreateExecutor::TestCreateExecutor() + : allocator_() +{ +} + +void TestCreateExecutor::SetUp() +{ + schema_service_.init(); + create_table_schema(&table_schema_, columns_); + schema_service_.add_table_schema(table_schema_, 1); +} + +void TestCreateExecutor::TearDown() +{ +} + +TEST_F(TestCreateExecutor, scan) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + // init ctx + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_get()); + for (int i = 0; i < 3; i++) { + ASSERT_EQ(columns_[i].get_column_id(), fake_ctx.select_col_ids_.at(i)); + ASSERT_EQ(columns_[i].get_meta_type(), fake_ctx.select_metas_.at(i)); + } + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(3, fake_ctx.get_all_exprs().get_expr_array().count()); + ASSERT_EQ(OB_SUCCESS, fake_ctx.classify_scan_exprs()); + ASSERT_EQ(3, fake_ctx.select_exprs_.count()); + ASSERT_EQ(1, fake_ctx.rowkey_exprs_.count()); + + ObTableApiSpec *root_spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ASSERT_EQ(OB_SUCCESS, ObTableSpecCgService::generate(allocator_, fake_ctx, root_spec)); + ASSERT_TRUE(nullptr != root_spec); + ASSERT_EQ(TABLE_API_EXEC_SCAN, root_spec->get_type()); + ASSERT_EQ(nullptr, root_spec->get_parent()); + ASSERT_EQ(nullptr, root_spec->get_child()); + + ASSERT_EQ(OB_SUCCESS, root_spec->create_executor(fake_ctx, executor)); + ASSERT_TRUE(nullptr != executor); + ObTableApiScanExecutor *scan_executor = dynamic_cast(executor); + ASSERT_TRUE(nullptr != scan_executor); + ASSERT_EQ(nullptr, executor->get_parent()); + ASSERT_EQ(nullptr, executor->get_child()); +} + +TEST_F(TestCreateExecutor, insert) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + // init ctx + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_insert()); + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(3, fake_ctx.get_all_exprs().get_expr_array().count()); + + ObTableApiSpec *root_spec = nullptr; + ObTableApiExecutor *executor = nullptr; + + // generate insert spec tree + ASSERT_EQ(OB_SUCCESS, ObTableSpecCgService::generate(allocator_, fake_ctx, root_spec)); + ASSERT_TRUE(nullptr != root_spec); + ASSERT_EQ(TABLE_API_EXEC_INSERT, root_spec->get_type()); + ASSERT_EQ(nullptr, root_spec->get_parent()); + ASSERT_EQ(nullptr, root_spec->get_child()); + + // create insert excutor tree; + ASSERT_EQ(OB_SUCCESS, root_spec->create_executor(fake_ctx, executor)); + ASSERT_TRUE(nullptr != executor); + ObTableApiInsertExecutor *ins_executor = dynamic_cast(executor); + ASSERT_TRUE(nullptr != ins_executor); + ASSERT_EQ(nullptr, executor->get_parent()); + ASSERT_EQ(nullptr, executor->get_child()); +} + +TEST_F(TestCreateExecutor, delete) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + // init ctx + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_delete()); + ASSERT_EQ(3, fake_ctx.select_col_ids_.count()); + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(3, fake_ctx.get_all_exprs().get_expr_array().count()); + ObTableApiSpec *root_spec = nullptr; + ObTableApiExecutor *executor = nullptr; + + // generate delete spec tree + ASSERT_EQ(OB_SUCCESS, ObTableSpecCgService::generate(allocator_, fake_ctx, root_spec)); + ASSERT_TRUE(nullptr != root_spec); + ASSERT_EQ(TABLE_API_EXEC_DELETE, root_spec->get_type()); + ASSERT_EQ(nullptr, root_spec->get_parent()); + ASSERT_TRUE(nullptr != root_spec->get_child()); + + // check child spec (scan) + const ObTableApiScanSpec *child_spec = dynamic_cast(root_spec->get_child()); + ASSERT_TRUE(nullptr != child_spec); + ASSERT_EQ(nullptr, child_spec->get_child()); + ASSERT_EQ(root_spec, child_spec->get_parent()); + + // create executor tree + ASSERT_EQ(OB_SUCCESS, root_spec->create_executor(fake_ctx, executor)); + ASSERT_TRUE(nullptr != executor); + ObTableApiDeleteExecutor *del_executor = dynamic_cast(executor); + ASSERT_TRUE(nullptr != del_executor); + ASSERT_EQ(nullptr, del_executor->get_parent()); + ASSERT_TRUE(nullptr != del_executor->get_child()); + + // check child executor (scan) + const ObTableApiScanExecutor *scan_executor = dynamic_cast(del_executor->get_child()); + ASSERT_TRUE(nullptr != scan_executor); + ASSERT_EQ(del_executor, scan_executor->get_parent()); + ASSERT_EQ(nullptr, scan_executor->get_child()); +} + +TEST_F(TestCreateExecutor, update) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + // prepare + ObTableEntity entity; + ObObj obj; + obj.set_int(1234); + entity.add_rowkey_value(obj); + entity.set_property(ObString::make_string("C2"), obj); + // init ctx + fake_ctx.set_entity(&entity); + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_update()); + + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(6, fake_ctx.get_all_exprs().get_expr_array().count()); + ObTableApiSpec *root_spec = nullptr; + ObTableApiExecutor *executor = nullptr; + // generate update spec tree + ASSERT_EQ(OB_SUCCESS, ObTableSpecCgService::generate(allocator_, fake_ctx, root_spec)); + ASSERT_TRUE(nullptr != root_spec); + ASSERT_EQ(TABLE_API_EXEC_UPDATE, root_spec->get_type()); + ASSERT_EQ(nullptr, root_spec->get_parent()); + ASSERT_TRUE(nullptr != root_spec->get_child()); + + // check child spec (scan) + const ObTableApiScanSpec *child_spec = dynamic_cast(root_spec->get_child()); + ASSERT_TRUE(nullptr != child_spec); + ASSERT_EQ(nullptr, child_spec->get_child()); + ASSERT_EQ(root_spec, child_spec->get_parent()); + + // create executor tree + ASSERT_EQ(OB_SUCCESS, root_spec->create_executor(fake_ctx, executor)); + ASSERT_TRUE(nullptr != executor); + ObTableApiUpdateExecutor *upd_executor = dynamic_cast(executor); + ASSERT_TRUE(nullptr != upd_executor); + ASSERT_EQ(nullptr, upd_executor->get_parent()); + ASSERT_TRUE(nullptr != upd_executor->get_child()); + + // check child executor (scan) + const ObTableApiScanExecutor *scan_executor = dynamic_cast(upd_executor->get_child()); + ASSERT_TRUE(nullptr != scan_executor); + ASSERT_EQ(upd_executor, scan_executor->get_parent()); + ASSERT_EQ(nullptr, scan_executor->get_child()); +} + +TEST_F(TestCreateExecutor, insertup) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + ObTableEntity entity; + ObObj obj; + obj.set_int(1234); + entity.add_rowkey_value(obj); + entity.set_property(ObString::make_string("C2"), obj); + // init ctx + fake_ctx.set_entity(&entity); + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_insert_up()); + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(6, fake_ctx.get_all_exprs().get_expr_array().count()); + ObTableApiSpec *root_spec = nullptr; + ObTableApiExecutor *executor = nullptr; + + // generate insertup spec tree + ASSERT_EQ(OB_SUCCESS, ObTableSpecCgService::generate(allocator_, fake_ctx, root_spec)); + ASSERT_TRUE(nullptr != root_spec); + ASSERT_EQ(TABLE_API_EXEC_INSERT_UP, root_spec->get_type()); + ASSERT_EQ(nullptr, root_spec->get_parent()); + ASSERT_EQ(nullptr, root_spec->get_child()); + + // create insertup excutor tree; + ASSERT_EQ(OB_SUCCESS, root_spec->create_executor(fake_ctx, executor)); + ASSERT_TRUE(nullptr != executor); + ObTableApiInsertUpExecutor *insup_executor = dynamic_cast(executor); + ASSERT_TRUE(nullptr != insup_executor); + ASSERT_EQ(nullptr, executor->get_parent()); + ASSERT_EQ(nullptr, executor->get_child()); +} + +TEST_F(TestCreateExecutor, replace) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + // init ctx + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_replace()); + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(3, fake_ctx.get_all_exprs().get_expr_array().count()); + ObTableApiSpec *root_spec = nullptr; + ObTableApiExecutor *executor = nullptr; + + // generate replace spec tree + ASSERT_EQ(OB_SUCCESS, ObTableSpecCgService::generate(allocator_, fake_ctx, root_spec)); + ASSERT_TRUE(nullptr != root_spec); + ASSERT_EQ(TABLE_API_EXEC_REPLACE, root_spec->get_type()); + ASSERT_EQ(nullptr, root_spec->get_parent()); + ASSERT_EQ(nullptr, root_spec->get_child()); + + // create replace excutor tree; + ASSERT_EQ(OB_SUCCESS, root_spec->create_executor(fake_ctx, executor)); + ASSERT_TRUE(nullptr != executor); + ObTableApiReplaceExecutor *replace_executor = dynamic_cast(executor); + ASSERT_TRUE(nullptr != replace_executor); + ASSERT_EQ(nullptr, executor->get_parent()); + ASSERT_EQ(nullptr, executor->get_child()); +} + +// refresh frame: init_datum_param_store + check_entity + refresh_rowkey_exprs_frame + refresh_properties_exprs_frame +TEST_F(TestCreateExecutor, refresh_exprs_frame) +{ + ObTableCtx fake_ctx(allocator_); + ObExprFrameInfo fake_expr_info(allocator_); + ObTableEntity entity; + ObStaticEngineCG cg; + // prepare data + ObObj obj; + obj.set_int(1234); + entity.add_rowkey_value(obj); + obj.set_int(1235); + entity.set_property(ObString::make_string("C2"), obj); + // init ctx + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.init_insert()); + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::alloc_exprs_memory(fake_ctx, fake_expr_info)); + fake_ctx.set_expr_info(&fake_expr_info); + ASSERT_EQ(3, fake_ctx.get_all_exprs().get_expr_array().count()); + ObArray rt_exprs; + ASSERT_EQ(OB_SUCCESS, cg.generate_rt_exprs(fake_ctx.get_all_exprs().get_expr_array(), rt_exprs)); + + ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::refresh_exprs_frame(fake_ctx, rt_exprs, entity)); + // verify the refresh value + ObEvalCtx eval_ctx(fake_ctx.get_exec_ctx()); + ObDatum *datum = nullptr; + ObExpr *rt_expr = nullptr; + ObObj objs[3]; + const ObIArray &all_exprs = fake_ctx.get_all_exprs().get_expr_array(); + for (int i = 0; i < all_exprs.count(); i++) { + ASSERT_EQ(OB_SUCCESS, cg.generate_rt_expr(*all_exprs.at(i), rt_expr)); + ASSERT_EQ(OB_SUCCESS, rt_expr->eval(eval_ctx, datum)); + ASSERT_EQ(OB_SUCCESS, datum->to_obj(objs[i], columns_[i].get_meta_type())); + } + ASSERT_EQ(1234, objs[0].get_int()); + ASSERT_EQ(1235, objs[1].get_int()); + // column default value + ASSERT_EQ(101, objs[2].get_int()); +} + +// table context +TEST_F(TestCreateExecutor, cons_column_type) +{ + ObColumnSchemaV2 col_schema; + // prepare data + col_schema.set_data_type(ObVarcharType); + col_schema.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ObAccuracy acc(1); + col_schema.set_accuracy(acc); + uint32_t res_flag = ObRawExprUtils::calc_column_result_flag(col_schema); + + ObExprResType column_type; + ObTableCtx fake_ctx(allocator_); + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + ASSERT_EQ(OB_SUCCESS, fake_ctx.cons_column_type(col_schema, column_type)); + ASSERT_EQ(ObVarcharType, column_type.get_type()); + ASSERT_EQ(res_flag, column_type.get_result_flag()); + ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, column_type.get_collation_type()); + ASSERT_EQ(CS_LEVEL_IMPLICIT, column_type.get_collation_level()); + ASSERT_EQ(1, column_type.get_accuracy().get_length()); +} + +TEST_F(TestCreateExecutor, check_column_type) +{ + ObExprResType column_type; + ObObj obj; + uint32_t res_flag = 0; + ObTableCtx fake_ctx(allocator_); + schema_service_.get_schema_guard(fake_ctx.schema_guard_, 1); + fake_ctx_init_common(fake_ctx, &table_schema_); + + // check nullable + obj.set_null(); + res_flag |= NOT_NULL_FLAG; + column_type.set_result_flag(res_flag); + ASSERT_EQ(OB_BAD_NULL_ERROR, fake_ctx.check_column_type(column_type, obj)); + // check data type mismatch + res_flag = 0; + obj.set_int(1); + column_type.set_result_flag(res_flag); + column_type.set_type(ObVarcharType); + ASSERT_EQ(OB_OBJ_TYPE_ERROR, fake_ctx.check_column_type(column_type, obj)); + // check collation + obj.set_binary("ttt"); + column_type.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ASSERT_EQ(OB_ERR_COLLATION_MISMATCH, fake_ctx.check_column_type(column_type, obj)); + // collation convert + obj.set_varchar("test"); + obj.set_collation_type(CS_TYPE_UTF8MB4_BIN); + ASSERT_EQ(OB_SUCCESS, fake_ctx.check_column_type(column_type, obj)); + ASSERT_EQ(CS_TYPE_UTF8MB4_GENERAL_CI, obj.get_collation_type()); +} + +TEST_F(TestCreateExecutor, generate_key_range) +{ + ObTableCtx fake_ctx(allocator_); + fake_ctx_init_common(fake_ctx, &table_schema_); + // prepare data + ObArray scan_ranges; + ObObj pk_objs_start[1]; + pk_objs_start[0].set_int(0); + ObObj pk_objs_end[1]; + pk_objs_end[0].set_max_value(); + ObNewRange range; + range.start_key_.assign(pk_objs_start, 1); + range.end_key_.assign(pk_objs_end, 1); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + scan_ranges.push_back(range); + ASSERT_EQ(OB_SUCCESS, fake_ctx.generate_key_range(scan_ranges)); + // primary key range + ASSERT_EQ(1, fake_ctx.get_key_ranges().count()); +} + +TEST_F(TestCreateExecutor, test_cache) +{ + ObAddr addr; + ObPCMemPctConf conf; + ObPlanCacheManager lib_cache_mgr; + ObLibCacheNameSpace ns = ObLibCacheNameSpace::NS_TABLEAPI; + uint64_t tenant_id = 1; + ObReqTimeGuard req_timeinfo_guard; + // register cache obj + ObLibCacheRegister::register_cache_objs(); + //init lib cache manager + ASSERT_EQ(OB_SUCCESS, lib_cache_mgr.init(addr)); + // get lib cache + ObPlanCache *lib_cache = lib_cache_mgr.get_or_create_plan_cache(tenant_id, conf); + ASSERT_TRUE(nullptr != lib_cache); + // construct cache key + ObTableApiCacheKey cache_key(1001, 1001, 1, ObTableOperationType::Type::INSERT); + // construct ctx + ObILibCacheCtx ctx; + // construct cache obj + { + ObCacheObjGuard guard(CacheRefHandleID::TABLEAPI_NODE_HANDLE); + ASSERT_EQ(OB_SUCCESS, lib_cache->alloc_cache_obj(guard, ns, tenant_id)); + ObILibCacheObject *cache_obj = guard.get_cache_obj(); + ASSERT_TRUE(nullptr != cache_obj); + ObTableApiCacheObj *table_cache_obj = static_cast(cache_obj); + ObIAllocator &cache_allocator = table_cache_obj->get_allocator(); + // construct spec + ObTableApiInsertSpec spec(cache_allocator, TABLE_API_EXEC_INSERT); + table_cache_obj->set_spec(&spec); + // add an cache obj + ASSERT_EQ(OB_SUCCESS, lib_cache->add_cache_obj(ctx, &cache_key , table_cache_obj)); + } + // get cache obj + { + ObCacheObjGuard guard(CacheRefHandleID::TABLEAPI_NODE_HANDLE); + ASSERT_EQ(OB_SUCCESS, lib_cache->get_cache_obj(ctx, &cache_key, guard)); + ObTableApiCacheObj *cache_obj = static_cast(guard.get_cache_obj()); + ASSERT_TRUE(nullptr != cache_obj); + ObTableApiSpec *spec = cache_obj->get_spec(); + ASSERT_TRUE(nullptr != spec); + ASSERT_EQ(TABLE_API_EXEC_INSERT, spec->type_); + } +} + +int main(int argc, char **argv) +{ + OB_LOGGER.set_log_level("INFO"); + OB_LOGGER.set_file_name("test_create_executor.log", true); + ::testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/test_hfilter_parser.cpp b/unittest/observer/table/test_hfilter_parser.cpp similarity index 96% rename from unittest/observer/test_hfilter_parser.cpp rename to unittest/observer/table/test_hfilter_parser.cpp index 3438a160ce..0f324ce201 100644 --- a/unittest/observer/test_hfilter_parser.cpp +++ b/unittest/observer/table/test_hfilter_parser.cpp @@ -101,9 +101,9 @@ void TestHFilterParser::do_parse(const char *filter_cstr, std::ofstream &of_resu TEST_F(TestHFilterParser, basic_test) { - const char* test_file = "./hfilter_parser.test"; - const char* result_file = "./hfilter_parser.result"; - const char* tmp_file = "./hfilter_parser.tmp"; + const char* test_file = "./table/hfilter_parser.test"; + const char* result_file = "./table/hfilter_parser.result"; + const char* tmp_file = "./table/hfilter_parser.tmp"; // run tests std::ifstream if_tests(test_file); ASSERT_TRUE(if_tests.is_open()); diff --git a/unittest/observer/table/test_table_sess_pool.cpp b/unittest/observer/table/test_table_sess_pool.cpp new file mode 100644 index 0000000000..f2e3246db1 --- /dev/null +++ b/unittest/observer/table/test_table_sess_pool.cpp @@ -0,0 +1,222 @@ +#include +#define private public // 获取私有成员 +#include "observer/table/ob_table_session_pool.h" + +using namespace oceanbase::common; +using namespace oceanbase::table; +using namespace oceanbase::sql; + +class TestTableSessPool: public ::testing::Test +{ +public: + const int64_t TENANT_CNT = 10; + const int64_t USER_CNT = 100; + const int64_t NODE_CNT = 100; + const int64_t SESS_CNT = 10; +public: + TestTableSessPool() {} + virtual ~TestTableSessPool() {} + void prepare_sess_pool(ObTableApiSessPoolMgr &mgr); +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(TestTableSessPool); +}; + +// 10租户，100用户，每个用户下面挂10个session +void TestTableSessPool::prepare_sess_pool(ObTableApiSessPoolMgr &mgr) +{ + ASSERT_EQ(OB_SUCCESS, mgr.init()); + uint64_t tenant_ids[TENANT_CNT]; + uint64_t user_ids[USER_CNT]; + ObTableApiSessPoolGuard pool_guards[TENANT_CNT]; + for (uint64_t i = 0; i < TENANT_CNT; i++) { + tenant_ids[i] = i + 1001; + ASSERT_EQ(OB_SUCCESS, mgr.extend_sess_pool(tenant_ids[i], pool_guards[i])); + ObTableApiSessPool *tmp_pool = pool_guards[i].get_sess_pool(); + ASSERT_NE(nullptr, tmp_pool); + for (uint64_t j = 0; j < USER_CNT; j++) { + user_ids[j] = j; + ASSERT_EQ(OB_SUCCESS, mgr.get_session_pool(tenant_ids[i], pool_guards[i])); + ObTableApiSessPool *pool = pool_guards[i].get_sess_pool(); + ASSERT_NE(nullptr, pool); + ObTableApiCredential credential; + credential.tenant_id_ = tenant_ids[i]; + credential.user_id_ = user_ids[j]; + ASSERT_EQ(OB_SUCCESS, pool->update_sess(credential)); + ObTableApiSessNode *node = nullptr; + ASSERT_EQ(OB_SUCCESS, tmp_pool->get_sess_node(user_ids[j], node)); + ASSERT_NE(nullptr, node); + for (int64_t k = 0; k < SESS_CNT; k++) { + void *buf = node->allocator_.alloc(sizeof(ObTableApiSessNodeVal)); + ASSERT_NE(nullptr, buf); + ObTableApiSessNodeVal *val = new (buf) ObTableApiSessNodeVal(tenant_ids[i], node); + val->is_inited_ = true; + ASSERT_EQ(true, node->sess_lists_.free_list_.add_last(val)); + } + } + } +} + +TEST_F(TestTableSessPool, get_session) +{ + uint64_t tenant_id = 1; + uint64_t user_id = 0; + uint64_t key = user_id; + ObTableApiSessPoolMgr mgr; + ASSERT_EQ(OB_SUCCESS, mgr.init()); + ObTableApiSessPoolGuard pool_guard; + ASSERT_EQ(OB_HASH_NOT_EXIST, mgr.get_session_pool(tenant_id, pool_guard)); + ASSERT_EQ(nullptr, pool_guard.get_sess_pool()); + ASSERT_EQ(OB_SUCCESS, mgr.extend_sess_pool(tenant_id, pool_guard)); + ObTableApiSessPool *pool = pool_guard.get_sess_pool(); + ASSERT_TRUE(nullptr != pool); + ASSERT_TRUE(pool->is_inited_); + ASSERT_EQ(1, pool->ref_count_); + ASSERT_EQ(false, pool->is_deleted_); + ASSERT_EQ(tenant_id, pool->tenant_id_); + ObTableApiSessGuard sess_guard; + ASSERT_EQ(OB_HASH_NOT_EXIST, pool->get_sess_info(key, sess_guard)); + ASSERT_EQ(OB_HASH_NOT_EXIST, pool->get_sess_info(key, sess_guard)); + ObTableApiCredential credential; + credential.tenant_id_ = tenant_id; + credential.user_id_ = user_id; + ASSERT_EQ(OB_SUCCESS, pool->update_sess(credential)); + ObTableApiSessNode *node = nullptr; + ASSERT_EQ(OB_SUCCESS, pool->get_sess_node(key, node)); + ASSERT_TRUE(nullptr != node); + ASSERT_TRUE(node->is_empty()); + void *buf = node->allocator_.alloc(sizeof(ObTableApiSessNodeVal)); + ASSERT_NE(nullptr, buf); + ObTableApiSessNodeVal *new_val = new (buf) ObTableApiSessNodeVal(tenant_id, node); + new_val->is_inited_ = true; + ASSERT_EQ(true, node->sess_lists_.free_list_.add_last(new_val)); +} + +TEST_F(TestTableSessPool, remove_session) +{ + int64_t tenant_id = 1001; + uint64_t user_id = 0; + ObTableApiCredential credential; + credential.tenant_id_ = tenant_id; + credential.user_id_ = user_id; + ObTableApiSessNode node(credential); + for (int64_t i = 0; i < SESS_CNT; i++) { + void *buf = node.allocator_.alloc(sizeof(ObTableApiSessNodeVal)); + ASSERT_NE(nullptr, buf); + ObTableApiSessNodeVal *val = new (buf) ObTableApiSessNodeVal(tenant_id, &node); + val->is_inited_ = true; + ASSERT_EQ(true, node.sess_lists_.free_list_.add_last(val)); + } + ASSERT_EQ(false, node.is_empty()); + ASSERT_EQ(SESS_CNT, node.sess_lists_.free_list_.get_size()); + node.remove_unused_sess(); + ASSERT_EQ(0, node.sess_lists_.free_list_.get_size()); +} + +TEST_F(TestTableSessPool, retire_session) +{ + int ret = 0; + ObTableApiSessPoolMgr mgr; + prepare_sess_pool(mgr); + ObTableApiSessPoolForeachOp op; + ASSERT_EQ(OB_SUCCESS, mgr.sess_pool_map_.foreach_refactored(op)); + const ObTableApiSessPoolForeachOp::TelantIdArray &tenant_ids = op.get_telant_id_array(); + ASSERT_EQ(TENANT_CNT, tenant_ids.count()); + const int64_t N = tenant_ids.count(); + // 1. 标记淘汰 + for (int64_t i = 0; i < N; i++) { + uint64_t tenant_id = tenant_ids.at(i); + ObTableApiSessPoolGuard pool_guard; + ASSERT_EQ(OB_SUCCESS, mgr.get_session_pool(tenant_id, pool_guard)); + ObTableApiSessPool *pool = pool_guard.get_sess_pool(); + ASSERT_NE(nullptr, pool); + ObTableApiSessForeachOp op; + ASSERT_EQ(OB_SUCCESS, pool->key_node_map_.foreach_refactored(op)); + const ObTableApiSessForeachOp::SessKvArray &kvs = op.get_key_value_array(); + ASSERT_EQ(NODE_CNT, kvs.count()); + for (int64_t j = 0; j < kvs.count(); j++) { + if (j % 2 == 0) { + const ObTableApiSessForeachOp::ObTableApiSessKV &kv = kvs.at(j); + ASSERT_EQ(OB_SUCCESS, pool->move_retired_sess(kv.key_)); + } + } + } + // 2. 触发淘汰 + ASSERT_EQ(OB_SUCCESS, mgr.elimination_task_.run_recycle_retired_sess_task()); + // 3. 检查 + for (int64_t i = 0; i < N; i++) { + uint64_t tenant_id = tenant_ids.at(i); + ObTableApiSessPoolGuard pool_guard; + ASSERT_EQ(OB_SUCCESS, mgr.get_session_pool(tenant_id, pool_guard)); + ObTableApiSessPool *pool = pool_guard.get_sess_pool(); + ASSERT_NE(nullptr, pool); + ObTableApiSessForeachOp op; + ASSERT_EQ(OB_SUCCESS, pool->key_node_map_.foreach_refactored(op)); + const ObTableApiSessForeachOp::SessKvArray &kvs = op.get_key_value_array(); + ASSERT_EQ(NODE_CNT/2, kvs.count()); + for (int64_t j = 0; j < kvs.count(); j++) { + const ObTableApiSessForeachOp::ObTableApiSessKV &kv = kvs.at(j); + ASSERT_EQ(SESS_CNT, kv.node_->sess_lists_.free_list_.get_size()); + } + } +} + +TEST_F(TestTableSessPool, reference_session) +{ + // prepare + ObTableApiSessPoolMgr mgr; + ASSERT_EQ(OB_SUCCESS, mgr.init()); + uint64_t tenant_id = 1001; + uint64_t user_id = 0; + ObTableApiSessPoolGuard pool_guard; + ASSERT_EQ(OB_SUCCESS, mgr.extend_sess_pool(tenant_id, pool_guard)); + ObTableApiSessPool *pool = pool_guard.get_sess_pool(); + ASSERT_NE(nullptr, pool); + ObTableApiCredential credential; + credential.tenant_id_ = tenant_id; + credential.user_id_ = user_id; + ASSERT_EQ(OB_SUCCESS, pool->update_sess(credential)); + ObTableApiSessNode *node = nullptr; + ASSERT_EQ(OB_SUCCESS, pool->get_sess_node(user_id, node)); + ASSERT_NE(nullptr, node); + void *buf = node->allocator_.alloc(sizeof(ObTableApiSessNodeVal)); + ASSERT_NE(nullptr, buf); + ObTableApiSessNodeVal *val = new (buf) ObTableApiSessNodeVal(tenant_id, node); + val->is_inited_ = true; + ASSERT_EQ(true, node->sess_lists_.free_list_.add_last(val)); + // get and retire + { + // get + ObTableApiSessGuard guard; + ASSERT_EQ(OB_SUCCESS, pool->get_sess_info(user_id, guard)); + ASSERT_NE(nullptr, guard.get_sess_node_val()); + // mark retire + ObTableApiSessForeachOp op; + ASSERT_EQ(OB_SUCCESS, pool->key_node_map_.foreach_refactored(op)); + const ObTableApiSessForeachOp::SessKvArray &kvs = op.get_key_value_array(); + ASSERT_EQ(1, kvs.count()); + const ObTableApiSessForeachOp::ObTableApiSessKV &kv = kvs.at(0); + // run retire task + ASSERT_EQ(OB_SUCCESS, mgr.elimination_task_.run_recycle_retired_sess_task()); + // check + op.reset(); + ASSERT_EQ(OB_SUCCESS, pool->key_node_map_.foreach_refactored(op)); + const ObTableApiSessForeachOp::SessKvArray &new_kvs = op.get_key_value_array(); + ASSERT_EQ(1, new_kvs.count()); + } + // retire after def ref + ASSERT_EQ(OB_SUCCESS, mgr.elimination_task_.run_recycle_retired_sess_task()); + // check + ObTableApiSessPoolForeachOp op; + ASSERT_EQ(OB_SUCCESS, mgr.sess_pool_map_.foreach_refactored(op)); + const ObTableApiSessPoolForeachOp::TelantIdArray &arr = op.get_telant_id_array(); + ASSERT_EQ(1, arr.count()); +} + +int main(int argc, char **argv) +{ + OB_LOGGER.set_log_level("INFO"); + OB_LOGGER.set_file_name("TestTableSessPool.log", true); + ::testing::InitGoogleTest(&argc,argv); + return RUN_ALL_TESTS(); +}