diff --git a/CMakeLists.txt b/CMakeLists.txt index e211d69093..a910d4442b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,11 @@ if (ENABLE_SANITY) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SANITY") endif() +if (OB_BUILD_RPM) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DOB_BUILD_RPM") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOB_BUILD_RPM") +endif() + message(STATUS "This is BINARY dir " ${PROJECT_BINARY_DIR}) message(STATUS "This is SOURCE dir " ${PROJECT_SOURCE_DIR}) diff --git a/README.md b/README.md index ad828ba258..d03aef1f75 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ English | [中文版](README_CN.md) - **Continuous Availability** - OceanBase Database adopts Paxos Consensus algorithm to achieve Zero RPO and less than 8 seconds of RTO. + OceanBase Database adopts Paxos Consensus algorithm to achieve Zero RPO and less than 8 seconds of RTO. Supports intra-city/remote disaster recovery, enabling multi-activity in multiple locations and zero data loss. - **MySQL Compatible** diff --git a/deps/oblib/src/common/object/ob_object.h b/deps/oblib/src/common/object/ob_object.h index c3a0c2348e..dbc40eff9f 100644 --- a/deps/oblib/src/common/object/ob_object.h +++ b/deps/oblib/src/common/object/ob_object.h @@ -1568,12 +1568,15 @@ public: OB_INLINE int get_json_print_data(ObString &json_data, char *buf, int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; json_data = get_string(); - if (!has_lob_header()) { + ObLobCommon* lob_comm = reinterpret_cast(json_data.ptr()); + bool bret = has_lob_header() || + (json_data.length() >= sizeof(ObLobCommon) && lob_comm->is_valid()); + if (!bret) { // if it called in log params, like K(json), it maybe a json with disk lob header only // cannot judge here is a plain json data or json data with disk lob header COMMON_LOG(DEBUG, "Lob: get json data without mem lob header", K(*this)); } else { - ObLobLocatorV2 loc(reinterpret_cast(v_.ptr_), val_len_, has_lob_header()); + ObLobLocatorV2 loc(reinterpret_cast(v_.ptr_), val_len_, bret); if (OB_UNLIKELY(!loc.is_valid(false))) { // do nothing, warn log inside COMMON_LOG(WARN, "Lob: invalid json lob", K(ret), K(json_data)); @@ -2035,21 +2038,25 @@ inline ObObj::ObObj() inline ObObj::ObObj(bool val) { set_bool(val); + val_len_ = 0; } inline ObObj::ObObj(int32_t val) { set_int32(val); + val_len_ = 0; } inline ObObj::ObObj(int64_t val) { set_int(val); + val_len_ = 0; } inline ObObj::ObObj(ObObjType type) { meta_.set_type(type); + val_len_ = 0; } inline ObObj::ObObj(const ObObj &other) diff --git a/deps/oblib/src/lib/alloc/object_mgr.cpp b/deps/oblib/src/lib/alloc/object_mgr.cpp index 33ef9175b5..e0fe0f0f59 100644 --- a/deps/oblib/src/lib/alloc/object_mgr.cpp +++ b/deps/oblib/src/lib/alloc/object_mgr.cpp @@ -63,7 +63,7 @@ void SubObjectMgr::free_block(ABlock *block) ObjectMgr::ObjectMgr(ObTenantCtxAllocator &allocator, uint64_t tenant_id, uint64_t ctx_id, uint32_t ablock_size, int parallel, IBlockMgr *blk_mgr) : IBlockMgr(tenant_id, ctx_id), ta_(allocator), - ablock_size_(ablock_size), parallel_(4), blk_mgr_(blk_mgr), sub_cnt_(1), + ablock_size_(ablock_size), parallel_(parallel), blk_mgr_(blk_mgr), sub_cnt_(1), root_mgr_(common::ObCtxIds::LOGGER_CTX_ID == ctx_id, tenant_id, ctx_id, ablock_size_, blk_mgr_), last_wash_ts_(0), last_washed_size_(0) { diff --git a/deps/oblib/src/lib/allocator/ob_mod_define.h b/deps/oblib/src/lib/allocator/ob_mod_define.h index 4e9fbf29e5..63b59ae265 100644 --- a/deps/oblib/src/lib/allocator/ob_mod_define.h +++ b/deps/oblib/src/lib/allocator/ob_mod_define.h @@ -41,6 +41,7 @@ CTX_ITEM_DEF(MAX_CTX_ID) #ifdef SUB_CTX_ITEM_DEF SUB_CTX_ITEM_DEF(TEST1) SUB_CTX_ITEM_DEF(TEST2) +SUB_CTX_ITEM_DEF(THREAD_LOCAL_DECODE_CTX_ID) SUB_CTX_ITEM_DEF(MAX_SUB_CTX_ID) #endif diff --git a/deps/oblib/src/lib/container/ob_array_array.h b/deps/oblib/src/lib/container/ob_array_array.h index 1ad6447c3f..fd7e374337 100644 --- a/deps/oblib/src/lib/container/ob_array_array.h +++ b/deps/oblib/src/lib/container/ob_array_array.h @@ -21,6 +21,7 @@ #include "lib/container/ob_se_array.h" #include "lib/utility/ob_template_utils.h" #include "lib/utility/ob_hang_fatal_error.h" +#include "lib/utility/serialization.h" namespace oceanbase { @@ -47,6 +48,9 @@ public: int reserve(const int64_t size); int at(const int64_t array_idx, ObIArray &array); int at(const int64_t array_idx, const int64_t idx, T &obj); + int assign(const ObArrayArray &other); + NEED_SERIALIZE_AND_DESERIALIZE; + OB_INLINE T &at(const int64_t array_idx, const int64_t idx) { if (OB_UNLIKELY(0 > array_idx || array_idx >= count_)) { @@ -71,6 +75,18 @@ public: } return *array_ptrs_[array_idx]; } + OB_INLINE const ObIArray &at(const int64_t array_idx) const + { + if (OB_UNLIKELY(0 > array_idx || array_idx >= count_)) { + LIB_LOG_RET(ERROR, common::OB_INVALID_ARGUMENT, "Unexpected array idx", K_(count), K_(capacity), K(array_idx)); + right_to_die_or_duty_to_live(); + } else if (OB_ISNULL(array_ptrs_) || OB_ISNULL(array_ptrs_[array_idx])) { + LIB_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "Unexpected null array array ptr", K_(count), K_(capacity), K(array_idx), + KP_(array_ptrs)); + right_to_die_or_duty_to_live(); + } + return *array_ptrs_[array_idx]; + } int push_back(const ObIArray &obj_array); int push_back(const int64_t array_idx, const T &obj); OB_INLINE int64_t count() const { return count_; } @@ -210,7 +226,7 @@ int ObArrayArray::reserv } if (OB_SUCC(ret)) { void *ptr = nullptr; - for (int64_t i = count_; OB_SUCC(ret) && i < capacity; i++) { + for (int64_t i = capacity_; OB_SUCC(ret) && i < capacity; i++) { if (OB_NOT_NULL(new_array_ptrs[i])) { ret = OB_ERR_UNEXPECTED; LIB_LOG(ERROR, "Unexpecte not null array array ptr", K(i), K_(count), K_(capacity), K(capacity), @@ -334,8 +350,80 @@ int64_t ObArrayArray::to return pos; } +template +int ObArrayArray::assign(const ObArrayArray &other) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &other)) { + this->reuse(); + int64_t N = other.count(); + ret = reserve(N); + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + if (OB_FAIL(array_ptrs_[i]->assign(other.at(i)))) { + LIB_LOG(WARN, "fail to assign array", K(ret)); + } + } + if (OB_SUCC(ret)) { + count_ = N; + } + } + return ret; +} +template +int ObArrayArray::serialize( + char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_SUCCESS != (ret = serialization::encode_vi64(buf, buf_len, pos, count()))) { + LIB_LOG(WARN, "fail to encode ob array count", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < count(); i++) { + const LocalArrayT &item = *array_ptrs_[i]; + if (OB_SUCCESS != (ret = serialization::encode(buf, buf_len, pos, item))) { + LIB_LOG(WARN, "fail to encode item", K(i), K(ret)); + } + } + return ret; +} + +template +int ObArrayArray::deserialize( + const char *buf, int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + int64_t count = 0; + reset(); + if (OB_SUCCESS != (ret = serialization::decode_vi64(buf, data_len, pos, &count))) { + LIB_LOG(WARN, "fail to decode ob array count", K(ret)); + } else if (OB_SUCCESS != (ret = reserve(count))) { + LIB_LOG(WARN, "fail to allocate space", K(ret), K(count)); + } else { + count_ = count; + } + for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { + LocalArrayT &item = *array_ptrs_[i]; + if (OB_SUCCESS != (ret = serialization::decode(buf, data_len, pos, item))) { + LIB_LOG(WARN, "fail to decode array item", K(ret), K(i), K(count)); + } + } + return ret; +} + +template +int64_t ObArrayArray::get_serialize_size() +const +{ + int64_t size = 0; + size += serialization::encoded_length_vi64(count()); + for (int64_t i = 0; i < count(); i++) { + const LocalArrayT &item = *array_ptrs_[i]; + size += serialization::encoded_length(item); + } + return size; +} + } // common } // oceanbase diff --git a/deps/oblib/src/lib/container/ob_vector.ipp b/deps/oblib/src/lib/container/ob_vector.ipp index f5c03d7de1..27160ab8ca 100644 --- a/deps/oblib/src/lib/container/ob_vector.ipp +++ b/deps/oblib/src/lib/container/ob_vector.ipp @@ -145,7 +145,7 @@ int ObVector::expand(int64_t size) copy(new_mem, mem_begin_, mem_end_); } - // deallocate old memeory + // deallocate old memory destroy(); mem_begin_ = new_mem; diff --git a/deps/oblib/src/lib/geo/ob_geo_func_box.cpp b/deps/oblib/src/lib/geo/ob_geo_func_box.cpp index 5cd6d6e815..325e756fba 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_box.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_box.cpp @@ -200,7 +200,7 @@ OB_GEO_UNARY_FUNC_BEGIN(ObGeoFuncBoxImpl, ObWkbGeogCollection, ObGeogBox *) if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, true, true, sub_g))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g->set_data(wkb_nosrid); sub_g->set_srid(g->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_func_correct.cpp b/deps/oblib/src/lib/geo/ob_geo_func_correct.cpp index 2607af4b61..c1947974a6 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_correct.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_correct.cpp @@ -92,7 +92,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, IsGeog, true, sub_g))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g->set_data(wkb_nosrid); sub_g->set_srid(g->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_func_covered_by.cpp b/deps/oblib/src/lib/geo/ob_geo_func_covered_by.cpp index 02630ba5d3..0140fd1349 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_covered_by.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_covered_by.cpp @@ -143,7 +143,7 @@ OB_GEO_CART_BINARY_FUNC_BEGIN(ObGeoFuncCoveredByImpl, ObWkbGeomPoint, ObWkbGeomC if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, false, true, sub_g2))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g2->set_data(wkb_nosrid); sub_g2->set_srid(g2->get_srid()); @@ -636,7 +636,7 @@ OB_GEO_GEOG_BINARY_FUNC_BEGIN(ObGeoFuncCoveredByImpl, ObWkbGeogPoint, ObWkbGeogC if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, true, true, sub_g2))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g2->set_data(wkb_nosrid); sub_g2->set_srid(g2->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_func_distance.cpp b/deps/oblib/src/lib/geo/ob_geo_func_distance.cpp index fbd5f57637..94d4b42288 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_distance.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_distance.cpp @@ -142,7 +142,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, is_geog, true, sub_g1))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g1->set_data(wkb_nosrid); sub_g1->set_srid(g1->get_srid()); @@ -163,7 +163,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, is_geog, true, sub_g2))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g2->set_data(wkb_nosrid); sub_g2->set_srid(g2->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_func_envelope.cpp b/deps/oblib/src/lib/geo/ob_geo_func_envelope.cpp index f9f36b83c9..60e6aacc5b 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_envelope.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_envelope.cpp @@ -74,7 +74,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, false, true, sub_g))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g->set_data(wkb_nosrid); sub_g->set_srid(g->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_func_intersects.cpp b/deps/oblib/src/lib/geo/ob_geo_func_intersects.cpp index 6a1c1f0341..a1e2fe0cd7 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_intersects.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_intersects.cpp @@ -154,7 +154,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, is_geog, true, sub_g1))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g1->set_data(wkb_nosrid); sub_g1->set_srid(g1->get_srid()); @@ -173,7 +173,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, is_geog, true, sub_g2))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g2->set_data(wkb_nosrid); sub_g2->set_srid(g2->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_func_isvalid.cpp b/deps/oblib/src/lib/geo/ob_geo_func_isvalid.cpp index f927a1ccb2..bee226d190 100644 --- a/deps/oblib/src/lib/geo/ob_geo_func_isvalid.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_func_isvalid.cpp @@ -101,7 +101,7 @@ private: if (OB_FAIL(ObGeoTypeUtil::create_geo_by_type(*allocator, sub_type, is_geog, true, sub_g))) { LOG_WARN("failed to create wkb", K(ret), K(sub_type)); } else { - // Length is not used, cannot get real length untill iter move to the next + // Length is not used, cannot get real length until iter move to the next ObString wkb_nosrid(WKB_COMMON_WKB_HEADER_LEN, reinterpret_cast(sub_ptr)); sub_g->set_data(wkb_nosrid); sub_g->set_srid(g->get_srid()); diff --git a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp index 8382e68a83..1015b13694 100644 --- a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp +++ b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp @@ -17,6 +17,15 @@ namespace oceanbase { namespace common { +bool ObWkbToS2Visitor::prepare(ObGeometry *geo) +{ + bool bret = true; + if (OB_ISNULL(geo)) { + bret = false; + } + return bret; +} + void ObWkbToS2Visitor::add_cell_from_point(S2Point point) { S2CellId cell_id = S2CellId(point).parent(options_.max_level()); @@ -33,8 +42,8 @@ template S2Cell* ObWkbToS2Visitor::MakeS2Point(T_IBIN *geo) { S2LatLng latlng = S2LatLng::FromDegrees(geo->y(), geo->x()); - mbr_.AddPoint(latlng); add_cell_from_point(latlng); + mbr_ = mbr_.is_empty() ? S2LatLngRect(latlng, latlng) : mbr_.Union(S2LatLngRect(latlng, latlng)); S2Cell* p = new S2Cell(latlng); return p; } @@ -105,7 +114,6 @@ S2Polyline* ObWkbToS2Visitor::MakeS2Polyline(T_IBIN *geo) for ( ; iter != line->end(); iter++) { S2LatLng latlng = S2LatLng::FromDegrees(iter->template get<1>(), iter->template get<0>()); - mbr_.AddPoint(latlng); add_cell_from_point(latlng); vertices.push_back(latlng); } @@ -142,7 +150,6 @@ S2Polygon* ObWkbToS2Visitor::MakeS2Polygon(T_IBIN *geo) typename T_BIN_RING::iterator iter = exterior.begin(); for (; iter != exterior.end(); ++iter) { S2LatLng latlng = S2LatLng::FromDegrees(iter->template get<1>(), iter->template get<0>()); - mbr_.AddPoint(latlng); add_cell_from_point(latlng); vertices.push_back(S2Point(latlng)); } @@ -157,7 +164,6 @@ S2Polygon* ObWkbToS2Visitor::MakeS2Polygon(T_IBIN *geo) typename T_BIN_RING::iterator iter = (*iterInnerRing).begin(); for (; iter != (*iterInnerRing).end(); ++iter) { S2LatLng latlng = S2LatLng::FromDegrees(iter->template get<1>(), iter->template get<0>()); - mbr_.AddPoint(latlng); add_cell_from_point(latlng); vertices.push_back(S2Point(latlng)); } @@ -213,7 +219,12 @@ S2Polygon* ObWkbToS2Visitor::MakeProjS2Polygon(T_IBIN *geo) int ObWkbToS2Visitor::visit(ObIWkbGeogPoint *geo) { INIT_SUCC(ret); - s2v_.emplace_back(MakeS2Point(geo)); + if (geo->length() < (WKB_GEO_BO_SIZE + WKB_GEO_TYPE_SIZE)) { + ret = OB_ERR_GIS_INVALID_DATA; + LOG_WARN("invalid swkb length", K(ret), K(geo->length())); + } else { + s2v_.emplace_back(MakeS2Point(geo)); + } return ret; } @@ -229,7 +240,14 @@ int ObWkbToS2Visitor::visit(ObIWkbGeomPoint *geo) int ObWkbToS2Visitor::visit(ObIWkbGeogLineString *geo) { INIT_SUCC(ret); - s2v_.emplace_back(MakeS2Polyline(geo)); + if (geo->length() < WKB_COMMON_WKB_HEADER_LEN) { + ret = OB_ERR_GIS_INVALID_DATA; + LOG_WARN("invalid swkb length", K(ret), K(geo->length())); + } else { + S2Polyline *polyline = MakeS2Polyline(geo); + s2v_.emplace_back(polyline); + mbr_ = mbr_.is_empty() ? polyline->GetRectBound() : mbr_.Union(polyline->GetRectBound()); + } return ret; } @@ -249,8 +267,10 @@ int ObWkbToS2Visitor::visit(ObIWkbGeogPolygon *geo) ret = OB_ERR_GIS_INVALID_DATA; LOG_WARN("invalid swkb length", K(ret), K(geo->length())); } else { - s2v_.emplace_back(MakeS2Polygon(geo)); + S2Polygon *polygon = MakeS2Polygon(geo); + s2v_.emplace_back(polygon); + mbr_ = mbr_.is_empty() ? polygon->GetRectBound() : mbr_.Union(polygon->GetRectBound()); } return ret; @@ -350,7 +370,8 @@ int64_t ObWkbToS2Visitor::get_mbr(S2LatLngRect &mbr, bool need_buffer, S1Angle d if (need_buffer) { mbr_ = mbr_.ExpandedByDistance(distance); } - mbr = mbr_; + // avoid rounding errors in mbr_ calculation + mbr = mbr_.ExpandedByDistance(S1Angle::Degrees(DBL_EPSILON)); } return ret; } diff --git a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h index 416cf75609..0fd0841a8d 100644 --- a/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h +++ b/deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h @@ -23,6 +23,7 @@ #include "s2/s2polygon.h" #include "s2/s2region_coverer.h" #include "s2/s2latlng_rect.h" +#include "s2/s2latlng_rect_bounder.h" #include @@ -49,7 +50,7 @@ public: { mbr_ = S2LatLngRect::Empty(); } - ~ObWkbToS2Visitor() {} + ~ObWkbToS2Visitor() { reset(); } template S2Cell* MakeS2Point(T_IBIN *geo); template @@ -66,7 +67,7 @@ public: typename T_BIN_RING, typename T_BIN_INNER_RING> S2Polygon* MakeProjS2Polygon(T_IBIN *geo); - bool prepare(ObGeometry *geo) { UNUSED(geo); return true; } + bool prepare(ObGeometry *geo); // wkb int visit(ObIWkbGeometry *geo) { INIT_SUCC(ret); return ret; } int visit(ObIWkbGeogPoint *geo); diff --git a/deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.cpp b/deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.cpp index d0dcd20322..66258f97f3 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.cpp @@ -12,6 +12,7 @@ #define USING_LOG_PREFIX LIB_MYSQLC #include "lib/mysqlclient/ob_dblink_error_trans.h" +#include "share/ob_errno.h" int __attribute__((weak)) get_oracle_errno(int index) { @@ -56,39 +57,19 @@ int sqlclient::ObDblinkErrorTrans::external_errno_to_ob_errno(bool is_oracle_err std::min(STRLEN(oracle_msg_prefix), STRLEN(external_errmsg)))))) { ob_errno = external_errno; // do not map, show user client errno directly. } else { - ob_errno = OB_ERR_DBLINK_REMOTE_ECODE; // default ob_errno, if external_errno can not map to any valid ob_errno - if (OB_ISNULL(external_errmsg) || 0 == STRLEN(external_errmsg)) { - for (int i = 0; i < oceanbase::common::OB_MAX_ERROR_CODE; ++i) { - if (external_errno == (is_oracle_err ? get_oracle_errno(i) : get_mysql_errno(i))) { - ob_errno = -i; - break; - } - } - } else { - ObEditDistance ed; - int64_t edit_dist = 0x7fffffffffffffff; - int64_t min_edit_dist = 0x7fffffffffffffff; - for (int i = 0; i < oceanbase::common::OB_MAX_ERROR_CODE; ++i) { - if (external_errno == (is_oracle_err ? get_oracle_errno(i) : get_mysql_errno(i))) { - const char *external_errstr = (is_oracle_err ? get_oracle_str_error(i) : get_mysql_str_error(i)); - if (OB_ISNULL(external_errstr)) { - // In the case of a null pointer boundary - edit_dist = 0x7ffffffffffffffe; - } else { - // The edit distance between the strings is used to measure their similarity. - // The smaller the edit distance, the greater the similarity, so as to find the most similar error message. - ObEditDistance::cal_edit_distance(external_errmsg, external_errstr, STRLEN(external_errmsg), STRLEN(external_errstr), edit_dist); - } - if (edit_dist < min_edit_dist) { - ob_errno = -i; - min_edit_dist = edit_dist; - if (0 == min_edit_dist) { - break; - } - } - } + int64_t match_count = 0; + for (int i = 0; i < oceanbase::common::OB_MAX_ERROR_CODE; ++i) { + if (external_errno == (is_oracle_err ? get_oracle_errno(i) : get_mysql_errno(i))) { + ob_errno = -i; + ++match_count; } } + if (1 != match_count) { + // default ob_errno, if external_errno can not map to any valid ob_errno + ob_errno = OB_ERR_DBLINK_REMOTE_ECODE; + int msg_len = STRLEN(external_errmsg); + LOG_USER_ERROR(OB_ERR_DBLINK_REMOTE_ECODE, external_errno, msg_len, external_errmsg); + } } } return ret; diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_client.h b/deps/oblib/src/lib/mysqlclient/ob_isql_client.h index e90e81791e..27012dd0a1 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_client.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_client.h @@ -53,10 +53,12 @@ public: virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) = 0; virtual int read(ReadResult &res, const int64_t cluster_id, const uint64_t tenant_id, const char *sql) = 0; virtual int read(ReadResult &res, const char *sql) { return this->read(res, OB_SYS_TENANT_ID, sql); } + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) = 0; // execute update sql virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) = 0; virtual int write(const char *sql, int64_t &affected_rows) { return this->write(OB_SYS_TENANT_ID, sql, affected_rows); } + virtual int write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) = 0; // executor execute int execute(const uint64_t tenant_id, sqlclient::ObIExecutor &executor) diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h index 431aa68d14..2d35b02711 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h @@ -204,6 +204,8 @@ public: virtual void set_use_external_session(bool v) { UNUSED(v); } virtual int64_t get_cluster_id() const { return common::OB_INVALID_ID; } void set_session_init_status(bool status) { is_inited_ = status;} + virtual void set_user_timeout(int64_t user_timeout) { UNUSED(user_timeout); } + virtual int64_t get_user_timeout() const { return 0; } int is_session_inited(const sqlclient::dblink_param_ctx ¶m_ctx, bool &is_inited) { int ret = OB_SUCCESS; diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_connection_pool.h b/deps/oblib/src/lib/mysqlclient/ob_isql_connection_pool.h index 520bf54e05..0fd2fb7201 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_connection_pool.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_connection_pool.h @@ -116,9 +116,9 @@ public: // acquired connection must be released virtual int acquire(ObISQLConnection *&conn, ObISQLClient *client_addr) { - return this->acquire(OB_INVALID_TENANT_ID, conn, client_addr); + return this->acquire(OB_INVALID_TENANT_ID, conn, client_addr, 0); } - virtual int acquire(const uint64_t tenant_id, ObISQLConnection *&conn, ObISQLClient *client_addr) = 0; + virtual int acquire(const uint64_t tenant_id, ObISQLConnection *&conn, ObISQLClient *client_addr, const int32_t group_id) = 0; virtual int release(ObISQLConnection *conn, const bool success) = 0; virtual int on_client_inactive(ObISQLClient *client_addr) = 0; virtual ObSQLConnPoolType get_type() = 0; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp index 7ea2e9397d..b18165ea85 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp @@ -560,8 +560,8 @@ int ObMySQLConnection::execute_read(const uint64_t tenant_id, const char *sql, LOG_WARN("create statement failed", KCSTRING(sql), K(ret)); } else if (OB_ISNULL(read_ctx->result_ = read_ctx->stmt_.execute_query(res.is_enable_use_result()))) { ret = get_last_error(); - const int ER_LOCK_WAIT_TIMEOUT = -1205; - if (ER_LOCK_WAIT_TIMEOUT == ret) { + //const int ER_LOCK_WAIT_TIMEOUT = -1205; + if (-1205 == ret) { LOG_INFO("query failed", K(get_server()), KCSTRING(sql), K(ret)); } else { LOG_WARN("query failed", K(get_server()), KCSTRING(sql), K(ret)); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp index 520be905db..969308ad83 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp @@ -698,10 +698,11 @@ void ObMySQLConnectionPool::runTimerTask() } } -int ObMySQLConnectionPool::acquire(const uint64_t tenant_id, ObISQLConnection *&conn, ObISQLClient *client_addr) +int ObMySQLConnectionPool::acquire(const uint64_t tenant_id, ObISQLConnection *&conn, ObISQLClient *client_addr, const int32_t group_id) { int ret = OB_SUCCESS; UNUSED(client_addr); + UNUSED(group_id); ObMySQLConnection *mysql_conn = NULL; conn = NULL; if (OB_FAIL(acquire(tenant_id, mysql_conn))) { diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h index 5289ab161d..ecec768c41 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h @@ -140,7 +140,7 @@ public: virtual int escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size); - virtual int acquire(const uint64_t tenant_id, ObISQLConnection *&conn, ObISQLClient *client_addr) override; + virtual int acquire(const uint64_t tenant_id, ObISQLConnection *&conn, ObISQLClient *client_addr, const int32_t group_id) override; virtual int release(ObISQLConnection *conn, const bool success) override; virtual int on_client_inactive(ObISQLClient *client_addr) override diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp index a2a7ad413c..04f1592b8e 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp @@ -56,11 +56,11 @@ void ObCommonSqlProxy::operator=(const ObCommonSqlProxy &o) pool_ = o.pool_; } -int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const char *sql) +int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const char *sql, const int32_t group_id) { int ret = OB_SUCCESS; ObISQLConnection *conn = NULL; - if (OB_FAIL(acquire(tenant_id, conn))) { + if (OB_FAIL(acquire(tenant_id, conn, group_id))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (OB_FAIL(read(conn, result, tenant_id, sql))) { LOG_WARN("read failed", K(ret)); @@ -76,7 +76,7 @@ int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const c if (OB_ISNULL(exec_sql_addr)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("read with typically exec addr failed", K(ret), K(exec_sql_addr)); - } else if (OB_FAIL(acquire(tenant_id, conn))) { + } else if (OB_FAIL(acquire(tenant_id, conn, 0/*group_id*/))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (OB_FAIL(read(conn, result, tenant_id, sql, exec_sql_addr))) { LOG_WARN("read failed", K(ret)); @@ -85,11 +85,11 @@ int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const c return ret; } -int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const char *sql, const ObSessionParam *session_param) +int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const char *sql, const ObSessionParam *session_param, int64_t user_set_timeout) { int ret = OB_SUCCESS; ObISQLConnection *conn = NULL; - if (OB_FAIL(acquire(tenant_id, conn))) { + if (OB_FAIL(acquire(tenant_id, conn, 0/*group_id*/))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (nullptr != session_param) { conn->set_ddl_info(&session_param->ddl_info_); @@ -104,6 +104,7 @@ int ObCommonSqlProxy::read(ReadResult &result, const uint64_t tenant_id, const c conn->set_force_remote_exec(true); } } + conn->set_user_timeout(user_set_timeout); if (OB_FAIL(ret)) { } else if (OB_FAIL(read(conn, result, tenant_id, sql))) { LOG_WARN("read failed", K(ret)); @@ -133,7 +134,7 @@ int ObCommonSqlProxy::read(ObISQLConnection *conn, ReadResult &result, return ret; } -int ObCommonSqlProxy::write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) +int ObCommonSqlProxy::write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) { int ret = OB_SUCCESS; int64_t start = ::oceanbase::common::ObTimeUtility::current_time(); @@ -141,7 +142,7 @@ int ObCommonSqlProxy::write(const uint64_t tenant_id, const char *sql, int64_t & if (OB_ISNULL(sql)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("empty sql"); - } else if (OB_FAIL(acquire(tenant_id, conn))) { + } else if (OB_FAIL(acquire(tenant_id, conn, group_id))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (OB_ISNULL(conn)) { ret = OB_INNER_STAT_ERROR; @@ -171,7 +172,7 @@ int ObCommonSqlProxy::write(const uint64_t tenant_id, const ObString sql, if (OB_UNLIKELY(sql.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("empty sql"); - } else if (OB_FAIL(acquire(tenant_id, conn))) { + } else if (OB_FAIL(acquire(tenant_id, conn, 0/*group_id*/))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (OB_ISNULL(conn)) { ret = OB_INNER_STAT_ERROR; @@ -254,7 +255,7 @@ int ObCommonSqlProxy::execute(const uint64_t tenant_id, ObIExecutor &executor) int ret = OB_SUCCESS; int64_t start = ::oceanbase::common::ObTimeUtility::current_time(); ObISQLConnection *conn = NULL; - if (OB_FAIL(acquire(tenant_id, conn))) { + if (OB_FAIL(acquire(tenant_id, conn, 0/*group_id*/))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (OB_ISNULL(conn)) { ret = OB_INNER_STAT_ERROR; @@ -336,13 +337,13 @@ int ObCommonSqlProxy::add_slashes(const char *from, const int64_t from_size, return ret; } -int ObCommonSqlProxy::acquire(const uint64_t tenant_id, sqlclient::ObISQLConnection *&conn) +int ObCommonSqlProxy::acquire(const uint64_t tenant_id, sqlclient::ObISQLConnection *&conn, const int32_t group_id) { int ret = OB_SUCCESS; if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("mysql proxy not inited", K(ret)); - } else if (OB_FAIL(pool_->acquire(tenant_id, conn, this))) { + } else if (OB_FAIL(pool_->acquire(tenant_id, conn, this, group_id))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; @@ -367,7 +368,7 @@ int ObCommonSqlProxy::read( } else if (NULL == sql) { ret = OB_INVALID_ARGUMENT; LOG_WARN("empty sql"); - } else if (OB_FAIL(acquire(tenant_id, conn))) { + } else if (OB_FAIL(acquire(tenant_id, conn, 0/*group_id*/))) { LOG_WARN("acquire connection failed", K(ret), K(conn)); } else if (NULL == conn) { ret = OB_INNER_STAT_ERROR; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index 7e572ffbab..99189ba7ac 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -47,13 +47,16 @@ public: bool is_dest_table_hidden() const { return is_dest_table_hidden_; } void set_heap_table_ddl(const bool flag) { is_heap_table_ddl_ = flag; } bool is_heap_table_ddl() const { return is_heap_table_ddl_; } + void set_ddl_check_default_value(const bool flag) { is_ddl_check_default_value_bit_ = flag; } + bool is_ddl_check_default_value() const { return is_ddl_check_default_value_bit_; } TO_STRING_KV(K_(ddl_info)); OB_UNIS_VERSION(1); public: static const int64_t IS_DDL_BIT = 1; static const int64_t IS_TABLE_HIDDEN_BIT = 1; static const int64_t IS_HEAP_TABLE_DDL_BIT = 1; - static const int64_t RESERVED_BIT = sizeof(int64_t) - IS_DDL_BIT - 2 * IS_TABLE_HIDDEN_BIT - IS_HEAP_TABLE_DDL_BIT; + static const int64_t IS_DDL_CHECK_DEFAULT_VALUE_BIT = 1; + static const int64_t RESERVED_BIT = sizeof(int64_t) - IS_DDL_BIT - 2 * IS_TABLE_HIDDEN_BIT - IS_HEAP_TABLE_DDL_BIT - IS_DDL_CHECK_DEFAULT_VALUE_BIT; union { uint64_t ddl_info_; struct { @@ -61,6 +64,7 @@ public: uint64_t is_source_table_hidden_: IS_TABLE_HIDDEN_BIT; uint64_t is_dest_table_hidden_: IS_TABLE_HIDDEN_BIT; uint64_t is_heap_table_ddl_: IS_HEAP_TABLE_DDL_BIT; + uint64_t is_ddl_check_default_value_bit_ : IS_DDL_CHECK_DEFAULT_VALUE_BIT; uint64_t reserved_bit : RESERVED_BIT; }; }; @@ -103,9 +107,10 @@ public: virtual int escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size) override; // execute query and return data result - virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override; - int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const ObSessionParam *session_param); + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override { return this->read(res, tenant_id, sql, 0/*group_id*/); } + int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const ObSessionParam *session_param, int64_t user_set_timeout = 0); int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const common::ObAddr *sql_exec_addr); + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) override; //only for across cluster //cluster_id can not GCONF.cluster_id virtual int read(ReadResult &res, @@ -114,7 +119,8 @@ public: const char *sql) override; using ObISQLClient::read; // execute update sql - virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override; + virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override { return this->write(tenant_id, sql, 0/**/, affected_rows); } + virtual int write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) override; int write(const uint64_t tenant_id, const ObString sql, int64_t &affected_rows, int64_t compatibility_mode, const ObSessionParam *session_param = nullptr, const common::ObAddr *sql_exec_addr = nullptr); @@ -133,8 +139,8 @@ public: int execute(const uint64_t tenant_id, sqlclient::ObIExecutor &executor); protected: - int acquire(sqlclient::ObISQLConnection *&conn) { return this->acquire(OB_INVALID_TENANT_ID, conn); } - int acquire(const uint64_t tenant_id, sqlclient::ObISQLConnection *&conn); + int acquire(sqlclient::ObISQLConnection *&conn) { return this->acquire(OB_INVALID_TENANT_ID, conn, 0); } + int acquire(const uint64_t tenant_id, sqlclient::ObISQLConnection *&conn, const int32_t group_id); int read(sqlclient::ObISQLConnection *conn, ReadResult &result, const uint64_t tenant_id, const char *sql, const common::ObAddr *sql_exec_addr = nullptr); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp index 4555911852..41ece19084 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp @@ -66,11 +66,12 @@ int ObMySQLTransaction::start_transaction( int ObMySQLTransaction::start( ObISQLClient *sql_client, const uint64_t tenant_id, - bool with_snapshot/* = false*/) + bool with_snapshot/* = false*/, + const int32_t group_id /* = 0*/) { int ret = OB_SUCCESS; start_time_ = ::oceanbase::common::ObTimeUtility::current_time(); - if (OB_FAIL(connect(tenant_id, sql_client))) { + if (OB_FAIL(connect(tenant_id, group_id, sql_client))) { LOG_WARN("failed to init", K(ret), K(tenant_id)); } else if (enable_query_stash_ && OB_FAIL(query_stash_desc_.create(1024, "BucketQueryS", "NodeQueryS"))) { LOG_WARN("failed to init map", K(ret), K(tenant_id)); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.h index 9a802a8d6d..5a64435b5b 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.h @@ -63,7 +63,8 @@ public: // start transaction virtual int start(ObISQLClient *proxy, const uint64_t tenant_id, - bool with_snapshot = false); + bool with_snapshot = false, + const int32_t group_id = 0); virtual int start(ObISQLClient *proxy, const uint64_t &tenant_id, const int64_t &refreshed_schema_version, diff --git a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp index 4c7c63d9d6..0542ee31d4 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp @@ -33,12 +33,12 @@ ObSingleConnectionProxy::~ObSingleConnectionProxy() (void)close(); } -int ObSingleConnectionProxy::connect(const uint64_t tenant_id, ObISQLClient *sql_client) +int ObSingleConnectionProxy::connect(const uint64_t tenant_id, const int32_t group_id, ObISQLClient *sql_client) { int ret = OB_SUCCESS; - if (NULL == sql_client || NULL == sql_client->get_pool()) { + if (NULL == sql_client || NULL == sql_client->get_pool() || group_id < 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(sql_client)); + LOG_WARN("invalid argument", K(sql_client), K(group_id)); } else if (NULL != pool_ || NULL != conn_) { ret = OB_INNER_STAT_ERROR; LOG_WARN("transaction can only be started once", K(tenant_id), K(pool_), K(conn_)); @@ -46,7 +46,7 @@ int ObSingleConnectionProxy::connect(const uint64_t tenant_id, ObISQLClient *sql oracle_mode_ = sql_client->is_oracle_mode(); pool_ = sql_client->get_pool(); - if (OB_FAIL(pool_->acquire(tenant_id, conn_, sql_client))) { + if (OB_FAIL(pool_->acquire(tenant_id, conn_, sql_client, group_id))) { LOG_WARN("acquire connection failed", K(ret), K(tenant_id), K(pool_)); } else if (NULL == conn_) { ret = OB_INNER_STAT_ERROR; @@ -72,10 +72,11 @@ int ObSingleConnectionProxy::connect(const uint64_t tenant_id, ObISQLClient *sql } int ObSingleConnectionProxy::read(ReadResult &res, - const uint64_t tenant_id, const char *sql) + const uint64_t tenant_id, const char *sql, const int32_t group_id) { int ret = OB_SUCCESS; res.reset(); + UNUSED(group_id); if (!check_inner_stat()) { ret = OB_INNER_STAT_ERROR; LOG_WARN("check inner stat failed"); @@ -92,6 +93,7 @@ int ObSingleConnectionProxy::read(ReadResult &res, LOG_TRACE("execute sql", KCSTRING(sql), K(ret)); return ret; } + int ObSingleConnectionProxy::read(ReadResult &res, const int64_t cluster_id, const uint64_t tenant_id, const char *sql) @@ -116,9 +118,10 @@ int ObSingleConnectionProxy::read(ReadResult &res, } int ObSingleConnectionProxy::write( - const uint64_t tenant_id, const char *sql, int64_t &affected_rows) + const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) { int ret = OB_SUCCESS; + UNUSED(group_id); if (!check_inner_stat()) { ret = OB_INNER_STAT_ERROR; LOG_WARN("check inner stat failed"); diff --git a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h index 5635fc22d4..a7f467601d 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h @@ -37,13 +37,15 @@ public: virtual int escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size) override; // %res should be destructed before execute other sql - virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override; + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override { return this->read(res, tenant_id, sql, 0/*group_id*/); } + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) override; virtual int read(ReadResult &res, const int64_t cluster_id, const uint64_t tenant_id, const char *sql) override; - virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override; + virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override { return this->write(tenant_id, sql, 0/*group_id*/, affected_rows); } + virtual int write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) override; using ObISQLClient::read; using ObISQLClient::write; - int connect(const uint64_t tenant_id, ObISQLClient *sql_client); + int connect(const uint64_t tenant_id, const int32_t group_id, ObISQLClient *sql_client); virtual sqlclient::ObISQLConnectionPool *get_pool() override { return pool_; } virtual sqlclient::ObISQLConnection *get_connection() override { return conn_; } diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index af55b53645..c971565aeb 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -529,6 +529,7 @@ STAT_EVENT_SET_DEF(CPU_TIME, "cpu time", ObStatClassIds::RESOURCE, 140013, false STAT_EVENT_SET_DEF(MEMORY_LIMIT, "effective observer memory limit", ObStatClassIds::RESOURCE, 140014, false, true) STAT_EVENT_SET_DEF(SYSTEM_MEMORY, "effective system memory", ObStatClassIds::RESOURCE, 140015, false, true) STAT_EVENT_SET_DEF(HIDDEN_SYS_MEMORY, "effective hidden sys memory", ObStatClassIds::RESOURCE, 140016, false, true) +STAT_EVENT_SET_DEF(MAX_SESSION_NUM, "max session num", ObStatClassIds::RESOURCE, 140017, false, true) //CLOG diff --git a/deps/oblib/src/lib/thread_local/thread_buffer.cpp b/deps/oblib/src/lib/thread_local/thread_buffer.cpp index 8b24297348..142225339f 100644 --- a/deps/oblib/src/lib/thread_local/thread_buffer.cpp +++ b/deps/oblib/src/lib/thread_local/thread_buffer.cpp @@ -80,7 +80,7 @@ ThreadSpecificBuffer::Buffer *ThreadSpecificBuffer::get_buffer() const ptr = ob_malloc(size_ + sizeof(Buffer), ObModIds::OB_THREAD_BUFFER); if (NULL == ptr) { // malloc failed; - _OB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "malloc thread specific memeory failed."); + _OB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "malloc thread specific memory failed."); } else { int ret = pthread_setspecific(key_, ptr); if (0 != ret) { diff --git a/deps/oblib/src/lib/thread_local/thread_buffer.h b/deps/oblib/src/lib/thread_local/thread_buffer.h index f37c33eab4..327ddaf3f6 100644 --- a/deps/oblib/src/lib/thread_local/thread_buffer.h +++ b/deps/oblib/src/lib/thread_local/thread_buffer.h @@ -21,7 +21,7 @@ namespace oceanbase namespace common { /** - * Provide a memeory allocate mechanism by allocate + * Provide a memory allocate mechanism by allocate * a buffer associate with specific thread, whenever * this buffer allocate by user function in a thread * and free when thread exit. diff --git a/deps/oblib/src/rpc/obmysql/ob_mysql_row.cpp b/deps/oblib/src/rpc/obmysql/ob_mysql_row.cpp index af6ce7e33c..0e58ee2ed7 100644 --- a/deps/oblib/src/rpc/obmysql/ob_mysql_row.cpp +++ b/deps/oblib/src/rpc/obmysql/ob_mysql_row.cpp @@ -53,7 +53,7 @@ int ObMySQLRow::serialize(char *buf, const int64_t len, int64_t &pos) const // } // } else if (OB_FAIL(encode_cell(cell_idx, buf, len, pos, bitmap))) { - //LOG_WARN("failed to encode cell", K(ret), K(cell_idx), K(len), K(pos), K(bitmap)); + LOG_WARN("failed to encode cell", K(ret), K(cell_idx), K(len), K(pos), K(bitmap)); } } } else { diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h index a469410726..9c22f1df74 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h @@ -150,7 +150,7 @@ public: static const uint16_t DISABLE_DEBUGSYNC_FLAG = 1 << 12; static const uint16_t CONTEXT_FLAG = 1 << 11; static const uint16_t UNNEED_RESPONSE_FLAG = 1 << 10; - static const uint16_t BAD_ROUTING_FLAG = 1 << 9; + static const uint16_t REQUIRE_REROUTING_FLAG = 1 << 9; static const uint16_t ENABLE_RATELIMIT_FLAG = 1 << 8; static const uint16_t BACKGROUND_FLOW_FLAG = 1 << 7; static const uint16_t TRACE_INFO_FLAG = 1 << 6; @@ -269,8 +269,8 @@ public: inline void unset_stream(); inline void set_unneed_response(); inline bool unneed_response() const; - inline void set_bad_routing(); - inline bool bad_routing() const; + inline void set_require_rerouting(); + inline bool require_rerouting() const; inline bool ratelimit_enabled() const; inline void enable_ratelimit(); @@ -536,14 +536,14 @@ bool ObRpcPacket::unneed_response() const return hdr_.flags_ & ObRpcPacketHeader::UNNEED_RESPONSE_FLAG; } -void ObRpcPacket::set_bad_routing() +void ObRpcPacket::set_require_rerouting() { - hdr_.flags_ |= ObRpcPacketHeader::BAD_ROUTING_FLAG; + hdr_.flags_ |= ObRpcPacketHeader::REQUIRE_REROUTING_FLAG; } -bool ObRpcPacket::bad_routing() const +bool ObRpcPacket::require_rerouting() const { - return hdr_.flags_ & ObRpcPacketHeader::BAD_ROUTING_FLAG; + return hdr_.flags_ & ObRpcPacketHeader::REQUIRE_REROUTING_FLAG; } bool ObRpcPacket::ratelimit_enabled() const diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index b23c4f69ab..753a0e12d0 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -600,6 +600,7 @@ PCODE_DEF(OB_TX_ROLLBACK_SAVEPOINT, 0x659) PCODE_DEF(OB_TX_KEEPALIVE, 0x65A) PCODE_DEF(OB_REGISTER_TX_DATA, 0x65B) PCODE_DEF(OB_TX_KEEPALIVE_RESP, 0x65C) +PCODE_DEF(OB_TX_ROLLBACK_SAVEPOINT_RESP, 0x65D) // trans id PCODE_DEF(OB_GET_GTI_REQUEST, 0x660) @@ -858,7 +859,6 @@ PCODE_DEF(OB_TTL_RESPONSE, 0x1109) // PCODE_DEF(OB_TABLE_API_LOAD_GET_TRANS_STATUS_PEER, 0x1121) // not supported on 4.2 PCODE_DEF(OB_TABLE_API_EXECUTE_GROUP_UPDATE, 0x1122) - PCODE_DEF(OB_TABLE_API_DIRECT_LOAD, 0x1123) PCODE_DEF(OB_TABLE_API_MOVE, 0x1124) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp index e2691b76f7..fe0d625536 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp @@ -311,8 +311,8 @@ int ObRpcProcessorBase::do_response(const Response &rsp) packet->set_stream_last(); } } - if (rsp.bad_routing_) { - packet->set_bad_routing(); + if (rsp.require_rerouting_) { + packet->set_require_rerouting(); } packet->set_unis_version(0); packet->calc_checksum(); @@ -480,7 +480,7 @@ int ObRpcProcessorBase::part_response(const int retcode, bool is_last) if (OB_SUCC(ret)) { const int64_t sessid = sc_ ? sc_->sessid() : 0; ObRpcPacket *pkt = new (pkt_buf) ObRpcPacket(); - Response rsp(sessid, is_stream_, is_last, bad_routing_, pkt); + Response rsp(sessid, is_stream_, is_last, require_rerouting_, pkt); if ((need_compressed) && NULL != tmp_buf) { // compress the serialized result buffer char *dst_buf = pkt_buf + sizeof(ObRpcPacket) + ez_rpc_header_size; @@ -528,7 +528,7 @@ int ObRpcProcessorBase::part_response_error(rpc::ObRequest* req, const int retco } else { ObRpcPacket pkt; pkt.set_content(tbuf, pos); - Response err_rsp(sessid, is_stream_, is_last, bad_routing_, &pkt); + Response err_rsp(sessid, is_stream_, is_last, require_rerouting_, &pkt); if (OB_FAIL(do_response(err_rsp))) { RPC_OBRPC_LOG(WARN, "response data fail", K(ret)); } diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.h b/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.h index 378907e254..ce30c3eec6 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.h @@ -34,7 +34,7 @@ public: public: ObRpcProcessorBase() : rpc_pkt_(NULL), sh_(NULL), sc_(NULL), is_stream_(false), is_stream_end_(false), - bad_routing_(false), preserve_recv_data_(false), preserved_buf_(NULL), + require_rerouting_(false), preserve_recv_data_(false), preserved_buf_(NULL), uncompressed_buf_(NULL), using_buffer_(NULL), send_timestamp_(0), pkt_size_(0), tenant_id_(0), result_compress_type_(common::INVALID_COMPRESSOR) {} @@ -82,12 +82,12 @@ protected: Response(int64_t sessid, bool is_stream, bool is_stream_last, - bool bad_routing, + bool require_rerouting, ObRpcPacket *pkt) : sessid_(sessid), is_stream_(is_stream), is_stream_last_(is_stream_last), - bad_routing_(bad_routing), + require_rerouting_(require_rerouting), pkt_(pkt) { } @@ -97,11 +97,11 @@ protected: bool is_stream_last_; // for routing check - bool bad_routing_; + bool require_rerouting_; ObRpcPacket *pkt_; - TO_STRING_KV(K(sessid_), K(is_stream_), K(is_stream_last_), K_(bad_routing)); + TO_STRING_KV(K_(sessid), K_(is_stream), K_(is_stream_last), K_(require_rerouting)); }; void reuse(); @@ -152,9 +152,8 @@ protected: // invalid, so the stream is end. bool is_stream_end_; - // If this request accidently should not been handled by this server, - // mark the flag so that the client can refresh location cache. - bool bad_routing_; + // For rerouting in obkv + bool require_rerouting_; // The flag marks received data must copy out from `easy buffer' // before we response packet back. Typical case is when we use diff --git a/deps/oblib/unittest/lib/container/test_array_array.cpp b/deps/oblib/unittest/lib/container/test_array_array.cpp index 1ffdd41513..49e72508ab 100644 --- a/deps/oblib/unittest/lib/container/test_array_array.cpp +++ b/deps/oblib/unittest/lib/container/test_array_array.cpp @@ -47,6 +47,7 @@ private: protected: // function members void setup(int64_t N, ObIArray &arr); + void setup(int64_t N, ObIArray &arr, ObArrayArray &arr_arr); void verify(int64_t N, const ObIArray &arr); void verify(const ObIArray &arr1, const ObIArray &arr2); void extend_to(const int64_t N, ObIArray &arr); @@ -80,6 +81,16 @@ void TestArrayArray::setup(int64_t N, ObIArray &arr) } // end for } +void TestArrayArray::setup(int64_t N, ObIArray &arr, ObArrayArray &arr_arr) +{ + arr.reset(); + arr_arr.reset(); + for (int64_t i = 0; i < N; ++i) { + setup(i, arr); + OK(arr_arr.push_back(arr)); + } // end for +} + void TestArrayArray::verify(int64_t N, const ObIArray &arr) { ASSERT_EQ(N, arr.count()); @@ -155,6 +166,36 @@ TEST_F(TestArrayArray, array_push) } +TEST_F(TestArrayArray, assign) +{ + int N = 10; + ObArrayArray src_arr_arr; + ObSEArray arr; + ObArrayArray dst_arr_arr; + for (int64_t i = 1; i < N; i++) { + setup(i, arr, src_arr_arr); + OK(dst_arr_arr.assign(src_arr_arr)); + ASSERT_EQ(dst_arr_arr.count(), src_arr_arr.count()); + for (int64_t j = 0; j < src_arr_arr.count(); j++) { + ASSERT_EQ(dst_arr_arr.count(j), src_arr_arr.count(j)); + verify(dst_arr_arr.at(j), src_arr_arr.at(j)); + } + } + COMMON_LOG(INFO, "print array array", K(dst_arr_arr)); + + N = 20; + for (int64_t i = 1; i < N; i++) { + setup(i, arr, src_arr_arr); + OK(dst_arr_arr.assign(src_arr_arr)); + ASSERT_EQ(dst_arr_arr.count(), src_arr_arr.count()); + for (int64_t j = 0; j < src_arr_arr.count(); j++) { + ASSERT_EQ(dst_arr_arr.count(j), src_arr_arr.count(j)); + verify(dst_arr_arr.at(j), src_arr_arr.at(j)); + } + } + COMMON_LOG(INFO, "print array array", K(dst_arr_arr)); +} + int main(int argc, char **argv) { //system("rm -rf test_array_array.log*"); diff --git a/mittest/logservice/test_ob_simple_log_restart.cpp b/mittest/logservice/test_ob_simple_log_restart.cpp index f636bdc8d1..24a84afe39 100644 --- a/mittest/logservice/test_ob_simple_log_restart.cpp +++ b/mittest/logservice/test_ob_simple_log_restart.cpp @@ -66,55 +66,55 @@ bool ObSimpleLogClusterTestBase::need_add_arb_server_ = false; constexpr int64_t timeout_ts_us = 3 * 1000 * 1000; -TEST_F(TestObSimpleLogClusterRestart, read_block_in_flashback) -{ - disable_hot_cache_ = true; - SET_CASE_LOG_FILE(TEST_NAME, "read_block_in_flashback"); - OB_LOGGER.set_log_level("TRACE"); - const int64_t id = ATOMIC_AAF(&palf_id_, 1); - int64_t leader_idx = 0; - PalfHandleImplGuard leader; - PalfEnv *palf_env = NULL; - EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); - - EXPECT_EQ(OB_SUCCESS, submit_log(leader, 2 * 32 + 2, id, MAX_LOG_BODY_SIZE)); - EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.get_palf_handle_impl()->get_max_lsn())); - - block_id_t min_block_id, max_block_id; - LogStorage *log_storage = &leader.get_palf_handle_impl()->log_engine_.log_storage_; - EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id)); - EXPECT_EQ(2, max_block_id); - SCN scn; - char block_name_tmp[OB_MAX_FILE_NAME_LENGTH]; - EXPECT_EQ(OB_SUCCESS, block_id_to_tmp_string(max_block_id, block_name_tmp, OB_MAX_FILE_NAME_LENGTH)); - char block_name[OB_MAX_FILE_NAME_LENGTH]; - EXPECT_EQ(OB_SUCCESS, block_id_to_string(max_block_id, block_name, OB_MAX_FILE_NAME_LENGTH)); - ::renameat(log_storage->block_mgr_.dir_fd_, block_name, log_storage->block_mgr_.dir_fd_, block_name_tmp); - EXPECT_EQ(-1, ::openat(log_storage->block_mgr_.dir_fd_, block_name, LOG_READ_FLAG)); - EXPECT_EQ(OB_NEED_RETRY, read_log(leader)); - EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn)); - - // 测试边界场景，read_log_tail_为文件中间，最后一个文件完全被flashback掉, 此时log_tail_是最后一个文件头 - log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE); - EXPECT_EQ(OB_NEED_RETRY, read_log(leader)); - EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn)); - - // 测试边界场景，read_log_tail_最后一个文件头，最后一个文件完全被flashback掉 - log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE); - log_storage->readable_log_tail_ = LSN(2*PALF_BLOCK_SIZE); - EXPECT_EQ(OB_ITER_END, read_log(leader)); - EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn)); - - // 测试边界场景，readable_log_tail_还没改变前检验是否可读通过，直接读文件时报错文件不存在。 - log_storage->log_tail_ = LSN(3*PALF_BLOCK_SIZE); - log_storage->readable_log_tail_ = LSN(3*PALF_BLOCK_SIZE); - // 设置max_block_id_为1是为了构造check_read_out_of_bound返回OB_ERR_OUT_OF_UPPER_BOUND的场景 - log_storage->block_mgr_.max_block_id_ = 1; - // log_storage返回OB_ERR_OUT_OF_UPPER_BOUND, iterator将其转换为OB_ITER_END - EXPECT_EQ(OB_ITER_END, read_log(leader)); - EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn)); -} - +//TEST_F(TestObSimpleLogClusterRestart, read_block_in_flashback) +//{ +// disable_hot_cache_ = true; +// SET_CASE_LOG_FILE(TEST_NAME, "read_block_in_flashback"); +// OB_LOGGER.set_log_level("TRACE"); +// const int64_t id = ATOMIC_AAF(&palf_id_, 1); +// int64_t leader_idx = 0; +// PalfHandleImplGuard leader; +// PalfEnv *palf_env = NULL; +// EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); +// +// EXPECT_EQ(OB_SUCCESS, submit_log(leader, 2 * 32 + 2, id, MAX_LOG_BODY_SIZE)); +// EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.get_palf_handle_impl()->get_max_lsn())); +// +// block_id_t min_block_id, max_block_id; +// LogStorage *log_storage = &leader.get_palf_handle_impl()->log_engine_.log_storage_; +// EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id)); +// EXPECT_EQ(2, max_block_id); +// SCN scn; +// char block_name_tmp[OB_MAX_FILE_NAME_LENGTH]; +// EXPECT_EQ(OB_SUCCESS, block_id_to_tmp_string(max_block_id, block_name_tmp, OB_MAX_FILE_NAME_LENGTH)); +// char block_name[OB_MAX_FILE_NAME_LENGTH]; +// EXPECT_EQ(OB_SUCCESS, block_id_to_string(max_block_id, block_name, OB_MAX_FILE_NAME_LENGTH)); +// ::renameat(log_storage->block_mgr_.dir_fd_, block_name, log_storage->block_mgr_.dir_fd_, block_name_tmp); +// EXPECT_EQ(-1, ::openat(log_storage->block_mgr_.dir_fd_, block_name, LOG_READ_FLAG)); +// EXPECT_EQ(OB_NEED_RETRY, read_log(leader)); +// EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn)); +// +// // 测试边界场景，read_log_tail_为文件中间，最后一个文件完全被flashback掉, 此时log_tail_是最后一个文件头 +// log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE); +// EXPECT_EQ(OB_NEED_RETRY, read_log(leader)); +// EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn)); +// +// // 测试边界场景，read_log_tail_最后一个文件头，最后一个文件完全被flashback掉 +// log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE); +// log_storage->readable_log_tail_ = LSN(2*PALF_BLOCK_SIZE); +// EXPECT_EQ(OB_ITER_END, read_log(leader)); +// EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn)); +// +// // 测试边界场景，readable_log_tail_还没改变前检验是否可读通过，直接读文件时报错文件不存在。 +// log_storage->log_tail_ = LSN(3*PALF_BLOCK_SIZE); +// log_storage->readable_log_tail_ = LSN(3*PALF_BLOCK_SIZE); +// // 设置max_block_id_为1是为了构造check_read_out_of_bound返回OB_ERR_OUT_OF_UPPER_BOUND的场景 +// log_storage->block_mgr_.max_block_id_ = 1; +// // log_storage返回OB_ERR_OUT_OF_UPPER_BOUND, iterator将其转换为OB_ITER_END +// EXPECT_EQ(OB_ITER_END, read_log(leader)); +// EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn)); +//} +// TEST_F(TestObSimpleLogClusterRestart, restart_when_first_log_block_is_empty) { SET_CASE_LOG_FILE(TEST_NAME, "restart_when_first_log_block_is_empty"); diff --git a/mittest/mtlenv/storage/CMakeLists.txt b/mittest/mtlenv/storage/CMakeLists.txt index f0f2c75e57..48149792db 100644 --- a/mittest/mtlenv/storage/CMakeLists.txt +++ b/mittest/mtlenv/storage/CMakeLists.txt @@ -20,6 +20,7 @@ storage_dml_unittest(test_multi_version_sstable_single_get) #storage_dml_unittest(test_multi_version_sstable_merge) TODO(dengzhi.ldz): fix it storage_dml_unittest(test_medium_info_reader test_medium_info_reader.cpp) storage_dml_unittest(test_tablet_mds_data test_tablet_mds_data.cpp) +storage_dml_unittest(test_mds_data_read_write test_mds_data_read_write.cpp) storage_unittest(test_physical_copy_task test_physical_copy_task.cpp) storage_unittest(test_shared_block_reader_writer) storage_dml_unittest(test_macro_ref_cnt test_macro_ref_cnt.cpp) diff --git a/mittest/mtlenv/storage/test_mds_data_read_write.cpp b/mittest/mtlenv/storage/test_mds_data_read_write.cpp new file mode 100644 index 0000000000..0be6608e50 --- /dev/null +++ b/mittest/mtlenv/storage/test_mds_data_read_write.cpp @@ -0,0 +1,488 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include + +#define protected public +#define private public + +#include "lib/ob_errno.h" +#include "lib/allocator/page_arena.h" +#include "lib/oblog/ob_log.h" +#include "mtlenv/mock_tenant_module_env.h" +#include "mtlenv/storage/medium_info_helper.h" +#include "storage/tablet/ob_tablet_mds_data.h" +#include "storage/tablet/ob_tablet_complex_addr.h" +#include "storage/tablet/ob_tablet_obj_load_helper.h" +#include "storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h" + +#define USING_LOG_PREFIX STORAGE + +using namespace oceanbase::common; + +namespace oceanbase +{ +namespace storage +{ +class TestMdsDataReadWrite : public::testing::Test +{ +public: + TestMdsDataReadWrite(); + virtual ~TestMdsDataReadWrite() = default; + static void SetUpTestCase(); + static void TearDownTestCase(); + //virtual void SetUp() override; + //virtual void TearDown() override; +private: + static int mock_tablet_status_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr); + static int mock_empty_tablet_status_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr); + static int mock_auto_inc_seq_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr); + static int mock_empty_auto_inc_seq_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr); +}; + +TestMdsDataReadWrite::TestMdsDataReadWrite() +{ +} + +void TestMdsDataReadWrite::SetUpTestCase() +{ + EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init()); +} + +void TestMdsDataReadWrite::TearDownTestCase() +{ + MockTenantModuleEnv::get_instance().destroy(); +} + +int TestMdsDataReadWrite::mock_tablet_status_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr) +{ + int ret = OB_SUCCESS; + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + mds::MdsDumpKV kv; + + { + ObTabletCreateDeleteMdsUserData user_data; + user_data.tablet_status_ = ObTabletStatus::NORMAL; + const int64_t size = user_data.get_serialize_size(); + int64_t pos = 0; + char *buf = static_cast(allocator.alloc(size)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(size)); + } else if (OB_FAIL(user_data.serialize(buf, size, pos))) { + LOG_WARN("failed to serialize", K(ret)); + } else { + ObString &str = kv.v_.user_data_; + str.assign(buf, size); + } + } + + + const int64_t size = kv.get_serialize_size(); + int64_t pos = 0; + + char *buf = static_cast(allocator.alloc(size)); + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(size)); + } else if (OB_FAIL(kv.serialize(buf, size, pos))) { + LOG_WARN("failed to serialize", K(ret)); + } else { + ObSharedBlockWriteInfo write_info; + write_info.buffer_ = buf; + write_info.offset_ = 0; + write_info.size_ = size; + write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); + + ObSharedBlockWriteHandle handle; + ObSharedBlocksWriteCtx write_ctx; + if (OB_FAIL(ckpt_slog_handler->get_shared_block_reader_writer().async_write(write_info, handle))) { + LOG_WARN("failed to do async write", K(ret)); + } else if (OB_FAIL(handle.get_write_ctx(write_ctx))) { + LOG_WARN("failed to get write ctx", K(ret)); + } else { + addr = write_ctx.addr_; + } + } + + if (nullptr != buf) { + allocator.free(buf); + } + + return ret; +} + +int TestMdsDataReadWrite::mock_empty_tablet_status_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr) +{ + int ret = OB_SUCCESS; + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + mds::MdsDumpKV kv; + const int64_t size = kv.get_serialize_size(); + int64_t pos = 0; + + char *buf = static_cast(allocator.alloc(size)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(size)); + } else if (OB_FAIL(kv.serialize(buf, size, pos))) { + LOG_WARN("failed to serialize", K(ret)); + } else { + ObSharedBlockWriteInfo write_info; + write_info.buffer_ = buf; + write_info.offset_ = 0; + write_info.size_ = size; + write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); + + ObSharedBlockWriteHandle handle; + ObSharedBlocksWriteCtx write_ctx; + if (OB_FAIL(ckpt_slog_handler->get_shared_block_reader_writer().async_write(write_info, handle))) { + LOG_WARN("failed to do async write", K(ret)); + } else if (OB_FAIL(handle.get_write_ctx(write_ctx))) { + LOG_WARN("failed to get write ctx", K(ret)); + } else { + addr = write_ctx.addr_; + } + } + + if (nullptr != buf) { + allocator.free(buf); + } + + return ret; +} + +int TestMdsDataReadWrite::mock_auto_inc_seq_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr) +{ + int ret = OB_SUCCESS; + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + share::ObTabletAutoincSeq auto_inc_seq; + + { + if (OB_FAIL(auto_inc_seq.set_autoinc_seq_value(allocator, 100))) { + LOG_WARN("failed to set auto inc seq", K(ret)); + } + } + + const int64_t size = auto_inc_seq.get_serialize_size(); + int64_t pos = 0; + + char *buf = static_cast(allocator.alloc(size)); + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(size)); + } else if (OB_FAIL(auto_inc_seq.serialize(buf, size, pos))) { + LOG_WARN("failed to serialize", K(ret)); + } else { + ObSharedBlockWriteInfo write_info; + write_info.buffer_ = buf; + write_info.offset_ = 0; + write_info.size_ = size; + write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); + + ObSharedBlockWriteHandle handle; + ObSharedBlocksWriteCtx write_ctx; + if (OB_FAIL(ckpt_slog_handler->get_shared_block_reader_writer().async_write(write_info, handle))) { + LOG_WARN("failed to do async write", K(ret)); + } else if (OB_FAIL(handle.get_write_ctx(write_ctx))) { + LOG_WARN("failed to get write ctx", K(ret)); + } else { + addr = write_ctx.addr_; + } + } + + if (nullptr != buf) { + allocator.free(buf); + } + + return ret; +} + +int TestMdsDataReadWrite::mock_empty_auto_inc_seq_disk_addr( + common::ObArenaAllocator &allocator, + ObMetaDiskAddr &addr) +{ + int ret = OB_SUCCESS; + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + share::ObTabletAutoincSeq auto_inc_seq; + const int64_t size = auto_inc_seq.get_serialize_size(); + int64_t pos = 0; + + char *buf = static_cast(allocator.alloc(size)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(size)); + } else if (OB_FAIL(auto_inc_seq.serialize(buf, size, pos))) { + LOG_WARN("failed to serialize", K(ret)); + } else { + ObSharedBlockWriteInfo write_info; + write_info.buffer_ = buf; + write_info.offset_ = 0; + write_info.size_ = size; + write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); + + ObSharedBlockWriteHandle handle; + ObSharedBlocksWriteCtx write_ctx; + if (OB_FAIL(ckpt_slog_handler->get_shared_block_reader_writer().async_write(write_info, handle))) { + LOG_WARN("failed to do async write", K(ret)); + } else if (OB_FAIL(handle.get_write_ctx(write_ctx))) { + LOG_WARN("failed to get write ctx", K(ret)); + } else { + addr = write_ctx.addr_; + } + } + + if (nullptr != buf) { + allocator.free(buf); + } + + return ret; +} + +TEST_F(TestMdsDataReadWrite, mds_dump_kv) +{ + int ret = OB_SUCCESS; + common::ObArenaAllocator allocator; + ObTabletComplexAddr src_addr; + ObTabletComplexAddr dst_addr; + + { + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + } + + { + src_addr.reset(); + dst_addr.reset(); + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, src_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + src_addr.addr_.set_none_addr(); + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } + + { + src_addr.reset(); + dst_addr.reset(); + + ObMetaDiskAddr addr; + ret = TestMdsDataReadWrite::mock_empty_tablet_status_disk_addr(allocator, addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(addr.is_block()); + src_addr.ptr_ = nullptr; + src_addr.addr_ = addr; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } + + { + src_addr.reset(); + dst_addr.reset(); + + ObMetaDiskAddr addr; + ret = TestMdsDataReadWrite::mock_tablet_status_disk_addr(allocator, addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(addr.is_block()); + src_addr.ptr_ = nullptr; + src_addr.addr_ = addr; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_memory_object()); + } +} + +TEST_F(TestMdsDataReadWrite, auto_inc_seq) +{ + int ret = OB_SUCCESS; + common::ObArenaAllocator allocator; + ObTabletComplexAddr src_addr; + ObTabletComplexAddr dst_addr; + + { + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + } + + { + src_addr.reset(); + dst_addr.reset(); + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, src_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + src_addr.addr_.set_none_addr(); + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } + + { + src_addr.reset(); + dst_addr.reset(); + + ObMetaDiskAddr addr; + ret = TestMdsDataReadWrite::mock_empty_auto_inc_seq_disk_addr(allocator, addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(addr.is_block()); + src_addr.ptr_ = nullptr; + src_addr.addr_ = addr; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } + + { + src_addr.reset(); + dst_addr.reset(); + + ObMetaDiskAddr addr; + ret = TestMdsDataReadWrite::mock_auto_inc_seq_disk_addr(allocator, addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(addr.is_block()); + src_addr.ptr_ = nullptr; + src_addr.addr_ = addr; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_memory_object()); + } +} + +TEST_F(TestMdsDataReadWrite, medium_info_list) +{ + int ret = OB_SUCCESS; + common::ObArenaAllocator allocator; + ObTabletComplexAddr src_addr; + ObTabletComplexAddr dst_addr; + + { + const int64_t finish_medium_scn = 0; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, 0, dst_addr); + ASSERT_EQ(OB_INVALID_ARGUMENT, ret); + } + + { + src_addr.reset(); + dst_addr.reset(); + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, src_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = src_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + src_addr.addr_.set_none_addr(); + + const int64_t finish_medium_scn = 0; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, finish_medium_scn, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } + + { + src_addr.reset(); + dst_addr.reset(); + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, src_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = src_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + src_addr.addr_.set_none_addr(); + + compaction::ObMediumCompactionInfo medium_info; + ret = MediumInfoHelper::build_medium_compaction_info(allocator, medium_info, 100); + ASSERT_EQ(OB_SUCCESS, ret); + ret = src_addr.ptr_->append(medium_info); + ASSERT_EQ(OB_SUCCESS, ret); + + int64_t finish_medium_scn = 80; + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, finish_medium_scn, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_memory_object()); + + finish_medium_scn = 200; + dst_addr.reset(); + ret = ObTabletMdsData::init_single_complex_addr(allocator, src_addr, finish_medium_scn, dst_addr); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } + + { + src_addr.reset(); + dst_addr.reset(); + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, src_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = src_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + src_addr.addr_.set_none_addr(); + + compaction::ObMediumCompactionInfo medium_info; + ret = MediumInfoHelper::build_medium_compaction_info(allocator, medium_info, 100); + ASSERT_EQ(OB_SUCCESS, ret); + ret = src_addr.ptr_->append(medium_info); + ASSERT_EQ(OB_SUCCESS, ret); + + ObTabletDumpedMediumInfo src_data; + ret = src_data.init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + medium_info.medium_snapshot_ = 120; + ret = src_data.append(medium_info); + ASSERT_EQ(OB_SUCCESS, ret); + + compaction::ObExtraMediumInfo src_addr_extra_info; + src_addr_extra_info.last_medium_scn_ = 10; + compaction::ObExtraMediumInfo src_data_extra_info; + src_data_extra_info.last_medium_scn_ = 20; + compaction::ObExtraMediumInfo dst_extra_info; + + int64_t finish_medium_scn = 80; + ret = ObTabletMdsData::init_single_complex_addr_and_extra_info(allocator, src_addr, src_addr_extra_info, + src_data, src_data_extra_info, finish_medium_scn, dst_addr, dst_extra_info); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_memory_object()); + ASSERT_EQ(2, dst_addr.ptr_->medium_info_list_.count()); + + finish_medium_scn = 110; + ret = ObTabletMdsData::init_single_complex_addr_and_extra_info(allocator, src_addr, src_addr_extra_info, + src_data, src_data_extra_info, finish_medium_scn, dst_addr, dst_extra_info); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_memory_object()); + ASSERT_EQ(1, dst_addr.ptr_->medium_info_list_.count()); + + finish_medium_scn = 130; + ret = ObTabletMdsData::init_single_complex_addr_and_extra_info(allocator, src_addr, src_addr_extra_info, + src_data, src_data_extra_info, finish_medium_scn, dst_addr, dst_extra_info); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(dst_addr.is_none_object()); + } +} +} // namespace storage +} // namespace oceanbase + +int main(int argc, char **argv) +{ + system("rm -f test_mds_data_read_write.log*"); + oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + OB_LOGGER.set_file_name("test_mds_data_read_write.log", true); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/mittest/mtlenv/storage/test_medium_info_reader.cpp b/mittest/mtlenv/storage/test_medium_info_reader.cpp index 78f16b469b..2f2aa3c4f2 100644 --- a/mittest/mtlenv/storage/test_medium_info_reader.cpp +++ b/mittest/mtlenv/storage/test_medium_info_reader.cpp @@ -351,7 +351,13 @@ TEST_F(TestMediumInfoReader, pure_dump_data) // insert data into tablet { - ObTabletDumpedMediumInfo &medium_info_list = *tablet->mds_data_.medium_info_list_.ptr_; + common::ObIAllocator &allocator = *tablet_handle.get_allocator(); + ObTabletComplexAddr &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_; + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_; constexpr uint8_t table_id = mds::TupleTypeIdx::value; constexpr uint8_t unit_id = mds::TupleTypeIdx>::value; @@ -417,7 +423,13 @@ TEST_F(TestMediumInfoReader, mds_table_dump_data_overlap) // insert data into mds data { - ObTabletDumpedMediumInfo &medium_info_list = *tablet->mds_data_.medium_info_list_.ptr_; + common::ObIAllocator &allocator = *tablet_handle.get_allocator(); + ObTabletComplexAddr &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_; + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_; constexpr uint8_t table_id = mds::TupleTypeIdx::value; constexpr uint8_t unit_id = mds::TupleTypeIdx>::value; @@ -529,7 +541,13 @@ TEST_F(TestMediumInfoReader, mds_table_dump_data_no_overlap) // insert data into mds data { - ObTabletDumpedMediumInfo &medium_info_list = *tablet->mds_data_.medium_info_list_.ptr_; + common::ObIAllocator &allocator = *tablet_handle.get_allocator(); + ObTabletComplexAddr &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_; + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_; constexpr uint8_t table_id = mds::TupleTypeIdx::value; constexpr uint8_t unit_id = mds::TupleTypeIdx>::value; @@ -631,7 +649,13 @@ TEST_F(TestMediumInfoReader, mds_table_dump_data_full_inclusion) // insert data into mds data { - ObTabletDumpedMediumInfo &medium_info_list = *tablet->mds_data_.medium_info_list_.ptr_; + common::ObIAllocator &allocator = *tablet_handle.get_allocator(); + ObTabletComplexAddr &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_; + ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_); + ASSERT_EQ(OB_SUCCESS, ret); + ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_; constexpr uint8_t table_id = mds::TupleTypeIdx::value; constexpr uint8_t unit_id = mds::TupleTypeIdx>::value; diff --git a/mittest/simple_server/CMakeLists.txt b/mittest/simple_server/CMakeLists.txt index 478297aa88..96f6d500b9 100644 --- a/mittest/simple_server/CMakeLists.txt +++ b/mittest/simple_server/CMakeLists.txt @@ -97,12 +97,12 @@ ob_unittest_observer(test_mds_recover test_mds_recover.cpp) ob_unittest_observer(test_keep_alive_min_start_scn test_keep_alive_min_start_scn.cpp) ob_unittest_observer(test_ls_replica test_ls_replica.cpp) # TODO(muwei.ym): open later -#ob_ha_unittest_observer(test_transfer_handler storage_ha/test_transfer_handler.cpp) -#ob_ha_unittest_observer(test_transfer_and_restart_basic storage_ha/test_transfer_and_restart_basic.cpp) -#ob_ha_unittest_observer(test_transfer_start_stage_restart_without_mds_flush storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp) -#ob_ha_unittest_observer(test_transfer_doing_stage_restart_without_mds_flush storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp) -#ob_ha_unittest_observer(test_transfer_complete_restart_without_mds_flush storage_ha/test_transfer_complete_restart_without_mds_flush.cpp) -#ob_ha_unittest_observer(test_transfer_doing_stage_restart_with_mds_flush storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp) -#ob_ha_unittest_observer(test_transfer_complete_restart_with_mds_flush storage_ha/test_transfer_complete_restart_with_mds_flush.cpp) -#ob_ha_unittest_observer(test_transfer_with_empty_shell storage_ha/test_transfer_with_empty_shell.cpp) +ob_ha_unittest_observer(test_transfer_handler storage_ha/test_transfer_handler.cpp) +ob_ha_unittest_observer(test_transfer_and_restart_basic storage_ha/test_transfer_and_restart_basic.cpp) +ob_ha_unittest_observer(test_transfer_start_stage_restart_without_mds_flush storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp) +ob_ha_unittest_observer(test_transfer_doing_stage_restart_without_mds_flush storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp) +ob_ha_unittest_observer(test_transfer_complete_restart_without_mds_flush storage_ha/test_transfer_complete_restart_without_mds_flush.cpp) +ob_ha_unittest_observer(test_transfer_doing_stage_restart_with_mds_flush storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp) +ob_ha_unittest_observer(test_transfer_complete_restart_with_mds_flush storage_ha/test_transfer_complete_restart_with_mds_flush.cpp) +ob_ha_unittest_observer(test_transfer_with_empty_shell storage_ha/test_transfer_with_empty_shell.cpp) errsim_ha_unittest_observer(errsim_test_transfer_handler errsim/storage_ha/errsim_test_transfer_handler.cpp) diff --git a/mittest/simple_server/storage_ha/test_transfer_and_restart_basic.cpp b/mittest/simple_server/storage_ha/test_transfer_and_restart_basic.cpp index 9029473c34..e17b007e5d 100644 --- a/mittest/simple_server/storage_ha/test_transfer_and_restart_basic.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_and_restart_basic.cpp @@ -29,7 +29,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -250,13 +250,13 @@ TEST_F(TestTransferRestart, observer_restart_basic) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); int ret = 0; int time_sec = 0; - ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME, - RESTART_CASE_NAME); + ObSimpleServerRestartHelper restart_helper(argc, argv, oceanbase::storage::TEST_FILE_NAME, oceanbase::storage::BORN_CASE_NAME, + oceanbase::storage::RESTART_CASE_NAME); restart_helper.set_sleep_sec(time_sec); restart_helper.run(); return ret; diff --git a/mittest/simple_server/storage_ha/test_transfer_complete_restart_with_mds_flush.cpp b/mittest/simple_server/storage_ha/test_transfer_complete_restart_with_mds_flush.cpp index 46f81d13a0..545778ba74 100644 --- a/mittest/simple_server/storage_ha/test_transfer_complete_restart_with_mds_flush.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_complete_restart_with_mds_flush.cpp @@ -29,7 +29,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -210,9 +210,14 @@ TEST_F(TestTransferHandler, prepare_valid_data) sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy2, sql, g_batch_part_list)); + + //create other ls by cluster table + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("create table part_table_with_dup(c1 int) duplicate_scope = 'CLUSTER' partition by hash(c1) partitions 4")); + ASSERT_EQ(OB_SUCCESS, sql_proxy2.write(sql.ptr(), affected_rows)); } -TEST_F(TestTransferHandler, test_transfer_1001_to_1_with_transfer_finish_without_mds_flush) +TEST_F(TestTransferHandler, test_transfer_1001_to_1002_with_transfer_finish_without_mds_flush) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -233,12 +238,12 @@ TEST_F(TestTransferHandler, test_transfer_1001_to_1_with_transfer_finish_without ObTenantTransferService *tenant_transfer = MTL(ObTenantTransferService*); ASSERT_TRUE(OB_NOT_NULL(tenant_transfer)); - //title: 1001 ls transfer to 1 ls + //title: 1001 ls transfer to 1002 ls // generate transfer task ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1002), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -272,13 +277,13 @@ TEST_F(TestTransferRestart, observer_restart_when_transfer_finish_without_flush) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); int ret = 0; int time_sec = 0; - ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME, - RESTART_CASE_NAME); + ObSimpleServerRestartHelper restart_helper(argc, argv, oceanbase::storage::TEST_FILE_NAME, oceanbase::storage::BORN_CASE_NAME, + oceanbase::storage::RESTART_CASE_NAME); restart_helper.set_sleep_sec(time_sec); restart_helper.run(); return ret; diff --git a/mittest/simple_server/storage_ha/test_transfer_complete_restart_without_mds_flush.cpp b/mittest/simple_server/storage_ha/test_transfer_complete_restart_without_mds_flush.cpp index 40691225e0..05e9212fcf 100644 --- a/mittest/simple_server/storage_ha/test_transfer_complete_restart_without_mds_flush.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_complete_restart_without_mds_flush.cpp @@ -29,7 +29,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -210,9 +210,14 @@ TEST_F(TestTransferHandler, prepare_valid_data) sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy2, sql, g_batch_part_list)); + + //create other ls by cluster table + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("create table part_table_with_dup(c1 int) duplicate_scope = 'CLUSTER' partition by hash(c1) partitions 4")); + ASSERT_EQ(OB_SUCCESS, sql_proxy2.write(sql.ptr(), affected_rows)); } -TEST_F(TestTransferHandler, test_transfer_1001_to_1_with_transfer_finish_without_mds_flush) +TEST_F(TestTransferHandler, test_transfer_1001_to_1002_with_transfer_finish_without_mds_flush) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -233,12 +238,12 @@ TEST_F(TestTransferHandler, test_transfer_1001_to_1_with_transfer_finish_without ObTenantTransferService *tenant_transfer = MTL(ObTenantTransferService*); ASSERT_TRUE(OB_NOT_NULL(tenant_transfer)); - //title: 1001 ls transfer to 1 ls + //title: 1001 ls transfer to 1002 ls // generate transfer task ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1002), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -271,13 +276,13 @@ TEST_F(TestTransferRestart, observer_restart_when_transfer_finish_without_flush) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); int ret = 0; int time_sec = 0; - ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME, - RESTART_CASE_NAME); + ObSimpleServerRestartHelper restart_helper(argc, argv, oceanbase::storage::TEST_FILE_NAME, oceanbase::storage::BORN_CASE_NAME, + oceanbase::storage::RESTART_CASE_NAME); restart_helper.set_sleep_sec(time_sec); restart_helper.run(); return ret; diff --git a/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp b/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp index 04bc64603c..76df827548 100644 --- a/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp @@ -29,7 +29,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -277,13 +277,13 @@ TEST_F(TestTransferRestart, observer_restart_when_transfer_start_without_flush) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); int ret = 0; int time_sec = 0; - ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME, - RESTART_CASE_NAME); + ObSimpleServerRestartHelper restart_helper(argc, argv, oceanbase::storage::TEST_FILE_NAME, oceanbase::storage::BORN_CASE_NAME, + oceanbase::storage::RESTART_CASE_NAME); restart_helper.set_sleep_sec(time_sec); restart_helper.run(); return ret; diff --git a/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp b/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp index c0e843820a..0697170683 100644 --- a/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp @@ -29,7 +29,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -209,9 +209,29 @@ TEST_F(TestTransferHandler, prepare_valid_data) sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy2, sql, g_batch_part_list)); + + //create other ls by cluster table + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("create table part_table_with_dup(c1 int) duplicate_scope = 'CLUSTER' partition by hash(c1) partitions 4")); + ASSERT_EQ(OB_SUCCESS, sql_proxy2.write(sql.ptr(), affected_rows)); + sql.reset(); + + ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _enable_balance_kill_transaction = true tenant = 'tt1';")); + ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); + usleep(100000); // wait for debug_sync_timeout to take effect + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _balance_wait_killing_transaction_end_threshold = '10s' tenant = 'tt1';")); + ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); + + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _enable_balance_kill_transaction = true tenant = 'tt1';")); + ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); + usleep(100000); // wait for debug_sync_timeout to take effect + sql.reset(); + } -TEST_F(TestTransferHandler, test_transfer_1_to_1001_with_transfer_doing_without_mds_flush) +TEST_F(TestTransferHandler, test_transfer_1001_to_1002_with_transfer_doing_without_mds_flush) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -238,12 +258,12 @@ TEST_F(TestTransferHandler, test_transfer_1_to_1001_with_transfer_doing_without_ ObTenantTransferService *tenant_transfer = MTL(ObTenantTransferService*); ASSERT_TRUE(OB_NOT_NULL(tenant_transfer)); - //title: 1001 ls transfer to 1 ls + //title: 1001 ls transfer to 1002 ls // generate transfer task ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1002), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -277,13 +297,13 @@ TEST_F(TestTransferRestart, observer_restart_when_transfer_doing_without_flush) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); int ret = 0; int time_sec = 0; - ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME, - RESTART_CASE_NAME); + ObSimpleServerRestartHelper restart_helper(argc, argv, oceanbase::storage::TEST_FILE_NAME, oceanbase::storage::BORN_CASE_NAME, + oceanbase::storage::RESTART_CASE_NAME); restart_helper.set_sleep_sec(time_sec); restart_helper.run(); return ret; diff --git a/mittest/simple_server/storage_ha/test_transfer_handler.cpp b/mittest/simple_server/storage_ha/test_transfer_handler.cpp index ba36a1b6eb..c77ffcca84 100644 --- a/mittest/simple_server/storage_ha/test_transfer_handler.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_handler.cpp @@ -27,7 +27,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -208,7 +208,8 @@ int TestTransferHandler::wait_transfer_out_deleted_tablet_gc( while (OB_SUCC(ret) && index < task.tablet_list_.count()) { ObTabletHandle tablet_handle; const ObTransferTabletInfo &transfer_info = task.tablet_list_.at(index); - if (OB_FAIL(ls->get_tablet(transfer_info.tablet_id_, tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + if (OB_FAIL(ls->get_tablet(transfer_info.tablet_id_, tablet_handle, + ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { if (OB_TABLET_NOT_EXIST == ret) { ret = OB_SUCCESS; index++; @@ -263,9 +264,32 @@ TEST_F(TestTransferHandler, prepare_valid_data) sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy, sql, g_batch_part_list)); + + //create other ls by cluster table + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("create table part_table_with_dup(c1 int) duplicate_scope = 'CLUSTER' partition by hash(c1) partitions 4")); + ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows)); + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); + ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy, sql, g_batch_part_list)); + + + ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _enable_balance_kill_transaction = true tenant = 'tt1';")); + ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); + usleep(100000); // wait for debug_sync_timeout to take effect + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _balance_wait_killing_transaction_end_threshold = '10s' tenant = 'tt1';")); + ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); + + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _enable_balance_kill_transaction = true tenant = 'tt1';")); + ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); + usleep(100000); // wait for debug_sync_timeout to take effect + sql.reset(); + } -TEST_F(TestTransferHandler, test_transfer_1001_to_1) +TEST_F(TestTransferHandler, test_transfer_1001_to_1002) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -286,12 +310,12 @@ TEST_F(TestTransferHandler, test_transfer_1001_to_1) ObTenantTransferService *tenant_transfer = MTL(ObTenantTransferService*); ASSERT_TRUE(OB_NOT_NULL(tenant_transfer)); - //title: 1001 ls transfer to 1 ls + //title: 1001 ls transfer to 1002 ls // generate transfer task ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1002), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -311,7 +335,7 @@ TEST_F(TestTransferHandler, test_transfer_1001_to_1) ASSERT_EQ(OB_SUCCESS, wait_transfer_out_deleted_tablet_gc(task)); } -TEST_F(TestTransferHandler, test_transfer_1_to_1001) +TEST_F(TestTransferHandler, test_transfer_1002_to_1001) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -325,12 +349,12 @@ TEST_F(TestTransferHandler, test_transfer_1_to_1001) ObTenantTransferService *tenant_transfer = MTL(ObTenantTransferService*); ASSERT_TRUE(OB_NOT_NULL(tenant_transfer)); - //title: 1001 ls transfer to 1 ls + //title: 1002 ls transfer to 1001 ls // generate transfer task ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1), ObLSID(1001), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1002), ObLSID(1001), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -353,7 +377,7 @@ TEST_F(TestTransferHandler, test_transfer_1_to_1001) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/storage_ha/test_transfer_lock_info_operator.cpp b/mittest/simple_server/storage_ha/test_transfer_lock_info_operator.cpp index 977a69d94b..2c078bf59f 100644 --- a/mittest/simple_server/storage_ha/test_transfer_lock_info_operator.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_lock_info_operator.cpp @@ -48,6 +48,7 @@ TEST_F(TestTransferLockInfoOperator, TransferLockInfo) src_ls_id_ = ObLSID(1001); dest_ls_id_ = ObLSID(1002); task_id_ = 1; + const int32_t group_id = 0; ObTransferLockStatus start_status = ObTransferLockStatus(ObTransferLockStatus::START); ObTransferLockStatus doing_status = ObTransferLockStatus(ObTransferLockStatus::DOING); int64_t start_src_lock_owner = 111; @@ -63,8 +64,8 @@ TEST_F(TestTransferLockInfoOperator, TransferLockInfo) common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); // insert - ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::insert(start_src_lock_info, sql_proxy)); - ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::insert(start_dest_lock_info, sql_proxy)); + ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::insert(start_src_lock_info, group_id, sql_proxy)); + ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::insert(start_dest_lock_info, group_id, sql_proxy)); // select ObTransferLockInfoRowKey src_row_key; @@ -77,8 +78,8 @@ TEST_F(TestTransferLockInfoOperator, TransferLockInfo) ObTransferTaskLockInfo new_start_src_lock_info; ObTransferTaskLockInfo new_start_dest_lock_info; - ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::get(src_row_key, task_id_, start_status, false, new_start_src_lock_info, sql_proxy)); - ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::get(dest_row_key, task_id_, start_status, false, new_start_dest_lock_info, sql_proxy)); + ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::get(src_row_key, task_id_, start_status, false, group_id, new_start_src_lock_info, sql_proxy)); + ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::get(dest_row_key, task_id_, start_status, false, group_id, new_start_dest_lock_info, sql_proxy)); LOG_INFO("[MITTEST]transfer_lock_info", K(new_start_src_lock_info)); ASSERT_EQ(new_start_src_lock_info.tenant_id_, start_src_lock_info.tenant_id_); @@ -95,11 +96,11 @@ TEST_F(TestTransferLockInfoOperator, TransferLockInfo) ASSERT_EQ(new_start_dest_lock_info.lock_owner_, start_dest_lock_info.lock_owner_); // remove - ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::remove(tenant_id_, src_ls_id_, task_id_, start_status, sql_proxy)); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferLockInfoOperator::get(src_row_key, task_id_, start_status, false, new_start_src_lock_info, sql_proxy)); + ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::remove(tenant_id_, src_ls_id_, task_id_, start_status, group_id, sql_proxy)); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferLockInfoOperator::get(src_row_key, task_id_, start_status, false, group_id, new_start_src_lock_info, sql_proxy)); - ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::remove(tenant_id_, dest_ls_id_, task_id_, start_status, sql_proxy)); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferLockInfoOperator::get(src_row_key, task_id_, start_status, false, new_start_dest_lock_info, sql_proxy)); + ASSERT_EQ(OB_SUCCESS, ObTransferLockInfoOperator::remove(tenant_id_, dest_ls_id_, task_id_, start_status, group_id, sql_proxy)); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferLockInfoOperator::get(src_row_key, task_id_, start_status, false, group_id, new_start_dest_lock_info, sql_proxy)); } } // namespace diff --git a/mittest/simple_server/storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp b/mittest/simple_server/storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp index 85c54bf340..aa2681c00a 100644 --- a/mittest/simple_server/storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp @@ -29,7 +29,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -209,6 +209,7 @@ TEST_F(TestTransferHandler, prepare_valid_data) sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy2, sql, g_batch_part_list)); + } TEST_F(TestTransferHandler, test_transfer_1001_to_1_with_transfer_start_without_mds_flush) @@ -276,13 +277,13 @@ TEST_F(TestTransferRestart, observer_restart_when_transfer_start_without_flush) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); int ret = 0; int time_sec = 0; - ObSimpleServerRestartHelper restart_helper(argc, argv, TEST_FILE_NAME, BORN_CASE_NAME, - RESTART_CASE_NAME); + ObSimpleServerRestartHelper restart_helper(argc, argv, oceanbase::storage::TEST_FILE_NAME, oceanbase::storage::BORN_CASE_NAME, + oceanbase::storage::RESTART_CASE_NAME); restart_helper.set_sleep_sec(time_sec); restart_helper.run(); return ret; diff --git a/mittest/simple_server/storage_ha/test_transfer_with_empty_shell.cpp b/mittest/simple_server/storage_ha/test_transfer_with_empty_shell.cpp index 6596203bfb..e059dd030b 100644 --- a/mittest/simple_server/storage_ha/test_transfer_with_empty_shell.cpp +++ b/mittest/simple_server/storage_ha/test_transfer_with_empty_shell.cpp @@ -27,7 +27,7 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; +using namespace oceanbase::unittest; namespace oceanbase { @@ -209,7 +209,8 @@ int TestTransferHandler::wait_transfer_out_deleted_tablet_become_empty_shell( ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; const ObTransferTabletInfo &transfer_info = task.tablet_list_.at(index); - if (OB_FAIL(ls->get_tablet(transfer_info.tablet_id_, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { + if (OB_FAIL(ls->get_tablet(transfer_info.tablet_id_, tablet_handle, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, + ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("failed to get tablet", K(ret), K(transfer_info)); } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -262,9 +263,14 @@ TEST_F(TestTransferHandler, prepare_valid_data) sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select object_id from oceanbase.DBA_OBJECTS where OBJECT_NAME='ttt2'")); ASSERT_EQ(OB_SUCCESS, read_sql(sql_proxy, sql, g_batch_part_list)); + + //create other ls by cluster table + sql.reset(); + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("create table part_table_with_dup(c1 int) duplicate_scope = 'CLUSTER' partition by hash(c1) partitions 4")); + ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows)); } -TEST_F(TestTransferHandler, test_transfer_1001_to_1) +TEST_F(TestTransferHandler, test_transfer_1001_to_1002) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -285,12 +291,12 @@ TEST_F(TestTransferHandler, test_transfer_1001_to_1) ObTenantTransferService *tenant_transfer = MTL(ObTenantTransferService*); ASSERT_TRUE(OB_NOT_NULL(tenant_transfer)); - //title: 1001 ls transfer to 1 ls + //title: 1001 ls transfer to 1002 ls // generate transfer task ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1001), ObLSID(1002), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -310,7 +316,7 @@ TEST_F(TestTransferHandler, test_transfer_1001_to_1) ASSERT_EQ(OB_SUCCESS, wait_transfer_out_deleted_tablet_become_empty_shell(task)); } -TEST_F(TestTransferHandler, test_transfer_1_to_1001) +TEST_F(TestTransferHandler, test_transfer_1002_to_1001) { int ret = OB_SUCCESS; ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy(); @@ -329,7 +335,7 @@ TEST_F(TestTransferHandler, test_transfer_1_to_1001) ObTransferTaskID task_id; ObMySQLTransaction trans; ASSERT_EQ(OB_SUCCESS, trans.start(&inner_sql_proxy, g_tenant_id)); - ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1), ObLSID(1001), + ASSERT_EQ(OB_SUCCESS, tenant_transfer->generate_transfer_task(trans, ObLSID(1002), ObLSID(1001), g_part_list, ObBalanceTaskID(123), task_id)); if (trans.is_started()) { int tmp_ret = OB_SUCCESS; @@ -350,7 +356,7 @@ TEST_F(TestTransferHandler, test_transfer_1_to_1001) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp b/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp index 16fc588a41..cc4e5c305d 100644 --- a/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp +++ b/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp @@ -56,6 +56,7 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) int64_t new_transfer_seq = 1; int64_t ls_id = ObLSID::INVALID_LS_ID; int64_t transfer_seq = -1; + const int32_t group_id = 0; info.reset(); info.init(t1, ls1, table_id, old_transfer_seq); ls_infos.push_back(info); @@ -80,7 +81,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: invalid old_transfer_seq ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -90,7 +92,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) -1, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: invalid new_transfer_seq ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -100,7 +103,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, -1, - ls2); + ls2, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: old_transfer_seq == new_transfer_seq ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -110,7 +114,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, old_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: invalid tablet_id ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -120,7 +125,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: invalid old_ls_id ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -130,7 +136,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ObLSID(), new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: invalid new_ls_id ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -140,7 +147,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ObLSID()); + ObLSID(), + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_INVALID_ARGUMENT: old_ls_id == new_ls_id ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -150,7 +158,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls1); + ls1, + group_id); ASSERT_EQ(OB_INVALID_ARGUMENT, ret); // test OB_ENTRY_NOT_EXIST: t1 is not in ls3 @@ -161,7 +170,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls3, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); // test OB_ENTRY_NOT_EXIST: t1's transfer_seq should be 0, but here it's 3 old_transfer_seq = 3; @@ -172,7 +182,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); // test OB_SUCCESS: transfer t1 from ls1 to ls2 @@ -184,7 +195,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_SUCCESS, ret); // test OB_ENTRY_NOT_EXIST: t1 is not in ls1 ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -194,7 +206,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); // test OB_SUCCESS: transfer t1 from ls2 to ls3 old_transfer_seq = new_transfer_seq; @@ -206,7 +219,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls2, new_transfer_seq, - ls3); + ls3, + group_id); ASSERT_EQ(OB_SUCCESS, ret); // test OB_SUCCESS: transfer t2 from ls2 to ls1 old_transfer_seq = 0; @@ -218,7 +232,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls2, new_transfer_seq, - ls1); + ls1, + group_id); ASSERT_EQ(OB_SUCCESS, ret); // test OB_ENTRY_NOT_EXIST: t2 is not in ls2 ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -228,7 +243,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls2, new_transfer_seq, - ls1); + ls1, + group_id); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); // test OB_SUCCESS: transfer t2 from ls1 to ls2 old_transfer_seq = new_transfer_seq; @@ -240,7 +256,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls1, new_transfer_seq, - ls2); + ls2, + group_id); ASSERT_EQ(OB_SUCCESS, ret); // test OB_SUCCESS: transfer t3 from ls3 to ls1 old_transfer_seq = 0; @@ -252,7 +269,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls3, new_transfer_seq, - ls1); + ls1, + group_id); ASSERT_EQ(OB_SUCCESS, ret); // test OB_ENTRY_NOT_EXIST: t3 is not in ls3 ret = ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( @@ -262,7 +280,8 @@ TEST_F(TestTabletToLSOperator, UpdateLSAndTransSeq) old_transfer_seq, ls3, new_transfer_seq, - ls1); + ls1, + group_id); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret); // test final result in table __all_tablet_to_ls diff --git a/mittest/simple_server/test_tenant_transfer_service.cpp b/mittest/simple_server/test_tenant_transfer_service.cpp index bcb25b8e36..0bdd557e4d 100644 --- a/mittest/simple_server/test_tenant_transfer_service.cpp +++ b/mittest/simple_server/test_tenant_transfer_service.cpp @@ -176,7 +176,7 @@ TEST_F(TestTenantTransferService, test_service) } } ObTransferTask task; - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_status().is_init_status() || task.get_status().is_start_status()); ASSERT_TRUE(task.get_part_list().count() == g_part_list.count()); ARRAY_FOREACH(g_part_list, idx) { @@ -191,7 +191,7 @@ TEST_F(TestTenantTransferService, test_service) task.reset(); ObArenaAllocator allocator; ObString tablet_list_str; - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task, 0/*group_id*/)); LOG_INFO("generate tablet list", K(task)); ASSERT_TRUE(task.is_valid()); const ObTransferTabletList &tablet_list = task.get_tablet_list(); @@ -213,11 +213,11 @@ TEST_F(TestTenantTransferService, test_service) ASSERT_EQ(OB_SUCCESS, tenant_transfer->try_cancel_transfer_task(ObTransferTaskID(555))); // task which does not exist will be canceled successfully ASSERT_EQ(OB_OP_NOT_ALLOW, tenant_transfer->try_cancel_transfer_task(aborted_task_id)); ASSERT_EQ(OB_SUCCESS, tenant_transfer->try_cancel_transfer_task(init_task_id)); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, init_task_id, false, init_task)); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, init_task_id, false, init_task, 0/*group_id*/)); // try clear transfer task task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task, 0/*group_id*/)); ASSERT_EQ(OB_SUCCESS, ret); sql.reset(); ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("update oceanbase.__all_transfer_task set status = 'COMPLETED' where task_id = %ld", task.get_task_id().id())); @@ -241,7 +241,7 @@ TEST_F(TestTenantTransferService, test_service) ASSERT_EQ(OB_SUCCESS, finished_part_list.to_display_str(allocator, finished_part_list_str)); LOG_WARN("finished_part_list", K(finished_part_list_str)); ASSERT_TRUE(0 == finished_part_list_str.case_compare("500002:500003,500002:500004,500016:500014,500016:500015")); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task)); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, task_id, false, task, 0/*group_id*/)); create_time = OB_INVALID_TIMESTAMP; finish_time = OB_INVALID_TIMESTAMP; ObTransferTask history_task; @@ -274,7 +274,7 @@ TEST_F(TestTenantTransferService, test_batch_part_list) } } ObTransferTask task; - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, batch_task_id, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(inner_sql_proxy, g_tenant_id, batch_task_id, false, task, 0/*group_id*/)); ASSERT_TRUE(ObTenantTransferService::PART_COUNT_IN_A_TRANSFER == task.get_part_list().count()); ARRAY_FOREACH(task.get_part_list(), idx) { ASSERT_TRUE(is_contain(g_batch_part_list, task.get_part_list().at(idx))); diff --git a/mittest/simple_server/test_transfer_task_operator.cpp b/mittest/simple_server/test_transfer_task_operator.cpp index 45164bb034..21cd6bee0c 100644 --- a/mittest/simple_server/test_transfer_task_operator.cpp +++ b/mittest/simple_server/test_transfer_task_operator.cpp @@ -237,16 +237,16 @@ TEST_F(TestTransferTaskOperator, test_operator) // get ObTransferTask task; - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_task_id() == task_id_); ASSERT_TRUE(task.get_tablet_list().empty()); ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "Task canceled")); LOG_INFO("get from table", K(task)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, true, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, true, task, 0/*group_id*/)); LOG_INFO("get from table", K(task)); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get(sql_proxy, tenant_id_, - ObTransferTaskID(555), false, task)); + ObTransferTaskID(555), false, task, 0/*group_id*/)); // get_task_with_time int64_t create_time = OB_INVALID_TIMESTAMP; @@ -266,66 +266,66 @@ TEST_F(TestTransferTaskOperator, test_operator) // get by dest_ls task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, dest_ls_, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, dest_ls_, task, 0/*group_id*/)); ASSERT_TRUE(task_id_ == task.get_task_id()); - ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, ObLSID(555), task)); + ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, ObLSID(555), task, 0/*group_id*/)); ObTransferTask dup_dest_ls_task; ASSERT_EQ(OB_SUCCESS, dup_dest_ls_task.init(ObTransferTaskID(2223), ObLSID(1003), ObLSID(1004), part_list_, ObTransferStatus(ObTransferStatus::INIT), trace_id_, ObBalanceTaskID(2))); ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::insert(sql_proxy, tenant_id_, dup_dest_ls_task)); task.reset(); - ASSERT_EQ(OB_ERR_UNEXPECTED, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, ObLSID(1004), task)); + ASSERT_EQ(OB_ERR_UNEXPECTED, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, ObLSID(1004), task, 0/*group_id*/)); // update_to_start_status ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::update_to_start_status(sql_proxy, tenant_id_,task_id_, ObTransferStatus(ObTransferStatus::INIT), part_list_, not_exist_part_list_, lock_conflict_part_list_, table_lock_tablet_list_, tablet_list_, ObTransferStatus(ObTransferStatus::START), lock_owner_id_)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task, 0/*group_id*/)); ASSERT_TRUE(!task.get_tablet_list().empty()); LOG_INFO("update to start status", K(task)); ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::update_to_start_status(sql_proxy, tenant_id_,task_id_, ObTransferStatus(ObTransferStatus::ABORTED), part_list_, not_exist_part_list_, lock_conflict_part_list_, table_lock_tablet_list_, tablet_list_, ObTransferStatus(ObTransferStatus::START), lock_owner_id_)); // update start_scn - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::update_start_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::START), start_scn_)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::update_start_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::START), start_scn_, 0/*group_id*/)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_start_scn() == start_scn_); //ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::update_start_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::ABORTED), start_scn_)); // update status ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::update_status_and_result(sql_proxy, tenant_id_, task_id_, - ObTransferStatus(ObTransferStatus::START), ObTransferStatus(ObTransferStatus::DOING), OB_SUCCESS)); + ObTransferStatus(ObTransferStatus::START), ObTransferStatus(ObTransferStatus::DOING), OB_SUCCESS, 0/*group_id*/)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_status().status() == ObTransferStatus::DOING); //ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::update_status(sql_proxy, tenant_id_, task_id_, // ObTransferStatus(ObTransferStatus::ABORTED), ObTransferStatus(ObTransferStatus::ABORTED))); // finish task ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::finish_task(sql_proxy, tenant_id_, other_task_id, - ObTransferStatus(ObTransferStatus::INIT), ObTransferStatus(ObTransferStatus::COMPLETED), OB_SUCCESS, ObTransferTaskComment::EMPTY_COMMENT)); + ObTransferStatus(ObTransferStatus::INIT), ObTransferStatus(ObTransferStatus::COMPLETED), OB_SUCCESS, ObTransferTaskComment::EMPTY_COMMENT, 0/*group_id*/)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, other_task_id, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, other_task_id, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_status().status() == ObTransferStatus::COMPLETED); ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::finish_task(sql_proxy, tenant_id_, other_task_id, - ObTransferStatus(ObTransferStatus::ABORTED), ObTransferStatus(ObTransferStatus::ABORTED), OB_SUCCESS, ObTransferTaskComment::EMPTY_COMMENT)); + ObTransferStatus(ObTransferStatus::ABORTED), ObTransferStatus(ObTransferStatus::ABORTED), OB_SUCCESS, ObTransferTaskComment::EMPTY_COMMENT, 0/*group_id*/)); // finish task from init ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::finish_task_from_init(sql_proxy, tenant_id_, dup_dest_ls_task.get_task_id(), ObTransferStatus(ObTransferStatus::INIT), part_list_, not_exist_part_list_, lock_conflict_part_list_, ObTransferStatus(ObTransferStatus::COMPLETED), OB_SUCCESS, ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, other_task_id, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, other_task_id, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_status().status() == ObTransferStatus::COMPLETED); ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::finish_task_from_init(sql_proxy, tenant_id_, dup_dest_ls_task.get_task_id(), ObTransferStatus(ObTransferStatus::ABORTED), part_list_, not_exist_part_list_, lock_conflict_part_list_, ObTransferStatus(ObTransferStatus::COMPLETED), OB_SUCCESS, ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION)); // update finish_scn - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::update_finish_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::DOING), finish_scn_)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::update_finish_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::DOING), finish_scn_, 0/*group_id*/)); task.reset(); - ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task)); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task, 0/*group_id*/)); ASSERT_TRUE(task.get_finish_scn() == finish_scn_); - ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::update_finish_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::ABORTED), finish_scn_)); + ASSERT_EQ(OB_STATE_NOT_MATCH, ObTransferTaskOperator::update_finish_scn(sql_proxy, tenant_id_, task_id_, ObTransferStatus(ObTransferStatus::ABORTED), finish_scn_, 0/*group_id*/)); // get_all_task_status task_status.reset(); diff --git a/src/diagnose/lua/ob_lua_api.cpp b/src/diagnose/lua/ob_lua_api.cpp index ec382bf4af..2f327ecd5a 100644 --- a/src/diagnose/lua/ob_lua_api.cpp +++ b/src/diagnose/lua/ob_lua_api.cpp @@ -1902,7 +1902,7 @@ int dump_thread_info(lua_State *L) struct iovec remote_iov = {thread_base + rpc_dest_addr_offset, sizeof(ObAddr)}; wait_event[0] = '\0'; if (0 != join_addr) { - IGNORE_RETURN snprintf(wait_event, BUF_LEN, "thread %u %ld", *(uint32_t*)(thread_base + tid_offset), tid_offset); + IGNORE_RETURN snprintf(wait_event, BUF_LEN, "thread %u %ld", *(uint32_t*)(join_addr + tid_offset), tid_offset); } else if (OB_NOT_NULL(wait_addr)) { uint32_t val = 0; struct iovec local_iov = {&val, sizeof(val)}; diff --git a/src/libtable/test/ob_batch_execute_test.cpp b/src/libtable/test/ob_batch_execute_test.cpp index b69e47e5aa..e47669c465 100644 --- a/src/libtable/test/ob_batch_execute_test.cpp +++ b/src/libtable/test/ob_batch_execute_test.cpp @@ -7170,425 +7170,426 @@ TEST_F(TestBatchExecute, multi_replace) } } -TEST_F(TestBatchExecute, htable_delete) -{ - // setup - ObTable *the_table = NULL; - int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_delete"), the_table); - ASSERT_EQ(OB_SUCCESS, ret); - the_table->set_entity_type(ObTableEntityType::ET_HKV); // important - ObTableEntityFactory entity_factory; - ObTableBatchOperation batch_operation; - ObITableEntity *entity = NULL; - DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_SIZE]; - ASSERT_TRUE(NULL != rows); - static constexpr int64_t VERSIONS_COUNT = 10; - static constexpr int64_t COLUMNS_SIZE = 10; - char qualifier[COLUMNS_SIZE][128]; - char qualifier2[COLUMNS_SIZE][128]; - for (int i = 0; i < COLUMNS_SIZE; ++i) - { - sprintf(qualifier[i], "cq%d", i); - } // end for - ObObj key1, key2, key3; - ObObj value; - for (int64_t i = 0; i < BATCH_SIZE; ++i) { - sprintf(rows[i], "row%ld", i); - key1.set_varbinary(ObString::make_string(rows[i])); - for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { - key2.set_varbinary(ObString::make_string(qualifier[j])); - for (int64_t k = 0; k < VERSIONS_COUNT; ++k) - { - key3.set_int(k); - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - switch (i % 4) { - case 0: - value.set_varbinary(ObString::make_string("string2")); - break; - case 1: - value.set_varbinary(ObString::make_string("string3")); - break; - case 2: // row50 - value.set_varbinary(ObString::make_string("string0")); - break; - case 3: - value.set_varbinary(ObString::make_string("string1")); - break; - default: - ASSERT_TRUE(0); - } - ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); - ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); - } // end for - } // end for - } // end for +// unstable test cases +// TEST_F(TestBatchExecute, htable_delete) +// { +// // setup +// ObTable *the_table = NULL; +// int ret = service_client_->alloc_table(ObString::make_string("htable1_cf1_delete"), the_table); +// ASSERT_EQ(OB_SUCCESS, ret); +// the_table->set_entity_type(ObTableEntityType::ET_HKV); // important +// ObTableEntityFactory entity_factory; +// ObTableBatchOperation batch_operation; +// ObITableEntity *entity = NULL; +// DefaultBuf *rows = new (std::nothrow) DefaultBuf[BATCH_SIZE]; +// ASSERT_TRUE(NULL != rows); +// static constexpr int64_t VERSIONS_COUNT = 10; +// static constexpr int64_t COLUMNS_SIZE = 10; +// char qualifier[COLUMNS_SIZE][128]; +// char qualifier2[COLUMNS_SIZE][128]; +// for (int i = 0; i < COLUMNS_SIZE; ++i) +// { +// sprintf(qualifier[i], "cq%d", i); +// } // end for +// ObObj key1, key2, key3; +// ObObj value; +// for (int64_t i = 0; i < BATCH_SIZE; ++i) { +// sprintf(rows[i], "row%ld", i); +// key1.set_varbinary(ObString::make_string(rows[i])); +// for (int64_t j = 0; j < COLUMNS_SIZE; ++j) { +// key2.set_varbinary(ObString::make_string(qualifier[j])); +// for (int64_t k = 0; k < VERSIONS_COUNT; ++k) +// { +// key3.set_int(k); +// entity = entity_factory.alloc(); +// ASSERT_TRUE(NULL != entity); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); +// switch (i % 4) { +// case 0: +// value.set_varbinary(ObString::make_string("string2")); +// break; +// case 1: +// value.set_varbinary(ObString::make_string("string3")); +// break; +// case 2: // row50 +// value.set_varbinary(ObString::make_string("string0")); +// break; +// case 3: +// value.set_varbinary(ObString::make_string("string1")); +// break; +// default: +// ASSERT_TRUE(0); +// } +// ASSERT_EQ(OB_SUCCESS, entity->set_property(V, value)); +// ASSERT_EQ(OB_SUCCESS, batch_operation.insert(*entity)); +// } // end for +// } // end for +// } // end for - ASSERT_TRUE(!batch_operation.is_readonly()); - ASSERT_TRUE(batch_operation.is_same_type()); - ASSERT_TRUE(batch_operation.is_same_properties_names()); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - OB_LOG(INFO, "batch execute result", K(result)); - ASSERT_EQ(BATCH_SIZE*COLUMNS_SIZE*VERSIONS_COUNT, result.count()); - for (int64_t i = 0; i < BATCH_SIZE*COLUMNS_SIZE*VERSIONS_COUNT; ++i) - { - const ObTableOperationResult &r = result.at(i); - ASSERT_EQ(OB_SUCCESS, r.get_errno()); - ASSERT_EQ(1, r.get_affected_rows()); - ASSERT_EQ(ObTableOperationType::INSERT, r.type()); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); - ASSERT_TRUE(result_entity->is_empty()); - } // end for +// ASSERT_TRUE(!batch_operation.is_readonly()); +// ASSERT_TRUE(batch_operation.is_same_type()); +// ASSERT_TRUE(batch_operation.is_same_properties_names()); +// ObTableBatchOperationResult result; +// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); +// OB_LOG(INFO, "batch execute result", K(result)); +// ASSERT_EQ(BATCH_SIZE*COLUMNS_SIZE*VERSIONS_COUNT, result.count()); +// for (int64_t i = 0; i < BATCH_SIZE*COLUMNS_SIZE*VERSIONS_COUNT; ++i) +// { +// const ObTableOperationResult &r = result.at(i); +// ASSERT_EQ(OB_SUCCESS, r.get_errno()); +// ASSERT_EQ(1, r.get_affected_rows()); +// ASSERT_EQ(ObTableOperationType::INSERT, r.type()); +// const ObITableEntity *result_entity = NULL; +// ASSERT_EQ(OB_SUCCESS, r.get_entity(result_entity)); +// ASSERT_TRUE(result_entity->is_empty()); +// } // end for - //////////////////////////////////////////////////////////////// - { - fprintf(stderr, "case: delete by row\n"); - const char* rowkey = "row1"; - batch_operation.reset(); - sprintf(rows[0], "%s", rowkey); - key1.set_varbinary(ObString::make_string(rows[0])); - key2.set_null(); // delete all qualifier - key3.set_int(-INT64_MAX); // delete all version - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); +// //////////////////////////////////////////////////////////////// +// { +// fprintf(stderr, "case: delete by row\n"); +// const char* rowkey = "row1"; +// batch_operation.reset(); +// sprintf(rows[0], "%s", rowkey); +// key1.set_varbinary(ObString::make_string(rows[0])); +// key2.set_null(); // delete all qualifier +// key3.set_int(-INT64_MAX); // delete all version +// entity = entity_factory.alloc(); +// ASSERT_TRUE(NULL != entity); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); +// ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); +// ObTableBatchOperationResult result; +// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - // verify - ObTableQuery query; - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - ObObj pk_objs_start[3]; - pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - ObObj pk_objs_end[3]; - pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); +// // verify +// ObTableQuery query; +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); +// ObObj pk_objs_start[3]; +// pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_start[1].set_min_value(); +// pk_objs_start[2].set_min_value(); +// ObObj pk_objs_end[3]; +// pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_end[1].set_max_value(); +// pk_objs_end[2].set_max_value(); +// ObNewRange range; +// range.start_key_.assign(pk_objs_start, 3); +// range.end_key_.assign(pk_objs_end, 3); +// range.border_flag_.set_inclusive_start(); +// range.border_flag_.set_inclusive_end(); +// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); +// ObHTableFilter &htable_filter = query.htable_filter(); +// htable_filter.set_valid(true); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - { - fprintf(stderr, "case: delete by qualifier: cq3, cq5\n"); - const char* rowkey = "row2"; - batch_operation.reset(); - sprintf(rows[0], "%s", rowkey); - key1.set_varbinary(ObString::make_string(rows[0])); - int cqids[] = {3, 5}; - for (int64_t j = 0; j < ARRAYSIZEOF(cqids); ++j) { - sprintf(qualifier2[j], "cq%d", cqids[j]); - key2.set_varbinary(ObString::make_string(qualifier2[j])); - key3.set_int(-INT64_MAX); // delete all version - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - } - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); +// ObTableEntityIterator *iter = nullptr; +// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); +// const ObITableEntity *result_entity = NULL; +// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); +// } +// { +// fprintf(stderr, "case: delete by qualifier: cq3, cq5\n"); +// const char* rowkey = "row2"; +// batch_operation.reset(); +// sprintf(rows[0], "%s", rowkey); +// key1.set_varbinary(ObString::make_string(rows[0])); +// int cqids[] = {3, 5}; +// for (int64_t j = 0; j < ARRAYSIZEOF(cqids); ++j) { +// sprintf(qualifier2[j], "cq%d", cqids[j]); +// key2.set_varbinary(ObString::make_string(qualifier2[j])); +// key3.set_int(-INT64_MAX); // delete all version +// entity = entity_factory.alloc(); +// ASSERT_TRUE(NULL != entity); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); +// ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); +// } +// ObTableBatchOperationResult result; +// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - // verify - ObTableQuery query; - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - ObObj pk_objs_start[3]; - pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - ObObj pk_objs_end[3]; - pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - htable_filter.clear_columns(); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int cqids_sorted[] = {0, 1, 2, 4, 6, 7, 8, 9}; - int64_t timestamps[] = {9}; - for (int64_t i = 0; i < 1; ++i) { - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { - // 4 qualifier - sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); - key2.set_varbinary(ObString::make_string(qualifier2[j])); - for (int64_t k = 0; k < 1; ++k) - { - key3.set_int(timestamps[k]); - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_EQ(key3, ts); - } // end for - } - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - { - fprintf(stderr, "case: delete by qualifier & version: cq3 & version5\n"); - const char* rowkey = "row3"; - const char* cq = "cq3"; - int64_t ts = 5; - batch_operation.reset(); - sprintf(rows[0], "%s", rowkey); - key1.set_varbinary(ObString::make_string(rows[0])); - key2.set_varbinary(ObString::make_string(cq)); - key3.set_int(ts); // delete the specified version - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); +// // verify +// ObTableQuery query; +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); +// ObObj pk_objs_start[3]; +// pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_start[1].set_min_value(); +// pk_objs_start[2].set_min_value(); +// ObObj pk_objs_end[3]; +// pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_end[1].set_max_value(); +// pk_objs_end[2].set_max_value(); +// ObNewRange range; +// range.start_key_.assign(pk_objs_start, 3); +// range.end_key_.assign(pk_objs_end, 3); +// range.border_flag_.set_inclusive_start(); +// range.border_flag_.set_inclusive_end(); +// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); +// ObHTableFilter &htable_filter = query.htable_filter(); +// htable_filter.set_valid(true); +// htable_filter.clear_columns(); +// ObTableEntityIterator *iter = nullptr; +// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); +// const ObITableEntity *result_entity = NULL; +// int cqids_sorted[] = {0, 1, 2, 4, 6, 7, 8, 9}; +// int64_t timestamps[] = {9}; +// for (int64_t i = 0; i < 1; ++i) { +// key1.set_varbinary(ObString::make_string(rowkey)); +// for (int64_t j = 0; j < ARRAYSIZEOF(cqids_sorted); ++j) { +// // 4 qualifier +// sprintf(qualifier2[j], "cq%d", cqids_sorted[j]); +// key2.set_varbinary(ObString::make_string(qualifier2[j])); +// for (int64_t k = 0; k < 1; ++k) +// { +// key3.set_int(timestamps[k]); +// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); +// ObObj rk, cq, ts, val; +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); +// //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); +// ASSERT_EQ(key1, rk); +// ASSERT_EQ(key2, cq); +// ASSERT_EQ(key3, ts); +// } // end for +// } +// } +// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); +// } +// { +// fprintf(stderr, "case: delete by qualifier & version: cq3 & version5\n"); +// const char* rowkey = "row3"; +// const char* cq = "cq3"; +// int64_t ts = 5; +// batch_operation.reset(); +// sprintf(rows[0], "%s", rowkey); +// key1.set_varbinary(ObString::make_string(rows[0])); +// key2.set_varbinary(ObString::make_string(cq)); +// key3.set_int(ts); // delete the specified version +// entity = entity_factory.alloc(); +// ASSERT_TRUE(NULL != entity); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); +// ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); +// ObTableBatchOperationResult result; +// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - // verify - ObTableQuery query; - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - ObObj pk_objs_start[3]; - pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - ObObj pk_objs_end[3]; - pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - htable_filter.clear_columns(); - ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(cq))); - htable_filter.set_max_versions(INT32_MAX); - htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); +// // verify +// ObTableQuery query; +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); +// ObObj pk_objs_start[3]; +// pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_start[1].set_min_value(); +// pk_objs_start[2].set_min_value(); +// ObObj pk_objs_end[3]; +// pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_end[1].set_max_value(); +// pk_objs_end[2].set_max_value(); +// ObNewRange range; +// range.start_key_.assign(pk_objs_start, 3); +// range.end_key_.assign(pk_objs_end, 3); +// range.border_flag_.set_inclusive_start(); +// range.border_flag_.set_inclusive_end(); +// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); +// ObHTableFilter &htable_filter = query.htable_filter(); +// htable_filter.set_valid(true); +// htable_filter.clear_columns(); +// ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(cq))); +// htable_filter.set_max_versions(INT32_MAX); +// htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int64_t timestamps[] = {9, 8, 7, 6, 4, 3, 2, 1, 0}; - for (int64_t i = 0; i < 1; ++i) { - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < 1; ++j) { - key2.set_varbinary(ObString::make_string(cq)); - for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) - { - key3.set_int(timestamps[k]); - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_EQ(key3, ts); - } // end for - } - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - { - fprintf(stderr, "case: delete by qualifier & version: cq3 & INT64_MAX\n"); - const char* rowkey = "row4"; - const char* cq = "cq3"; - batch_operation.reset(); - sprintf(rows[0], "%s", rowkey); - key1.set_varbinary(ObString::make_string(rows[0])); - key2.set_varbinary(ObString::make_string(cq)); - key3.set_int(INT64_MAX); // delete the latest version - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); +// ObTableEntityIterator *iter = nullptr; +// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); +// const ObITableEntity *result_entity = NULL; +// int64_t timestamps[] = {9, 8, 7, 6, 4, 3, 2, 1, 0}; +// for (int64_t i = 0; i < 1; ++i) { +// key1.set_varbinary(ObString::make_string(rowkey)); +// for (int64_t j = 0; j < 1; ++j) { +// key2.set_varbinary(ObString::make_string(cq)); +// for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) +// { +// key3.set_int(timestamps[k]); +// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); +// ObObj rk, cq, ts, val; +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); +// //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); +// ASSERT_EQ(key1, rk); +// ASSERT_EQ(key2, cq); +// ASSERT_EQ(key3, ts); +// } // end for +// } +// } +// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); +// } +// { +// fprintf(stderr, "case: delete by qualifier & version: cq3 & INT64_MAX\n"); +// const char* rowkey = "row4"; +// const char* cq = "cq3"; +// batch_operation.reset(); +// sprintf(rows[0], "%s", rowkey); +// key1.set_varbinary(ObString::make_string(rows[0])); +// key2.set_varbinary(ObString::make_string(cq)); +// key3.set_int(INT64_MAX); // delete the latest version +// entity = entity_factory.alloc(); +// ASSERT_TRUE(NULL != entity); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); +// ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); +// ObTableBatchOperationResult result; +// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - // verify - ObTableQuery query; - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - ObObj pk_objs_start[3]; - pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - ObObj pk_objs_end[3]; - pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - htable_filter.clear_columns(); - ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(cq))); - htable_filter.set_max_versions(INT32_MAX); - htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); +// // verify +// ObTableQuery query; +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); +// ObObj pk_objs_start[3]; +// pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_start[1].set_min_value(); +// pk_objs_start[2].set_min_value(); +// ObObj pk_objs_end[3]; +// pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_end[1].set_max_value(); +// pk_objs_end[2].set_max_value(); +// ObNewRange range; +// range.start_key_.assign(pk_objs_start, 3); +// range.end_key_.assign(pk_objs_end, 3); +// range.border_flag_.set_inclusive_start(); +// range.border_flag_.set_inclusive_end(); +// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); +// ObHTableFilter &htable_filter = query.htable_filter(); +// htable_filter.set_valid(true); +// htable_filter.clear_columns(); +// ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(cq))); +// htable_filter.set_max_versions(INT32_MAX); +// htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int64_t timestamps[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; - for (int64_t i = 0; i < 1; ++i) { - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < 1; ++j) { - key2.set_varbinary(ObString::make_string(cq)); - for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) - { - key3.set_int(timestamps[k]); - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_EQ(key3, ts); - } // end for - } - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); +// ObTableEntityIterator *iter = nullptr; +// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); +// const ObITableEntity *result_entity = NULL; +// int64_t timestamps[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; +// for (int64_t i = 0; i < 1; ++i) { +// key1.set_varbinary(ObString::make_string(rowkey)); +// for (int64_t j = 0; j < 1; ++j) { +// key2.set_varbinary(ObString::make_string(cq)); +// for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) +// { +// key3.set_int(timestamps[k]); +// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); +// ObObj rk, cq, ts, val; +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); +// //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); +// ASSERT_EQ(key1, rk); +// ASSERT_EQ(key2, cq); +// ASSERT_EQ(key3, ts); +// } // end for +// } +// } +// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - { - fprintf(stderr, "case: delete by qualifier & version: cq3 & version<=5\n"); - const char* rowkey = "row5"; - const char* cq = "cq3"; - int64_t ts = 5; - batch_operation.reset(); - sprintf(rows[0], "%s", rowkey); - key1.set_varbinary(ObString::make_string(rows[0])); - key2.set_varbinary(ObString::make_string(cq)); - key3.set_int(-ts); // delete the versions < 5 - entity = entity_factory.alloc(); - ASSERT_TRUE(NULL != entity); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); - ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); - ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); +// } +// { +// fprintf(stderr, "case: delete by qualifier & version: cq3 & version<=5\n"); +// const char* rowkey = "row5"; +// const char* cq = "cq3"; +// int64_t ts = 5; +// batch_operation.reset(); +// sprintf(rows[0], "%s", rowkey); +// key1.set_varbinary(ObString::make_string(rows[0])); +// key2.set_varbinary(ObString::make_string(cq)); +// key3.set_int(-ts); // delete the versions < 5 +// entity = entity_factory.alloc(); +// ASSERT_TRUE(NULL != entity); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key1)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key2)); +// ASSERT_EQ(OB_SUCCESS, entity->add_rowkey_value(key3)); +// ASSERT_EQ(OB_SUCCESS, batch_operation.del(*entity)); - ObTableBatchOperationResult result; - ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); +// ObTableBatchOperationResult result; +// ASSERT_EQ(OB_SUCCESS, the_table->batch_execute(batch_operation, result)); - // verify - ObTableQuery query; - ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); - ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); - ObObj pk_objs_start[3]; - pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_start[1].set_min_value(); - pk_objs_start[2].set_min_value(); - ObObj pk_objs_end[3]; - pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); - pk_objs_end[1].set_max_value(); - pk_objs_end[2].set_max_value(); - ObNewRange range; - range.start_key_.assign(pk_objs_start, 3); - range.end_key_.assign(pk_objs_end, 3); - range.border_flag_.set_inclusive_start(); - range.border_flag_.set_inclusive_end(); - ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); - ObHTableFilter &htable_filter = query.htable_filter(); - htable_filter.set_valid(true); - ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(cq))); - htable_filter.set_max_versions(INT32_MAX); - htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); +// // verify +// ObTableQuery query; +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(K)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(Q)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(T)); +// ASSERT_EQ(OB_SUCCESS, query.add_select_column(V)); +// ObObj pk_objs_start[3]; +// pk_objs_start[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_start[1].set_min_value(); +// pk_objs_start[2].set_min_value(); +// ObObj pk_objs_end[3]; +// pk_objs_end[0].set_varbinary(ObString::make_string(rowkey)); +// pk_objs_end[1].set_max_value(); +// pk_objs_end[2].set_max_value(); +// ObNewRange range; +// range.start_key_.assign(pk_objs_start, 3); +// range.end_key_.assign(pk_objs_end, 3); +// range.border_flag_.set_inclusive_start(); +// range.border_flag_.set_inclusive_end(); +// ASSERT_EQ(OB_SUCCESS, query.add_scan_range(range)); +// ObHTableFilter &htable_filter = query.htable_filter(); +// htable_filter.set_valid(true); +// ASSERT_EQ(OB_SUCCESS, htable_filter.add_column(ObString::make_string(cq))); +// htable_filter.set_max_versions(INT32_MAX); +// htable_filter.set_time_range(ObHTableConstants::INITIAL_MIN_STAMP, ObHTableConstants::INITIAL_MAX_STAMP); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - int64_t timestamps[] = {9, 8, 7, 6}; - for (int64_t i = 0; i < 1; ++i) { - key1.set_varbinary(ObString::make_string(rowkey)); - for (int64_t j = 0; j < 1; ++j) { - key2.set_varbinary(ObString::make_string(cq)); - for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) - { - key3.set_int(timestamps[k]); - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ObObj rk, cq, ts, val; - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_EQ(key3, ts); - } // end for - } - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); - } - //////////////////////////////////////////////////////////////// - // teardown - service_client_->free_table(the_table); - the_table = NULL; - delete [] rows; -} +// ObTableEntityIterator *iter = nullptr; +// ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); +// const ObITableEntity *result_entity = NULL; +// int64_t timestamps[] = {9, 8, 7, 6}; +// for (int64_t i = 0; i < 1; ++i) { +// key1.set_varbinary(ObString::make_string(rowkey)); +// for (int64_t j = 0; j < 1; ++j) { +// key2.set_varbinary(ObString::make_string(cq)); +// for (int64_t k = 0; k < ARRAYSIZEOF(timestamps); ++k) +// { +// key3.set_int(timestamps[k]); +// ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); +// ObObj rk, cq, ts, val; +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); +// ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); +// //fprintf(stderr, "(%s,%s,%s,%s)\n", S(rk), S(cq), S(ts), S(val)); +// ASSERT_EQ(key1, rk); +// ASSERT_EQ(key2, cq); +// ASSERT_EQ(key3, ts); +// } // end for +// } +// } +// ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); +// } +// ////////////////////////////////////////////////////////////// +// teardown +// service_client_->free_table(the_table); +// the_table = NULL; +// delete [] rows; +// } TEST_F(TestBatchExecute, complex_batch_execute) { @@ -11952,28 +11953,29 @@ TEST_F(TestBatchExecute, htable_check_and_put) ASSERT_EQ(0, result.affected_rows_); } } + // unstable test cases // execute query and verify result - htable_filter.reset(); - htable_filter.add_column(ObString::make_string(qualifier)); - htable_filter.set_max_versions(INT32_MAX); // get all versions - htable_filter.set_valid(true); - ObTableEntityIterator *iter = nullptr; - ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); - const ObITableEntity *result_entity = NULL; - ObObj rk, cq, ts, val; - ObString str; - for (int j = 0; j < 2; j++) { - ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); - ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); - ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); - ASSERT_EQ(key1, rk); - ASSERT_EQ(key2, cq); - ASSERT_TRUE(str.compare(values[1-j]) == 0); - } - ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); + // htable_filter.reset(); + // htable_filter.add_column(ObString::make_string(qualifier)); + // htable_filter.set_max_versions(INT32_MAX); // get all versions + // htable_filter.set_valid(true); + // ObTableEntityIterator *iter = nullptr; + // ASSERT_EQ(OB_SUCCESS, the_table->execute_query(query, iter)); + // const ObITableEntity *result_entity = NULL; + // ObObj rk, cq, ts, val; + // ObString str; + // for (int j = 0; j < 2; j++) { + // ASSERT_EQ(OB_SUCCESS, iter->get_next_entity(result_entity)); + // ASSERT_EQ(OB_SUCCESS, result_entity->get_property(K, rk)); + // ASSERT_EQ(OB_SUCCESS, result_entity->get_property(Q, cq)); + // ASSERT_EQ(OB_SUCCESS, result_entity->get_property(T, ts)); + // ASSERT_EQ(OB_SUCCESS, result_entity->get_property(V, val)); + // ASSERT_EQ(OB_SUCCESS, val.get_varbinary(str)); + // ASSERT_EQ(key1, rk); + // ASSERT_EQ(key2, cq); + // ASSERT_TRUE(str.compare(values[1-j]) == 0); + // } + // ASSERT_EQ(OB_ITER_END, iter->get_next_entity(result_entity)); //////////////////////////////////////////////////////////////// // teardown service_client_->free_table(the_table); diff --git a/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp b/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp index 20e08b2709..1667b1d0e1 100644 --- a/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_sql_client.cpp @@ -14,12 +14,13 @@ #include "ob_data_dict_sql_client.h" -#include "lib/mysqlclient/ob_isql_client.h" // ObISQLClient +#include "lib/mysqlclient/ob_mysql_proxy.h" // ObMySQLProxy #include "lib/string/ob_sql_string.h" // ObSqlString #include "common/ob_smart_var.h" // SMART_VAR #include "share/inner_table/ob_inner_table_schema_constants.h" #include "share/ob_define.h" #include "share/ob_ls_id.h" +#include "share/ls/ob_ls_operator.h" // ObLSAttrOperator #define IF_CLIENT_VALID \ if (IS_NOT_INIT) { \ @@ -39,8 +40,6 @@ namespace oceanbase namespace datadict { -const char *ObDataDictSqlClient::query_ls_info_sql_format = - "SELECT LS_ID FROM %s AS OF SNAPSHOT %lu WHERE STATUS = \"NORMAL\""; const char *ObDataDictSqlClient::query_tenant_schema_version_sql_format = "SELECT MAX(SCHEMA_VERSION) AS SCHEMA_VERSION FROM %s AS OF SNAPSHOT %lu"; const char *ObDataDictSqlClient::report_data_dict_persist_info_sql_format = @@ -51,7 +50,7 @@ ObDataDictSqlClient::ObDataDictSqlClient() sql_proxy_(NULL) {} -int ObDataDictSqlClient::init(ObISQLClient *mysql_client) +int ObDataDictSqlClient::init(ObMySQLProxy *mysql_client) { int ret = OB_SUCCESS; @@ -80,28 +79,22 @@ int ObDataDictSqlClient::get_ls_info( int ret = OB_SUCCESS; IF_CLIENT_VALID { - if (OB_UNLIKELY(! snapshot_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - DDLOG(WARN, "invalid snapshot_scn to get_ls_info", KR(ret), K(snapshot_scn)); - } else { - ObSqlString sql; - int64_t record_count; - uint64_t gts_ts = snapshot_scn.get_val_for_inner_table_field(); + share::ObLSAttrOperator ls_attr_op(tenant_id, sql_proxy_); + share::ObLSAttrArray ls_attr_arr; - SMART_VAR(ObISQLClient::ReadResult, result) { - if (OB_FAIL(sql.assign_fmt(query_ls_info_sql_format, - OB_ALL_LS_TNAME, gts_ts))) { - DDLOG(WARN, "assign_fmt to sql_string failed", KR(ret), - K(tenant_id), K(snapshot_scn), K(gts_ts)); - } else if (OB_FAIL(sql_proxy_->read(result, tenant_id, sql.ptr()))) { - DDLOG(WARN, "read from sql_proxy_ for ls_info failed", KR(ret), - K(tenant_id), "sql", sql.ptr()); - } else if (OB_ISNULL(result.get_result())) { - ret = OB_ERR_UNEXPECTED; - DDLOG(WARN, "get sql result failed", KR(ret), "sql", sql.ptr()); - } else if (OB_FAIL(parse_record_from_row_(*result.get_result(), record_count, ls_array))) { - DDLOG(WARN, "parse_record_from_row_ for ls_array failed", KR(ret), - K(tenant_id), K(snapshot_scn), "sql", sql.ptr()); + if (OB_FAIL(ls_attr_op.load_all_ls_and_snapshot( + snapshot_scn, + ls_attr_arr, + true/*only_existing_ls*/))) { + DDLOG(WARN, "load_all_ls_and_snapshot failed", KR(ret), K(tenant_id), K(snapshot_scn)); + } else { + ARRAY_FOREACH(ls_attr_arr, ls_attr_idx) { + share::ObLSAttr &ls_attr = ls_attr_arr[ls_attr_idx]; + if (ls_attr.is_valid() + && ! ls_attr.ls_is_creating() // load_all_ls_and_snapshot will filter abort and dropped ls + && OB_FAIL(ls_array.push_back(ls_attr.get_ls_id()))) { + DDLOG(WARN, "push_back normal ls into ls_array failed", KR(ret), + K(tenant_id), K(snapshot_scn), K(ls_attr), K(ls_attr_idx), K(ls_attr_arr)); } } } @@ -187,43 +180,6 @@ int ObDataDictSqlClient::report_data_dict_persist_info( return ret; } -int ObDataDictSqlClient::parse_record_from_row_( - common::sqlclient::ObMySQLResult &result, - int64_t &record_count, - share::ObLSArray &ls_array) -{ - int ret = OB_SUCCESS; - - IF_CLIENT_VALID { - while (OB_SUCC(ret)) { - if (OB_FAIL(result.next())) { - if (OB_ITER_END != ret) { - DDLOG(WARN, "get next result failed", KR(ret), K(record_count)); - } - } else { - int64_t ls_id = share::ObLSID::INVALID_LS_ID; - - (void)GET_COL_IGNORE_NULL(result.get_int, "LS_ID", ls_id); - - if (OB_UNLIKELY(share::ObLSID::INVALID_LS_ID == ls_id)) { - ret = OB_ERR_UNEXPECTED; - DDLOG(WARN, "invalid ls_id fetched from DBA_OB_LS", KR(ret), K(ls_id), K(record_count)); - } else if (OB_FAIL(ls_array.push_back(ObLSID(ls_id)))) { - DDLOG(WARN, "push_back ls_id into ls_array failed", KR(ret), K(record_count)); - } else { - record_count++; - } - } - } // while - - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - } - - return ret; -} - int ObDataDictSqlClient::parse_record_from_row_( common::sqlclient::ObMySQLResult &result, int64_t &record_count, diff --git a/src/logservice/data_dictionary/ob_data_dict_sql_client.h b/src/logservice/data_dictionary/ob_data_dict_sql_client.h index 0164a9b941..77a587a06c 100644 --- a/src/logservice/data_dictionary/ob_data_dict_sql_client.h +++ b/src/logservice/data_dictionary/ob_data_dict_sql_client.h @@ -23,7 +23,7 @@ namespace oceanbase { namespace common { - class ObISQLClient; + class ObMySQLProxy; } namespace datadict { @@ -34,7 +34,7 @@ public: ObDataDictSqlClient(); ~ObDataDictSqlClient() { destroy(); } public: - int init(common::ObISQLClient *sql_client); + int init(common::ObMySQLProxy *sql_client); void destroy(); public: int get_ls_info( @@ -56,17 +56,13 @@ private: static const char *query_tenant_schema_version_sql_format; static const char *report_data_dict_persist_info_sql_format; private: - int parse_record_from_row_( - common::sqlclient::ObMySQLResult &result, - int64_t &record_count, - share::ObLSArray &ls_array); int parse_record_from_row_( common::sqlclient::ObMySQLResult &result, int64_t &record_count, int64_t &schema_version); private: bool is_inited_; - common::ObISQLClient *sql_proxy_; + common::ObMySQLProxy *sql_proxy_; }; } // namespace datadict diff --git a/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.cpp b/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.cpp index c3c93e1e08..a05c0a9945 100644 --- a/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.cpp +++ b/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX OBLOG #include "ob_cdc_lob_aux_meta_storager.h" +#include "ob_log_store_service.h" // IObStoreService using namespace oceanbase::common; @@ -24,7 +25,11 @@ int LobAuxMetaKey::compare(const LobAuxMetaKey &other) const { int cmp_ret = 0; - if (tenant_id_ > other.tenant_id_) { + if (commit_version_ > other.commit_version_) { + cmp_ret = 1; + } else if (commit_version_ < other.commit_version_) { + cmp_ret = -1; + } else if (tenant_id_ > other.tenant_id_) { cmp_ret = 1; } else if (tenant_id_ < other.tenant_id_) { cmp_ret = -1; @@ -49,10 +54,45 @@ int LobAuxMetaKey::compare(const LobAuxMetaKey &other) const return cmp_ret; } +int LobAuxMetaKey::get_key(std::string &key) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("lob store key is not valid", KR(ret), KPC(this)); + } else { + key.append(std::to_string(commit_version_)); + // to isolate tenant resources, each tenant data store in separate column famliy + // so no need to add tenant_id_ in key + // key.append(std::to_string(tenant_id_)); + key.append("_"); + key.append(std::to_string(trans_id_.get_id())); + key.append("_"); + key.append(std::to_string(aux_lob_meta_tid_)); + key.append("_"); + key.append(std::to_string(lob_id_.tablet_id_)); + key.append("_"); + key.append(std::to_string(lob_id_.lob_id_)); + key.append("_"); + key.append(std::to_string(seq_no_.cast_to_int())); + } + return ret; +} + +void ObCDCLobAuxDataCleanTask::reset() +{ + is_task_push_ = false; + commit_version_ = OB_INVALID_VERSION; + tenant_id_ = OB_INVALID_TENANT_ID; +} + ObCDCLobAuxMetaStorager::ObCDCLobAuxMetaStorager() : is_inited_(false), lob_aux_meta_map_(), - lob_aux_meta_allocator_() + lob_aux_meta_allocator_(), + store_service_(nullptr), + memory_limit_(0), + enable_memory_(true) { } @@ -62,25 +102,33 @@ ObCDCLobAuxMetaStorager::~ObCDCLobAuxMetaStorager() destroy(); } -int ObCDCLobAuxMetaStorager::init() +int ObCDCLobAuxMetaStorager::init(IObStoreService *store_service) { int ret = OB_SUCCESS; - + enable_memory_ = TCONF.enable_lob_data_storage_memory; + memory_limit_ = TCONF.lob_data_storage_memory_limit.get(); if (IS_INIT) { ret = OB_INIT_TWICE; LOG_ERROR("ObCDCLobAuxMetaStorager has been initialized", KR(ret)); - } else if (OB_FAIL(lob_aux_meta_map_.init("LobAuxMetaMap"))) { - LOG_ERROR("lob_aux_meta_map_ init failed", KR(ret)); - } else if (OB_FAIL(lob_aux_meta_allocator_.init(LOB_AUX_META_ALLOCATOR_TOTAL_LIMIT, - LOB_AUX_META_ALLOCATOR_HOLD_LIMIT, - LOB_AUX_META_ALLOCATOR_PAGE_SIZE))) { - LOG_ERROR("init lob_aux_meta_allocator_ fail", KR(ret)); - } else { - lob_aux_meta_allocator_.set_label("LobAuxMetaData"); - + } else if (enable_memory_) { + if (memory_limit_ <= 0) { + LOG_ERROR("invalid arguments", KR(ret), K(memory_limit_)); + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(lob_aux_meta_map_.init("LobAuxMetaMap"))) { + LOG_ERROR("lob_aux_meta_map_ init failed", KR(ret)); + } else if (OB_FAIL(lob_aux_meta_allocator_.init(memory_limit_, + LOB_AUX_META_ALLOCATOR_HOLD_LIMIT, + LOB_AUX_META_ALLOCATOR_PAGE_SIZE))) { + LOG_ERROR("init lob_aux_meta_allocator_ fail", KR(ret)); + } else { + lob_aux_meta_allocator_.set_label("LobAuxMetaData"); + } + } + if (OB_SUCC(ret)) { + store_service_ = store_service; is_inited_ = true; - LOG_INFO("ObCDCLobAuxMetaStorager init succ"); + LOG_INFO("ObCDCLobAuxMetaStorager init succ", K(enable_memory_), K(memory_limit_)); } return ret; @@ -89,12 +137,22 @@ int ObCDCLobAuxMetaStorager::init() void ObCDCLobAuxMetaStorager::destroy() { if (is_inited_) { - lob_aux_meta_map_.destroy(); - lob_aux_meta_allocator_.destroy(); + if (enable_memory_) { + lob_aux_meta_map_.destroy(); + lob_aux_meta_allocator_.destroy(); + memory_limit_ = 0; + enable_memory_ = false; + } + store_service_ = nullptr; is_inited_ = false; } } +bool ObCDCLobAuxMetaStorager::is_need_to_disk_(int64_t alloc_size) +{ + return lob_aux_meta_allocator_.allocated() + alloc_size > memory_limit_; +} + int ObCDCLobAuxMetaStorager::put( const LobAuxMetaKey &key, const char *key_type, @@ -110,28 +168,95 @@ int ObCDCLobAuxMetaStorager::put( || OB_ISNULL(lob_data) || OB_UNLIKELY(lob_data_len <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_ERROR("invalid argument", KR(ret), K(key), K(lob_data), K(lob_data_len)); - } else { - void *alloc_lob_data = lob_aux_meta_allocator_.alloc(lob_data_len); - - if (OB_ISNULL(alloc_lob_data)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("alloc memory failed", KR(ret), K(key), K(lob_data), K(lob_data_len)); - } else { - memcpy(alloc_lob_data, lob_data, lob_data_len); - LobAuxMetaValue lob_aux_meta_value(static_cast(alloc_lob_data), lob_data_len); - - if (OB_FAIL(lob_aux_meta_map_.insert(key, lob_aux_meta_value))) { - LOG_ERROR("lob_aux_meta_map_ insert failed", KR(ret), K(key), K(lob_data), K(lob_data_len)); - } else { - LOG_DEBUG("lob_aux_meta_map_ insert succ", K(key), K(lob_aux_meta_value)); - } + } else if (! enable_memory_ || is_need_to_disk_(lob_data_len)) { + if (OB_FAIL(disk_put_(key, key_type, lob_data, lob_data_len))) { + LOG_ERROR("disk_put_ failed", KR(ret), K(key), K(lob_data), K(lob_data_len)); + } + } else if (OB_FAIL(memory_put_(key, key_type, lob_data, lob_data_len))) { + if (OB_ALLOCATE_MEMORY_FAILED != ret) { + LOG_ERROR("memory_put_ failed", KR(ret), K(key), K(lob_data), K(lob_data_len)); + } else if (OB_FAIL(disk_put_(key, key_type, lob_data, lob_data_len))) { + LOG_ERROR("disk_put_ failed", KR(ret), K(key), K(lob_data), K(lob_data_len)); } } + return ret; +} +int ObCDCLobAuxMetaStorager::memory_put_( + const LobAuxMetaKey &key, + const char *key_type, + const char *lob_data, + const int64_t lob_data_len) +{ + int ret = OB_SUCCESS; + void *alloc_lob_data = lob_aux_meta_allocator_.alloc(lob_data_len); + if (OB_ISNULL(alloc_lob_data)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", KR(ret), K(key), K(lob_data), K(lob_data_len)); + } else { + memcpy(alloc_lob_data, lob_data, lob_data_len); + LobAuxMetaValue value(static_cast(alloc_lob_data), lob_data_len); + if (OB_FAIL(lob_aux_meta_map_.insert(key, value))) { + LOG_ERROR("lob_aux_meta_map_ insert failed", KR(ret), KCSTRING(key_type), K(key), K(lob_data), K(lob_data_len)); + } else { + LOG_DEBUG("lob_aux_meta_map_ insert succ", KCSTRING(key_type), K(key), K(value)); + } + } + return ret; +} + +int ObCDCLobAuxMetaStorager::disk_put_( + const LobAuxMetaKey &key, + const char *key_type, + const char *lob_data, + const int64_t lob_data_len) +{ + int ret = OB_SUCCESS; + std::string key_str; + void *cf_handle = nullptr; + if (OB_FAIL(get_cf_handle(key.tenant_id_, cf_handle))) { + LOG_ERROR("get_cf_handle fail", KR(ret), KCSTRING(key_type), K(key)); + } else if (OB_FAIL(key.get_key(key_str))) { + LOG_ERROR("get key_str fail", KR(ret), KCSTRING(key_type), K(key)); + } else if (OB_FAIL(store_service_->put(cf_handle, key_str, ObSlice(lob_data, lob_data_len)))) { + if (OB_IN_STOP_STATE != ret) { + LOG_ERROR("store_service_ put fail", KR(ret), KCSTRING(key_type), K(key), KCSTRING(key_str.c_str()), K(lob_data_len)); + } + } else { + LOG_DEBUG("store_service_ insert succ", KCSTRING(key_type), K(key), KCSTRING(key_str.c_str()), K(lob_data_len)); + } return ret; } int ObCDCLobAuxMetaStorager::get( + ObIAllocator &allocator, + const LobAuxMetaKey &key, + const char *&lob_data, + int64_t &lob_data_len) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObCDCLobAuxMetaStorager has not been initialized", KR(ret)); + } else if (! enable_memory_) { + if (OB_FAIL(disk_get_(allocator, key, lob_data, lob_data_len))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("disk_get_ failed", KR(ret), K(key)); + } + } + } else if (OB_FAIL(memory_get_(key, lob_data, lob_data_len))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("memory_get_ failed", KR(ret), K(key)); + } else if (OB_FAIL(disk_get_(allocator, key, lob_data, lob_data_len))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("disk_get_ failed", KR(ret), K(key)); + } + } + } + return ret; +} + +int ObCDCLobAuxMetaStorager::memory_get_( const LobAuxMetaKey &key, const char *&lob_data, int64_t &lob_data_len) @@ -139,22 +264,52 @@ int ObCDCLobAuxMetaStorager::get( int ret = OB_SUCCESS; lob_data = nullptr; lob_data_len = 0; - LobAuxMetaValue lob_aux_meta_value; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_ERROR("ObCDCLobAuxMetaStorager has not been initialized", KR(ret)); - } else if (OB_FAIL(lob_aux_meta_map_.get(key, lob_aux_meta_value))) { + LobAuxMetaValue value; + if (OB_FAIL(lob_aux_meta_map_.get(key, value))) { if (OB_ENTRY_NOT_EXIST != ret) { LOG_ERROR("lob_aux_meta_map_ get failed", KR(ret), K(key)); } else if (REACH_TIME_INTERVAL(10 * _SEC_)) { LOG_WARN("lob_aux_meta_map_ get not exist, need retry", KR(ret), K(key)); } } else { - lob_data = lob_aux_meta_value.lob_data_; - lob_data_len = lob_aux_meta_value.lob_data_len_; + lob_data = value.lob_data_; + lob_data_len = value.lob_data_len_; } + return ret; +} +int ObCDCLobAuxMetaStorager::disk_get_( + ObIAllocator &allocator, + const LobAuxMetaKey &key, + const char *&lob_data, + int64_t &lob_data_len) +{ + int ret = OB_SUCCESS; + std::string key_str; + std::string value; + char *lob_data_ptr = nullptr; + void *cf_handle = nullptr; + if (OB_FAIL(get_cf_handle(key.tenant_id_, cf_handle))) { + LOG_ERROR("get_cf_handle fail", KR(ret), K(key)); + } else if (OB_FAIL(key.get_key(key_str))) { + LOG_ERROR("get key_str fail", KR(ret), K(key)); + } else if (OB_FAIL(store_service_->get(cf_handle, key_str, value))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("get failed", KR(ret), K(key)); + } else if (REACH_TIME_INTERVAL(10 * _SEC_)) { + LOG_WARN("get not exist, need retry", KR(ret), K(key)); + } + } else if (value.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("data is emptry", KR(ret), K(key)); + } else if (OB_ISNULL(lob_data_ptr = static_cast(allocator.alloc(value.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc fail", KR(ret), "size", value.length(), K(key)); + } else { + MEMCPY(lob_data_ptr, value.c_str(), value.length()); + lob_data = lob_data_ptr; + lob_data_len = value.length(); + } return ret; } @@ -164,6 +319,8 @@ int ObCDCLobAuxMetaStorager::del( { int ret = OB_SUCCESS; LOG_DEBUG("ObCDCLobAuxMetaStorager del", K(lob_data_out_row_ctx_list)); + const PartTransTask &part_trans_task = lob_data_out_row_ctx_list.get_dml_stmt_task()->get_host(); + const int64_t commit_version = part_trans_task.get_trans_commit_version(); const uint64_t tenant_id = lob_data_out_row_ctx_list.get_tenant_id(); const transaction::ObTransID &trans_id = lob_data_out_row_ctx_list.get_trans_id(); const uint64_t aux_lob_meta_tid = lob_data_out_row_ctx_list.get_aux_lob_meta_table_id(); @@ -171,7 +328,7 @@ int ObCDCLobAuxMetaStorager::del( ObLobDataGetCtx *cur_lob_data_get_ctx = lob_data_get_ctx_list.head_; while (OB_SUCC(ret) && ! stop_flag && cur_lob_data_get_ctx) { - if (OB_FAIL(del_lob_col_value_(tenant_id, trans_id, aux_lob_meta_tid, *cur_lob_data_get_ctx, stop_flag))) { + if (OB_FAIL(del_lob_col_value_(commit_version, tenant_id, trans_id, aux_lob_meta_tid, *cur_lob_data_get_ctx, stop_flag))) { LOG_ERROR("del_lob_col_value_ failed", KR(ret), K(tenant_id), K(trans_id), K(aux_lob_meta_tid)); } else { cur_lob_data_get_ctx = cur_lob_data_get_ctx->get_next(); @@ -186,6 +343,7 @@ int ObCDCLobAuxMetaStorager::del( } int ObCDCLobAuxMetaStorager::del_lob_col_value_( + const int64_t commit_version, const uint64_t tenant_id, const transaction::ObTransID &trans_id, const uint64_t aux_lob_meta_tid, @@ -207,24 +365,9 @@ int ObCDCLobAuxMetaStorager::del_lob_col_value_( const ObLobId &lob_id = lob_data_get_ctx.get_new_lob_data()->id_; for (int64_t idx = 0; OB_SUCC(ret) && idx < seq_no_cnt; ++idx, ++seq_no) { - LobAuxMetaKey lob_aux_meta_key(tenant_id, trans_id, aux_lob_meta_tid, lob_id, seq_no); - LobAuxMetaValue lob_aux_meta_value; - - if (OB_FAIL(lob_aux_meta_map_.erase(lob_aux_meta_key, lob_aux_meta_value))) { - if (OB_ENTRY_NOT_EXIST != ret) { - LOG_ERROR("lob_aux_meta_map_ erase failed", KR(ret), K(lob_aux_meta_value)); - } else { - ret = OB_SUCCESS; - } - } else { - char *lob_data_ptr = lob_aux_meta_value.lob_data_; - - if (OB_ISNULL(lob_data_ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("lob_data_ptr is nullptr", KR(ret), K(lob_data_get_ctx)); - } else { - lob_aux_meta_allocator_.free(lob_data_ptr); - } + LobAuxMetaKey key(commit_version, tenant_id, trans_id, aux_lob_meta_tid, lob_id, seq_no); + if (OB_FAIL(del(key))) { + LOG_ERROR("del fail", KR(ret), K(key)); } } // for } @@ -232,22 +375,152 @@ int ObCDCLobAuxMetaStorager::del_lob_col_value_( return ret; } -int ObCDCLobAuxMetaStorager::del( - const uint64_t tenant_id, - const transaction::ObTransID &trans_id) +int ObCDCLobAuxMetaStorager::del(const LobAuxMetaKey &key) { int ret = OB_SUCCESS; - struct LobAuxMetaDataPurger lob_aux_meata_purger(*this, tenant_id, trans_id); - - if (OB_FAIL(lob_aux_meta_map_.remove_if(lob_aux_meata_purger))) { - LOG_ERROR("lob_aux_meta_map_ remove_if failed", KR(ret), K(tenant_id), K(trans_id)); - } else { - if (lob_aux_meata_purger.purge_count_ > 0) { - LOG_INFO("ObCDCLobAuxMetaStorager del", K(tenant_id), K(trans_id), "purge_count", - lob_aux_meata_purger.purge_count_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_ERROR("ObCDCLobAuxMetaStorager has not been initialized", KR(ret)); + } else if (! enable_memory_) { + if (OB_FAIL(disk_del_(key))) { + LOG_ERROR("disk_del_ fail", KR(ret), K(key)); + } + } else if (OB_FAIL(memory_del_(key))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("memory_get_ fail", KR(ret), K(key)); + } else if (OB_FAIL(disk_del_(key))) { + LOG_ERROR("disk_del_ fail", KR(ret), K(key)); } } + return ret; +} +int ObCDCLobAuxMetaStorager::memory_del_(const LobAuxMetaKey &key) +{ + int ret = OB_SUCCESS; + LobAuxMetaValue value; + if (OB_FAIL(lob_aux_meta_map_.erase(key, value))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("lob_aux_meta_map_ erase failed", KR(ret), K(key)); + } + } else { + char *lob_data_ptr = value.lob_data_; + if (OB_ISNULL(lob_data_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("lob_data_ptr is nullptr", KR(ret), K(key)); + } else { + lob_aux_meta_allocator_.free(lob_data_ptr); + } + } + return ret; +} + +int ObCDCLobAuxMetaStorager::disk_del_(const LobAuxMetaKey &key) +{ + int ret = OB_SUCCESS; + std::string key_str; + void *cf_handle = nullptr; + if (OB_FAIL(get_cf_handle(key.tenant_id_, cf_handle))) { + LOG_ERROR("get_cf_handle fail", KR(ret), K(key)); + } else if (OB_FAIL(key.get_key(key_str))) { + LOG_ERROR("get key_str fail", KR(ret), K(key)); + } else if (OB_FAIL(store_service_->del(cf_handle, key_str))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_ERROR("del failed", KR(ret), K(key)); + } else { + ret = OB_SUCCESS; + } + } else { + LOG_DEBUG("store_service_ del succ", K(key), KCSTRING(key_str.c_str())); + } + return ret; +} + +int ObCDCLobAuxMetaStorager::clean_unused_data(ObCDCLobAuxDataCleanTask *task) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(do_clean_unused_data_(task))) { + LOG_WARN("do_clean_unused_data_ fail", KR(ret)); + } else { + ATOMIC_STORE(&task->is_task_push_, false); + } + return ret; +} + +int ObCDCLobAuxMetaStorager::do_clean_unused_data_(ObCDCLobAuxDataCleanTask *task) +{ + int ret = OB_SUCCESS; + void *cf_handle = nullptr; + if (enable_memory_ && OB_FAIL(memory_del_(task->tenant_id_, task->commit_version_))) { + LOG_ERROR("memory_del_ failed", KR(ret), KPC(task)); + } else if (OB_FAIL(get_cf_handle(task->tenant_id_, cf_handle))) { + LOG_ERROR("get_cf_handle fail", KR(ret), KPC(task)); + } else if (OB_FAIL(disk_del_(cf_handle, task->commit_version_))) { + LOG_ERROR("disk_del_ failed", KR(ret), KPC(task)); + } + return ret; +} + +int ObCDCLobAuxMetaStorager::memory_del_(const uint64_t tenant_id, const int64_t commit_version) +{ + int ret = OB_SUCCESS; + LobAuxMetaDataPurger lob_aux_meata_purger(*this, tenant_id, commit_version); + if (OB_FAIL(lob_aux_meta_map_.remove_if(lob_aux_meata_purger))) { + LOG_ERROR("lob_aux_meta_map_ remove_if failed", KR(ret), K(tenant_id), K(commit_version)); + } else { + LOG_INFO("ObCDCLobAuxMetaStorager del", K(tenant_id), K(commit_version), "purge_count", + lob_aux_meata_purger.purge_count_, "map_count", lob_aux_meta_map_.count(), + "memory_used", lob_aux_meta_allocator_.allocated()); + } + return ret; +} + +int ObCDCLobAuxMetaStorager::disk_del_(void* cf_family, const int64_t commit_version) +{ + int ret = OB_SUCCESS; + std::string begin_key; + std::string end_key; + // some trans may have same commit_version, + // to avoid accidentally deleting uncommitted transactions, + // we set end_key to "commit_version - 1". + end_key.append(std::to_string(commit_version - 1)); + end_key.append("_}"); + if (OB_FAIL(store_service_->del_range(cf_family, begin_key, end_key))) { + if (OB_IN_STOP_STATE != ret) { + LOG_ERROR("store_service_ del fail", KR(ret), "begin_key", begin_key.c_str(), + "end_key", end_key.c_str(), K(commit_version)); + } + } else { + LOG_INFO("store_service_ del_range succ", K(commit_version), KCSTRING(end_key.c_str())); + } + return ret; +} + +int ObCDCLobAuxMetaStorager::get_cf_handle(const uint64_t tenant_id, void *&cf_handle) +{ + int ret = OB_SUCCESS; + ObLogTenantGuard guard; + ObLogTenant *tenant = nullptr; + if (OB_FAIL(TCTX.get_tenant_guard(tenant_id, guard))) { + LOG_ERROR("get_tenant_guard fail", KR(ret), K(tenant_id)); + } else { + tenant = guard.get_tenant(); + cf_handle = tenant->get_lob_storage_cf_handle(); + } + return ret; +} + +int ObCDCLobAuxMetaStorager::get_clean_task(const uint64_t tenant_id, ObCDCLobAuxDataCleanTask *&task) +{ + int ret = OB_SUCCESS; + ObLogTenantGuard guard; + ObLogTenant *tenant = nullptr; + if (OB_FAIL(TCTX.get_tenant_guard(tenant_id, guard))) { + LOG_ERROR("get_tenant_guard fail", KR(ret), K(tenant_id)); + } else { + tenant = guard.get_tenant(); + task = &tenant->get_lob_storage_clean_task(); + } return ret; } @@ -257,7 +530,7 @@ bool ObCDCLobAuxMetaStorager::LobAuxMetaDataPurger::operator()( { int ret = OB_SUCCESS; const bool need_purge = (tenant_id_ == lob_aux_meta_key.tenant_id_) - && (trans_id_ == lob_aux_meta_key.trans_id_); + && (commit_version_ > lob_aux_meta_key.commit_version_); if (need_purge) { char *lob_data_ptr = lob_aux_meta_value.lob_data_; @@ -268,6 +541,7 @@ bool ObCDCLobAuxMetaStorager::LobAuxMetaDataPurger::operator()( } else { host_.lob_aux_meta_allocator_.free(lob_data_ptr); ++purge_count_; + LOG_DEBUG("purge succ", K(lob_aux_meta_key), K(purge_count_)); } } diff --git a/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.h b/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.h index 6f62a26fbb..9ca4c3a9ed 100644 --- a/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.h +++ b/src/logservice/libobcdc/src/ob_cdc_lob_aux_meta_storager.h @@ -27,11 +27,13 @@ namespace libobcdc struct LobAuxMetaKey { LobAuxMetaKey( + const int64_t commit_version, const uint64_t tenant_id, const transaction::ObTransID &trans_id, const uint64_t aux_lob_meta_tid, const common::ObLobId &lob_id, const transaction::ObTxSEQ &seq_no) : + commit_version_(commit_version), tenant_id_(tenant_id), trans_id_(trans_id), aux_lob_meta_tid_(aux_lob_meta_tid), @@ -50,6 +52,7 @@ struct LobAuxMetaKey uint64_t hash() const { uint64_t hash_val = 0; + hash_val = common::murmurhash(&commit_version_, sizeof(tenant_id_), hash_val); hash_val = common::murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); hash_val = common::murmurhash(&aux_lob_meta_tid_, sizeof(aux_lob_meta_tid_), hash_val); hash_val = common::murmurhash(&lob_id_, sizeof(lob_id_), hash_val); @@ -69,20 +72,25 @@ struct LobAuxMetaKey bool operator<(const LobAuxMetaKey &other) const { return -1 == compare(other); } bool operator==(const LobAuxMetaKey &other) const { - return tenant_id_ == other.tenant_id_ + return commit_version_ == other.commit_version_ + && tenant_id_ == other.tenant_id_ && trans_id_ == other.trans_id_ && aux_lob_meta_tid_ == other.aux_lob_meta_tid_ && lob_id_ == other.lob_id_ && seq_no_ == other.seq_no_; } + int get_key(std::string &key) const; + TO_STRING_KV( + K_(commit_version), K_(tenant_id), K_(trans_id), "table_id", aux_lob_meta_tid_, K_(lob_id), K_(seq_no)); + int64_t commit_version_; uint64_t tenant_id_; transaction::ObTransID trans_id_; uint64_t aux_lob_meta_tid_; @@ -113,12 +121,32 @@ struct LobAuxMetaValue int64_t lob_data_len_; }; +class ObCDCLobAuxDataCleanTask : public ObLogResourceRecycleTask +{ +public: + ObCDCLobAuxDataCleanTask(): + ObLogResourceRecycleTask(ObLogResourceRecycleTask::LOB_DATA_CLEAN_TASK), + tenant_id_(OB_INVALID_TENANT_ID), + commit_version_(OB_INVALID_VERSION), + is_task_push_(false) + {} + + void reset(); + TO_STRING_KV( + K_(tenant_id), K_(commit_version), K_(is_task_push)); +public: + uint64_t tenant_id_; + int64_t commit_version_; + bool is_task_push_; +}; + +class IObStoreService; class ObCDCLobAuxMetaStorager { public: ObCDCLobAuxMetaStorager(); ~ObCDCLobAuxMetaStorager(); - int init(); + int init(IObStoreService *store_service); void destroy(); int put( @@ -128,6 +156,7 @@ public: const int64_t lob_data_len); int get( + ObIAllocator &allocator, const LobAuxMetaKey &key, const char *&lob_data, int64_t &lob_data_len); @@ -135,28 +164,58 @@ public: int del( ObLobDataOutRowCtxList &lob_data_out_row_ctx_list, volatile bool &stop_flag); - - int del( - const uint64_t tenant_id, - const transaction::ObTransID &trans_id); + int del(const LobAuxMetaKey &key); + int get_clean_task(const uint64_t tenant_id, ObCDCLobAuxDataCleanTask *&task); + int clean_unused_data(ObCDCLobAuxDataCleanTask *task); + int get_cf_handle(const uint64_t tenant_id, void *&cf_handle); private: int del_lob_col_value_( + const int64_t commit_version, const uint64_t tenant_id, const transaction::ObTransID &trans_id, const uint64_t aux_lob_meta_tid, ObLobDataGetCtx &lob_data_get_ctx, volatile bool &stop_flag); + bool is_need_to_disk_(int64_t alloc_size); + + int memory_put_( + const LobAuxMetaKey &key, + const char *key_type, + const char *lob_data, + const int64_t lob_data_len); + int memory_get_( + const LobAuxMetaKey &key, + const char *&lob_data, + int64_t &lob_data_len); + int memory_del_(const LobAuxMetaKey &key); + int memory_del_(const uint64_t tenant_id, const int64_t commit_version); + + int disk_put_( + const LobAuxMetaKey &key, + const char *key_type, + const char *lob_data, + const int64_t lob_data_len); + int disk_get_( + ObIAllocator &allocator, + const LobAuxMetaKey &key, + const char *&lob_data, + int64_t &lob_data_len); + int disk_del_(const LobAuxMetaKey &key); + int disk_del_(void* cf_family, const int64_t commit_version); + + int do_clean_unused_data_(ObCDCLobAuxDataCleanTask *task); + struct LobAuxMetaDataPurger { LobAuxMetaDataPurger( ObCDCLobAuxMetaStorager &host, const uint64_t tenant_id, - const transaction::ObTransID &trans_id) : + const int64_t commit_version) : host_(host), tenant_id_(tenant_id), - trans_id_(trans_id), + commit_version_(commit_version), purge_count_(0) {} @@ -164,7 +223,7 @@ private: ObCDCLobAuxMetaStorager &host_; uint64_t tenant_id_; - transaction::ObTransID trans_id_; + int64_t commit_version_; int64_t purge_count_; }; @@ -184,6 +243,9 @@ private: bool is_inited_; LobAuxMetaMap lob_aux_meta_map_; LobAuxMetaAllocator lob_aux_meta_allocator_; + IObStoreService *store_service_; + int64_t memory_limit_; + bool enable_memory_; DISALLOW_COPY_AND_ASSIGN(ObCDCLobAuxMetaStorager); }; diff --git a/src/logservice/libobcdc/src/ob_cdc_lob_data_merger.cpp b/src/logservice/libobcdc/src/ob_cdc_lob_data_merger.cpp index 730a6e957a..0ff7b03bcc 100644 --- a/src/logservice/libobcdc/src/ob_cdc_lob_data_merger.cpp +++ b/src/logservice/libobcdc/src/ob_cdc_lob_data_merger.cpp @@ -370,6 +370,8 @@ int ObCDCLobDataMerger::handle_task_( ObCDCLobAuxMetaStorager &lob_aux_meta_storager = TCTX.lob_aux_meta_storager_; ObLobDataGetCtx &lob_data_get_ctx = task.host_; ObLobDataOutRowCtxList *lob_data_out_row_ctx_list = static_cast(lob_data_get_ctx.host_); + const PartTransTask &part_trans_task = lob_data_out_row_ctx_list->get_dml_stmt_task()->get_host(); + const int64_t commit_version = part_trans_task.get_trans_commit_version(); const ObLobData *new_lob_data = lob_data_get_ctx.new_lob_data_; const bool is_new_col = task.is_new_col_; ObString **fragment_cb_array= lob_data_get_ctx.get_fragment_cb_array(is_new_col); @@ -384,13 +386,13 @@ int ObCDCLobDataMerger::handle_task_( const uint64_t aux_lob_meta_tid = lob_data_out_row_ctx_list->get_aux_lob_meta_table_id(); const ObLobId &lob_id = new_lob_data->id_; const uint32_t idx = task.idx_; - LobAuxMetaKey lob_aux_meta_key(tenant_id, trans_id, aux_lob_meta_tid, lob_id, task.seq_no_); + LobAuxMetaKey lob_aux_meta_key(commit_version, tenant_id, trans_id, aux_lob_meta_tid, lob_id, task.seq_no_); const char *lob_data_ptr = nullptr; int64_t lob_data_len = 0; - + ObIAllocator &allocator = lob_data_out_row_ctx_list->get_allocator(); // We need retry to get the lob data based on lob_aux_meta_key when return OB_ENTRY_NOT_EXIST, // because LobAuxMeta table data and primary table data are processed concurrently. - RETRY_FUNC_ON_ERROR_WITH_USLEEP_MS(OB_ENTRY_NOT_EXIST, 1 * _MSEC_, stop_flag, lob_aux_meta_storager, get, lob_aux_meta_key, + RETRY_FUNC_ON_ERROR_WITH_USLEEP_MS(OB_ENTRY_NOT_EXIST, 1 * _MSEC_, stop_flag, lob_aux_meta_storager, get, allocator, lob_aux_meta_key, lob_data_ptr, lob_data_len); if (OB_SUCC(ret)) { diff --git a/src/logservice/libobcdc/src/ob_log_config.h b/src/logservice/libobcdc/src/ob_log_config.h index f203889f2f..b20f28f2a3 100644 --- a/src/logservice/libobcdc/src/ob_log_config.h +++ b/src/logservice/libobcdc/src/ob_log_config.h @@ -565,6 +565,12 @@ public: DEF_STR(tenant_password, OB_CLUSTER_PARAMETER, "|", "login password for tenant_user"); // tenatn level sync config end // + // lob data storage + // enable_lob_data_storage_memory : default is enable + T_DEF_BOOL(enable_lob_data_storage_memory, OB_CLUSTER_PARAMETER, 1, "0:disabled, 1:enabled"); + // lob_data_storage_memory_limit takes effect only when enable_lob_data_storage_memory is true. + DEF_CAP(lob_data_storage_memory_limit, OB_CLUSTER_PARAMETER, "1G", "[128M,]", "lob data storage memory limit"); + #undef OB_CLUSTER_PARAMETER private: diff --git a/src/logservice/libobcdc/src/ob_log_formatter.cpp b/src/logservice/libobcdc/src/ob_log_formatter.cpp index 775b32bd61..f92006f618 100644 --- a/src/logservice/libobcdc/src/ob_log_formatter.cpp +++ b/src/logservice/libobcdc/src/ob_log_formatter.cpp @@ -2108,6 +2108,8 @@ int ObLogFormatter::parse_aux_lob_meta_table_insert_( ret = OB_ERR_UNEXPECTED; LOG_ERROR("stmt_task is not INSERT operation", KR(ret)); } else { + const PartTransTask &part_trans_task = stmt_task.get_host(); + const int64_t commit_version = part_trans_task.get_trans_commit_version(); const uint64_t tenant_id = log_entry_task.get_tenant_id(); const transaction::ObTransID &trans_id = log_entry_task.get_trans_id(); const uint64_t aux_lob_meta_table_id = stmt_task.get_table_id(); @@ -2120,7 +2122,7 @@ int ObLogFormatter::parse_aux_lob_meta_table_insert_( if (OB_FAIL(ObCDCLobAuxMetaParse::parse_aux_lob_meta_table_row(new_cols, lob_id, lob_data, lob_data_len))) { LOG_ERROR("parse_aux_lob_meta_table_row_ failed", KR(ret), K(tenant_id)); } else { - LobAuxMetaKey lob_aux_meta_key(tenant_id, trans_id, aux_lob_meta_table_id, lob_id, row_seq_no); + LobAuxMetaKey lob_aux_meta_key(commit_version, tenant_id, trans_id, aux_lob_meta_table_id, lob_id, row_seq_no); if (OB_FAIL(lob_aux_meta_storager.put(lob_aux_meta_key, "insert", lob_data, lob_data_len))) { LOG_ERROR("lob_aux_meta_storager put failed", KR(ret), K(lob_aux_meta_key)); @@ -2142,6 +2144,8 @@ int ObLogFormatter::parse_aux_lob_meta_table_delete_( ret = OB_ERR_UNEXPECTED; LOG_ERROR("stmt_task is not DELETE operation", KR(ret)); } else { + const PartTransTask &part_trans_task = stmt_task.get_host(); + const int64_t commit_version = part_trans_task.get_trans_commit_version(); const uint64_t tenant_id = log_entry_task.get_tenant_id(); const transaction::ObTransID &trans_id = log_entry_task.get_trans_id(); const uint64_t aux_lob_meta_table_id = stmt_task.get_table_id(); @@ -2154,7 +2158,7 @@ int ObLogFormatter::parse_aux_lob_meta_table_delete_( if (OB_FAIL(ObCDCLobAuxMetaParse::parse_aux_lob_meta_table_row(old_cols, lob_id, lob_data, lob_data_len))) { LOG_ERROR("parse_aux_lob_meta_table_row_ failed", KR(ret), K(tenant_id)); } else { - LobAuxMetaKey lob_aux_meta_key(tenant_id, trans_id, aux_lob_meta_table_id, lob_id, row_seq_no); + LobAuxMetaKey lob_aux_meta_key(commit_version, tenant_id, trans_id, aux_lob_meta_table_id, lob_id, row_seq_no); if (OB_FAIL(lob_aux_meta_storager.put(lob_aux_meta_key, "delete", lob_data, lob_data_len))) { LOG_ERROR("lob_aux_meta_storager put failed", KR(ret), K(lob_aux_meta_key)); diff --git a/src/logservice/libobcdc/src/ob_log_instance.cpp b/src/logservice/libobcdc/src/ob_log_instance.cpp index cc74a7d8e4..33fbdb707a 100644 --- a/src/logservice/libobcdc/src/ob_log_instance.cpp +++ b/src/logservice/libobcdc/src/ob_log_instance.cpp @@ -937,7 +937,7 @@ int ObLogInstance::init_components_(const uint64_t start_tstamp_ns) INIT(lob_data_merger_, ObCDCLobDataMerger, TCONF.lob_data_merger_thread_num, DEFAULT_QUEUE_SIZE, *err_handler); if (OB_SUCC(ret)) { - if (OB_FAIL(lob_aux_meta_storager_.init())) { + if (OB_FAIL(lob_aux_meta_storager_.init(store_service_))) { LOG_ERROR("lob_aux_meta_storager_ init failed", KR(ret)); } } diff --git a/src/logservice/libobcdc/src/ob_log_part_trans_parser.cpp b/src/logservice/libobcdc/src/ob_log_part_trans_parser.cpp index 98b418e8de..f5f5ff768a 100644 --- a/src/logservice/libobcdc/src/ob_log_part_trans_parser.cpp +++ b/src/logservice/libobcdc/src/ob_log_part_trans_parser.cpp @@ -787,6 +787,7 @@ int ObLogPartTransParser::parse_ddl_lob_aux_stmts_( } else if (OB_FAIL(stmt_task->get_cols(&rowkey_cols, &new_cols, &old_cols, nullptr/*new_lob_ctx_cols*/))) { LOG_ERROR("get_cols fail", KR(ret), K(stmt_task)); } else { + const int64_t commit_version = part_trans_task.get_trans_commit_version(); const uint64_t tenant_id = part_trans_task.get_tenant_id(); const transaction::ObTransID &trans_id = part_trans_task.get_trans_id(); ObLobId lob_id; @@ -798,7 +799,7 @@ int ObLogPartTransParser::parse_ddl_lob_aux_stmts_( if (OB_FAIL(ObCDCLobAuxMetaParse::parse_aux_lob_meta_table_row(*new_cols, lob_id, lob_data, lob_data_len))) { LOG_ERROR("parse_aux_lob_meta_table_row failed", KR(ret)); } else { - LobAuxMetaKey lob_aux_meta_key(tenant_id, trans_id, table_id, lob_id, row_seq_no); + LobAuxMetaKey lob_aux_meta_key(commit_version, tenant_id, trans_id, table_id, lob_id, row_seq_no); if (OB_FAIL(lob_aux_meta_storager.put(lob_aux_meta_key, "insert", lob_data, lob_data_len))) { LOG_ERROR("lob_aux_meta_storager put failed", KR(ret), K(lob_aux_meta_key)); diff --git a/src/logservice/libobcdc/src/ob_log_resource_collector.cpp b/src/logservice/libobcdc/src/ob_log_resource_collector.cpp index 98423cf462..2dfbdf4101 100644 --- a/src/logservice/libobcdc/src/ob_log_resource_collector.cpp +++ b/src/logservice/libobcdc/src/ob_log_resource_collector.cpp @@ -511,6 +511,7 @@ int ObLogResourceCollector::handle(void *data, // Copy the Trans ID to avoid invalidating the Trans ID when the PartTransTask is recycled uint64_t tenant_id = task->get_tenant_id(); ObTransID trans_id = task->get_trans_id(); + int64_t commit_version = task->get_trans_commit_version(); if (OB_FAIL(trans_ctx_mgr_->get_trans_ctx(tenant_id, trans_id, trans_ctx, enable_create))) { LOG_ERROR("get trans_ctx fail", KR(ret), K(tenant_id), K(trans_id), K(*task)); @@ -540,12 +541,9 @@ int ObLogResourceCollector::handle(void *data, // do nothing } - // TODO optimize if (OB_SUCC(ret)) { - ObCDCLobAuxMetaStorager &lob_aux_meta_storager = TCTX.lob_aux_meta_storager_; - - if (OB_FAIL(lob_aux_meta_storager.del(tenant_id, trans_id))) { - LOG_ERROR("lob_aux_meta_storager del failed", K(tenant_id), K(trans_id)); + if (OB_FAIL(push_lob_data_clean_task_(tenant_id, commit_version))) { + LOG_ERROR("push_lob_data_clean_task_ fail", KR(ret), K(tenant_id), K(trans_id), K(commit_version)); } } @@ -592,6 +590,12 @@ int ObLogResourceCollector::handle(void *data, ATOMIC_DEC(&br_count_); task = NULL; } + } else if (recycle_task->is_lob_data_clean_task()) { + ObCDCLobAuxMetaStorager &lob_aux_meta_storager = TCTX.lob_aux_meta_storager_; + ObCDCLobAuxDataCleanTask *task = static_cast(recycle_task); + if (OB_FAIL(lob_aux_meta_storager.clean_unused_data(task))) { + LOG_ERROR("clean lob data fail", KR(ret)); + } } else { LOG_ERROR("task type not supported", K(recycle_task), K(thread_index), "task_type", ObLogResourceRecycleTask::print_task_type(task_type)); @@ -609,6 +613,33 @@ int ObLogResourceCollector::handle(void *data, return ret; } +int ObLogResourceCollector::push_lob_data_clean_task_(const uint64_t tenant_id, const int64_t commit_version) +{ + int ret = OB_SUCCESS; + ObCDCLobAuxMetaStorager &lob_aux_meta_storager = TCTX.lob_aux_meta_storager_; + ObCDCLobAuxDataCleanTask *clean_task = nullptr; + if (OB_FAIL(lob_aux_meta_storager.get_clean_task(tenant_id, clean_task))){ + LOG_ERROR("lob_aux_meta_storager get_clean_task failed", KR(ret), K(tenant_id)); + } else { + bool is_task_push = ATOMIC_LOAD(&clean_task->is_task_push_); + if (is_task_push || ! REACH_TIME_INTERVAL(5 * _SEC_)) { + LOG_DEBUG("no need push clean task", K(is_task_push), K(tenant_id), K(commit_version)); + // try set flag by cas, oldv is false, newv is ture + // expect return oldv (false). if return true, means cas fail, just skip + } else if (ATOMIC_CAS(&clean_task->is_task_push_, false, true)) { + LOG_DEBUG("no need push clean task", K(is_task_push), K(tenant_id), K(commit_version)); + } else { + ATOMIC_STORE(&clean_task->commit_version_, commit_version); + if (OB_FAIL(push_task_into_queue_(*clean_task))) { + LOG_ERROR("push_task_into_queue_ failed", KR(ret), KPC(clean_task)); + } else { + LOG_INFO("push lob data clean task succ", KPC(clean_task)); + } + } + } + return ret; +} + int ObLogResourceCollector::revert_dml_binlog_record_(ObLogBR &br, volatile bool &stop_flag) { int ret = OB_SUCCESS; diff --git a/src/logservice/libobcdc/src/ob_log_resource_collector.h b/src/logservice/libobcdc/src/ob_log_resource_collector.h index 2a72b885d8..37d2af70b5 100644 --- a/src/logservice/libobcdc/src/ob_log_resource_collector.h +++ b/src/logservice/libobcdc/src/ob_log_resource_collector.h @@ -135,7 +135,7 @@ private: int dec_ref_cnt_and_try_to_revert_task_(PartTransTask *part_trans_task); int del_trans_(const uint64_t tenant_id, const ObString &trans_id_str); - + int push_lob_data_clean_task_(const uint64_t tenant_id, const int64_t commit_version); int dec_ref_cnt_and_try_to_recycle_log_entry_task_(ObLogBR &br); int revert_log_entry_task_(ObLogEntryTask *log_entry_task); int del_store_service_data_(const uint64_t tenant_id, diff --git a/src/logservice/libobcdc/src/ob_log_resource_recycle_task.h b/src/logservice/libobcdc/src/ob_log_resource_recycle_task.h index 030244ef66..e1f776f413 100644 --- a/src/logservice/libobcdc/src/ob_log_resource_recycle_task.h +++ b/src/logservice/libobcdc/src/ob_log_resource_recycle_task.h @@ -26,11 +26,13 @@ public: { UNKNOWN_TASK = 0, PART_TRANS_TASK = 1, - BINLOG_RECORD_TASK = 2 + BINLOG_RECORD_TASK = 2, + LOB_DATA_CLEAN_TASK = 3, }; bool is_unknown_task() const { return UNKNOWN_TASK == task_type_; } bool is_part_trans_task() const { return PART_TRANS_TASK == task_type_; } bool is_binlog_record_task() const { return BINLOG_RECORD_TASK == task_type_; } + bool is_lob_data_clean_task() const { return LOB_DATA_CLEAN_TASK == task_type_; } TaskType get_task_type() const { return task_type_; } static const char *print_task_type(TaskType task) @@ -44,6 +46,8 @@ public: case BINLOG_RECORD_TASK: str = "BinlogRecordTask"; break; + case LOB_DATA_CLEAN_TASK: + str = "LobDataCleanTask"; default: str = "UNKNOWN_TASK"; break; diff --git a/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp b/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp index e5604df701..9d7ac905b1 100644 --- a/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp +++ b/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp @@ -270,7 +270,9 @@ int RocksDbStoreService::get(void *cf_handle, const std::string &key, std::strin } else { rocksdb::Status s = m_db_->Get(rocksdb::ReadOptions(), column_family_handle, key, &value); - if (!s.ok()) { + if (s.IsNotFound()) { + ret = OB_ENTRY_NOT_EXIST; + } else if (!s.ok()) { _LOG_ERROR("RocksDbStoreService get value from rocksdb failed, error %s, key:%s", s.ToString().c_str(), key.c_str()); ret = OB_ERR_UNEXPECTED; diff --git a/src/logservice/libobcdc/src/ob_log_tenant.cpp b/src/logservice/libobcdc/src/ob_log_tenant.cpp index 5d097a2a7e..60017ec52c 100644 --- a/src/logservice/libobcdc/src/ob_log_tenant.cpp +++ b/src/logservice/libobcdc/src/ob_log_tenant.cpp @@ -54,7 +54,9 @@ ObLogTenant::ObLogTenant() : committer_global_heartbeat_(OB_INVALID_VERSION), committer_cur_schema_version_(OB_INVALID_VERSION), committer_next_trans_schema_version_(OB_INVALID_VERSION), - cf_handle_(NULL) + cf_handle_(NULL), + lob_storage_cf_handle_(nullptr), + lob_storage_clean_task_() { tenant_name_[0] = '\0'; global_seq_and_schema_version_.lo = 0; @@ -73,6 +75,7 @@ int ObLogTenant::init( const int64_t start_seq, const int64_t start_schema_version, void *cf_handle, + void *lob_storage_cf_handle, ObLogTenantMgr &tenant_mgr) { int ret = OB_SUCCESS; @@ -134,6 +137,8 @@ int ObLogTenant::init( committer_cur_schema_version_ = start_schema_version; committer_next_trans_schema_version_ = start_schema_version; cf_handle_ = cf_handle; + lob_storage_cf_handle_ = lob_storage_cf_handle; + lob_storage_clean_task_.tenant_id_ = tenant_id; inited_ = true; @@ -191,6 +196,8 @@ void ObLogTenant::reset() committer_cur_schema_version_ = OB_INVALID_VERSION; committer_next_trans_schema_version_ = OB_INVALID_VERSION; cf_handle_ = NULL; + lob_storage_cf_handle_ = nullptr; + lob_storage_clean_task_.reset(); ObMallocAllocator::get_instance()->recycle_tenant_allocator(tenant_id); } diff --git a/src/logservice/libobcdc/src/ob_log_tenant.h b/src/logservice/libobcdc/src/ob_log_tenant.h index 04e30a7d67..896f692f97 100644 --- a/src/logservice/libobcdc/src/ob_log_tenant.h +++ b/src/logservice/libobcdc/src/ob_log_tenant.h @@ -23,6 +23,7 @@ #include "ob_log_part_mgr.h" // ObLogPartMgr #include "ob_log_ls_mgr.h" // ObLogLSMgr #include "ob_log_ref_state.h" // RefState +#include "ob_cdc_lob_aux_meta_storager.h" // ObCDCLobAuxDataCleanTask #include namespace oceanbase @@ -96,6 +97,7 @@ public: const int64_t start_seq, const int64_t start_schema_version, void *cf_handle, + void *lob_storage_cf_handle, ObLogTenantMgr &tenant_mgr); void reset(); bool is_inited() const { return inited_; } @@ -131,6 +133,8 @@ public: int64_t get_global_schema_version() const { return global_seq_and_schema_version_.hi; } int64_t get_global_seq() const { return global_seq_and_schema_version_.lo; } void *get_cf() { return cf_handle_; } + void *get_lob_storage_cf_handle() { return lob_storage_cf_handle_; } + ObCDCLobAuxDataCleanTask& get_lob_storage_clean_task() { return lob_storage_clean_task_; } public: // Print statistics @@ -324,6 +328,8 @@ private: int64_t committer_next_trans_schema_version_ CACHE_ALIGNED; void *cf_handle_; + void *lob_storage_cf_handle_; + ObCDCLobAuxDataCleanTask lob_storage_clean_task_; private: DISALLOW_COPY_AND_ASSIGN(ObLogTenant); diff --git a/src/logservice/libobcdc/src/ob_log_tenant_mgr.cpp b/src/logservice/libobcdc/src/ob_log_tenant_mgr.cpp index 649de7d94b..9c53887a16 100644 --- a/src/logservice/libobcdc/src/ob_log_tenant_mgr.cpp +++ b/src/logservice/libobcdc/src/ob_log_tenant_mgr.cpp @@ -284,6 +284,7 @@ int ObLogTenantMgr::do_add_tenant_(const uint64_t tenant_id, int64_t start_seq = DEFAULT_START_SEQUENCE_NUM; IObStoreService *store_service = TCTX.store_service_; void *column_family_handle = NULL; + void *lob_storage_cf_handle = nullptr; const int64_t start_tstamp_usec = start_tstamp_ns / NS_CONVERSION; int64_t tenant_start_serve_ts_ns = start_tstamp_ns; bool use_add_tenant_start_ddl_commit_version = false; @@ -337,10 +338,16 @@ int ObLogTenantMgr::do_add_tenant_(const uint64_t tenant_id, if (OB_IN_STOP_STATE != ret) { LOG_ERROR("create_column_family fail", KR(ret), K(tenant_id), K(tenant_name), K(column_family_handle)); } + } else if (OB_FAIL(store_service->create_column_family( + std::to_string(tenant_id) + ":" + std::string(tenant_name) + ":lob", + lob_storage_cf_handle))) { + if (OB_IN_STOP_STATE != ret) { + LOG_ERROR("create_column_family fail", KR(ret), K(tenant_id), K(tenant_name), K(lob_storage_cf_handle)); + } } // init tenant else if (OB_FAIL(tenant->init(tenant_id, tenant_name, tenant_start_serve_ts_ns, start_seq, - tenant_start_schema_version, column_family_handle, *this))) { + tenant_start_schema_version, column_family_handle, lob_storage_cf_handle, *this))) { LOG_ERROR("tenant init fail", KR(ret), K(tenant_id), K(tenant_name), K(start_tstamp_ns), K(start_seq), K(tenant_start_schema_version), K(tenant_start_serve_ts_ns), K(use_add_tenant_start_ddl_commit_version)); } @@ -1428,6 +1435,7 @@ bool ObLogTenantMgr::TenantPrinter::operator()(const TenantID &tid, ObLogTenant } (void)tenant_ids_.push_back(tid.tenant_id_); (void)cf_handles_.push_back(tenant->get_cf()); + (void)lob_storage_cf_handles_.push_back(tenant->get_lob_storage_cf_handle()); } return true; } @@ -1446,6 +1454,7 @@ void ObLogTenantMgr::print_stat_info() if (NULL != store_service) { store_service->get_mem_usage(printer.tenant_ids_, printer.cf_handles_); + store_service->get_mem_usage(printer.tenant_ids_, printer.lob_storage_cf_handles_); } } diff --git a/src/logservice/libobcdc/src/ob_log_tenant_mgr.h b/src/logservice/libobcdc/src/ob_log_tenant_mgr.h index d1a16f8613..b93fc52ae5 100644 --- a/src/logservice/libobcdc/src/ob_log_tenant_mgr.h +++ b/src/logservice/libobcdc/src/ob_log_tenant_mgr.h @@ -332,8 +332,9 @@ private: int64_t offline_tenant_count_; std::vector tenant_ids_; std::vector cf_handles_; + std::vector lob_storage_cf_handles_; - TenantPrinter() : serving_tenant_count_(0), offline_tenant_count_(0), cf_handles_() {} + TenantPrinter() : serving_tenant_count_(0), offline_tenant_count_(0), cf_handles_(), lob_storage_cf_handles_() {} bool operator()(const TenantID &tid, ObLogTenant *tenant); }; diff --git a/src/logservice/ob_garbage_collector.cpp b/src/logservice/ob_garbage_collector.cpp index 780f8dd4d3..a622354071 100644 --- a/src/logservice/ob_garbage_collector.cpp +++ b/src/logservice/ob_garbage_collector.cpp @@ -226,7 +226,11 @@ int ObGarbageCollector::QueryLSIsValidMemberFunctor::handle_rpc_response_(const ret = OB_ERR_UNEXPECTED; CLOG_LOG(WARN, "log stream service is NULL", K(ret)); } else if (OB_SUCCESS != (tmp_ret = ls_service_->get_ls(id, handle, ObLSGetMod::OBSERVER_MOD))) { - CLOG_LOG(WARN, "get log stream failed", K(id), K(tmp_ret)); + if (OB_LS_NOT_EXIST == tmp_ret) { + CLOG_LOG(INFO, "ls does not exist anymore, maybe removed", K(tmp_ret), K(id)); + } else { + CLOG_LOG(WARN, "get log stream failed", K(id), K(tmp_ret)); + } } else if (OB_ISNULL(ls = handle.get_ls())) { tmp_ret = OB_ERR_UNEXPECTED; CLOG_LOG(WARN, " log stream not exist", K(id), K(tmp_ret)); @@ -415,8 +419,8 @@ int ObGCHandler::execute_pre_remove() if (OB_SUCC(ret) && need_check_readonly_tx) { //follower or not in member list replica need block_tx here if (OB_INVALID_TIMESTAMP == block_tx_ts_) { - if (OB_FAIL(ls_->block_tx_start())) { - CLOG_LOG(WARN, "failed to block_tx_start", K(ls_id), KPC(this)); + if (OB_FAIL(ls_->block_all())) { + CLOG_LOG(WARN, "failed to block_all", K(ls_id), KPC(this)); } else { block_tx_ts_ = ObClockGenerator::getClock(); } @@ -629,16 +633,16 @@ bool ObGCHandler::is_valid_ls_gc_state(const LSGCState &state) && LSGCState::INVALID_LS_GC_STATE < state; } +bool ObGCHandler::is_ls_offline_gc_state(const LSGCState &state) +{ + return LSGCState::LS_OFFLINE == state; +} + bool ObGCHandler::is_ls_blocked_state_(const LSGCState &state) { return LSGCState::LS_BLOCKED == state; } -bool ObGCHandler::is_ls_offline_state_(const LSGCState &state) -{ - return LSGCState::LS_OFFLINE == state; -} - bool ObGCHandler::is_ls_wait_gc_state_(const LSGCState &state) { return LSGCState::WAIT_GC == state; @@ -983,9 +987,8 @@ void ObGCHandler::block_ls_transfer_in_(const SCN &block_scn) CLOG_LOG(WARN, "ls check gc state invalid", K(ls_id), K(gc_state)); } else if (is_ls_blocked_finished_(gc_state)) { CLOG_LOG(INFO, "ls already blocked, ignore", K(ls_id), K(gc_state), K(block_scn)); - //TODO: @keqing.llt transfer功能完成之前,先用杀事务代替transfer out - } else if (OB_FAIL(ls_->block_tx_start())) { - CLOG_LOG(WARN, "block_tx_start failed", K(ls_id), K(ret)); + } else if (OB_FAIL(ls_->block_all())) { + CLOG_LOG(WARN, "block_all failed", K(ls_id), K(ret)); } else if (FALSE_IT(block_tx_ts_ = ObClockGenerator::getClock())) { } else if (OB_FAIL(ls_->set_gc_state(LSGCState::LS_BLOCKED))) { CLOG_LOG(WARN, "set_gc_state block failed", K(ls_id), K(ret)); @@ -1011,12 +1014,8 @@ void ObGCHandler::offline_ls_(const SCN &offline_scn) } else { CLOG_LOG(INFO, "ls already offline, ignore", K(ls_id), K(offline_scn), K(gc_state), K(pre_offline_scn)); } - } else if (OB_FAIL(ls_->set_gc_state(LSGCState::LS_OFFLINE))) { - //TODO: @yanyuan 需要调用ObLS的offline_ls_接口 - //offline_scn依赖gc_state写slog, 顺序必须为先设置offline_scn再设置gc状态 - CLOG_LOG(WARN, "set_gc_state failed", K(ls_->get_ls_id()), K(ret)); - } else if (OB_FAIL(ls_->set_offline_scn(offline_scn))) { - CLOG_LOG(WARN, "set_gc_state failed", K(ls_->get_ls_id()), K(ret)); + } else if (OB_FAIL(ls_->set_gc_state(LSGCState::LS_OFFLINE, offline_scn))) { + CLOG_LOG(WARN, "set_gc_state failed", K(ls_->get_ls_id()), K(offline_scn)); } else { CLOG_LOG(INFO, "offline_ls success", K(ls_->get_ls_id()), K(offline_scn)); } } @@ -1115,7 +1114,7 @@ void ObGCHandler::handle_gc_ls_offline_(ObGarbageCollector::LSStatus &ls_status) } else if (is_ls_wait_gc_state_(gc_state)) { ls_status = ObGarbageCollector::LSStatus::LS_NEED_DELETE_ENTRY; CLOG_LOG(INFO, "handle_gc_ls_offline need delete entry", K(ls_id), K(gc_state)); - } else if (is_ls_offline_state_(gc_state)) { + } else if (is_ls_offline_gc_state(gc_state)) { (void)try_check_and_set_wait_gc_(ls_status); } else { if (OB_FAIL(submit_log_(ObGCLSLOGType::OFFLINE_LS, is_success))) { @@ -1609,7 +1608,11 @@ void ObGarbageCollector::execute_gc_(ObGCCandidateArray &gc_candidates) ret = OB_NOT_INIT; CLOG_LOG(ERROR, "ls service is NULL", K(ret)); } else if (OB_SUCCESS != (tmp_ret = ls_service_->get_ls(id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - CLOG_LOG(ERROR, "get ls failed", K(tmp_ret), K(id)); + if (OB_LS_NOT_EXIST == tmp_ret) { + CLOG_LOG(INFO, "ls does not exist anymore, maybe removed", K(tmp_ret), K(id)); + } else { + CLOG_LOG(ERROR, "get ls failed", K(tmp_ret), K(id)); + } } else if (OB_ISNULL(ls = ls_handle.get_ls())) { tmp_ret = OB_ERR_UNEXPECTED; CLOG_LOG(ERROR, "ls not exist", K(tmp_ret), K(id)); diff --git a/src/logservice/ob_garbage_collector.h b/src/logservice/ob_garbage_collector.h index 52c6adea0d..91afebf4e9 100644 --- a/src/logservice/ob_garbage_collector.h +++ b/src/logservice/ob_garbage_collector.h @@ -251,6 +251,7 @@ public: int check_ls_can_offline(const share::ObLSStatus &ls_status); int gc_check_invalid_member_seq(const int64_t gc_seq, bool &need_gc); static bool is_valid_ls_gc_state(const LSGCState &state); + static bool is_ls_offline_gc_state(const LSGCState &state); int diagnose(GCDiagnoseInfo &diagnose_info) const; @@ -314,10 +315,10 @@ private: const int64_t GET_GTS_TIMEOUT_US = 10L * 1000 * 1000; //10s int get_gts_(const int64_t timeout_us, share::SCN >s_scn); bool is_ls_blocked_state_(const LSGCState &state); - bool is_ls_offline_state_(const LSGCState &state); bool is_ls_wait_gc_state_(const LSGCState &state); bool is_ls_blocked_finished_(const LSGCState &state); bool is_ls_offline_finished_(const LSGCState &state); + bool is_tablet_clear_(const ObGarbageCollector::LSStatus &ls_status); void try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_status); int try_check_and_set_wait_gc_when_log_archive_is_off_( diff --git a/src/logservice/palf/log_io_worker_wrapper.cpp b/src/logservice/palf/log_io_worker_wrapper.cpp index 6f8259f949..dbbc356826 100644 --- a/src/logservice/palf/log_io_worker_wrapper.cpp +++ b/src/logservice/palf/log_io_worker_wrapper.cpp @@ -115,8 +115,6 @@ int LogIOWorkerWrapper::notify_need_writing_throttling(const bool &need_throttli int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (!is_user_tenant_) { - //need no nothing } else { throttle_.notify_need_writing_throttling(need_throttling); if (need_throttling) { @@ -156,8 +154,8 @@ int LogIOWorkerWrapper::create_and_init_log_io_workers_(const LogIOWorkerConfig for (int64_t i = 0; i < log_writer_parallelism && OB_SUCC(ret); i++) { LogIOWorker *iow = log_io_workers_ + i; iow = new(iow)LogIOWorker(); - // NB: the first LogIOWorker need ignoring throttling - bool need_ignoring_throttling = (i == SYS_LOG_IO_WORKER_INDEX); + // NB:only sys log streams of user tenants need to ignore throtting + bool need_ignoring_throttling = (i == SYS_LOG_IO_WORKER_INDEX && is_user_tenant(tenant_id)); if (OB_FAIL(iow->init(config, tenant_id, cb_thread_pool_tg_id, allocator, &throttle_, need_ignoring_throttling, palf_env_impl))) { PALF_LOG(WARN, "init LogIOWorker failed", K(i), K(config), K(tenant_id), diff --git a/src/logservice/palf/log_storage.cpp b/src/logservice/palf/log_storage.cpp index 10bd318850..58719f1e65 100644 --- a/src/logservice/palf/log_storage.cpp +++ b/src/logservice/palf/log_storage.cpp @@ -43,6 +43,7 @@ LogStorage::LogStorage() : accum_read_io_count_(0), accum_read_log_size_(0), accum_read_cost_ts_(0), + flashback_version_(OB_INVALID_TIMESTAMP), is_inited_(false) {} @@ -108,6 +109,7 @@ int LogStorage::load_manifest_for_meta_storage(block_id_t &expected_next_block_i void LogStorage::destroy() { is_inited_ = false; + flashback_version_ = 0; logical_block_size_ = 0; palf_id_ = INVALID_PALF_ID; need_append_block_header_ = false; @@ -441,6 +443,13 @@ int LogStorage::end_flashback(const LSN &start_lsn_of_block) { int ret = OB_SUCCESS; const block_id_t block_id = lsn_2_block(start_lsn_of_block, logical_block_size_); + // to ensure the integrity of each read data, before delete blocks, + // reset readable_log_tail_ and inc flashback_version_ firstly. + { + ObSpinLockGuard guard(tail_info_lock_); + readable_log_tail_ = log_tail_; + flashback_version_++; + } // constriaints: 'expected_next_block_id' is used to check whether blocks on disk are integral, // we make sure that the content in each block_id which is greater than or equal to // 'expected_next_block_id' are not been used. @@ -457,8 +466,6 @@ int LogStorage::end_flashback(const LSN &start_lsn_of_block) PALF_LOG(ERROR, "LogBlockMgr rename_tmp_block_handler_to_normal failed", K(ret), KPC(this), K(start_lsn_of_block)); } else { - ObSpinLockGuard guard(tail_info_lock_); - readable_log_tail_ = log_tail_; PALF_EVENT("[END STORAGE FLASHBACK]", palf_id_, KPC(this), K(start_lsn_of_block)); } return ret; @@ -639,6 +646,7 @@ int LogStorage::do_init_(const char *base_dir, plugins_ = plugins; hot_cache_ = hot_cache; last_accum_read_statistic_time_ = ObTimeUtility::fast_current_time(); + flashback_version_ = 0; is_inited_ = true; } if (OB_FAIL(ret) && OB_INIT_TWICE != ret) { @@ -647,25 +655,57 @@ int LogStorage::do_init_(const char *base_dir, return ret; } -int LogStorage::check_read_out_of_bound_(const block_id_t &block_id) const +// To ensure the integrity of each read data, we need check the 'block_id' whether is integrity +// after read data successfully. +// +// To ensure the integrity of log blocks, we need check the 'block_id' which opened failed whether +// is deleted by others. +// +// 1. For delete block, LogBlockMgr will inc 'min_block_id_' firstly, and then reuse the block, +// after reading, if 'block_id' is smaller than 'min_block_id_', means that the data is not integrity. +// +// 2. For flashback block, LogBlockMgr will dec 'max_block_id_' firstly, and then reuse the block. +// compare with 'min_block_id_', the 'max_block_id_' will be advanced after writing data, therefore, +// we can not check the data whether is integrity according to 'max_block_id_'. to solve this problem, +// double check 'flashback_version_' which will be advanced after flashback. +// +int LogStorage::check_read_out_of_bound_(const block_id_t &block_id, + const int64_t flashback_version, + const bool no_such_block) const { int ret = OB_SUCCESS; block_id_t min_block_id = LOG_INVALID_BLOCK_ID; block_id_t max_block_id = LOG_INVALID_BLOCK_ID; - if (OB_FAIL(get_block_id_range(min_block_id, max_block_id)) - && OB_ENTRY_NOT_EXIST != ret) { - PALF_LOG(WARN, "get_block_id_range failed", K(ret), K(min_block_id), K(max_block_id)); - } else if (max_block_id == block_id) { + LSN readable_log_tail; + int64_t curr_flashback_version = OB_INVALID_TIMESTAMP; + get_readable_log_tail_guarded_by_lock_(readable_log_tail, curr_flashback_version); + block_id_t readable_end_block_id = lsn_2_block(readable_log_tail, logical_block_size_); + // if read data is concurrently with flashback, return OB_NEED_RETRY. + // to avoid unnecessary failure, only check flashback_version when read block need to be overwriting. + // NB: update 'reabable_log_tail_' and 'flashback_version_' is atomic, and updating is performed before + // overwriting. + if (block_id >= readable_end_block_id && flashback_version != curr_flashback_version) { ret = OB_NEED_RETRY; - PALF_LOG(WARN, "the block to be read is in flashback, need read retry", K(min_block_id), K(max_block_id), K(block_id)); - } else if (max_block_id < block_id) { - ret = OB_ERR_OUT_OF_UPPER_BOUND; - PALF_LOG(WARN, "read something out of upper bound, the blocks may be deleted by flashback", - K(min_block_id), K(max_block_id), K(block_id)); + PALF_LOG(WARN, "there is flashbacking during read data, need read retry", + KPC(this), K(flashback_version), K(curr_flashback_version), + K(min_block_id), K(max_block_id), K(block_id)); + } else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id)) + && OB_ENTRY_NOT_EXIST != ret) { + PALF_LOG(ERROR, "get_block_id_range failed", K(ret), K(min_block_id), K(max_block_id)); } else if (min_block_id > block_id) { ret = OB_ERR_OUT_OF_LOWER_BOUND; - PALF_LOG(TRACE, "read something out of lower bound", K(min_block_id), K(max_block_id), K(block_id)); - } else { + PALF_LOG(INFO, "read something out of lower bound, the block may be deleted by GC or rebuild", + K(min_block_id), K(max_block_id), K(block_id)); + // there is no possibility read data out of upper bound because we have checked flashback_version. + } else if (block_id > max_block_id) { + ret = OB_ERR_UNEXPECTED; + PALF_LOG(ERROR, "unexpected error, the block to be read is greater than max_block_id", + K(min_block_id), K(max_block_id), K(block_id)); + } + // if there is no block whose names with 'block_id' and 'block_id' is in range of [min_block_id, max_block_id] + // return OB_ERR_UNEXPECTED. + if (OB_SUCC(ret) && no_such_block + && min_block_id <= block_id && block_id <= max_block_id) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "unexpected error, the block may be deleted by human", K(min_block_id), K(max_block_id), K(block_id)); } @@ -764,13 +804,15 @@ void LogStorage::update_log_tail_guarded_by_lock_(const LSN &lsn) const LSN &LogStorage::get_log_tail_guarded_by_lock_() const { ObSpinLockGuard guard(tail_info_lock_); - return log_tail_; + return readable_log_tail_; } -const LSN &LogStorage::get_readable_log_tail_guarded_by_lock_() const +void LogStorage::get_readable_log_tail_guarded_by_lock_(LSN &readable_log_tail, + int64_t &flashback_version) const { ObSpinLockGuard guard(tail_info_lock_); - return readable_log_tail_; + readable_log_tail = readable_log_tail_; + flashback_version = flashback_version_; } offset_t LogStorage::get_phy_offset_(const LSN &lsn) const @@ -788,35 +830,46 @@ int LogStorage::read_block_header_(const block_id_t block_id, ReadBufGuard read_buf_guard("LogStorage", in_read_size); ReadBuf &read_buf = read_buf_guard.read_buf_; - // 'log_tail' and 'block_header' are snapshot, we can read valid data even if the block - // is deleted. NB: we need ensure that the lsn_2_block('log_tail') is smaller than or + // 'readable_log_tail' and 'block_header' are snapshot, we can read valid data even if the block + // is deleted. NB: we need ensure that the lsn_2_block('readable_log_tail') is smaller than or // equal to 'max_block_id'. - LSN log_tail = get_readable_log_tail_guarded_by_lock_(); - block_id_t max_block_id = lsn_2_block(log_tail, logical_block_size_); - bool last_block_has_data = (0 == lsn_2_offset(log_tail, logical_block_size_) ? false : true); + LSN readable_log_tail; + int64_t flashback_version = -1; + get_readable_log_tail_guarded_by_lock_(readable_log_tail, flashback_version); + block_id_t max_block_id = lsn_2_block(readable_log_tail, logical_block_size_); + bool last_block_has_data = (0 == lsn_2_offset(readable_log_tail, logical_block_size_) ? false : true); if (!read_buf.is_valid()) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(WARN, "allocate memory failed"); } else if (block_id > max_block_id || (block_id == max_block_id && false == last_block_has_data)) { ret = OB_ERR_OUT_OF_UPPER_BOUND; PALF_LOG(WARN, "block_id is large than max_block_id", K(ret), K(block_id), - K(log_tail), K(max_block_id), K(log_block_header)); - } else if (OB_FAIL( - log_reader_.pread(block_id, 0, in_read_size, read_buf, out_read_size))) { - PALF_LOG(WARN, "read info block failed", K(ret), K(read_buf)); - } else if (OB_FAIL(log_block_header.deserialize(read_buf.buf_, out_read_size, pos))) { - PALF_LOG(WARN, "deserialize info block failed", K(ret), K(read_buf), - K(out_read_size)); - } else if (false == log_block_header.check_integrity()) { - ret = OB_INVALID_DATA; - PALF_LOG(ERROR, "info block has been corrupted!!!", K(log_block_header), K(block_id)); + K(readable_log_tail), K(max_block_id), K(log_block_header)); } else { - PALF_LOG(TRACE, "read_block_header_ success", K(ret), K(block_id), - K(log_block_header)); - } - - if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { - ret = check_read_out_of_bound_(block_id); + if (OB_FAIL(log_reader_.pread(block_id, 0, in_read_size, read_buf, out_read_size))) { + PALF_LOG(WARN, "read info block failed", K(ret), K(read_buf)); + } else if (OB_FAIL(log_block_header.deserialize(read_buf.buf_, out_read_size, pos))) { + PALF_LOG(WARN, "deserialize info block failed", K(ret), K(read_buf), + K(out_read_size)); + } else if (false == log_block_header.check_integrity()) { + ret = OB_INVALID_DATA; + PALF_LOG(ERROR, "info block has been corrupted!!!", K(log_block_header), K(block_id)); + } else { + PALF_LOG(TRACE, "read_block_header_ success", K(ret), K(block_id), + K(log_block_header)); + } + // to ensure the data integrity, we should check 'block_id' whether has integrity data. + int tmp_ret = check_read_out_of_bound_(block_id, flashback_version, OB_NO_SUCH_FILE_OR_DIRECTORY == ret); + // overwrite ret code: + // 1. if ret is OB_NO_SUCH_FILE_OR_DIRECTORY, the block may be recycled or overwriting(i.e. flashback). + // 2. if ret is OB_INVALID_DATA, the block may be being recycled or overwriting(i.e. flashback). + // 3. if ret is OB_SUCCESS, we should check the data has been read whether is integrity because the block + // may be being recycled or overwriting(i.e. flashback). + if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret + || OB_INVALID_DATA == ret + || OB_SUCC(ret)) { + ret = tmp_ret; + } } return ret; } @@ -849,55 +902,65 @@ int LogStorage::inner_pread_(const LSN &read_lsn, { int ret = OB_SUCCESS; // NB: don't support read data from diffent file. - const LSN log_tail = get_readable_log_tail_guarded_by_lock_(); + LSN readable_log_tail; + int64_t flashback_version = -1; + get_readable_log_tail_guarded_by_lock_(readable_log_tail, flashback_version); const block_id_t read_block_id = lsn_2_block(read_lsn, logical_block_size_); const LSN curr_block_end_lsn = LSN((read_block_id + 1) * logical_block_size_); - const LSN &max_readable_lsn = MIN(log_tail, curr_block_end_lsn); + const LSN &max_readable_lsn = MIN(readable_log_tail, curr_block_end_lsn); const int64_t real_in_read_size = MIN(max_readable_lsn - read_lsn, in_read_size); const offset_t read_offset = lsn_2_offset(read_lsn, logical_block_size_); const offset_t real_read_offset = read_offset == 0 && true == need_read_log_block_header ? 0 : get_phy_offset_(read_lsn); const int64_t start_ts = ObTimeUtility::fast_current_time(); - if (read_lsn >= log_tail) { + if (read_lsn >= readable_log_tail) { ret = OB_ERR_OUT_OF_UPPER_BOUND; PALF_LOG(WARN, "read something out of upper bound", K(ret), K(read_lsn), K(log_tail_)); - } else if (OB_FAIL(log_reader_.pread(read_block_id, - real_read_offset, - real_in_read_size, - read_buf, - out_read_size))) { - PALF_LOG( - WARN, "LogReader pread failed", K(ret), K(read_lsn), K(log_tail_), K(real_in_read_size)); } else { - PALF_LOG(TRACE, - "inner_pread success", - K(ret), - K(read_lsn), - K(in_read_size), - K(real_in_read_size), - K(read_lsn), - K(out_read_size), - K(log_tail)); - const int64_t cost_ts = ObTimeUtility::fast_current_time() - start_ts; - EVENT_TENANT_INC(ObStatEventIds::PALF_READ_IO_COUNT_FROM_DISK, MTL_ID()); - EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_DISK, out_read_size); - EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_DISK, cost_ts); - ATOMIC_INC(&accum_read_io_count_); - ATOMIC_AAF(&accum_read_log_size_, out_read_size); - ATOMIC_AAF(&accum_read_cost_ts_, cost_ts); - if (palf_reach_time_interval(PALF_IO_STAT_PRINT_INTERVAL_US, last_accum_read_statistic_time_)) { - const int64_t avg_pread_cost = accum_read_cost_ts_ / accum_read_io_count_; - PALF_LOG(INFO, "[PALF STAT READ LOG INFO FROM DISK]", KPC(this), K_(accum_read_io_count), - K_(accum_read_log_size), K(avg_pread_cost)); - ATOMIC_STORE(&accum_read_io_count_, 0); - ATOMIC_STORE(&accum_read_log_size_, 0); - ATOMIC_STORE(&accum_read_cost_ts_, 0); + if (OB_FAIL(log_reader_.pread(read_block_id, + real_read_offset, + real_in_read_size, + read_buf, + out_read_size))) { + PALF_LOG( + WARN, "LogReader pread failed", K(ret), K(read_lsn), K(log_tail_), K(real_in_read_size)); + } else { + PALF_LOG(TRACE, + "inner_pread success", + K(ret), + K(read_lsn), + K(in_read_size), + K(real_in_read_size), + K(read_lsn), + K(out_read_size), + K(readable_log_tail)); + const int64_t cost_ts = ObTimeUtility::fast_current_time() - start_ts; + EVENT_TENANT_INC(ObStatEventIds::PALF_READ_IO_COUNT_FROM_DISK, MTL_ID()); + EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_DISK, out_read_size); + EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_DISK, cost_ts); + ATOMIC_INC(&accum_read_io_count_); + ATOMIC_AAF(&accum_read_log_size_, out_read_size); + ATOMIC_AAF(&accum_read_cost_ts_, cost_ts); + if (palf_reach_time_interval(PALF_IO_STAT_PRINT_INTERVAL_US, last_accum_read_statistic_time_)) { + const int64_t avg_pread_cost = accum_read_cost_ts_ / accum_read_io_count_; + PALF_LOG(INFO, "[PALF STAT READ LOG INFO FROM DISK]", KPC(this), K_(accum_read_io_count), + K_(accum_read_log_size), K(avg_pread_cost)); + ATOMIC_STORE(&accum_read_io_count_, 0); + ATOMIC_STORE(&accum_read_log_size_, 0); + ATOMIC_STORE(&accum_read_cost_ts_, 0); + } + } + // to ensure the data integrity, we should check 'read_block_id' whether has integrity data. + int tmp_ret = check_read_out_of_bound_(read_block_id, flashback_version, OB_NO_SUCH_FILE_OR_DIRECTORY == ret); + // overwrite ret code: + // 1. if ret is OB_NO_SUCH_FILE_OR_DIRECTORY, the block may be recycled or overwriting(i.e. flashback). + // 2. if ret is OB_SUCCESS, we should check the data has been read whether is integrity because the block + // may be being recycled or overwriting(i.e. flashback). + if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret + || OB_SUCC(ret)) { + ret = tmp_ret; } - } - - if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { - ret = check_read_out_of_bound_(lsn_2_block(read_lsn, logical_block_size_)); } return ret; } @@ -928,5 +991,6 @@ int LogStorage::get_logical_block_size(int64_t &logical_block_size) const } return ret; } + } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_storage.h b/src/logservice/palf/log_storage.h index 6a6261bdfe..1efd6fea7b 100644 --- a/src/logservice/palf/log_storage.h +++ b/src/logservice/palf/log_storage.h @@ -123,7 +123,8 @@ public: K_(block_mgr), K(logical_block_size_), K(curr_block_writable_size_), - KP(block_header_serialize_buf_)); + KP(block_header_serialize_buf_), + K_(flashback_version)); private: int do_init_(const char *log_dir, @@ -140,12 +141,15 @@ private: // @ret val: // OB_SUCCESS // OB_ERR_OUT_OF_LOWER_BOUND + // the block has been recycled. // OB_ERR_OUT_OF_UPPER_BOUND // in flashback, (flashback_block_id, max_block_id] may be deleted, however, fetch log may read // some blocks in range of (flashback_block_id, max_block_id]. - // OB_NEED_RETRY + // OB_NEED_RETRY, open the block need to be flashbacked failed or there is flashbacking during read data. // OB_ERR_UNEXPECTED - int check_read_out_of_bound_(const block_id_t &block_id) const; + int check_read_out_of_bound_(const block_id_t &block_id, + const int64_t flashback_version, + const bool no_such_block) const; int inner_switch_block_(); int append_block_header_used_for_meta_storage_(); int append_block_header_(const LSN &block_min_lsn, const share::SCN &block_min_scn); @@ -166,7 +170,8 @@ private: void update_log_tail_guarded_by_lock_(const int64_t log_size); void update_log_tail_guarded_by_lock_(const LSN &lsn); const LSN &get_log_tail_guarded_by_lock_() const; - const LSN &get_readable_log_tail_guarded_by_lock_() const; + void get_readable_log_tail_guarded_by_lock_(LSN &readable_log_tail, + int64_t &flashback_version) const; offset_t get_phy_offset_(const LSN &lsn) const; int read_block_header_(const block_id_t block_id, LogBlockHeader &block_header) const; bool check_last_block_is_full_(const block_id_t max_block_id) const; @@ -178,6 +183,7 @@ private: int64_t &out_read_size); void reset_log_tail_for_last_block_(const LSN &lsn, bool last_block_exist); int update_manifest_(const block_id_t expected_next_block_id, const bool in_restart = false); + int check_read_integrity_(const block_id_t &block_id); private: // Used to perform IO tasks in the background LogBlockMgr block_mgr_; @@ -203,6 +209,7 @@ private: int64_t accum_read_io_count_; int64_t accum_read_log_size_; int64_t accum_read_cost_ts_; + int64_t flashback_version_; bool is_inited_; }; diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 6d06b1ae57..591bc0a5a9 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -919,6 +919,7 @@ int PalfHandleImpl::replace_learners(const common::ObMemberList &added_learners, return ret; } +ERRSIM_POINT_DEF(ERRSIM_REPLACE_MEMBER_NOT_REMOVE_ERROR); int PalfHandleImpl::replace_member_with_learner(const common::ObMember &added_member, const common::ObMember &removed_member, const palf::LogConfigVersion &config_version, @@ -944,6 +945,8 @@ int PalfHandleImpl::replace_member_with_learner(const common::ObMember &added_me PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), KPC(this)); } else if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { PALF_LOG(WARN, "add_member in replace_member_with_learner failed", KR(ret), KPC(this), K(args)); + } else if (OB_UNLIKELY(ERRSIM_REPLACE_MEMBER_NOT_REMOVE_ERROR)) { + // do nothing } else if (FALSE_IT(args.server_ = removed_member)) { } else if (FALSE_IT(args.type_ = REMOVE_MEMBER_AND_NUM)) { } else if (OB_FAIL(one_stage_config_change_(args, timeout_us + begin_time_us - common::ObTimeUtility::current_time()))) { diff --git a/src/logservice/palf/palf_iterator.h b/src/logservice/palf/palf_iterator.h index 344c9e4fe2..96fbc16c06 100644 --- a/src/logservice/palf/palf_iterator.h +++ b/src/logservice/palf/palf_iterator.h @@ -90,7 +90,7 @@ public: // OB_INVALID_DATA. // OB_ITER_END, has iterated to the end of block. // OB_NEED_RETRY, the data in cache is not integrity, and the integrity data has been truncate from disk, - // need read data from storage eagin.(data in cache will not been clean up, therefore, + // need read data from storage eagain.(data in cache will not been clean up, therefore, // user need used a new iterator to read data again) // OB_ERR_OUT_LOWER_BOUND, block has been recycled // OB_PARTIAL_LOG, this replica has not finished flashback, and iterator start lsn is not the header of LogGroupEntry. @@ -116,7 +116,7 @@ public: // OB_INVALID_DATA. // OB_ITER_END, has iterated to the end of block. // OB_NEED_RETRY, the data in cache is not integrity, and the integrity data has been truncate from disk, - // need read data from storage eagin.(data in cache will not been clean up, therefore, + // need read data from storage eagain.(data in cache will not been clean up, therefore, // user need used a new iterator to read data again) // OB_ERR_OUT_LOWER_BOUND, block has been recycled // OB_PARTIAL_LOG, this replica has not finished flashback, and iterator start lsn is not the header of LogGroupEntry. @@ -142,9 +142,10 @@ public: // OB_SUCCESS. // OB_INVALID_DATA. // OB_ITER_END, has iterated to the end of block. - // OB_NEED_RETRY, the data in cache is not integrity, and the integrity data has been truncate from disk, - // need read data from storage eagin.(data in cache will not been clean up, therefore, - // user need used a new iterator to read data again) + // OB_NEED_RETRY: + // 1. the data in cache is not integrity, and the integrity data has been truncate from disk, + // need read data from storage eagain. + // 2. during read data from disk, there is a concurrently flashback. // OB_ERR_OUT_LOWER_BOUND, block has been recycled // OB_PARTIAL_LOG, this replica has not finished flashback, and iterator start lsn is not the header of LogGroupEntry. int next(const share::SCN &replayable_point_scn, diff --git a/src/objit/src/expr/ob_expr_info_flag.h b/src/objit/src/expr/ob_expr_info_flag.h index 729add75f4..c47de5bc76 100644 --- a/src/objit/src/expr/ob_expr_info_flag.h +++ b/src/objit/src/expr/ob_expr_info_flag.h @@ -22,7 +22,7 @@ enum ObExprInfoFlag { IS_CONST = 0x0, // const expression, e.g. 1.2, 'abc' IS_COLUMN, // column expression, e.g. C1, T1.C1 - IS_STATIC_PARAM, // vaiable expression, e.g. ? + IS_STATIC_PARAM, // variable expression, e.g. ? IS_AGG, // aggregate function, e.g. max, avg IS_FUNC, // system function IS_IN, @@ -87,7 +87,7 @@ enum ObExprInfoFlag IS_JOIN_COND, // column = column of different tables IS_COLUMNLIZED, // for code generator, @see ObExprGenerator IS_ROOT, //for code generator, @see ObExprGenerator - IS_DYNAMIC_PARAM, //for opt to distinguish normal paramaterized const and exec-params + IS_DYNAMIC_PARAM, //for opt to distinguish normal parameterized const and exec-params IS_NEW_AGG_EXPR, IS_WITH_ALL, IS_WITH_ANY, @@ -167,5 +167,5 @@ inline const char* get_expr_info_flag_str(const ObExprInfoFlag flag) return ret; } } // namespace sql -} // namespace oceabase +} // namespace oceanbase #endif /* OCEANBASE_SRC_SQL_RESOLVER_EXPR_OB_EXPR_INFO_FLAG_H_ */ diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 2e41316f34..adabca15e7 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -161,6 +161,7 @@ ob_set_subtarget(ob_server table table/ttl/ob_tenant_tablet_ttl_mgr.cpp table/ttl/ob_table_ttl_task.cpp table/ttl/ob_table_ttl_executor.cpp + table/ob_table_move_response.cpp ) ob_set_subtarget(ob_server table_load diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp index 50151e98b6..36f80373e3 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp @@ -298,7 +298,7 @@ int ObDBMSSchedJobMaster::start() } else if (OB_FAIL(register_check_tenant_job())) { LOG_WARN("fail to load all dbms sched jobs", K(ret)); } - LOG_WARN("dbms sched job master started", K(ret)); + LOG_INFO("dbms sched job master started", K(ret)); return ret; } @@ -313,7 +313,7 @@ int ObDBMSSchedJobMaster::stop() ready_queue_.clear(); alive_jobs_.clear(); stoped_ = false; - LOG_INFO("dbms sched job master stoped", K(ret), K(lbt())); + LOG_INFO("dbms sched job master stoped", K(ret)); return ret; } @@ -360,7 +360,7 @@ int ObDBMSSchedJobMaster::scheduler() return ret; } -int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key, bool is_retry) +int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key) { int ret = OB_SUCCESS; @@ -368,13 +368,12 @@ int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key, bool is_retr ObDBMSSchedJobInfo job_info; bool can_running = false; - UNUSED(is_retry); - CK (OB_LIKELY(inited_)); CK (OB_NOT_NULL(job_key)); CK (OB_LIKELY(job_key->is_valid())); if (OB_FAIL(ret)) { + LOG_WARN("fail to scheduler job", K(ret), KPC(job_key)); } else if (job_key->is_check_new_tenant()) { OZ (load_and_register_all_jobs(job_key)); } else if (job_key->is_check_new()) { @@ -389,13 +388,15 @@ int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key, bool is_retr OZ (table_operator_.get_dbms_sched_job_info( job_key->get_tenant_id(), job_key->is_oracle_tenant(), job_key->get_job_id(), allocator, job_info)); - if (OB_FAIL(ret)) { + if (OB_FAIL(ret) || !job_info.valid()) { int tmp = alive_jobs_.erase_refactored(job_key->get_job_id_with_tenant()); if (tmp != OB_SUCCESS) { - LOG_ERROR("failed delete valid job from hash set", K(ret), K(job_info)); + LOG_ERROR("failed delete invalid job from hash set", K(tmp), K(ret), K(job_info), KPC(job_key)); + } else { + LOG_INFO("delete invalid job from hash set", K(tmp), K(ret), K(job_info), KPC(job_key)); } allocator_.free(job_key); // sql proxy error - } else if (job_info.valid()) { + } else{ bool ignore_nextdate = false; if (!job_key->is_check() && !job_info.is_running() && !job_info.is_broken() && !job_info.is_disabled()) { bool can_running = false; @@ -406,6 +407,8 @@ int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key, bool is_retr job_info.get_tenant_id(), job_info)); OZ (job_rpc_proxy_->run_dbms_sched_job( job_key->get_tenant_id(), job_key->is_oracle_tenant(), job_key->get_job_id(), execute_addr, self_addr_)); + } else { + LOG_INFO("avoid duplicate job", K(ret), K(job_info), K(can_running)); } ignore_nextdate = true; } @@ -414,14 +417,7 @@ int ObDBMSSchedJobMaster::scheduler_job(ObDBMSSchedJobKey *job_key, bool is_retr if (OB_SUCCESS != (tmp_ret = register_job(job_info, job_key, ignore_nextdate))) { LOG_WARN("failed to register job to job queue", K(tmp_ret)); } - } else { - int tmp = alive_jobs_.erase_refactored(job_key->get_job_id_with_tenant()); - if (tmp != OB_SUCCESS) { - LOG_ERROR("failed delete valid job from hash set", K(ret), K(job_info)); - } - allocator_.free(job_key); // job deleted! } - LOG_DEBUG("scheduler A real JOB!", K(ret), KPC(job_key)); } return ret; } @@ -453,7 +449,7 @@ int ObDBMSSchedJobMaster::alloc_job_key( execute_at, delay, check_job, check_new, check_new_tenant))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to init scheduler job id", K(ret)); + LOG_WARN("fail to init scheduler job id", K(ret), K(tenant_id)); } return ret; } @@ -545,7 +541,7 @@ int ObDBMSSchedJobMaster::server_random_pick(int64_t tenant_id, ObString &pick_z ret = OB_ERR_UNEXPECTED; LOG_WARN("can not find a alive server", K(ret), K(cnt), K(total_server.count())); } else { - LOG_INFO("GET A ADDR FOR EXECUTE", K(ret), K(pick)); + LOG_INFO("get a addr for execute", K(ret), K(tenant_id), K(pick)); server = pick; } } @@ -585,6 +581,7 @@ int ObDBMSSchedJobMaster::register_purge_run_detail_job(int64_t tenant_id, bool } OZ (scheduler_task_.scheduler(job_key)); + LOG_INFO("register purge run detail job", K(ret), K(tenant_id), K(is_oracle_tenant)); return ret; } @@ -615,15 +612,14 @@ int ObDBMSSchedJobMaster::load_and_register_all_jobs(ObDBMSSchedJobKey *job_key) uint64_t data_version = 0; if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_ids.at(i), data_version))) { LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); - } else if (DATA_VERSION_4_2_1_0 <= data_version) { + } else if (MOCK_DATA_VERSION <= data_version) { //add default job class OZ (table_operator_.register_default_job_class(tenant_ids.at(i))); //add purge run detail job OZ (register_purge_run_detail_job(tenant_ids.at(i), tenant_schema->is_oracle_tenant())); } OZ (load_and_register_new_jobs(tenant_ids.at(i), tenant_schema->is_oracle_tenant())); - LOG_INFO("register single tenant", - K(ret), K(tenant_ids.at(i)), K(tenant_schema->get_compatibility_mode())); + LOG_INFO("register single tenant", K(ret), K(tenant_ids.at(i)), K(tenant_schema->get_compatibility_mode())); } OX (max_tenant_id = max_tenant_id > tenant_ids.at(i) ? max_tenant_id : tenant_ids.at(i)); } @@ -650,7 +646,7 @@ int ObDBMSSchedJobMaster::load_and_register_new_jobs(uint64_t tenant_id, ObArenaAllocator allocator; OZ (table_operator_.get_dbms_sched_job_infos_in_tenant(tenant_id, is_oracle_tenant, allocator, job_infos)); - LOG_INFO("load and register new jobs", K(ret), KPC(job_key), K(job_key), K(job_infos)); + LOG_INFO("load and register new jobs", K(ret), K(tenant_id), K(is_oracle_tenant), KPC(job_key), K(job_infos)); OZ (register_jobs(tenant_id, is_oracle_tenant, job_infos, job_key)); return ret; } @@ -665,7 +661,7 @@ int ObDBMSSchedJobMaster::register_jobs(uint64_t tenant_id, uint64_t max_job_id = (NULL != job_key) ? job_key->get_job_id() : 0; int64_t now = ObTimeUtility::current_time(); - int64_t delay = MIN_SCHEDULER_INTERVAL; // 5s + int64_t delay = MIN_SCHEDULER_INTERVAL; for (int64_t i = 0; OB_SUCC(ret) && i < job_infos.count(); i++) { job_info = job_infos.at(i); @@ -676,6 +672,7 @@ int ObDBMSSchedJobMaster::register_jobs(uint64_t tenant_id, } else if (OB_HASH_NOT_EXIST) { OZ (register_job(job_info)); OZ (alive_jobs_.set_refactored(job_info.get_job_id_with_tenant())); + LOG_INFO("register new dbms sched job", K(ret), K(tenant_id), K(job_info), KPC(job_key)); } else { LOG_ERROR("dbms sched job master check job exist failed", K(ret), K(job_info)); } @@ -760,7 +757,7 @@ int ObDBMSSchedJobMaster::register_job( if (OB_FAIL(ret) && OB_NOT_NULL(job_key)) { allocator_.free(job_key); } - LOG_INFO("register new dbms sched job", K(ret), K(job_info), KPC(job_key), K(job_key)); + LOG_INFO("register dbms sched job", K(ret), K(job_info), KPC(job_key), K(ignore_nextdate)); return ret; } diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h index 3167822524..99d023a340 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h @@ -53,7 +53,7 @@ public: ObDBMSSchedJobKey( uint64_t tenant_id, bool is_oracle_tenant, uint64_t job_id, uint64_t execute_at, uint64_t delay, - bool check_job, bool check_new, bool check_new_tenant) + bool check_job, bool check_new, bool check_new_tenant, bool purge_run_detail = false) : tenant_id_(tenant_id), is_oracle_tenant_(is_oracle_tenant), job_id_(job_id), @@ -61,7 +61,8 @@ public: delay_(delay), check_job_(check_job), check_new_(check_new), - check_new_tenant_(check_new_tenant) {} + check_new_tenant_(check_new_tenant), + purge_run_detail_(purge_run_detail) {} virtual ~ObDBMSSchedJobKey() {} @@ -102,8 +103,15 @@ public: bool is_oracle_tenant() { return is_oracle_tenant_; } TO_STRING_KV( - K_(check_job), K_(check_new), K_(check_new_tenant), - K_(execute_at), K_(delay), K_(job_id), K_(tenant_id)); + K_(tenant_id), + K_(is_oracle_tenant), + K_(job_id), + K_(execute_at), + K_(delay), + K_(check_job), + K_(check_new), + K_(check_new_tenant), + K_(purge_run_detail)); private: uint64_t tenant_id_; @@ -213,7 +221,7 @@ public: ObDBMSSchedJobKey *job_key = NULL); int register_job(ObDBMSSchedJobInfo &job_info, ObDBMSSchedJobKey *job_key = NULL, bool ignore_nextdate = false); - int scheduler_job(ObDBMSSchedJobKey *job_key, bool is_retry = false); + int scheduler_job(ObDBMSSchedJobKey *job_key); private: const static int MAX_READY_JOBS_CAPACITY = 1024 * 1024; diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h index 2975a0cf7e..4037facc74 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_utils.h @@ -19,6 +19,9 @@ #include "lib/mysqlclient/ob_isql_client.h" #include "lib/container/ob_iarray.h" + +#define MOCK_DATA_VERSION (oceanbase::common::cal_version(9, 9, 9, 9)) + namespace oceanbase { namespace common @@ -91,11 +94,14 @@ public: K(powner_), K(cowner_), K(last_modify_), + K(start_date_), K(last_date_), K(this_date_), K(next_date_), + K(end_date_), K(total_), K(interval_), + K(repeat_interval_), K(failures_), K(flag_), K(what_), @@ -104,6 +110,7 @@ public: K(field1_), K(scheduler_flags_), K(enabled_), + K(auto_drop_), K(max_run_duration_), K(interval_ts_)); diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp index a0aa30d826..7343d426ba 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_table_operator.cpp @@ -128,14 +128,14 @@ int ObDBMSSchedTableOperator::update_for_end( if (OB_SUCC(ret)) { if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); - } else if (DATA_VERSION_4_2_1_0 <= data_version) { + } else if (MOCK_DATA_VERSION <= data_version) { CK (OB_LIKELY(!job_info.job_class_.empty())); } } ObDBMSSchedJobClassInfo job_class_info; ObArenaAllocator allocator; - if (DATA_VERSION_4_2_1_0 <= data_version) { + if (MOCK_DATA_VERSION <= data_version) { OZ (get_dbms_sched_job_class_info(tenant_id, job_info.is_oracle_tenant(), job_info.get_job_class(), allocator, job_class_info)); } // when if failures > 16 then set broken flag. @@ -177,7 +177,7 @@ int ObDBMSSchedTableOperator::update_for_end( //If a non-existent JOB CLASS is entered when creating a JOB, //job_run_detail still needs to be recorded. bool need_write_job_run_detail = true; - if (DATA_VERSION_4_2_1_0 <= data_version) { + if (MOCK_DATA_VERSION <= data_version) { ObString logging_level = job_class_info.get_logging_level(); if (logging_level.empty()) { LOG_WARN("logging_level may not assigned"); @@ -199,7 +199,7 @@ int ObDBMSSchedTableOperator::update_for_end( OZ (dml2.add_column("code", err)); OZ (dml2.add_column( "message", ObHexEscapeSqlStr(errmsg.empty() ? ObString("SUCCESS") : errmsg))); - if (DATA_VERSION_4_2_1_0 <= data_version) { + if (MOCK_DATA_VERSION <= data_version) { OZ (dml2.add_column("job_class", job_info.job_class_)); } OZ (dml2.splice_insert_sql(OB_ALL_TENANT_SCHEDULER_JOB_RUN_DETAIL_TNAME, sql2)); @@ -231,8 +231,6 @@ int ObDBMSSchedTableOperator::check_job_timeout(ObDBMSSchedJobInfo &job_info) } else if (ObTimeUtility::current_time() > (job_info.get_this_date() + TO_TS(job_info.get_max_run_duration()))) { OZ(update_for_end(job_info.get_tenant_id(), job_info, 0, "check job timeout")); LOG_WARN("job is timeout, force update for end", K(job_info), K(ObTimeUtility::current_time())); - } else { - LOG_DEBUG("job is still running, not timeout", K(job_info)); } return ret; } @@ -246,8 +244,6 @@ int ObDBMSSchedTableOperator::check_auto_drop(ObDBMSSchedJobInfo &job_info) (true == job_info.auto_drop_)) { OZ(update_for_end(job_info.get_tenant_id(), job_info, 0, "check auto drop expired job")); LOG_WARN("auto drop miss out job", K(job_info), K(ObTimeUtility::current_time())); - } else { - LOG_DEBUG("job no need to drop", K(job_info)); } return ret; } @@ -516,13 +512,15 @@ int ObDBMSSchedTableOperator::calc_execute_at( execute_at = job_info.get_next_date(); delay = job_info.get_next_date() - now; } else if (now - job_info.get_next_date() < TO_TS(job_info.get_max_run_duration())) { - LOG_WARN("job maybe missed, retry it", K(now), K(job_info)); + LOG_WARN("job maybe missed, retry it", K(now), K(job_info), K(execute_at), K(delay), K(ignore_nextdate), K(lbt())); execute_at = now; delay = 0; } else if (last_sub_next < 5 && last_sub_next >= -5) { + LOG_WARN("job maybe missed, retry it", K(last_sub_next), K(now), K(job_info), K(execute_at), K(delay), K(ignore_nextdate), K(lbt())); execute_at = now; delay = 0; } else { + LOG_WARN("job maybe missed, ignore it", K(last_sub_next), K(now), K(job_info), K(execute_at), K(delay), K(ignore_nextdate), K(lbt())); delay = -1; } } else { @@ -555,7 +553,6 @@ int ObDBMSSchedTableOperator::calc_execute_at( if (OB_SUCC(ret)) { execute_at = sysdate + job_info.get_interval_ts(); } - LOG_INFO("interval date is", K(sysdate), K(execute_at), K(job_info)); if (OB_FAIL(ret)) { } else if (job_info.get_next_date() > execute_at) { execute_at = job_info.get_next_date(); @@ -570,9 +567,11 @@ int ObDBMSSchedTableOperator::calc_execute_at( } } } + LOG_INFO("repeat job update nextdate", K(job_info), K(execute_at), K(delay), K(ignore_nextdate)); } else if (delay < 0 && job_info.get_interval_ts() == 0) { OX (job_info.next_date_ = 64060560000000000); // 4000-01-01 OZ (update_nextdate(job_info.get_tenant_id(), job_info)); + LOG_INFO("once job update nextdate", K(job_info), K(execute_at), K(delay), K(ignore_nextdate)); } return ret; @@ -603,6 +602,7 @@ int ObDBMSSchedTableOperator::register_default_job_class(uint64_t tenant_id) ret = OB_SUCC(ret) ? tmp_ret : ret; } } + LOG_INFO("register default job class", K(ret), K(tenant_id)); return ret; } @@ -637,6 +637,7 @@ int ObDBMSSchedTableOperator::purge_run_detail_histroy(uint64_t tenant_id) ret = OB_SUCC(ret) ? tmp_ret : ret; } } + LOG_INFO("purge run detail history", K(ret), K(tenant_id), K(sql.ptr())); return ret; } diff --git a/src/observer/mysql/obmp_base.cpp b/src/observer/mysql/obmp_base.cpp index b8cf700946..7bd61f70d7 100644 --- a/src/observer/mysql/obmp_base.cpp +++ b/src/observer/mysql/obmp_base.cpp @@ -317,7 +317,7 @@ int ObMPBase::free_session() LOG_INFO("free session successfully", K(ctx)); conn->is_sess_free_ = true; if (OB_UNLIKELY(OB_FAIL(sql::ObSQLSessionMgr::is_need_clear_sessid(conn, is_need_clear)))) { - LOG_ERROR("fail to jugde need clear", K(ret), "sessid", conn->sessid_, "server_id", GCTX.server_id_); + LOG_ERROR("fail to judge need clear", K(ret), "sessid", conn->sessid_, "server_id", GCTX.server_id_); } else if (is_need_clear) { if (OB_FAIL(GCTX.session_mgr_->mark_sessid_unused(conn->sessid_))) { LOG_WARN("mark session id unused failed", K(ret), "sessid", conn->sessid_); diff --git a/src/observer/mysql/obmp_change_user.cpp b/src/observer/mysql/obmp_change_user.cpp index 6deeb7c76b..cd128d9887 100644 --- a/src/observer/mysql/obmp_change_user.cpp +++ b/src/observer/mysql/obmp_change_user.cpp @@ -330,6 +330,7 @@ int ObMPChangeUser::load_privilege_info(ObSQLSessionInfo *session) OB_LOG(WARN, "user connect failed", K(ret), K(session_priv)); } else { uint64_t db_id = OB_INVALID_ID; + const ObSysVariableSchema *sys_variable_schema = NULL; session->set_user(session_priv.user_name_, session_priv.host_name_, session_priv.user_id_); session->set_user_priv_set(session_priv.user_priv_set_); session->set_db_priv_set(session_priv.db_priv_set_); @@ -340,8 +341,13 @@ int ObMPChangeUser::load_privilege_info(ObSQLSessionInfo *session) OB_LOG(WARN, "failed to set default database", K(ret), K(database_)); } else if (OB_FAIL(session->set_real_client_ip(login_info.client_ip_))) { LOG_WARN("failed to set_real_client_ip", K(ret)); - } else if (OB_FAIL(session->load_default_sys_variable(false, true))) { - LOG_WARN("failed to load system variables", K(ret)); + } else if (OB_FAIL(schema_guard.get_sys_variable_schema(session_priv.tenant_id_, sys_variable_schema))) { + LOG_WARN("get sys variable schema failed", K(ret)); + } else if (OB_ISNULL(sys_variable_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys variable schema is null", K(ret)); + } else if (OB_FAIL(session->load_all_sys_vars(*sys_variable_schema, false))) { + LOG_WARN("load system variables failed", K(ret)); } else if (OB_FAIL(session->update_database_variables(&schema_guard))) { OB_LOG(WARN, "failed to update database variables", K(ret)); } else if (OB_FAIL(schema_guard.get_database_id(session->get_effective_tenant_id(), @@ -350,6 +356,7 @@ int ObMPChangeUser::load_privilege_info(ObSQLSessionInfo *session) OB_LOG(WARN, "failed to get database id", K(ret)); } else { session->set_database_id(db_id); + session->reset_user_var(); } } } diff --git a/src/observer/mysql/obmp_connect.cpp b/src/observer/mysql/obmp_connect.cpp index ea4891ae4f..1d9ca1222a 100644 --- a/src/observer/mysql/obmp_connect.cpp +++ b/src/observer/mysql/obmp_connect.cpp @@ -1826,10 +1826,19 @@ int ObMPConnect::verify_connection(const uint64_t tenant_id) const LOG_WARN("failed to verify_ip_white_list", K(ret)); } else { const int64_t tenant_id = conn->tenant_id_; - if (OB_SYS_TENANT_ID == tenant_id || - 0 == user_name_.compare(OB_SYS_USER_NAME)) { - // sys tenant or root user is considered as vip - } else { + // sys tenant or root(SYS) user is considered as vip + bool check_max_sess = tenant_id != OB_SYS_TENANT_ID; + if (check_max_sess) { + lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID; + if (OB_FAIL(ObCompatModeGetter::get_tenant_mode(tenant_id, compat_mode))) { + LOG_WARN("get_compat_mode failed", K(ret), K(tenant_id)); + } else if (Worker::CompatMode::MYSQL == compat_mode) { + check_max_sess = user_name_.compare(OB_SYS_USER_NAME) != 0; + } else if (Worker::CompatMode::ORACLE == compat_mode) { + check_max_sess = user_name_.compare(OB_ORA_SYS_USER_NAME) != 0; + } + } + if (OB_SUCC(ret) && check_max_sess) { omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); if (tenant_config.is_valid()) { int64_t max_sess_num = 0; @@ -1848,7 +1857,7 @@ int ObMPConnect::verify_connection(const uint64_t tenant_id) const cur_connections))) { LOG_WARN("fail to get session count", K(ret)); } else if (tenant_exists && cur_connections >= max_sess_num) { - ret = OB_RESOURCE_OUT; + ret = OB_ERR_CON_COUNT_ERROR; LOG_WARN("too much sessions", K(ret), K(tenant_id), K(cur_connections), K(max_sess_num), K(tenant_name_), K(user_name_)); } diff --git a/src/observer/mysql/obmp_disconnect.cpp b/src/observer/mysql/obmp_disconnect.cpp index a0100ed2ac..2a005b8f1d 100644 --- a/src/observer/mysql/obmp_disconnect.cpp +++ b/src/observer/mysql/obmp_disconnect.cpp @@ -81,7 +81,7 @@ int ObMPDisconnect::run() EVENT_INC(SQL_USER_LOGOUTS_CUMULATIVE); LOG_INFO("free session successfully", "sessid", ctx_.sessid_); if (OB_UNLIKELY(OB_FAIL(sql::ObSQLSessionMgr::is_need_clear_sessid(&conn, is_need_clear)))) { - LOG_ERROR("fail to jugde need clear", K(ret), "sessid", conn.sessid_, "server_id", GCTX.server_id_); + LOG_ERROR("fail to judge need clear", K(ret), "sessid", conn.sessid_, "server_id", GCTX.server_id_); } else if (is_need_clear) { if (OB_FAIL(GCTX.session_mgr_->mark_sessid_unused(conn.sessid_))) { LOG_WARN("mark session id unused failed", K(ret), "sessid", conn.sessid_); diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 1fc713aee5..34a9ce846e 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -2476,7 +2476,7 @@ int ObMPStmtExecute::parse_param_value(ObIAllocator &allocator, TypeInfo *type_info, ObObjParam ¶m, const char *bitmap, - int16_t param_id) + int64_t param_id) { int ret = OB_SUCCESS; uint64_t length = 0; diff --git a/src/observer/mysql/obmp_stmt_execute.h b/src/observer/mysql/obmp_stmt_execute.h index a9f1f619b8..1d5b8a5b1e 100644 --- a/src/observer/mysql/obmp_stmt_execute.h +++ b/src/observer/mysql/obmp_stmt_execute.h @@ -307,7 +307,7 @@ private: sql::TypeInfo *type_info, ObObjParam ¶m, const char *bitmap, - int16_t param_id = OB_INVALID_INDEX); + int64_t param_id); int parse_complex_param_value(ObIAllocator &allocator, const ObCharsetType charset, const ObCollationType cs_type, diff --git a/src/observer/mysql/obmp_stmt_send_piece_data.cpp b/src/observer/mysql/obmp_stmt_send_piece_data.cpp index 524cd60bf3..bc67b58afc 100644 --- a/src/observer/mysql/obmp_stmt_send_piece_data.cpp +++ b/src/observer/mysql/obmp_stmt_send_piece_data.cpp @@ -74,7 +74,7 @@ int ObMPStmtSendPieceData::before_process() const char* pos = pkt.get_cdata(); // stmt_id ObMySQLUtil::get_int4(pos, stmt_id_); - ObMySQLUtil::get_int2(pos, param_id_); + ObMySQLUtil::get_uint2(pos, param_id_); ObMySQLUtil::get_int1(pos, piece_mode_); int8_t is_null = 0; ObMySQLUtil::get_int1(pos, is_null); @@ -358,7 +358,7 @@ int64_t ObPieceBuffer::to_string(char *buffer, int64_t len) const int ObPiece::piece_init(ObSQLSessionInfo &session, int32_t stmt_id, - int16_t param_id) { + uint16_t param_id) { int ret = OB_SUCCESS; set_stmt_id(stmt_id); set_param_id(param_id); @@ -408,7 +408,7 @@ int ObPieceCache::init_piece_cache(ObSQLSessionInfo &session) } int ObPieceCache::make_piece(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, ObPiece *&piece, ObSQLSessionInfo &session) { @@ -508,7 +508,7 @@ int ObPieceCache::close_all(ObSQLSessionInfo &session) return ret; } -int ObPieceCache::get_piece(int32_t stmt_id, int16_t param_id, ObPiece *&piece) +int ObPieceCache::get_piece(int32_t stmt_id, uint16_t param_id, ObPiece *&piece) { int ret = OB_SUCCESS; piece = NULL; @@ -537,7 +537,7 @@ int ObPieceCache::get_piece(int32_t stmt_id, int16_t param_id, ObPiece *&piece) */ // for fetch int ObPieceCache::get_piece_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, int32_t offset, uint64_t piece_size, ObPieceBuffer &piece_buf, @@ -605,7 +605,7 @@ int ObPieceCache::get_piece_buffer(int32_t stmt_id, // for execute // buf needs to allocate memory in the outer layer ！！！ int ObPieceCache::get_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, uint64_t count, uint64_t &length, common::ObFixedArray &str_buf, @@ -617,7 +617,7 @@ int ObPieceCache::get_buffer(int32_t stmt_id, } int ObPieceCache::get_oracle_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, uint64_t count, uint64_t &length, common::ObFixedArray &str_buf, @@ -651,7 +651,7 @@ int ObPieceCache::get_oracle_buffer(int32_t stmt_id, } int ObPieceCache::get_mysql_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, uint64_t &length, ObSqlString &str_buf) { diff --git a/src/observer/mysql/obmp_stmt_send_piece_data.h b/src/observer/mysql/obmp_stmt_send_piece_data.h index fdc0815820..3c05ea473b 100644 --- a/src/observer/mysql/obmp_stmt_send_piece_data.h +++ b/src/observer/mysql/obmp_stmt_send_piece_data.h @@ -74,7 +74,7 @@ private: int64_t exec_start_timestamp_; int64_t exec_end_timestamp_; int32_t stmt_id_; - int16_t param_id_; + uint16_t param_id_; int64_t buffer_len_; common::ObString buffer_; int8_t piece_mode_; @@ -188,8 +188,8 @@ public: } void set_stmt_id(int32_t stmt_id) { stmt_id_ = stmt_id; } int32_t get_stmt_id() { return stmt_id_; } - void set_param_id(int16_t param_id) { param_id_ = param_id; } - int16_t get_param_id() { return param_id_; } + void set_param_id(uint16_t param_id) { param_id_ = param_id; } + uint16_t get_param_id() { return param_id_; } void set_position(uint64_t pos) { pos_ = pos; } uint64_t get_position() { return pos_; } void add_position() { pos_++; } @@ -205,12 +205,12 @@ public: } ObPieceBufferArray *get_buffer_array() { return buffer_array_; } void set_buffer_array(ObPieceBufferArray *array) { buffer_array_ = array; } - int piece_init(sql::ObSQLSessionInfo &session, int32_t stmt_id, int16_t param_id); + int piece_init(sql::ObSQLSessionInfo &session, int32_t stmt_id, uint16_t param_id); void set_error_ret(int err_ret) { err_ret_ = err_ret; } int get_error_ret() { return err_ret_; } private: int32_t stmt_id_; - int16_t param_id_; + uint16_t param_id_; uint64_t pos_; ObPieceBufferArray *buffer_array_; ObIAllocator *allocator_; @@ -250,36 +250,36 @@ class ObPieceCache { } // piece int make_piece(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, ObPiece *&piece, sql::ObSQLSessionInfo &session); int remove_piece(int64_t key, sql::ObSQLSessionInfo &session); int add_piece(ObPiece *piece); - int get_piece(int32_t stmt_id, int16_t param_id, ObPiece *&piece); + int get_piece(int32_t stmt_id, uint16_t param_id, ObPiece *&piece); int get_piece_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, int32_t offset, uint64_t piece_size, ObPieceBuffer &piece_buf, sql::ObSQLSessionInfo &session); //merge int get_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, uint64_t count, uint64_t &length, common::ObFixedArray &str_buf, char *is_null_map); int get_oracle_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, uint64_t count, uint64_t &length, common::ObFixedArray &str_buf, char *is_null_map); int get_mysql_buffer(int32_t stmt_id, - int16_t param_id, + uint16_t param_id, uint64_t &length, ObSqlString &str_buf); - inline int64_t get_piece_key(int32_t stmt_id, int16_t param_id) + inline int64_t get_piece_key(int32_t stmt_id, uint16_t param_id) { return (((static_cast(stmt_id)) << 32) | param_id); } diff --git a/src/observer/mysql/obsm_conn_callback.cpp b/src/observer/mysql/obsm_conn_callback.cpp index 7427adf430..171ceabbec 100644 --- a/src/observer/mysql/obsm_conn_callback.cpp +++ b/src/observer/mysql/obsm_conn_callback.cpp @@ -214,7 +214,7 @@ void ObSMConnectionCallback::destroy(ObSMConnection& conn) } } else { if (OB_UNLIKELY(OB_FAIL(sql::ObSQLSessionMgr::is_need_clear_sessid(&conn, is_need_clear)))) { - LOG_ERROR("fail to jugde need clear", K(ret)); + LOG_ERROR("fail to judge need clear", K(ret)); } else if (is_need_clear) { if (OB_FAIL(GCTX.session_mgr_->mark_sessid_unused(conn.sessid_))) { LOG_ERROR("fail to mark sessid unused", K(ret), K(conn.sessid_), diff --git a/src/observer/mysql/obsm_handler.cpp b/src/observer/mysql/obsm_handler.cpp index 55ae9269fa..8546d82410 100644 --- a/src/observer/mysql/obsm_handler.cpp +++ b/src/observer/mysql/obsm_handler.cpp @@ -240,7 +240,7 @@ int ObSMHandler::on_close(easy_connection_t *c) } } else { if (OB_UNLIKELY(OB_FAIL(sql::ObSQLSessionMgr::is_need_clear_sessid(conn, is_need_clear)))) { - LOG_ERROR("fail to jugde need clear", K(ret)); + LOG_ERROR("fail to judge need clear", K(ret)); } else if (is_need_clear) { if (OB_UNLIKELY(OB_FAIL(gctx_.session_mgr_->mark_sessid_unused(conn->sessid_)))) { LOG_ERROR("fail to mark sessid unused", K(ret), K(conn->sessid_), diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index e4128f4eae..f96e608f60 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -156,7 +156,9 @@ ObInnerSQLConnection::ObInnerSQLConnection() last_query_timestamp_(0), force_remote_execute_(false), force_no_reuse_(false), - use_external_session_(false) + use_external_session_(false), + group_id_(0), + user_timeout_(0) { } @@ -187,7 +189,8 @@ int ObInnerSQLConnection::init(ObInnerSQLConnectionPool *pool, ObISQLClient *client_addr, /* = NULL */ ObRestoreSQLModifier *sql_modifier /* = NULL */, const bool use_static_engine /* = false */, - const bool is_oracle_mode /* = false */) + const bool is_oracle_mode /* = false */, + const int32_t group_id /* = 0*/) { int ret = OB_SUCCESS; if (inited_) { @@ -222,6 +225,7 @@ int ObInnerSQLConnection::init(ObInnerSQLConnectionPool *pool, if (OB_FAIL(init_session(extern_session, use_static_engine))) { LOG_WARN("init session failed", K(ret)); } else { + group_id_ = group_id; inited_ = true; } } @@ -248,6 +252,7 @@ int ObInnerSQLConnection::destroy() config_ = NULL; associated_client_ = NULL; ref_ctx_ = NULL; + user_timeout_ = 0; } return ret; } @@ -679,6 +684,7 @@ int ObInnerSQLConnection::do_query(sqlclient::ObIExecutor &executor, ObInnerSQLR LOG_WARN("executor execute failed", K(ret)); } else { ObSQLSessionInfo &session = res.result_set().get_session(); + session.set_expect_group_id(group_id_); if (OB_ISNULL(res.sql_ctx().schema_guard_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("schema guard is null"); @@ -1044,7 +1050,7 @@ int ObInnerSQLConnection::start_transaction_inner( LOG_WARN("fail to set tz info wrap", K(ret)); } else if (FALSE_IT(handler->get_result()->set_conn_id(OB_INVALID_ID))) { } else if (OB_FAIL(GCTX.inner_sql_rpc_proxy_->to(resource_server_addr).by(tenant_id). - timeout(query_timeout). + timeout(query_timeout).group_id(group_id_). inner_sql_sync_transmit( arg, *(handler->get_result()), handler->get_handle()))) { LOG_WARN("inner_sql_sync_transmit process failed", K(ret), K(tenant_id)); @@ -1224,6 +1230,7 @@ int ObInnerSQLConnection::forward_request_(const uint64_t tenant_id, } else if (OB_FAIL(GCTX.inner_sql_rpc_proxy_->to(get_resource_svr()) .by(tenant_id) .timeout(query_timeout) + .group_id(group_id_) .inner_sql_sync_transmit(arg, *(handler->get_result()), handler->get_handle()))) { @@ -1294,6 +1301,7 @@ int ObInnerSQLConnection::rollback() LOG_WARN("fail to set tz info wrap", K(ret)); } else if (OB_FAIL(GCTX.inner_sql_rpc_proxy_->to(get_resource_svr()).by(OB_SYS_TENANT_ID). timeout(query_timeout). + group_id(group_id_). inner_sql_sync_transmit( arg, *(handler->get_result()), handler->get_handle()))) { LOG_WARN("inner_sql_sync_transmit process failed", @@ -1364,7 +1372,7 @@ int ObInnerSQLConnection::commit() } else if (OB_FAIL(arg.set_tz_info_wrap(get_session().get_tz_info_wrap()))) { LOG_WARN("fail to set tz info wrap", K(ret)); } else if (OB_FAIL(GCTX.inner_sql_rpc_proxy_->to(get_resource_svr()).by(OB_SYS_TENANT_ID). - timeout(query_timeout). + timeout(query_timeout).group_id(group_id_). inner_sql_sync_transmit( arg, *(handler->get_result()), handler->get_handle()))) { LOG_WARN("inner_sql_sync_transmit process failed", @@ -1503,13 +1511,14 @@ int ObInnerSQLConnection::execute_write_inner(const uint64_t tenant_id, const Ob get_session().get_local_nls_timestamp_format(), get_session().get_local_nls_timestamp_tz_format()); ObInnerSqlRpcStreamHandle *handler = res.remote_result_set().get_stream_handler(); + if (OB_ISNULL(handler)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("handler is null ptr", K(ret)); } else if (OB_FAIL(arg.set_tz_info_wrap(get_session().get_tz_info_wrap()))) { LOG_WARN("fail to set tz info wrap", K(ret)); } else if (OB_FAIL(GCTX.inner_sql_rpc_proxy_->to(get_resource_svr()).by(tenant_id). - timeout(query_timeout).inner_sql_sync_transmit( + timeout(query_timeout).group_id(group_id_).inner_sql_sync_transmit( arg, *(handler->get_result()), handler->get_handle()))) { // complement data for offline ddl may exceed rpc default timeout, thus need to set it to a bigger value for this proxy. LOG_WARN("inner_sql_sync_transmit process failed", K(ret), K(tenant_id)); @@ -1618,6 +1627,7 @@ int ObInnerSQLConnection::execute_read_inner(const int64_t cluster_id, static_assert(ctx_size <= ObISQLClient::ReadResult::BUF_SIZE, "buffer not enough"); ObSqlQueryExecutor executor(sql); const bool local_execute = is_local_execute(cluster_id, tenant_id); + if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("connection not inited", K(ret)); @@ -1704,7 +1714,7 @@ int ObInnerSQLConnection::execute_read_inner(const int64_t cluster_id, } else if (OB_FAIL(arg.set_tz_info_wrap(get_session().get_tz_info_wrap()))) { LOG_WARN("fail to set tz info wrap", K(ret)); } else if (OB_FAIL(GCTX.inner_sql_rpc_proxy_->to(get_resource_svr()). - dst_cluster_id(cluster_id).by(tenant_id).timeout(query_timeout). + dst_cluster_id(cluster_id).by(tenant_id).timeout(query_timeout).group_id(group_id_). inner_sql_sync_transmit( arg, *(handler->get_result()), handler->get_handle()))) { LOG_WARN("inner_sql_sync_transmit process failed", K(ret), K(tenant_id), K(cluster_id)); @@ -1889,7 +1899,7 @@ int ObInnerSQLConnection::set_timeout(int64_t &abs_timeout_us) if (OB_SUCC(ret)) { if (0 == abs_timeout_us) { - timeout = GCONF.internal_sql_execute_timeout; + timeout = (user_timeout_ > 0) ? user_timeout_ : GCONF.internal_sql_execute_timeout; trx_timeout = timeout; abs_timeout_us = now + timeout; } diff --git a/src/observer/ob_inner_sql_connection.h b/src/observer/ob_inner_sql_connection.h index 8d2a9ab532..1e3ac7b9e0 100644 --- a/src/observer/ob_inner_sql_connection.h +++ b/src/observer/ob_inner_sql_connection.h @@ -141,7 +141,8 @@ public: ObISQLClient *client_addr = NULL, ObRestoreSQLModifier *sql_modifer = NULL, const bool use_static_engine = false, - const bool is_oracle_mode = false); + const bool is_oracle_mode = false, + const int32_t group_id = 0); int destroy(void); inline void reset() { destroy(); } virtual int execute_read(const uint64_t tenant_id, const char *sql, @@ -189,7 +190,8 @@ public: virtual void set_force_remote_exec(bool v) { force_remote_execute_ = v; } virtual void set_use_external_session(bool v) { use_external_session_ = v; } bool is_nested_conn(); - + virtual void set_user_timeout(int64_t timeout) { user_timeout_ = timeout; } + virtual int64_t get_user_timeout() const { return user_timeout_; } void ref(); // when ref count decrease to zero, revert connection to connection pool. void unref(); @@ -414,7 +416,9 @@ private: // ask the inner sql connection to use external session instead of internal one // this enables show session / kill session using sql query command bool use_external_session_; - + int32_t group_id_; + //support set user timeout of stream rpc but not depend on internal_sql_execute_timeout + int64_t user_timeout_; DISABLE_COPY_ASSIGN(ObInnerSQLConnection); }; diff --git a/src/observer/ob_inner_sql_connection_pool.cpp b/src/observer/ob_inner_sql_connection_pool.cpp index 3662bbf997..1dce1ffe4f 100644 --- a/src/observer/ob_inner_sql_connection_pool.cpp +++ b/src/observer/ob_inner_sql_connection_pool.cpp @@ -91,7 +91,7 @@ int ObInnerSQLConnectionPool::init(ObMultiVersionSchemaService *schema_service, return ret; } -int ObInnerSQLConnectionPool::acquire(const uint64_t tenant_id, common::sqlclient::ObISQLConnection *&conn, ObISQLClient *client_addr) +int ObInnerSQLConnectionPool::acquire(const uint64_t tenant_id, common::sqlclient::ObISQLConnection *&conn, ObISQLClient *client_addr, const int32_t group_id) { int ret = OB_SUCCESS; UNUSED(tenant_id); @@ -102,7 +102,8 @@ int ObInnerSQLConnectionPool::acquire(const uint64_t tenant_id, common::sqlclien } else if (OB_FAIL(alloc_conn(inner_sql_conn))) { LOG_WARN("alloc connection from pool failed", K(ret)); } else if (OB_FAIL(inner_sql_conn->init(this, schema_service_, ob_sql_, vt_iter_creator_, - config_, nullptr /* session_info */, client_addr, nullptr/*sql modifer*/, is_ddl_))) { + config_, nullptr /* session_info */, client_addr, nullptr/*sql modifer*/, is_ddl_, + false /*is_oracle_mode*/, group_id))) { LOG_WARN("init connection failed", K(ret)); } else if (OB_FAIL(add_to_used_conn_list(inner_sql_conn))) { LOG_WARN("add_to_used_conn_list failed", K(ret)); diff --git a/src/observer/ob_inner_sql_connection_pool.h b/src/observer/ob_inner_sql_connection_pool.h index 4a4de39f47..a09a3d4540 100644 --- a/src/observer/ob_inner_sql_connection_pool.h +++ b/src/observer/ob_inner_sql_connection_pool.h @@ -74,7 +74,7 @@ public: char *to, const int64_t to_size, int64_t &out_size); // acquired connection must be released - virtual int acquire(const uint64_t tenant_id, common::sqlclient::ObISQLConnection *&conn, ObISQLClient *client_addr) override; + virtual int acquire(const uint64_t tenant_id, common::sqlclient::ObISQLConnection *&conn, ObISQLClient *client_addr, const int32_t group_id) override; virtual int release(common::sqlclient::ObISQLConnection *conn, const bool success); int acquire_spi_conn(sql::ObSQLSessionInfo *session_info, observer::ObInnerSQLConnection *&conn); int acquire(sql::ObSQLSessionInfo *session_info, diff --git a/src/observer/ob_resource_inner_sql_connection_pool.cpp b/src/observer/ob_resource_inner_sql_connection_pool.cpp index 0537453830..3007b8d297 100644 --- a/src/observer/ob_resource_inner_sql_connection_pool.cpp +++ b/src/observer/ob_resource_inner_sql_connection_pool.cpp @@ -118,7 +118,13 @@ int ObResourceInnerSQLConnectionPool::acquire( } } else { // get conn with conn_id from id_conn_map_ directly if (OB_FAIL(id_conn_map_.get_refactored(conn_id, conn))) { - LOG_WARN("fail to get from id_conn_map_", K(ret)); + // Failed to get the connection by conn_id. the connection may not have received + // any requests from the source observer for more than ten minutes, and was then + // released by the connection leak check mechanism. Please check first. + ret = OB_SESSION_NOT_FOUND; + LOG_WARN("failed to get the connection by conn_id. the connection may not have received " + "any requests from the source observer for more than ten minutes, and was then " + "released by the connection leak check mechanism.", K(ret)); } else if (FALSE_IT(inner_conn = static_cast(conn))) { } else if (OB_ISNULL(inner_conn)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/observer/ob_signal_handle.cpp b/src/observer/ob_signal_handle.cpp index ca2c5c04a0..078aec4d66 100644 --- a/src/observer/ob_signal_handle.cpp +++ b/src/observer/ob_signal_handle.cpp @@ -114,6 +114,9 @@ int ObSignalHandle::deal_signals(int signum) break; } case SIGTERM: { +#ifdef OB_BUILD_RPM + raise(SIGKILL); +#endif ObServer::get_instance().prepare_stop(); ObServer::get_instance().set_stop(); break; diff --git a/src/observer/ob_sql_client_decorator.cpp b/src/observer/ob_sql_client_decorator.cpp index ec78a9e256..57c3a6d14a 100644 --- a/src/observer/ob_sql_client_decorator.cpp +++ b/src/observer/ob_sql_client_decorator.cpp @@ -39,30 +39,30 @@ int ObSQLClientRetry::read(ReadResult &res, const int64_t cluster_id, const uint return OB_NOT_SUPPORTED; } -int ObSQLClientRetry::read(ReadResult &res, const uint64_t tenant_id, const char *sql) +int ObSQLClientRetry::read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) { int ret = OB_SUCCESS; if (OB_ISNULL(sql_client_)) { ret = OB_INNER_STAT_ERROR; } else { - ret = sql_client_->read(res, tenant_id, sql); + ret = sql_client_->read(res, tenant_id, sql, group_id); if (OB_FAIL(ret)) { for (int32_t retry = 0; retry < retry_limit_ && OB_SUCCESS != ret; retry++) { LOG_WARN("retry execute query when failed", K(ret), K(retry), K_(retry_limit), K(sql)); - ret = sql_client_->read(res, tenant_id, sql); + ret = sql_client_->read(res, tenant_id, sql, group_id); } } } return ret; } -int ObSQLClientRetry::write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) +int ObSQLClientRetry::write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) { int ret = OB_SUCCESS; if (OB_ISNULL(sql_client_)) { ret = OB_INNER_STAT_ERROR; } else { - ret = sql_client_->write(tenant_id, sql, affected_rows); + ret = sql_client_->write(tenant_id, sql, group_id, affected_rows); } return ret; } @@ -139,7 +139,7 @@ int ObSQLClientRetryWeak::read(ReadResult &res, const int64_t cluster_id, const return OB_NOT_SUPPORTED; } -int ObSQLClientRetryWeak::read(ReadResult &res, const uint64_t tenant_id, const char *sql) +int ObSQLClientRetryWeak::read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) { int ret = OB_SUCCESS; if (OB_ISNULL(sql_client_)) { @@ -147,13 +147,13 @@ int ObSQLClientRetryWeak::read(ReadResult &res, const uint64_t tenant_id, const } else { // normal read if (check_sys_variable_) { - ret = sql_client_->read(res, tenant_id, sql); + ret = sql_client_->read(res, tenant_id, sql, group_id); } else { sqlclient::ObISQLConnection *conn = sql_client_->get_connection(); ObSingleConnectionProxy single_conn_proxy; if (OB_NOT_NULL(conn)) { // for transaction - } else if (OB_FAIL(single_conn_proxy.connect(tenant_id, sql_client_))) { + } else if (OB_FAIL(single_conn_proxy.connect(tenant_id, group_id, sql_client_))) { LOG_WARN("failed to get mysql connect", KR(ret), K(tenant_id)); } else { conn = single_conn_proxy.get_connection(); @@ -166,13 +166,13 @@ int ObSQLClientRetryWeak::read(ReadResult &res, const uint64_t tenant_id, const return ret; } -int ObSQLClientRetryWeak::write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) +int ObSQLClientRetryWeak::write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) { int ret = OB_SUCCESS; if (OB_ISNULL(sql_client_)) { ret = OB_INNER_STAT_ERROR; } else { - ret = sql_client_->write(tenant_id, sql, affected_rows); + ret = sql_client_->write(tenant_id, sql, group_id, affected_rows); } return ret; } diff --git a/src/observer/ob_sql_client_decorator.h b/src/observer/ob_sql_client_decorator.h index 0c1a2cab2b..dbaab24d87 100644 --- a/src/observer/ob_sql_client_decorator.h +++ b/src/observer/ob_sql_client_decorator.h @@ -30,9 +30,12 @@ public: virtual int escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size) override; - virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override; + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override { return this->read(res, tenant_id, sql, 0 /*group_id*/); } + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) override; virtual int read(ReadResult &res, const int64_t cluster_id, const uint64_t tenant_id, const char *sql) override; - virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override; + virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override { return this->write(tenant_id, sql, 0/*group_id*/, affected_rows); } + virtual int write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) override; + virtual sqlclient::ObISQLConnectionPool *get_pool() override; virtual sqlclient::ObISQLConnection *get_connection() override; using ObISQLClient::read; @@ -86,9 +89,11 @@ public: virtual int escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size) override; - virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override; + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql) override { return this->read(res, tenant_id, sql, 0 /*group_id*/); } + virtual int read(ReadResult &res, const uint64_t tenant_id, const char *sql, const int32_t group_id) override; virtual int read(ReadResult &res, const int64_t cluster_id, const uint64_t tenant_id, const char *sql) override; - virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override; + virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override { return this->write(tenant_id, sql, 0/*group_id*/, affected_rows); } + virtual int write(const uint64_t tenant_id, const char *sql, const int32_t group_id, int64_t &affected_rows) override; using ObISQLClient::read; using ObISQLClient::write; diff --git a/src/observer/ob_srv_xlator_primary.cpp b/src/observer/ob_srv_xlator_primary.cpp index 09832f128e..504753f97c 100644 --- a/src/observer/ob_srv_xlator_primary.cpp +++ b/src/observer/ob_srv_xlator_primary.cpp @@ -133,6 +133,7 @@ void oceanbase::observer::init_srv_xlator_for_transaction(ObSrvRpcXlator *xlator RPC_PROCESSOR(ObTxCommitRespP); RPC_PROCESSOR(ObTxAbortP); RPC_PROCESSOR(ObTxRollbackSPP); + RPC_PROCESSOR(ObTxRollbackSPRespP); RPC_PROCESSOR(ObTxKeepaliveP); RPC_PROCESSOR(ObTxKeepaliveRespP); //for dup_table diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index ea3032e9a5..c479d1c9e5 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -853,7 +853,11 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons tenant_allocator_created = true; } if (OB_SUCC(ret)) { - if (OB_FAIL(update_tenant_memory(tenant_id, meta.unit_.config_.memory_size(), allowed_mem_limit))) { + int64_t memory_size = meta.unit_.config_.memory_size(); + if (is_sys_tenant(tenant_id) && !meta.super_block_.is_hidden_) { + memory_size += GMEMCONF.get_extra_memory(); + } + if (OB_FAIL(update_tenant_memory(tenant_id, memory_size, allowed_mem_limit))) { LOG_WARN("fail to update tenant memory", K(ret), K(tenant_id)); } } diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 51caad67a9..7be02559d0 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -584,7 +584,7 @@ ObTenant::ObTenant(const int64_t id, shrink_(0), total_worker_cnt_(0), gc_thread_(0), - stopped_(true), + stopped_(0), wait_mtl_finished_(false), req_queue_(), multi_level_queue_(nullptr), @@ -697,7 +697,7 @@ int ObTenant::init(const ObTenantMeta &meta) if (OB_FAIL(ret)) { LOG_ERROR("fail to create tenant module", K(ret)); } else { - ATOMIC_STORE(&stopped_, false); + start(); } return ret; @@ -858,6 +858,15 @@ int ObTenant::create_tenant_module() return ret; } +void ObTenant::sleep_and_warn(ObTenant* tenant) +{ + ob_usleep(10_ms); + const int64_t ts = ObTimeUtility::current_time() - tenant->stopped_; + if (ts >= 3_min && TC_REACH_TIME_INTERVAL(3_min)) { + LOG_ERROR_RET(OB_SUCCESS, "tenant destructed for too long time.", K_(tenant->id), K(ts)); + } +} + void* ObTenant::wait(void* t) { ObStackHeaderGuard stack_header_guard; @@ -869,7 +878,7 @@ void* ObTenant::wait(void* t) tenant->handle_retry_req(true); while (tenant->req_queue_.size() > 0 || (tenant->multi_level_queue_ != nullptr && tenant->multi_level_queue_->get_total_size() > 0)) { - ob_usleep(10L * 1000L); + sleep_and_warn(tenant); } while (tenant->workers_.get_size() > 0) { if (OB_SUCC(tenant->workers_lock_.trylock())) { @@ -879,16 +888,16 @@ void* ObTenant::wait(void* t) destroy_worker(w); } IGNORE_RETURN tenant->workers_lock_.unlock(); - if (REACH_TIME_INTERVAL(10 * 1000L * 1000L)) { + if (REACH_TIME_INTERVAL(10_s)) { LOG_INFO( "Tenant has some workers need stop", K_(tenant->id), "workers", tenant->workers_.get_size(), K_(tenant->req_queue)); } } - ob_usleep(10L * 1000L); + sleep_and_warn(tenant); } - LOG_WARN_RET(OB_SUCCESS,"start remove nesting", K(tenant->nesting_workers_.get_size()), K_(tenant->id)); + LOG_INFO("start remove nesting", K(tenant->nesting_workers_.get_size()), K_(tenant->id)); while (tenant->nesting_workers_.get_size() > 0) { int ret = OB_SUCCESS; if (OB_SUCC(tenant->workers_lock_.trylock())) { @@ -898,7 +907,7 @@ void* ObTenant::wait(void* t) destroy_worker(w); } IGNORE_RETURN tenant->workers_lock_.unlock(); - if (REACH_TIME_INTERVAL(10 * 1000L * 1000L)) { + if (REACH_TIME_INTERVAL(10_s)) { LOG_INFO( "Tenant has some nesting workers need stop", K_(tenant->id), @@ -906,12 +915,12 @@ void* ObTenant::wait(void* t) K_(tenant->req_queue)); } } - ob_usleep(10L * 1000L); + sleep_and_warn(tenant); } - LOG_WARN_RET(OB_SUCCESS, "finish remove nesting", K(tenant->nesting_workers_.get_size()), K_(tenant->id)); - LOG_WARN_RET(OB_SUCCESS, "start remove group_map", K_(tenant->id)); + LOG_INFO("finish remove nesting", K(tenant->nesting_workers_.get_size()), K_(tenant->id)); + LOG_INFO("start remove group_map", K_(tenant->id)); tenant->group_map_.wait_group(); - LOG_WARN_RET(OB_SUCCESS, "finish remove group_map", K_(tenant->id)); + LOG_INFO("finish remove group_map", K_(tenant->id)); if (!is_virtual_tenant_id(tenant->id_) && !tenant->wait_mtl_finished_) { ObTenantSwitchGuard guard(tenant); tenant->stop_mtl_module(); @@ -920,6 +929,7 @@ void* ObTenant::wait(void* t) tenant->wait_mtl_module(); tenant->wait_mtl_finished_ = true; } + LOG_INFO("finish waiting", K_(tenant->id)); return nullptr; } @@ -934,6 +944,8 @@ int ObTenant::try_wait() ret = OB_ERR_UNEXPECTED; LOG_ERROR("tenant gc thread create failed", K(tmp), K(errno), K(id_)); } else { + // it may takes too much time for killing session after remove_tenant, we should recalculate. + ATOMIC_STORE(&stopped_, ObTimeUtility::current_time()); ret = OB_EAGAIN; LOG_INFO("tenant pthread_create gc thread successfully", K(id_), K(gc_thread_)); } @@ -949,6 +961,11 @@ int ObTenant::try_wait() ret = OB_ERR_UNEXPECTED; LOG_ERROR("pthread_tryjoin_np failed", K(tmp), K(errno), K(id_)); } + const int64_t ts = ObTimeUtility::current_time() - stopped_; + // only warn for one time in all tenant. + if (ts >= 3_min && REACH_TIME_INTERVAL(3_min)) { + LOG_ERROR_RET(OB_SUCCESS, "tenant destructed for too long time.", K_(id), K(ts)); + } } return ret; } @@ -1190,8 +1207,8 @@ int ObTenant::recv_request(ObRequest &req) { int ret = OB_SUCCESS; int req_level = 0; - if (ATOMIC_LOAD(&stopped_)) { - ret = OB_IN_STOP_STATE; + if (has_stopped()) { + ret = OB_TENANT_NOT_IN_SERVER; LOG_WARN("receive request but tenant has already stopped", K(ret), K(id_)); } else if (0 != req.get_group_id()) { if (OB_FAIL(recv_group_request(req, req.get_group_id()))) { @@ -1325,7 +1342,7 @@ int ObTenant::recv_large_request(rpc::ObRequest &req) int ObTenant::push_retry_queue(rpc::ObRequest &req, const uint64_t timestamp) { int ret = OB_SUCCESS; - if (ATOMIC_LOAD(&stopped_)) { + if (has_stopped()) { ret = OB_IN_STOP_STATE; LOG_WARN("receive retry request but tenant has already stopped", K(ret), K(id_)); } else if (OB_FAIL(retry_queue_.push(req, timestamp))) { @@ -1338,9 +1355,9 @@ int ObTenant::timeup() { int ret = OB_SUCCESS; ObLDHandle handle; - if (!stopped_ && OB_SUCC(try_rdlock(handle))) { + if (!has_stopped() && OB_SUCC(try_rdlock(handle))) { // it may fail during drop tenant, try next time. - if (!stopped_) { + if (!has_stopped()) { check_group_worker_count(); check_worker_count(); update_token_usage(); diff --git a/src/observer/omt/ob_tenant.h b/src/observer/omt/ob_tenant.h index 428e20dea1..96f2566869 100644 --- a/src/observer/omt/ob_tenant.h +++ b/src/observer/omt/ob_tenant.h @@ -383,11 +383,11 @@ public: int init_ctx(); int init_multi_level_queue(); int init(const ObTenantMeta &meta); - void stop() { ATOMIC_STORE(&stopped_, true); } - void start() { ATOMIC_STORE(&stopped_, false); } + void stop() { ATOMIC_STORE(&stopped_, ObTimeUtility::current_time()); } + void start() { ATOMIC_STORE(&stopped_, 0); } int try_wait(); void destroy(); - bool has_stopped() const { return ATOMIC_LOAD(&stopped_); } + bool has_stopped() const { return ATOMIC_LOAD(&stopped_) != 0; } ObTenantMeta get_tenant_meta(); bool is_hidden(); @@ -498,6 +498,7 @@ public: // OB_INLINE bool has_normal_request() const { return req_queue_.size() != 0; } // OB_INLINE bool has_level_request() const { return OB_NOT_NULL(multi_level_queue_) && multi_level_queue_->get_total_size() != 0; } private: + static void sleep_and_warn(ObTenant* tenant); static void* wait(void* tenant); // update CPU usage void update_token_usage(); @@ -528,7 +529,6 @@ private: int construct_mtl_init_ctx(const ObTenantMeta &meta, share::ObTenantModuleInitCtx *&ctx); int recv_group_request(rpc::ObRequest &req, int64_t group_id); - protected: mutable common::TCRWLock meta_lock_; @@ -540,7 +540,7 @@ protected: volatile bool shrink_ CACHE_ALIGNED; int64_t total_worker_cnt_; pthread_t gc_thread_; - bool stopped_; + int64_t stopped_; bool wait_mtl_finished_; /// tenant task queue, @@ -609,6 +609,8 @@ OB_INLINE int64_t ObResourceGroup::min_worker_cnt() const cnt = std::max(cnt, 8L); } else if (share::OBCG_WR == group_id_) { cnt = 2; // one for take snapshot, one for purge + } else if (share::OBCG_DBA_COMMAND == group_id_) { + cnt = 1; } return cnt; } @@ -623,6 +625,8 @@ OB_INLINE int64_t ObResourceGroup::max_worker_cnt() const cnt = std::max(cnt, tenant_->max_worker_cnt()); } else if (share::OBCG_WR == group_id_) { cnt = 2; // one for take snapshot, one for purge + } else if (share::OBCG_DBA_COMMAND == group_id_) { + cnt = 1; } return cnt; } diff --git a/src/observer/report/ob_tablet_table_updater.cpp b/src/observer/report/ob_tablet_table_updater.cpp index e45b71ef1a..a227030dff 100644 --- a/src/observer/report/ob_tablet_table_updater.cpp +++ b/src/observer/report/ob_tablet_table_updater.cpp @@ -496,7 +496,8 @@ int ObTabletTableUpdater::batch_process_tasks( update_tablet_checksums, update_tablet_tasks, remove_tablet_tasks))) { - LOG_ERROR("generate_tasks failed", KR(ret), "batch_tasks count", batch_tasks.count(), + //There is a situation where there are too many tablet holds and cannot be obtained + LOG_WARN("generate_tasks failed", KR(ret), "batch_tasks count", batch_tasks.count(), "update_tablet_replicas", update_tablet_replicas.count(), "remove_tablet_replicas", remove_tablet_replicas.count(), "update_tablet_checksums", update_tablet_checksums.count(), diff --git a/src/observer/table/ob_table_batch_execute_processor.cpp b/src/observer/table/ob_table_batch_execute_processor.cpp index 770d1a0f95..de46dd2a1e 100644 --- a/src/observer/table/ob_table_batch_execute_processor.cpp +++ b/src/observer/table/ob_table_batch_execute_processor.cpp @@ -21,6 +21,7 @@ #include "ob_htable_utils.h" #include "ob_table_cg_service.h" #include "observer/ob_req_time_service.h" +#include "ob_table_move_response.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -30,6 +31,7 @@ using namespace oceanbase::sql; ObTableBatchExecuteP::ObTableBatchExecuteP(const ObGlobalContext &gctx) :ObTableRpcProcessor(gctx), + default_entity_factory_("TableBatchEntFac", MTL_ID()), allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), tb_ctx_(allocator_), need_rollback_trans_(false), @@ -118,9 +120,9 @@ uint64_t ObTableBatchExecuteP::get_request_checksum() int ObTableBatchExecuteP::response(const int retcode) { int ret = OB_SUCCESS; - if (!need_retry_in_queue_ && !did_async_end_trans()) { + if (!need_retry_in_queue_ && !had_do_response()) { // For HKV table, modify the value of timetamp to be positive - if (OB_SUCC(ret) && ObTableEntityType::ET_HKV == arg_.entity_type_) { + if (ObTableEntityType::ET_HKV == arg_.entity_type_) { const int64_t N = result_.count(); for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i) { @@ -132,7 +134,21 @@ int ObTableBatchExecuteP::response(const int retcode) } } // end for } - if (OB_SUCC(ret)) { + + // return the package even if negate_htable_timestamp fails + const obrpc::ObRpcPacket *rpc_pkt = &reinterpret_cast(req_->get_packet()); + if (is_require_rerouting_err(retcode) && rpc_pkt->require_rerouting()) { + // response rerouting packet + ObTableMoveResponseSender sender(req_, retcode); + if (OB_FAIL(sender.init(arg_.table_id_, arg_.tablet_id_, *gctx_.schema_service_))) { + LOG_WARN("fail to init move response sender", K(ret), K_(arg)); + } else if (OB_FAIL(sender.response())) { + LOG_WARN("fail to do move response", K(ret)); + } + if (OB_FAIL(ret)) { + ret = ObRpcProcessor::response(retcode); // do common response when do move response failed + } + } else { ret = ObRpcProcessor::response(retcode); } } @@ -160,8 +176,10 @@ int ObTableBatchExecuteP::try_process() LOG_WARN("no operation in the batch", K(ret)); } else if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(check_table_index_supported(table_id, is_index_supported))) { - LOG_WARN("fail to check index supported", K(ret), K(table_id)); + } else if (FALSE_IT(table_id_ = arg_.table_id_)) { + } else if (FALSE_IT(tablet_id_ = arg_.tablet_id_)) { + } else if (OB_FAIL(check_table_index_supported(table_id_, is_index_supported))) { + LOG_WARN("fail to check index supported", K(ret), K(table_id_)); } else if (OB_UNLIKELY(!is_index_supported)) { ret = OB_NOT_SUPPORTED; LOG_WARN("index type is not supported by table api", K(ret)); diff --git a/src/observer/table/ob_table_cache.cpp b/src/observer/table/ob_table_cache.cpp index 32fcf0b817..7e4637b0b7 100644 --- a/src/observer/table/ob_table_cache.cpp +++ b/src/observer/table/ob_table_cache.cpp @@ -142,9 +142,13 @@ int ObTableApiCacheGuard::create_cache_key(ObTableCtx *tb_ctx) || operation_type == ObTableOperationType::Type::INSERT_OR_UPDATE || operation_type == ObTableOperationType::Type::INCREMENT || operation_type == ObTableOperationType::Type::APPEND) { - const ObTableCtx::ObAssignIds &assign_ids = tb_ctx->get_assign_ids(); - for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids.count(); i++) { - if (OB_FAIL(cache_key_.op_column_ids_.push_back(assign_ids.at(i).column_id_))) { + const ObIArray &assigns = tb_ctx->get_assignments(); + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + const ObTableAssignment &tmp_assign = assigns.at(i); + if (OB_ISNULL(tmp_assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(tmp_assign)); + } else if (OB_FAIL(cache_key_.op_column_ids_.push_back(tmp_assign.column_item_->column_id_))) { LOG_WARN("fail to add assign column id", K(ret), K(i)); } } diff --git a/src/observer/table/ob_table_cache.h b/src/observer/table/ob_table_cache.h index 9ad0e9f596..f1ecb4e840 100644 --- a/src/observer/table/ob_table_cache.h +++ b/src/observer/table/ob_table_cache.h @@ -55,7 +55,7 @@ struct ObTableApiCacheKey: public ObILibCacheKey int64_t schema_version_; ObTableOperationType::Type operation_type_; bool is_ttl_table_; - common::ObArray op_column_ids_; + common::ObSEArray op_column_ids_; }; class ObTableApiCacheNode: public ObILibCacheNode diff --git a/src/observer/table/ob_table_cg_service.cpp b/src/observer/table/ob_table_cg_service.cpp index 87814cc49a..55a268066b 100644 --- a/src/observer/table/ob_table_cg_service.cpp +++ b/src/observer/table/ob_table_cg_service.cpp @@ -17,274 +17,87 @@ #include "share/system_variable/ob_system_variable.h" // for ObBinlogRowImage::FULL #include "sql/engine/expr/ob_expr_autoinc_nextval.h" // for ObAutoincNextvalExtra +using namespace oceanbase::common; +using namespace oceanbase::share; +using namespace oceanbase::sql; + namespace oceanbase { namespace table { - -ObRawExpr* ObTableExprCgService::get_ref_raw_expr(const ObIArray &all_exprs, - const ObString &col_name) -{ - bool found = false; - ObColumnRefRawExpr *expr = nullptr; - for (int64_t i = 0; i < all_exprs.count() && !found; i++) { - if (all_exprs.at(i)->is_column_ref_expr()) { - expr = static_cast(all_exprs.at(i)); - if (0 == col_name.case_compare(expr->get_column_name())) { - found = true; - } - } - } - - return expr; -} - -int ObTableExprCgService::build_generated_column_expr(ObTableCtx &ctx, - ObColumnRefRawExpr &col_expr, - const ObString &expr_str, - const ObIArray &exprs) +/* + generate column ref exprs. + 1. generate column reference expr which order is same as schema. + 2. generate stored generated column assign item expr + when update stored generated column directly or update its reference exprs. such as: + create table t(`c1` int primary key, `c2` varchar(10), `c3` varchar(10) generated always as (substring(`c2`, 1, 4) stored))); + - update t set `c3`='abc' where `c1`=1; + - update t set `c2`='abc' where `c1`=1; + both need assign `c3` +*/ +int ObTableExprCgService::generate_all_column_exprs(ObTableCtx &ctx) { int ret = OB_SUCCESS; + ObIArray &items = ctx.get_column_items(); const ObTableSchema *table_schema = ctx.get_table_schema(); + const uint64_t column_cnt = table_schema->get_column_count(); if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret), K(ctx)); - } else { - ObArray columns; - ObSchemaChecker schema_checker; - ObSchemaGetterGuard &schema_guard = ctx.get_schema_guard(); - ObRawExprFactory &expr_factory = ctx.get_expr_factory(); - ObSQLSessionInfo &sess_info = ctx.get_session_info(); - ObRawExpr *gen_expr = nullptr; - if (OB_FAIL(schema_checker.init(schema_guard))) { - LOG_WARN("fail to init schema checker", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_generated_column_expr(expr_str, - expr_factory, - sess_info, - gen_expr, - columns, - table_schema, - false, /* allow_sequence */ - nullptr, - &schema_checker))) { - LOG_WARN("fail to build generated expr", K(ret), K(expr_str), K(ctx)); - } else if (OB_ISNULL(gen_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("generated expr is null", K(ret)); - } else { - ObRawExpr *real_ref_expr = nullptr; - bool is_inc_or_append_replace = !ctx.get_delta_exprs().empty(); - if (is_inc_or_append_replace && 2 != columns.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid column for increment or append", K(ret), K(columns.count())); - } - ObIArray &gen_dependants_pairs = ctx.get_gen_dependants_pairs(); - ObTableCtx::ObGenDenpendantsPair pair; - pair.first = &col_expr; // 记录生成列 - const ObIArray *src_exprs = &exprs; - for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { - if (1 == i && is_inc_or_append_replace) { - src_exprs = &ctx.get_delta_exprs(); - } - if (OB_ISNULL(real_ref_expr = get_ref_raw_expr(*src_exprs, - columns.at(i).col_name_))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get ref expr", K(ret), K(columns.at(i).ref_expr_->get_column_id())); - } else if (OB_FAIL(ObRawExprUtils::replace_ref_column(gen_expr, - columns.at(i).ref_expr_, - real_ref_expr))) { - LOG_WARN("fail to replace column reference expr", K(ret)); - } else if (OB_FAIL(add_var_to_array_no_dup(pair.second, real_ref_expr))) { // 记录依赖的列 - LOG_WARN("fail to add expr to array", K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(gen_expr->formalize(&sess_info))) { - LOG_WARN("fail to formailize column reference expr", K(ret)); - } else if (ObRawExprUtils::need_column_conv(col_expr.get_result_type(), *gen_expr)) { - if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(expr_factory, - ctx.get_allocator(), - col_expr, - gen_expr, - &sess_info))) { - LOG_WARN("fail to build column convert expr", K(ret)); - } - } - } - - if (OB_SUCC(ret)) { - gen_expr->set_for_generated_column(); - col_expr.set_dependant_expr(gen_expr); - if (OB_FAIL(gen_dependants_pairs.push_back(pair))) { - LOG_WARN("fail to push back generate ref expr pair", K(ret), K(gen_dependants_pairs)); - } - } - } - } - - return ret; -} - -int ObTableExprCgService::resolve_generated_column_expr(ObTableCtx &ctx) -{ - int ret = OB_SUCCESS; - const ObIArray &exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); - - for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); i++) { - if (exprs.at(i)->is_column_ref_expr()) { - ObColumnRefRawExpr *ref_expr = static_cast(exprs.at(i)); - if (ref_expr->is_generated_column()) { - const ObColumnSchemaV2 *gen_col_schema = nullptr; - if (OB_ISNULL(gen_col_schema = ctx.get_table_schema()->get_column_schema(ref_expr->get_column_id()))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get generated column schema", K(ret), K(ref_expr->get_column_id())); - } else if (OB_FAIL(build_generated_column_expr(ctx, - *ref_expr, - gen_col_schema->get_cur_default_value().get_string(), - exprs))) { - LOG_WARN("fail to build generated raw expr", K(ret), K(*ref_expr)); - } - } - } - } - - return ret; -} - -int ObTableExprCgService::generate_column_ref_raw_expr(ObTableCtx &ctx, - const ObColumnSchemaV2 &col_schema, - ObRawExpr *&expr) -{ - int ret = OB_SUCCESS; - - ObColumnRefRawExpr *col_ref_expr = NULL; - // 存储最开始生成的列原生表达式, 用于后续生成主键冲突 - ObIArray &all_column_exprs = ctx.get_all_column_ref_exprs(); - // 在update场景下, 会多次调用该函数，我们只需要存最开始生成的列引用表达式，记录下是否存储满 - bool is_full = (ctx.get_table_schema()->get_column_count() == all_column_exprs.count()); - - if (OB_FAIL(ObRawExprUtils::build_column_expr(ctx.get_expr_factory(), - col_schema, - col_ref_expr))) { - LOG_WARN("fail to build column expr", K(ret), K(col_schema)); - } else if (OB_ISNULL(col_ref_expr)) { + LOG_WARN("table schema is null", K(ret)); + } else if (items.count() != column_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("expr is null", K(ret)); + LOG_WARN("column item count not equal to column count", K(ret), K(items), K(column_cnt)); } else { - expr = col_ref_expr; - if (!is_full && OB_FAIL(all_column_exprs.push_back(col_ref_expr))) { - LOG_WARN("fail to push back column ref expr to all column exprs", K(ret)); - } else { - // delete/update/get/scan操作只需要生成列引用表达式 - // 自增表达式结构为: con_conv_expr - auto_inc_expr - con_conv_expr - if (col_schema.is_autoincrement() && ctx.need_auto_inc_expr()) { - if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(ctx.get_expr_factory(), - ctx.get_allocator(), - *col_ref_expr, - expr, - &ctx.get_session_info()))) { - LOG_WARN("fail to build column conv expr", K(ret), K(*col_ref_expr)); - } else if (OB_FAIL(generate_autoinc_nextval_expr(ctx, expr, col_schema))) { - LOG_WARN("fail to generate auto inc next val expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(ctx.get_expr_factory(), - ctx.get_allocator(), - *col_ref_expr, - expr, - &ctx.get_session_info()))) { - LOG_WARN("fail to build column conv expr", K(ret)); - } - } - } - } - - return ret; -} - -int ObTableExprCgService::genreate_filter_exprs(ObTableCtx &ctx) -{ - int ret = OB_SUCCESS; - - // won't put filter exprs into all exprs here - if (ctx.is_ttl_table()) { - ObIArray &filter_exprs = ctx.get_filter_exprs(); - ObRawExpr *expire_expr = nullptr; - if (OB_FAIL(build_expire_expr(ctx, expire_expr))) { - LOG_WARN("fail to build expire expr", K(ret), K(ctx)); - } else if(OB_FAIL(filter_exprs.push_back(expire_expr))) { - LOG_WARN("fail to push back expire expr", K(ret), K(filter_exprs)); - } - } - - return ret; -} - -int ObTableExprCgService::generate_exprs(ObTableCtx &ctx, - oceanbase::common::ObIAllocator &allocator, - oceanbase::sql::ObExprFrameInfo &expr_frame_info) -{ - int ret = OB_SUCCESS; - if (ctx.is_for_update() || ctx.is_for_insertup()) { - if (OB_FAIL(generate_update_raw_exprs(ctx))) { - LOG_WARN("fail to generate update raw exprs", K(ret), K(ctx)); - } - } else if (OB_FAIL(generate_column_raw_exprs(ctx))) { - LOG_WARN("fail to generate column raw exprs", K(ret), K(ctx)); - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(genreate_filter_exprs(ctx))) { - LOG_WARN("fail to generate filter exprs", K(ret), K(ctx)); - } else if (OB_FAIL(generate_expr_frame_info(ctx, allocator, expr_frame_info))) { - LOG_WARN("fail to generate expr frame info", K(ret), K(ctx)); - } - } - - return ret; -} - -int ObTableExprCgService::generate_column_raw_exprs(ObTableCtx &ctx) -{ - int ret = OB_SUCCESS; - const ObTableSchema *table_schema = ctx.get_table_schema(); - ObIArray *exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? - &ctx.get_old_row_exprs() : const_cast *>(&ctx.get_all_exprs().get_expr_array()); - - if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret), K(ctx)); - } else { - ObRawExpr *col_ref_expr = nullptr; - const ObColumnSchemaV2 *column_schema = nullptr; - const uint64_t column_cnt = table_schema->get_column_count(); + const ObColumnSchemaV2 *col_schema = nullptr; for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; i++) { - if (OB_ISNULL(column_schema = table_schema->get_column_schema_by_idx(i))) { + ObTableColumnItem &item = items.at(i); + if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(i))) { ret = OB_SCHEMA_ERROR; LOG_WARN("fail to get column schema", K(ret)); - } else if (OB_FAIL(generate_column_ref_raw_expr(ctx, *column_schema, col_ref_expr))) { - LOG_WARN("fail to generate column ref raw expr", K(ret), K(*column_schema)); - } else if (OB_FAIL(exprs->push_back(col_ref_expr))) { - LOG_WARN("fail to push back column ref raw expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_column_expr(ctx.get_expr_factory(), + *col_schema, + item.expr_))) { + LOG_WARN("fail to build column expr", K(ret), K(*col_schema)); } } } - if (OB_SUCC(ret) && table_schema->has_generated_column()) { - if (OB_FAIL(resolve_generated_column_expr(ctx))) { - LOG_WARN("fail to resolver generated column expr", K(ret)); + // generate stored generated column assign item expr + if (OB_SUCC(ret)) { + const ObColumnSchemaV2 *col_schema = nullptr; + ObIArray &assigns = ctx.get_assignments(); + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + ObTableAssignment &assign = assigns.at(i); + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (!assign.column_item_->is_stored_generated_column_) { + // do nothing + } else if (OB_ISNULL(col_schema = table_schema->get_column_schema(assign.column_item_->column_id_))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(assign)); + } else if (OB_FAIL(ObRawExprUtils::build_column_expr(ctx.get_expr_factory(), + *col_schema, + assign.column_item_->expr_))) { + LOG_WARN("fail to build column expr", K(ret), K(*col_schema)); + } } } return ret; } -int ObTableExprCgService::build_expire_expr(ObTableCtx &ctx, ObRawExpr *&expire_expr) +/* + expr tree: + T_OP_LE + / \ + ttl_gen_expr T_FUN_SYS_CUR_TIMESTAMP +*/ +int ObTableExprCgService::generate_expire_expr(ObTableCtx &ctx, + ObRawExpr *&expr) { int ret = OB_SUCCESS; - const ObIArray &exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); const ObTableSchema *table_schema = ctx.get_table_schema(); ObRawExprFactory &expr_factory = ctx.get_expr_factory(); ObSysFunRawExpr *now_func_expr = nullptr; @@ -318,15 +131,18 @@ int ObTableExprCgService::build_expire_expr(ObTableCtx &ctx, ObRawExpr *&expire_ LOG_WARN("fail to build expire expr", K(ret), K(ttl_definition)); } else { // 找到生成列引用的列并替换为真正的列 - ObRawExpr *real_ref_expr = nullptr; - const ObIArray *src_exprs = &exprs; for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { - if (OB_ISNULL(real_ref_expr = get_ref_raw_expr(*src_exprs, columns.at(i).col_name_))) { + const ObQualifiedName &tmp_column = columns.at(i); + const ObString &col_name = tmp_column.col_name_; + ObRawExpr *tmp_expr = nullptr; + if (OB_FAIL(ctx.get_expr_from_column_items(col_name, tmp_expr))) { + LOG_WARN("fail to get expr from column items", K(ret), K(col_name)); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(tmp_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get ref expr", K(ret), K(columns.at(i).ref_expr_->get_column_id())); - } else if (OB_FAIL(ObRawExprUtils::replace_ref_column(ttl_gen_expr, - columns.at(i).ref_expr_, - real_ref_expr))) { + LOG_WARN("expr to replace is null", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::replace_ref_column(ttl_gen_expr, tmp_column.ref_expr_, tmp_expr))) { LOG_WARN("fail to replace column reference expr", K(ret)); } } @@ -337,7 +153,7 @@ int ObTableExprCgService::build_expire_expr(ObTableCtx &ctx, ObRawExpr *&expire_ } else if (OB_FAIL(expire_expr_tmp->formalize(&sess_info))) { LOG_WARN("fail to formailize expire expr", K(ret)); } else { - expire_expr = expire_expr_tmp; + expr = expire_expr_tmp; } } } @@ -346,114 +162,198 @@ int ObTableExprCgService::build_expire_expr(ObTableCtx &ctx, ObRawExpr *&expire_ return ret; } +/* + expr tree: + autoinc_nextval expr + | + column_conv expr +*/ +int ObTableExprCgService::generate_autoinc_nextval_expr(ObTableCtx &ctx, + const ObTableColumnItem &item, + ObRawExpr *&expr) +{ + int ret = OB_SUCCESS; + ObRawExpr *column_cnv_expr = item.expr_; -int ObTableExprCgService::generate_full_assign_raw_exprs(ObTableCtx &ctx) + if (!item.is_auto_increment_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid column item", K(ret), K(item)); + } else if (OB_ISNULL(item.expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("column item expr is null", K(ret), K(item)); + } else if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(ctx.get_expr_factory(), + ctx.get_allocator(), + *item.expr_, + column_cnv_expr, + &ctx.get_session_info()))) { + LOG_WARN("fail to build column conv expr", K(ret), K(item)); + } else { + ObSysFunRawExpr *autoinc_nextval_expr = NULL; + if (OB_FAIL(ctx.get_expr_factory().create_raw_expr(T_FUN_SYS_AUTOINC_NEXTVAL, autoinc_nextval_expr))) { + LOG_WARN("fail to create nextval expr", K(ret)); + } else { + autoinc_nextval_expr->set_func_name(ObString::make_string(N_AUTOINC_NEXTVAL)); + if (OB_FAIL(autoinc_nextval_expr->add_param_expr(column_cnv_expr))) { + LOG_WARN("fail to add collumn conv expr to function param", K(ret)); + } else if (OB_FAIL(autoinc_nextval_expr->formalize(&ctx.get_session_info()))) { + LOG_WARN("fail to extract info", K(ret)); + } else if (OB_FAIL(ObAutoincNextvalExtra::init_autoinc_nextval_extra(&ctx.get_allocator(), + reinterpret_cast(autoinc_nextval_expr), + item.table_id_, + item.column_id_, + ctx.get_table_name(), + item.column_name_))) { + LOG_WARN("fail to init autoinc_nextval_extra", K(ret), K(ctx.get_table_name()), K(item)); + } else { + expr = autoinc_nextval_expr; + } + } + } + + return ret; +} + +/* + expr tree: + column conv expr + | + cur timestamp expr +*/ +int ObTableExprCgService::generate_current_timestamp_expr(ObTableCtx &ctx, + const ObTableColumnItem &item, + ObRawExpr *&expr) +{ + int ret = OB_SUCCESS; + ObSysFunRawExpr *tmp_expr = NULL; + + if (!IS_DEFAULT_NOW_OBJ(item.default_value_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid column item", K(ret), K(item)); + } else if (OB_ISNULL(item.expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("column item expr is null", K(ret), K(item)); + } else if (OB_FAIL(ctx.get_expr_factory().create_raw_expr(T_FUN_SYS_CUR_TIMESTAMP, tmp_expr))) { + LOG_WARN("fail to create cur timestamp expr", K(ret)); + } else { + tmp_expr->set_data_type(ObTimestampType); + tmp_expr->set_accuracy(item.expr_->get_accuracy()); + tmp_expr->set_func_name(ObString::make_string(N_CUR_TIMESTAMP)); + if (OB_FAIL(tmp_expr->formalize(&ctx.get_session_info()))) { + LOG_WARN("fail to formalize cur_timestamp_expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_column_conv_expr(ctx.get_expr_factory(), + ctx.get_allocator(), + *item.expr_, + reinterpret_cast(tmp_expr), + &ctx.get_session_info()))) { + LOG_WARN("fail to build column conv expr", K(ret), K(item)); + } else { + // need clear IS_CONST_EXPR flag, cause StoredRow.to_expr will skip const expr in insertUp/replace executor + tmp_expr->clear_flag(IS_CONST_EXPR); + expr = tmp_expr; + } + } + + return ret; +} + +/* + build generate column expr + - delta_expr is for increment or append operation + - item.expr_ is column ref expr, gen_expr is real calculate expr. +*/ +int ObTableExprCgService::build_generated_column_expr(ObTableCtx &ctx, + ObTableColumnItem &item, + const ObString &expr_str, + ObRawExpr *&expr, + sql::ObRawExpr *delta_expr/* = nullptr*/) { int ret = OB_SUCCESS; const ObTableSchema *table_schema = ctx.get_table_schema(); - ObIArray &full_assign_exprs = ctx.get_full_assign_exprs(); - bool is_inc_or_append = ctx.is_inc_or_append(); - bool has_stored_gen_col = false; if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret), K(ctx)); - } else { - ObRawExpr *col_ref_expr = nullptr; - const ObColumnSchemaV2 *column_schema = nullptr; - const uint64_t column_cnt = table_schema->get_column_count(); - for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; i++) { - bool is_virtual_col = false; - if (OB_ISNULL(column_schema = table_schema->get_column_schema_by_idx(i))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(i)); - } else if (column_schema->is_virtual_generated_column()) { - is_virtual_col = true; - } else if (column_schema->is_stored_generated_column()) { - has_stored_gen_col = true; - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(!is_virtual_col && generate_column_ref_raw_expr(ctx, *column_schema, col_ref_expr))) { - LOG_WARN("fail to generate column ref raw expr", K(ret), K(*column_schema)); - } else if (OB_FAIL(!is_virtual_col && full_assign_exprs.push_back(col_ref_expr))) { - LOG_WARN("fail to push back column ref raw expr", K(ret)); - } - } - } - - if (OB_SUCC(ret) && has_stored_gen_col) { - for (int64_t i = 0; OB_SUCC(ret) && i < full_assign_exprs.count(); i++) { - if (full_assign_exprs.at(i)->is_column_ref_expr()) { - ObColumnRefRawExpr *ref_expr = static_cast(full_assign_exprs.at(i)); - if (ref_expr->is_stored_generated_column()) { - const ObColumnSchemaV2 *gen_col_schema = nullptr; - if (OB_ISNULL(gen_col_schema = ctx.get_table_schema()->get_column_schema(ref_expr->get_column_id()))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get generated column schema", K(ret), K(ref_expr->get_column_id())); - } else if (OB_FAIL(build_generated_column_expr(ctx, - *ref_expr, - gen_col_schema->get_cur_default_value().get_string(), - full_assign_exprs))) { - LOG_WARN("fail to build generated column expr", K(ret), K(*ref_expr)); - } - } - } - } - } - - // 构造delta expr，作为生成列表达式的参数表达式 - // 针对increment和append场景，将对应的列表达式设置为stored生成列表达式，并且构造生成列 - if (OB_SUCC(ret) && is_inc_or_append) { - const ObColumnSchemaV2 *column_schema = nullptr; - ObRawExpr *col_ref_expr = nullptr; - ObIArray &delta_exprs = ctx.get_delta_exprs(); - const ObIArray &expr_strs = ctx.get_expr_strs(); - const ObTableCtx::ObAssignIds &assign_ids = ctx.get_assign_ids(); - const int64_t N = assign_ids.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - uint64_t idx = assign_ids.at(i).idx_; - uint64_t column_id = assign_ids.at(i).column_id_; - if (OB_ISNULL(column_schema = table_schema->get_column_schema(column_id))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(column_id)); - } else if (column_schema->is_autoincrement()) { // do not support delta on auto increment column currently - ret = OB_NOT_SUPPORTED; - LOG_WARN("auto increment column do not support delta", K(ret), K(*column_schema)); - } else if (OB_FAIL(generate_column_ref_raw_expr(ctx, *column_schema, col_ref_expr))) { - LOG_WARN("fail to generate column ref raw expr", K(ret), K(*column_schema)); - } else if (OB_FAIL(delta_exprs.push_back(col_ref_expr))) { - LOG_WARN("fail to push back column ref raw expr", K(ret)); - } else { - ObColumnRefRawExpr *ref_expr = static_cast(full_assign_exprs.at(idx)); - ref_expr->set_column_flags(ref_expr->get_column_flags() | STORED_GENERATED_COLUMN_FLAG); - if (OB_FAIL(build_generated_column_expr(ctx, - *ref_expr, - expr_strs.at(i), - ctx.get_old_row_exprs()))) { - LOG_WARN("fail to build generated column expr", K(ret), K(*ref_expr), K(expr_strs.at(i))); - } - } - } - } - - return ret; -} - -int ObTableExprCgService::generate_assign_exprs(ObTableCtx &ctx, - const ObTableCtx::ObAssignIds &assign_ids, - common::ObIArray &assign_exprs) -{ - int ret = OB_SUCCESS; - const ObIArray &full_exprs = ctx.get_full_assign_exprs(); - - if (assign_ids.count() > full_exprs.count()) { + LOG_WARN("table schema is null", K(ret)); + } else if (ctx.is_inc_or_append() && OB_ISNULL(delta_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid assign ids count", K(ret), K(assign_ids.count()), K(full_exprs.count())); + LOG_WARN("delta expr should not be null when do append or increment", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids.count(); i++) { - const ObTableCtx::ObAssignId &assign_id = assign_ids.at(i); - if (OB_FAIL(assign_exprs.push_back(full_exprs.at(assign_id.idx_)))) { - LOG_WARN("fail to push back assign expr", K(ret), K(i), K(assign_id)); + ObArray columns; + ObSchemaChecker schema_checker; + ObSchemaGetterGuard &schema_guard = ctx.get_schema_guard(); + ObRawExprFactory &expr_factory = ctx.get_expr_factory(); + ObSQLSessionInfo &sess_info = ctx.get_session_info(); + ObRawExpr *gen_expr = nullptr; + + if (OB_FAIL(schema_checker.init(schema_guard))) { + LOG_WARN("fail to init schema checker", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_generated_column_expr(expr_str, + expr_factory, + sess_info, + gen_expr, + columns, + table_schema, + false, /* allow_sequence */ + nullptr, + &schema_checker))) { + LOG_WARN("fail to build generated expr", K(ret), K(expr_str), K(ctx)); + } else { + /* + 1 replace ref columns and add exprs to dependant_exprs. + such as: + `K` varbinary(1024), + `G` varbinary(1024) generated always as (substring(`K`, 1, 4)) + 1.1. gen_expr is substring(`K`, 1, 4) which is reference to `K` + 1.2. `G` is depend on `K`, so we record it. + 2. replace second expr with delta expr when do increment or append. + 2.1 increment expr: (IFNULL(`%s`, 0) + `%s`) - %s is column name + 2.2 append expr:(concat_ws('', `%s`, `%s`)) - %s is column name + 3. get replace expr from assignments firstly. + such as: + `c2` varchar(20), + `c3` varchar(20), + `g` varchar(30) generated always as (concat(`c2`,`c3`)) stored + 3.1 when only update `c2`, 'g' should update as well. + 3.2 `g` expr should be concat(`c2_assign`,`c3`) + */ + for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); i++) { + const ObQualifiedName &tmp_column = columns.at(i); + const ObString &col_name = tmp_column.col_name_; + ObRawExpr *tmp_expr = nullptr; + if (1 == i && ctx.is_inc_or_append()) { + tmp_expr = delta_expr; + } else if (OB_FAIL(ctx.get_expr_from_assignments(col_name, tmp_expr))) { + LOG_WARN("fail to get expr from assignments", K(ret), K(col_name)); + } else if (OB_ISNULL(tmp_expr) && OB_FAIL(ctx.get_expr_from_column_items(col_name, tmp_expr))) { + LOG_WARN("fail to get expr from column items", K(ret), K(col_name)); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(tmp_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr to replace is null", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::replace_ref_column(gen_expr, tmp_column.ref_expr_, tmp_expr))) { + LOG_WARN("fail to replace column reference expr", K(ret)); + } else if (OB_FAIL(add_var_to_array_no_dup(item.dependant_exprs_, tmp_expr))) { + LOG_WARN("fail to add expr to array", K(ret), K(item), K(*tmp_expr)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(gen_expr->formalize(&sess_info))) { + LOG_WARN("fail to formailize column reference expr", K(ret)); + } else if (ObRawExprUtils::need_column_conv(item.expr_->get_result_type(), *gen_expr) + && OB_FAIL(ObRawExprUtils::build_column_conv_expr(expr_factory, + ctx.get_allocator(), + *item.expr_, + gen_expr, + &sess_info))) { + LOG_WARN("fail to build column conv expr", K(ret)); + } else if (ctx.is_inc_or_append()) { + expr = gen_expr; // expr should be a calculate expr in increment or append operation + } else { + gen_expr->set_for_generated_column(); + item.expr_->set_dependant_expr(gen_expr); + expr = item.expr_; + } } } } @@ -461,45 +361,271 @@ int ObTableExprCgService::generate_assign_exprs(ObTableCtx &ctx, return ret; } -int ObTableExprCgService::generate_update_raw_exprs(ObTableCtx &ctx) +/* + construct exprs, include column ref expr and calculate expr. + calculate expr: + - auto increment expr + - current timestamp expr + - generate expr +*/ +int ObTableExprCgService::resolve_exprs(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = ctx.get_table_schema(); + ObIArray &items = ctx.get_column_items(); + bool is_dml = ctx.is_dml(); + + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (items.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column items is empty", K(ret)); + } else { + ObIArray &all_exprs = ctx.get_all_exprs_array(); + ObRawExpr *expr = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < items.count(); i++) { + ObTableColumnItem &item = items.at(i); + if (is_dml) { + if (item.is_auto_increment_ && ctx.need_auto_inc_expr()) { + if (OB_FAIL(generate_autoinc_nextval_expr(ctx, item, expr))) { + LOG_WARN("fail to generate autoinc nextval expr", K(ret)); + } + } else if (IS_DEFAULT_NOW_OBJ(item.default_value_)) { // defualt current time + if (OB_FAIL(generate_current_timestamp_expr(ctx, item, expr))) { + LOG_WARN("fail to generate autoinc nextval expr", K(ret)); + } + } else if (item.is_generated_column_) { + const ObString &expr_str = item.default_value_.get_string(); + if (OB_FAIL(build_generated_column_expr(ctx, item, expr_str, expr))) { + LOG_WARN("fail to build generated column expr", K(ret), K(item), K(expr_str)); + } + } else { + expr = item.expr_; + } + } else { + if (item.is_generated_column_) { + const ObString &expr_str = item.default_value_.get_string(); + if (OB_FAIL(build_generated_column_expr(ctx, item, expr_str, expr))) { + LOG_WARN("fail to build generated column expr", K(ret), K(item), K(expr_str)); + } + } else { + expr = item.expr_; + } + } + + if (OB_SUCC(ret)) { + item.raw_expr_ = expr; + if (OB_FAIL(all_exprs.push_back(expr))) { + LOG_WARN("fail to push back expr to all exprs", K(ret)); + } + } + } + } + + return ret; +} + + +// generate assign expr(column ref expr or calculate expr) for assignment. +int ObTableExprCgService::generate_assign_expr(ObTableCtx &ctx, ObTableAssignment &assign) +{ + int ret = OB_SUCCESS; + ObRawExpr *tmp_expr = nullptr; + ObTableColumnItem *item = assign.column_item_; + + if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item is null", K(ret)); + } else if (FALSE_IT(assign.column_expr_ = item->expr_)) { + } else if (item->is_auto_increment_ && ctx.need_auto_inc_expr()) { + if (OB_FAIL(generate_autoinc_nextval_expr(ctx, *item, tmp_expr))) { + LOG_WARN("fail to generate autoinc nextval expr", K(ret)); + } + } else if (IS_DEFAULT_NOW_OBJ(item->default_value_)) { // defualt current time + if (OB_FAIL(generate_current_timestamp_expr(ctx, *item, tmp_expr))) { + LOG_WARN("fail to generate autoinc nextval expr", K(ret)); + } + } else if (item->is_generated_column_) { + if (!item->is_stored_generated_column_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("assign virtual generated column is not supported", K(ret)); + } else { + if (OB_FAIL(build_generated_column_expr(ctx, *item, item->generated_expr_str_, tmp_expr))) { + LOG_WARN("fail to build generated column expr", K(ret), K(*item)); + } + } + } else if (assign.is_inc_or_append_) { + if (OB_FAIL(build_generated_column_expr(ctx, *item, item->generated_expr_str_, tmp_expr, assign.delta_expr_))) { + LOG_WARN("fail to build generated column expr", K(ret), K(*item)); + } + } else { + // generate column ref expr + const ObTableSchema *table_schema = ctx.get_table_schema(); + const ObColumnSchemaV2 *col_schema = nullptr; + ObColumnRefRawExpr *tmp_ref_expr = nullptr; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (OB_ISNULL(col_schema = table_schema->get_column_schema(item->column_id_))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(*item)); + } else if (OB_FAIL(ObRawExprUtils::build_column_expr(ctx.get_expr_factory(), *col_schema, tmp_ref_expr))) { + LOG_WARN("fail to build column expr", K(ret)); + } + tmp_expr = tmp_ref_expr; + } + + if (OB_SUCC(ret)) { + assign.expr_ = tmp_expr; + } + + return ret; +} + +/* + generate delta expr for increment or append operation. + increment expr: IFNULL(`c1`, 0) + `c1_delta` + append expr: concat_ws('', `c1`, `c1_delta`) +*/ +int ObTableExprCgService::generate_delta_expr(ObTableCtx &ctx, ObTableAssignment &assign) +{ + int ret = OB_SUCCESS; + ObTableColumnItem *item = assign.column_item_; + + if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item is null", K(ret)); + } else if (!assign.is_inc_or_append_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid assignment", K(ret), K(assign)); + } else { + // generate column ref expr + const ObTableSchema *table_schema = ctx.get_table_schema(); + const ObColumnSchemaV2 *col_schema = nullptr; + ObColumnRefRawExpr *tmp_ref_expr = nullptr; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else if (OB_ISNULL(col_schema = table_schema->get_column_schema(item->column_id_))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("fail to get column schema", K(ret), K(*item)); + } else if (OB_FAIL(ObRawExprUtils::build_column_expr(ctx.get_expr_factory(), *col_schema, tmp_ref_expr))) { + LOG_WARN("fail to build column expr", K(ret)); + } + assign.delta_expr_ = tmp_ref_expr; + assign.column_expr_ = item->expr_; // column_expr_ ref to old expr + } + + return ret; +} + +/* + generate assign exprs for update or insertup operation. + - increment and append operations use insertup executor. + - generate delta expr for increment and append operations. + - generate assign expr. + - push back assign expr to all exprs, assign expr need alloc frame as well. +*/ +int ObTableExprCgService::generate_assignments(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + ObIArray &assigns = ctx.get_assignments(); + ObIArray &all_exprs = ctx.get_all_exprs_array(); + + if (!ctx.is_for_update() && !ctx.is_for_insertup()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected operation", K(ret), K(ctx)); + } else if (assigns.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assigns is empty", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + ObTableAssignment &assign = assigns.at(i); + if (assign.is_inc_or_append_ && OB_FAIL(generate_delta_expr(ctx, assign))) { + LOG_WARN("fail to generate delta expr", K(ret), K(assign)); + } else if (OB_FAIL(generate_assign_expr(ctx, assign))) { + LOG_WARN("fail to generate assign expr", K(ret), K(assign)); + } else if (OB_FAIL(all_exprs.push_back(assign.expr_))) { + LOG_WARN("fail to push back expr to all exprs", K(ret)); + } else if (assign.is_inc_or_append_ && OB_FAIL(all_exprs.push_back(assign.expr_))) { + LOG_WARN("fail to push back delta expr to all exprs", K(ret)); + } + } + } + + return ret; +} + +// generate filter exprs and push back to all exprs. +// currently there is only expire expr in filter exprs. +int ObTableExprCgService::generate_filter_exprs(ObTableCtx &ctx) { int ret = OB_SUCCESS; - if (OB_FAIL(generate_column_raw_exprs(ctx))) { - LOG_WARN("fail to generate column raw exprs", K(ret), K(ctx)); - } else if (OB_FAIL(generate_full_assign_raw_exprs(ctx))) { - LOG_WARN("fail to generate assign raw exprs", K(ret), K(ctx)); - } else { - ObRawExprUniqueSet &all_exprs = ctx.get_all_exprs(); - ObIArray &old_row_exprs = ctx.get_old_row_exprs(); - ObIArray &assign_exprs = ctx.get_full_assign_exprs(); - ObIArray &delta_exprs = ctx.get_delta_exprs(); - - // 将 old row exprs加到all exprs里面 - for (int64_t i = 0; OB_SUCC(ret) && i < old_row_exprs.count(); i++) { - if (OB_FAIL(all_exprs.append(old_row_exprs.at(i)))) { - LOG_WARN("fail to append old row expr to all exprs", K(ret), K(i)); - } - } - - // 将 assign row exprs加到all exprs里面 - for (int64_t i = 0; OB_SUCC(ret) && i < assign_exprs.count(); i++) { - if (OB_FAIL(all_exprs.append(assign_exprs.at(i)))) { - LOG_WARN("fail to append assign expr to all exprs", K(ret), K(i)); - } - } - - // 将 delta exprs加到all exprs里面 - for (int64_t i = 0; OB_SUCC(ret) && i < delta_exprs.count(); i++) { - if (OB_FAIL(all_exprs.append(delta_exprs.at(i)))) { - LOG_WARN("fail to append delta expr to all exprs", K(ret), K(i)); - } + if (ctx.is_ttl_table()) { + ObIArray &all_exprs = ctx.get_all_exprs_array(); + ObIArray &filter_exprs = ctx.get_filter_exprs(); + ObRawExpr *expire_expr = nullptr; + if (OB_FAIL(generate_expire_expr(ctx, expire_expr))) { + LOG_WARN("fail to generate expire expr", K(ret), K(ctx)); + } else if(OB_FAIL(filter_exprs.push_back(expire_expr))) { + LOG_WARN("fail to push back expire expr", K(ret), K(filter_exprs)); + } else if (OB_FAIL(all_exprs.push_back(expire_expr))) { + LOG_WARN("fail to push back expire expr to all exprs", K(ret)); } } return ret; } +/* + generate all expressions. + - generate all column exprs firstly. + - resolve_exprs is for generate calculate expr if need, and push back expr to all_exprs. + - generate assign expr when update. + - generate expr frame info finally. +*/ +int ObTableExprCgService::generate_exprs(ObTableCtx &ctx, + oceanbase::common::ObIAllocator &allocator, + oceanbase::sql::ObExprFrameInfo &expr_frame_info) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(generate_all_column_exprs(ctx))) { // 1. generate all column exprs and add to column item array + LOG_WARN("fail to generate all column exprs", K(ret), K(ctx)); + } else if (OB_FAIL(resolve_exprs(ctx))) { // 2. resolve exprs, such as generate expr. + LOG_WARN("fail to resolve exprs", K(ret), K(ctx)); + } else if ((ctx.is_for_update() || ctx.is_for_insertup()) && OB_FAIL(generate_assignments(ctx))) { + LOG_WARN("fail to generate assign infos", K(ret), K(ctx)); + } else if (OB_FAIL(generate_filter_exprs(ctx))) { + LOG_WARN("fail to generate filer exprs", K(ret), K(ctx)); + } else if (OB_FAIL(generate_expr_frame_info(ctx, allocator, expr_frame_info))) { + LOG_WARN("fail to generate expr frame info", K(ret), K(ctx)); + } + + return ret; +} + +// we need to make sure all column ref exprs have beed added, cause scan need column ref exprs. +int ObTableExprCgService::add_extra_column_exprs(ObTableCtx &ctx) +{ + int ret = OB_SUCCESS; + ObIArray &all_exprs = ctx.get_all_exprs_array(); + ObIArray &items = ctx.get_column_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < items.count(); i++) { + const ObTableColumnItem &item = items.at(i); + if (item.is_auto_increment_) { + // do nothing, auto_increment column ref expr has been add in column conv expr param + } else if (OB_FAIL(add_var_to_array_no_dup(all_exprs, static_cast(item.expr_)))) { + LOG_WARN("fail to add column expr", K(ret), K(all_exprs), K(item)); + } + } + + return ret; +} + +// generate expr frame info, expr frame info represents the memory layout of the expr. int ObTableExprCgService::generate_expr_frame_info(ObTableCtx &ctx, common::ObIAllocator &allocator, ObExprFrameInfo &expr_frame_info) @@ -511,25 +637,16 @@ int ObTableExprCgService::generate_expr_frame_info(ObTableCtx &ctx, 0, 0, ctx.get_cur_cluster_version()); - - if (ctx.get_filter_exprs().empty()) { - if (OB_FAIL(expr_cg.generate(ctx.get_all_exprs(), expr_frame_info))) { - LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); - } - } else { - ObRawExprUniqueSet exprs(false/* need_unique */); - if (OB_FAIL(exprs.append(ctx.get_all_exprs().get_expr_array()))) { - LOG_WARN("fail to append all exprs", K(ret), K(ctx.get_all_exprs().get_expr_array())); - } else if (OB_FAIL(exprs.append(ctx.get_filter_exprs()))) { - LOG_WARN("fail to append filter exprs", K(ret), K(ctx.get_filter_exprs())); - } else if (OB_FAIL(expr_cg.generate(exprs, expr_frame_info))) { - LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); - } + if (OB_FAIL(add_extra_column_exprs(ctx))) { + LOG_WARN("fail to add extra column exprs", K(ret), K(ctx)); + } else if (OB_FAIL(expr_cg.generate(ctx.get_all_exprs(), expr_frame_info))) { + LOG_WARN("fail to generate expr frame info by expr cg", K(ret), K(ctx)); } return ret; } +// alloc expr's memory according to expr_frame_info. int ObTableExprCgService::alloc_exprs_memory(ObTableCtx &ctx, ObExprFrameInfo &expr_frame_info) { int ret = OB_SUCCESS; @@ -622,24 +739,10 @@ int ObTableLocCgService::generate_table_loc_meta(const ObTableCtx &ctx, } int ObTableExprCgService::refresh_update_exprs_frame(ObTableCtx &ctx, - const ObIArray &old_row, const ObIArray &new_row, - const ObIArray &full_assign_row, - const ObTableCtx::ObAssignIds &assign_ids, const ObTableEntity &entity) { - int ret = OB_SUCCESS; - - if (OB_FAIL(refresh_assign_exprs_frame(ctx, - old_row, - new_row, - full_assign_row, - assign_ids, - entity))) { - LOG_WARN("fail to refresh assign exprs frame", K(ret), K(assign_ids)); - } - - return ret; + return refresh_assign_exprs_frame(ctx, new_row, entity); } int ObTableExprCgService::refresh_ttl_exprs_frame(ObTableCtx &ctx, @@ -652,7 +755,7 @@ int ObTableExprCgService::refresh_ttl_exprs_frame(ObTableCtx &ctx, int ObTableExprCgService::refresh_insert_up_exprs_frame(ObTableCtx &ctx, const ObIArray &ins_new_row, - const ObIArray &delta_exprs, + const ObIArray &delta_row, const ObTableEntity &entity) { int ret = OB_SUCCESS; @@ -661,9 +764,9 @@ int ObTableExprCgService::refresh_insert_up_exprs_frame(ObTableCtx &ctx, if (OB_FAIL(refresh_rowkey_exprs_frame(ctx, ins_new_row, rowkey))) { LOG_WARN("fail to init rowkey exprs frame", K(ret), K(ctx), K(rowkey)); } else if (OB_FAIL(refresh_properties_exprs_frame(ctx, ins_new_row, entity))) { - LOG_WARN("fail to init properties exprs frame", K(ret), K(ctx)); - } else if (ctx.is_inc_or_append() && OB_FAIL(refresh_delta_exprs_frame(ctx, delta_exprs, entity))) { - LOG_WARN("fail to init delta exprs frame", K(ret), K(ctx)); + LOG_WARN("fail to init properties exprs frame", K(ret), K(ctx), K(ins_new_row), K(entity)); + } else if (ctx.is_inc_or_append() && OB_FAIL(refresh_delta_exprs_frame(ctx, delta_row, entity))) { + LOG_WARN("fail to init delta exprs frame", K(ret), K(ctx), K(delta_row), K(entity)); } return ret; @@ -721,6 +824,48 @@ int ObTableExprCgService::refresh_delete_exprs_frame(ObTableCtx &ctx, return ret; } +/* + write auto increment expr datum. + - auto increment expr tree: + auto increment expr + | + column conv expr + - specific value from user should fill to column conv expr. +*/ +int ObTableExprCgService::write_autoinc_datum(ObTableCtx &ctx, + const ObExpr &expr, + ObEvalCtx &eval_ctx, + const ObObj &obj) +{ + int ret = OB_SUCCESS; + + if (T_FUN_SYS_AUTOINC_NEXTVAL != expr.type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid expr type", K(ret), K(expr)); + } else if (expr.arg_cnt_ != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid arg count for auto inc expr", K(ret), K(expr)); + } else if (expr.get_eval_info(eval_ctx).evaluated_ == true) { + // do nothing + } else { + const ObExpr *conlumn_conv_expr = expr.args_[0]; + if (OB_ISNULL(conlumn_conv_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column conv expr below auto inc expr is null", K(ret)); + } else { + ObDatum &datum = conlumn_conv_expr->locate_datum_for_write(eval_ctx); + if (OB_FAIL(datum.from_obj(obj))) { + LOG_WARN("fail to convert object from datum", K(ret), K(obj)); + } else { + conlumn_conv_expr->get_eval_info(eval_ctx).evaluated_ = true; + conlumn_conv_expr->get_eval_info(eval_ctx).projected_ = true; + } + } + } + + return ret; +} + int ObTableExprCgService::write_datum(ObTableCtx &ctx, ObIAllocator &allocator, const ObExpr &expr, @@ -728,57 +873,17 @@ int ObTableExprCgService::write_datum(ObTableCtx &ctx, const ObObj &obj) { int ret = OB_SUCCESS; - const ObExpr *write_expr = &expr; if (is_lob_storage(obj.get_type()) && (obj.has_lob_header() != expr.obj_meta_.has_lob_header())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to check lob header", K(ret), K(expr), K(obj)); - } else if (expr.type_ == T_FUN_COLUMN_CONV && ctx.has_auto_inc()) { // 为列转换表达式, 且为自增场景下 - if (expr.arg_cnt_ != 6) { - ret = OB_ERR_UNDEFINED; - LOG_WARN("invalid arg count for auto inc expr", K(ret), K(expr)); - } else { - const ObExpr *auto_inc_expr = expr.args_[4]; // 取出中间的列自增表达式 - if (OB_ISNULL(auto_inc_expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("auto inc expr is null", K(ret)); - } else if (auto_inc_expr->get_eval_info(eval_ctx).evaluated_ == true) { - // do nothing - } else if (auto_inc_expr->arg_cnt_ != 1) { - ret = OB_ERR_UNDEFINED; - LOG_WARN("invalid arg count for auto inc expr", K(ret), K(*auto_inc_expr)); - } else { - write_expr = auto_inc_expr->args_[0]; // 取出底部的列转换表达式 - if (OB_ISNULL(write_expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("column conv expr below auto inc expr is null", K(ret)); - } else { - // 用于生成自增主键冲突的列引用表达式，在write_datum时刷值 - ObExpr *auto_inc_ref_col_expr = write_expr->args_[4]; - if (OB_ISNULL(auto_inc_ref_col_expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("auto_inc_ref_col_expr is null", K(ret)); - } else { - ObDatum &datum = auto_inc_ref_col_expr->locate_datum_for_write(eval_ctx); - if (OB_FAIL(datum.from_obj(obj))) { - LOG_WARN("fail to convert object from datum", K(ret), K(obj)); - } else { - auto_inc_ref_col_expr->get_eval_info(eval_ctx).evaluated_ = true; - auto_inc_ref_col_expr->get_eval_info(eval_ctx).projected_ = true; - } - } - } - } - } - } - - if (OB_SUCC(ret)) { - ObDatum &datum = write_expr->locate_datum_for_write(eval_ctx); + } else { + ObDatum &datum = expr.locate_datum_for_write(eval_ctx); if (OB_FAIL(datum.from_obj(obj))) { LOG_WARN("fail to convert object from datum", K(ret), K(obj)); } else { - write_expr->get_eval_info(eval_ctx).evaluated_ = true; - write_expr->get_eval_info(eval_ctx).projected_ = true; + expr.get_eval_info(eval_ctx).evaluated_ = true; + expr.get_eval_info(eval_ctx).projected_ = true; } } @@ -801,217 +906,130 @@ int ObTableExprCgService::refresh_exprs_frame(ObTableCtx &ctx, return ret; } -// exprs必须是按照schema序的完整行 +/* + refresh rowkey frame + 1. The number of entity's rowkey may not equal the number of schema's rowkey + when there is auto_increment column in primary keys. + 2. auto_increment expr tree is autoinc_nextval_expr - column_conv_expr, + we need to fill value to column_conv_expr when user had set value. + 3. "IS_DEFAULT_NOW_OBJ(item.default_value_)" means default current_timestamp in column, + we need to fill eval current timestamp when user not fill value. +*/ int ObTableExprCgService::refresh_rowkey_exprs_frame(ObTableCtx &ctx, const ObIArray &exprs, const ObIArray &rowkey) { int ret = OB_SUCCESS; + const ObIArray &items = ctx.get_column_items(); const int64_t schema_rowkey_cnt = ctx.get_table_schema()->get_rowkey_column_num(); const int64_t entity_rowkey_cnt = rowkey.count(); + bool is_full_filled = (schema_rowkey_cnt == entity_rowkey_cnt); // did user fill all rowkey columns or not ObEvalCtx eval_ctx(ctx.get_exec_ctx()); - bool is_full_filled = schema_rowkey_cnt == entity_rowkey_cnt; - // 不存在自增列的情况下，按照用户的值填datum - // 存在自增列，用户填了值的情况下，使用用户的值刷datum - // 存在自增列，用户没有填值的情况下，使用null obj刷datum - // idx 是为了在主键中存在自增列是为了解决使用i访问obj_ptr会出现访问越界的问题，因为在主键存在自增列场景下，用户可以 - // 不填自增列值，这时rowkey中的obj数量少于schema上的rowkey数量 - for (int64_t i = 0, idx = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { + if (exprs.count() < schema_rowkey_cnt) { + ret = OB_ERR_UNDEFINED; + LOG_WARN("invalid expr count", K(ret), K(exprs), K(schema_rowkey_cnt)); + } else if (items.count() < schema_rowkey_cnt) { + ret = OB_ERR_UNDEFINED; + LOG_WARN("invalid column item count", K(ret), K(items), K(schema_rowkey_cnt)); + } + + for (int64_t i = 0, rowkey_idx = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { + const ObTableColumnItem &item = items.at(i); const ObExpr *expr = exprs.at(i); - if (OB_ISNULL(expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("expr is null", K(ret)); - } else if (ctx.has_auto_inc() && !is_full_filled && expr->type_ == T_FUN_COLUMN_CONV) { // 自增列，用户未填值 + if (T_FUN_SYS_AUTOINC_NEXTVAL == expr->type_) { ObObj null_obj; null_obj.set_null(); - if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, null_obj))) { - LOG_WARN("fail to write datum", K(ret), K(rowkey.at(i)), K(*expr)); + const ObObj *tmp_obj = is_full_filled ? &rowkey.at(rowkey_idx) : &null_obj; + if (OB_FAIL(write_autoinc_datum(ctx, *expr, eval_ctx, *tmp_obj))) { + LOG_WARN("fail to write auto increment datum", K(ret), K(is_full_filled), K(*expr), K(*tmp_obj)); + } + if (is_full_filled) { + rowkey_idx++; + } + } else if (!is_full_filled && IS_DEFAULT_NOW_OBJ(item.default_value_)) { + ObDatum *tmp_datum = nullptr; + if (OB_FAIL(expr->eval(eval_ctx, tmp_datum))) { + LOG_WARN("fail to eval current timestamp expr", K(ret)); } - } else if (idx >= entity_rowkey_cnt) { - ret = OB_INDEX_OUT_OF_RANGE; - LOG_WARN("idx out of range", K(ret), K(idx), K(entity_rowkey_cnt)); - } else if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, rowkey.at(idx)))) { - LOG_WARN("fail to write datum", K(ret), K(rowkey.at(i)), K(*expr)); } else { - idx++; + if (rowkey_idx >= entity_rowkey_cnt) { + ret = OB_INDEX_OUT_OF_RANGE; + LOG_WARN("idx out of range", K(ret), K(i), K(entity_rowkey_cnt)); + } else if (OB_ISNULL(expr)) { + ret = OB_ERR_UNDEFINED; + LOG_WARN("expr is null", K(ret)); + } else if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, rowkey.at(rowkey_idx)))) { + LOG_WARN("fail to write datum", K(ret), K(rowkey_idx), K(rowkey.at(rowkey_idx)), K(*expr)); + } else { + rowkey_idx++; + } } } return ret; } -// exprs必须是按照schema序的完整行 +/* + refresh properties's exprs frame. + - generate column expr eval. + - auto increment expr fill user value or null obj to child expr(column conv expr). + - current timestamp expr eval if user not fill value. + - column ref expr fill user value or default value. +*/ int ObTableExprCgService::refresh_properties_exprs_frame(ObTableCtx &ctx, const ObIArray &exprs, const ObTableEntity &entity) { int ret = OB_SUCCESS; + const ObIArray &items = ctx.get_column_items(); const ObTableSchema *table_schema = ctx.get_table_schema(); ObEvalCtx eval_ctx(ctx.get_exec_ctx()); if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", K(ret)); + } else if (items.count() < table_schema->get_column_count()) { + ret = OB_ERR_UNDEFINED; + LOG_WARN("invalid column item count", K(ret), K(items), K(table_schema->get_column_count())); + } else if (exprs.count() < table_schema->get_column_count()) { + ret = OB_ERR_UNDEFINED; + LOG_WARN("invalid expr count", K(ret), K(exprs), K(table_schema->get_column_count())); } else { ObObj prop_value; - ObObj *obj = nullptr; + const ObObj *obj = nullptr; const int64_t rowkey_column_cnt = table_schema->get_rowkey_column_num(); - for (int64_t i = rowkey_column_cnt; OB_SUCC(ret) && i < exprs.count(); i++) { + for (int64_t i = rowkey_column_cnt; OB_SUCC(ret) && i < items.count(); i++) { + const ObTableColumnItem &item = items.at(i); const ObExpr *expr = exprs.at(i); - const ObColumnSchemaV2 *col_schema = nullptr; - if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(i))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(i), K(*table_schema)); - } else if (col_schema->is_generated_column()) { // 生成列需要eval一次 + if (item.is_generated_column_) { // generate column need eval first ObDatum *tmp_datum = nullptr; if (OB_FAIL(expr->eval(eval_ctx, tmp_datum))) { LOG_WARN("fail to eval generate expr", K(ret)); } } else { - const ObString &col_name = col_schema->get_column_name_str(); // 这里使用schema的列名在entity中查找property，有可能出现本身entity中的prop_name是不对的，导致找不到 - bool use_default = (OB_SEARCH_NOT_FOUND == entity.get_property(col_name, prop_value)); - if (use_default) { - obj = const_cast(&col_schema->get_cur_default_value()); + bool not_found = (OB_SEARCH_NOT_FOUND == entity.get_property(item.column_name_, prop_value)); + if (not_found) { + obj = &item.default_value_; } else { obj = &prop_value; } - if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, *obj))) { - LOG_WARN("fail to write datum", K(ret), K(*obj), K(*expr)); - } - } - } - } - - return ret; -} - -// 将生成列引用的，并且entity没有填充的列刷成旧值 -// eg: create table t(c1 int primary key, -// c2 varchar(10), -// c3 varchar(10), -// gen varchar(30) generated always as (concat(c2,c3)) stored); -// update t set c3 = ? where c1 = ?; -// full row: c1(old) | c2(old) | c3(old) | gen(old) concat(c1(old), c2(old)) -// c1(new) | c2(new) | c3(new) | gen(new) concat(c1(new), c2(new)) -// 这个时候，只是更新了c3的值，gen引用的是c2,c3的值，这里将c2(old)填到c2(new)中 -int ObTableExprCgService::refresh_generated_column_related_frame(ObTableCtx &ctx, - const ObIArray &old_row, - const ObIArray &full_assign_row, - const ObTableCtx::ObAssignIds &assign_ids, - const ObColumnSchemaV2 &col_schema) -{ - int ret = OB_SUCCESS; - const ObTableSchema *table_schema = ctx.get_table_schema(); - ObEvalCtx eval_ctx(ctx.get_exec_ctx()); - ObArray schema_column_ids; - ObArray ref_column_ids; - - if (!col_schema.is_stored_generated_column()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid expr", K(ret), K(col_schema)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("table schema is null", K(ret)); - } else if (OB_FAIL(table_schema->get_column_ids(schema_column_ids))) { - LOG_WARN("fail to get schema column ids", K(ret)); - } else if (OB_FAIL(col_schema.get_cascaded_column_ids(ref_column_ids))) { - LOG_WARN("fail to get cascade column ids", K(ret)); - } else { - ObArray not_upd_col_ids; - // 查询生成列依赖的列是否都被更新，记下未被更新的列到not_upd_col_ids - for (int64_t i = 0; OB_SUCC(ret) && i < ref_column_ids.count(); i++) { - bool found = false; - uint64_t col_id = ref_column_ids.at(i); - for (int64_t j = 0; OB_SUCC(ret) && j < assign_ids.count() && !found; j++) { - if (col_id == assign_ids.at(j).column_id_) { - found = true; - } - } - if (!found && OB_FAIL(not_upd_col_ids.push_back(col_id))) { - LOG_WARN("fail to push back not update column id", K(ret), K(col_id)); - } - } - - // 通过column id找到old row对应的列 - for (int64_t i = 0; OB_SUCC(ret) && i < not_upd_col_ids.count(); i++) { - for (int64_t j = 0; OB_SUCC(ret) && j < schema_column_ids.count(); j++) { - if (schema_column_ids.at(j) == not_upd_col_ids.at(i)) { - if (j >= old_row.count()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid column id", K(ret), K(j), K(old_row.count())); - } else if (j >= full_assign_row.count()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid column id", K(ret), K(j), K(full_assign_row.count())); - } else { - ObExpr *old_expr = old_row.at(j); - ObExpr *new_expr = full_assign_row.at(j); - new_expr->locate_expr_datum(eval_ctx) = old_expr->locate_expr_datum(eval_ctx); - new_expr->get_eval_info(eval_ctx).evaluated_ = true; - new_expr->get_eval_info(eval_ctx).projected_ = true; + if (T_FUN_SYS_AUTOINC_NEXTVAL == expr->type_) { + ObObj null_obj; + null_obj.set_null(); + obj = not_found ? &null_obj : &prop_value; + if (OB_FAIL(write_autoinc_datum(ctx, *expr, eval_ctx, *obj))) { + LOG_WARN("fail to write auto increment datum", K(ret), K(not_found), K(*expr), K(*obj)); } - } - } - } - } - - return ret; -} - -int ObTableExprCgService::refresh_assign_exprs_frame(ObTableCtx &ctx, - const ObIArray &old_row, - const ObIArray &new_row, - const ObIArray &full_assign_row, - const ObTableCtx::ObAssignIds &assign_ids, - const ObTableEntity &entity) -{ - int ret = OB_SUCCESS; - const ObTableSchema *table_schema = ctx.get_table_schema(); - ObEvalCtx eval_ctx(ctx.get_exec_ctx()); - - if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret)); - } else { - ObObj prop_value; - const int64_t N = assign_ids.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - const ObColumnSchemaV2 *col_schema = nullptr; - uint64_t assign_id = assign_ids.at(i).idx_; - if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(assign_id))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(assign_id), K(*table_schema)); - } else if (col_schema->is_virtual_generated_column()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("should not have virtual generated expr", K(ret)); - } else if (col_schema->is_stored_generated_column()) { - if (OB_FAIL(refresh_generated_column_related_frame(ctx, - old_row, - full_assign_row, - assign_ids, - *col_schema))) { - LOG_WARN("fail to refresh generated column related frame", K(ret)); - } else { // 计算一次 - ObExpr *expr = full_assign_row.at(assign_id); + } else if (not_found && IS_DEFAULT_NOW_OBJ(item.default_value_)) { ObDatum *tmp_datum = nullptr; if (OB_FAIL(expr->eval(eval_ctx, tmp_datum))) { - LOG_WARN("fail to eval datum", K(ret), K(*expr)); + LOG_WARN("fail to eval current timestamp expr", K(ret)); } - } - } else if (OB_FAIL(entity.get_property(col_schema->get_column_name_str(), prop_value))) { - LOG_WARN("fail to get assign propertity value", K(ret), K(col_schema->get_column_name_str())); - } else { - if (assign_id >= new_row.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid assign idx", K(ret), K(assign_id), K(new_row.count())); } else { - const ObExpr *expr = new_row.at(assign_id); - if (OB_ISNULL(expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("expr is null", K(ret)); - } else if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, prop_value))) { - LOG_WARN("fail to write datum", K(ret), K(prop_value), K(*expr)); + if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, *obj))) { + LOG_WARN("fail to write datum", K(ret), K(*obj), K(*expr)); } } } @@ -1021,34 +1039,101 @@ int ObTableExprCgService::refresh_assign_exprs_frame(ObTableCtx &ctx, return ret; } + +// refresh delta expr's frame with user specific value. int ObTableExprCgService::refresh_delta_exprs_frame(ObTableCtx &ctx, - const ObIArray &delta_exprs, + const ObIArray &delta_row, const ObTableEntity &entity) { int ret = OB_SUCCESS; - const ObTableSchema *table_schema = ctx.get_table_schema(); + const ObIArray &assigns = ctx.get_assignments(); ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + ObObj prop_value; if (!ctx.is_inc_or_append()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid operation type", K(ret), K(ctx)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret)); + } else if (delta_row.count() > assigns.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid delta row length", K(ret), K(delta_row), K(assigns)); } else { - ObObj prop_value; - const ObTableCtx::ObAssignIds &assign_ids = ctx.get_assign_ids(); - const int64_t N = assign_ids.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - const ObColumnSchemaV2 *col_schema = nullptr; - uint64_t column_id = assign_ids.at(i).column_id_; - if (OB_ISNULL(col_schema = table_schema->get_column_schema(column_id))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(column_id), K(*table_schema)); - } else if (OB_FAIL(entity.get_property(col_schema->get_column_name_str(), prop_value))) { - LOG_WARN("fail to get assign propertity value", K(ret), K(col_schema->get_column_name_str())); + for (int64_t i = 0, idx = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + const ObTableAssignment &assign = assigns.at(i); + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (assign.column_item_->auto_filled_timestamp_) { + // do nothing + } else if (idx >= delta_row.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("index out of range", K(ret), K(assign), K(delta_row)); } else { - const ObExpr *expr = delta_exprs.at(i); + bool not_found = (OB_SEARCH_NOT_FOUND == entity.get_property(assign.column_item_->column_name_, prop_value)); + const ObExpr *expr = delta_row.at(idx); + if (OB_ISNULL(expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("expr is null", K(ret)); + } else if (not_found) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not found delta value", K(ret), K(assign)); + } else if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, prop_value))) { + LOG_WARN("fail to write datum", K(ret), K(prop_value), K(*expr)); + } else { + idx++; + } + } + } + } + + return ret; +} + +/* + refresh assign expr's frame when do update. + - assign virtual generated column is not support. + - eval current timestamp expr if user not assign specific value. + - fill other expr's datum with user value. +*/ +int ObTableExprCgService::refresh_assign_exprs_frame(ObTableCtx &ctx, + const ObIArray &new_row, + const ObTableEntity &entity) +{ + int ret = OB_SUCCESS; + const ObIArray &assigns = ctx.get_assignments(); + ObEvalCtx eval_ctx(ctx.get_exec_ctx()); + ObObj prop_value; + + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + const ObTableAssignment &assign = assigns.at(i); + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (new_row.count() < assign.column_item_->col_idx_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assign projector_index_", K(ret), K(new_row), K(assign.column_item_)); + } else if (assign.column_item_->is_virtual_generated_column_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("virtual generated column not support to update", K(ret), K(assign)); + } else { + // on update current timestamp will not find value + bool not_found = (OB_SEARCH_NOT_FOUND == entity.get_property(assign.column_item_->column_name_, prop_value)); + const ObExpr *expr = new_row.at(assign.column_item_->col_idx_); + if (OB_ISNULL(expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("expr is null", K(ret)); + } else if (not_found) { + if (!assign.column_item_->auto_filled_timestamp_ && !assign.column_item_->is_stored_generated_column_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get assign propertity value", K(ret), K(assign)); + } else if (assign.column_item_->is_stored_generated_column_) { + // do nothing, stored generated column not need to fill + } else { // on update current timestamp + ObDatum *tmp_datum = nullptr; + if (OB_FAIL(expr->eval(eval_ctx, tmp_datum))) { + LOG_WARN("fail to eval current timestamp expr", K(ret)); + } + } + } else { // found if (OB_FAIL(write_datum(ctx, ctx.get_allocator(), *expr, eval_ctx, prop_value))) { LOG_WARN("fail to write datum", K(ret), K(prop_value), K(*expr)); } @@ -1059,35 +1144,53 @@ int ObTableExprCgService::refresh_delta_exprs_frame(ObTableCtx &ctx, return ret; } -int ObTableDmlCgService::replace_exprs_with_dependant(const ObIArray &src_exprs, +/* + create table t(c1 int primary key, + c2 varchar(10), + c3 varchar(10), + c4 varchar(30) generated always as (concat(c2, c3))); + generated expr: c4 + dependant_expr: concat(`c2`, `c3`) + DAS need dependant_expr to calculate result, so we use dependant_expr. +*/ +int ObTableDmlCgService::replace_exprs_with_dependant(ObTableCtx &ctx, ObIArray &dst_exprs) { int ret = OB_SUCCESS; - for (int64_t i = 0; i < src_exprs.count() && OB_SUCC(ret); i++) { - ObRawExpr *expr = src_exprs.at(i); - if (OB_ISNULL(expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("expr is null", K(ret)); - } else if (expr->is_column_ref_expr()) { - ObColumnRefRawExpr *col_ref_expr = static_cast(expr); - if (col_ref_expr->is_generated_column()) { - expr = col_ref_expr->get_dependant_expr(); - } - if (OB_FAIL(dst_exprs.push_back(expr))) { - LOG_WARN("fail to push back expr", K(ret), K(dst_exprs)); - } - } else if (expr->get_expr_type() == T_FUN_COLUMN_CONV) { // 兼容自增场景下的列转换表达式 - if (OB_FAIL(dst_exprs.push_back(expr))) { - LOG_WARN("fail to push back expr", K(ret), K(dst_exprs)); - } + const ObIArray &all_exprs = ctx.get_all_exprs_array(); + ObIArray &items = ctx.get_column_items(); + + if (all_exprs.count() < items.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid expr count", K(ret), K(all_exprs), K(items)); + } + + for (int64_t i = 0; i < items.count() && OB_SUCC(ret); i++) { + const ObTableColumnItem &item = items.at(i); + ObRawExpr *tmp_expr = nullptr; + if (item.is_generated_column_) { + ObColumnRefRawExpr *col_ref_expr = static_cast(all_exprs.at(i)); + tmp_expr = col_ref_expr->get_dependant_expr(); } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid expr type", K(ret), K(*expr)); + tmp_expr = all_exprs.at(i); + } + if (OB_FAIL(dst_exprs.push_back(tmp_expr))) { + LOG_WARN("fail to push back expr", K(ret)); } } + return ret; } +/* + genreate insert ctdef + - replace exprs with depenedant expr if there are generated column. + - construct new row. + - old row is empty in insert ctdef + - generate base ctdef which include column_ids, old_row and new_row. + - generate das insert ctdef which include projector, table_id and so on. + - generate related(index) insert ctdef. +*/ int ObTableDmlCgService::generate_insert_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, ObTableInsCtDef &ins_ctdef) @@ -1095,17 +1198,13 @@ int ObTableDmlCgService::generate_insert_ctdef(ObTableCtx &ctx, int ret = OB_SUCCESS; ObSEArray old_row; ObSEArray new_row; - const ObIArray &exprs = ctx.is_for_insertup() ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); ObSEArray tmp_exprs; - if (OB_FAIL(replace_exprs_with_dependant(exprs, tmp_exprs))) { - LOG_WARN("fail to replace exprs with dependant", K(ret), K(exprs)); + + if (OB_FAIL(replace_exprs_with_dependant(ctx, tmp_exprs))) { + LOG_WARN("fail to replace exprs with dependant", K(ret), K(ctx)); } else if (OB_FAIL(new_row.assign(tmp_exprs))) { LOG_WARN("fail to assign new row", K(ret)); - } else if (OB_FAIL(generate_base_ctdef(ctx, - ins_ctdef, - old_row, - new_row))) { + } else if (OB_FAIL(generate_base_ctdef(ctx, ins_ctdef, old_row, new_row))) { LOG_WARN("fail to generate dml base ctdef", K(ret)); } else if (OB_FAIL(generate_das_ins_ctdef(ctx, ctx.get_ref_table_id(), @@ -1122,76 +1221,59 @@ int ObTableDmlCgService::generate_insert_ctdef(ObTableCtx &ctx, return ret; } +/* + create table t(c1 int primary key, c2 int default null, c3 int default null); + insert into t values(1,1,1); + update t set c3=2 where c1=1; + + assign expr: c3' + old row: c1, c2, c3 + new row: c1, c2, c3' + full row: c1, c2, c3, c3' +*/ int ObTableDmlCgService::generate_update_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, - const ObTableCtx::ObAssignIds &assign_ids, ObTableUpdCtDef &upd_ctdef) { int ret = OB_SUCCESS; ObSEArray old_row; ObSEArray new_row; ObSEArray full_row; - ObSEArray assign_exprs; + ObSEArray delta_row; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); - ObIArray &old_exprs = ctx.get_old_row_exprs(); ObSEArray tmp_old_exprs; ObSEArray tmp_full_assign_exprs; - if (OB_FAIL(replace_exprs_with_dependant(old_exprs, tmp_old_exprs))) { - LOG_WARN("fail to replace exprs with dependant", K(ret), K(old_exprs)); + if (OB_FAIL(replace_exprs_with_dependant(ctx, tmp_old_exprs))) { + LOG_WARN("fail to replace exprs with dependant", K(ret)); } else if (OB_FAIL(old_row.assign(tmp_old_exprs))) { LOG_WARN("fail to assign old row expr", K(ret)); } else if (OB_FAIL(new_row.assign(old_row))) { LOG_WARN("fail to assign new row", K(ret)); } else if (OB_FAIL(append(full_row, old_row))) { LOG_WARN("fail to append old row expr to full row", K(ret), K(old_row)); - } else if (OB_FAIL(ObTableExprCgService::generate_assign_exprs(ctx, assign_ids, assign_exprs))) { - LOG_WARN("fail to generate assign exprs", K(ret), K(ctx), K(assign_ids)); } else { - ObColumnRefRawExpr *col_ref_expr = nullptr; - for (int64_t i = 0; OB_SUCC(ret) && i < assign_exprs.count(); i++) { - ObRawExpr *expr = assign_exprs.at(i); - if (OB_ISNULL(expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("expr is null", K(ret)); - } else if (expr->get_expr_type() == T_FUN_COLUMN_CONV && ctx.has_auto_inc()) { // 兼容自增场景下的列转换表达式 - if (OB_FAIL(full_row.push_back(expr))) { - LOG_WARN("fail to add assign expr to full row", K(ret), K(i)); - } - for (int64_t j = 0; OB_SUCC(ret) && j < old_exprs.count(); j++) { - if (old_exprs.at(j)->get_expr_type() == T_FUN_COLUMN_CONV) { - new_row.at(j) = expr; - } - } + ObRawExpr *tmp_expr = nullptr; + ObIArray &assigns = ctx.get_assignments(); + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + const ObTableAssignment &assign = assigns.at(i); + if (assign.column_expr_->is_generated_column()) { + tmp_expr = assign.column_expr_->get_dependant_expr(); } else { - if (!expr->is_column_ref_expr()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected expr type", K(ret), K(*expr)); - } else if (FALSE_IT(col_ref_expr = static_cast(expr))) { - } else if (col_ref_expr->is_generated_column()) { - expr = col_ref_expr->get_dependant_expr(); - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(full_row.push_back(expr))) { - LOG_WARN("fail to add assign expr to full row", K(ret), K(i)); - } else { - ObColumnRefRawExpr *old_col_expr = nullptr; - for (int64_t j = 0; OB_SUCC(ret) && j < old_exprs.count(); j++) { - ObRawExpr *old_expr = old_exprs.at(j); - if (OB_ISNULL(old_expr)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("old_expr is null", K(ret)); - } else if (!old_expr->is_column_ref_expr() && (old_expr->get_expr_type() != T_FUN_COLUMN_CONV)) { // 兼容自增场景下的列转换表达式 - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected expr type", K(ret), K(*old_exprs.at(i))); - } else if (old_expr->get_expr_type() == T_FUN_COLUMN_CONV) { - // do nothing - } else if (FALSE_IT(old_col_expr = static_cast(old_expr))) { - } else if (old_col_expr->get_column_id() == col_ref_expr->get_column_id()) { - new_row.at(j) = expr; - } - } - } + tmp_expr = assign.expr_; + } + if (OB_FAIL(full_row.push_back(tmp_expr))) { + LOG_WARN("fail to add assign expr to full row", K(ret), K(i)); + } else if (assign.is_inc_or_append_ && OB_FAIL(delta_row.push_back(assign.delta_expr_))) { + LOG_WARN("fail to add delta expr to delta row", K(ret), K(assign)); + } else if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (assign.column_item_->col_idx_ >= new_row.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column index", K(ret), K(assign), K(new_row)); + } else { + new_row.at(assign.column_item_->col_idx_) = tmp_expr; } } } @@ -1205,15 +1287,10 @@ int ObTableDmlCgService::generate_update_ctdef(ObTableCtx &ctx, LOG_WARN("fail to generate dml base ctdef", K(ret)); } else if (OB_FAIL(cg.generate_rt_exprs(full_row, upd_ctdef.full_row_))) { LOG_WARN("fail to generate dml update full row exprs", K(ret), K(full_row)); - } else if (OB_FAIL(replace_exprs_with_dependant(ctx.get_full_assign_exprs(), tmp_full_assign_exprs))) { - LOG_WARN("fail to replace exprs with dependant", K(ret)); - } else if (OB_FAIL(cg.generate_rt_exprs(tmp_full_assign_exprs, upd_ctdef.full_assign_row_))) { - LOG_WARN("fail to generate full assign row exprs", K(ret)); - } else if (ctx.is_inc_or_append() && OB_FAIL(cg.generate_rt_exprs(ctx.get_delta_exprs(), upd_ctdef.delta_exprs_))) { - LOG_WARN("fail to generate delta exprs", K(ret)); + } else if (!delta_row.empty() && OB_FAIL(cg.generate_rt_exprs(delta_row, upd_ctdef.delta_row_))) { + LOG_WARN("fail to generate dml update delta row exprs", K(ret), K(delta_row)); } else if (OB_FAIL(generate_das_upd_ctdef(ctx, ctx.get_ref_table_id(), - assign_exprs, upd_ctdef.das_ctdef_, old_row, new_row, @@ -1221,19 +1298,16 @@ int ObTableDmlCgService::generate_update_ctdef(ObTableCtx &ctx, LOG_WARN("fail to generate das upd ctdef", K(ret)); } else if (OB_FAIL(generate_related_upd_ctdef(ctx, allocator, - assign_exprs, old_row, new_row, full_row, upd_ctdef.related_ctdefs_))) { LOG_WARN("fail to generate related upd ctdef", K(ret)); - } else if (OB_FAIL(generate_upd_assign_infos(ctx, allocator, assign_exprs, upd_ctdef))) { + } else if (OB_FAIL(generate_upd_assign_infos(ctx, allocator, upd_ctdef))) { LOG_WARN("fail to generate related upd assign info", K(ret)); } - if (OB_FAIL(ret)) { - // do nothing - } else if (ctx.is_for_insertup()) { + if (OB_SUCC(ret) && ctx.is_for_insertup()) { ObDMLCtDefAllocator ddel_allocator(allocator); ObDMLCtDefAllocator dins_allocator(allocator); if (OB_ISNULL(upd_ctdef.ddel_ctdef_ = ddel_allocator.alloc())) { @@ -1270,7 +1344,6 @@ int ObTableDmlCgService::generate_update_ctdef(ObTableCtx &ctx, int ObTableDmlCgService::generate_das_upd_ctdef(ObTableCtx &ctx, uint64_t index_tid, - const ObIArray &assign_exprs, ObDASUpdCtDef &das_upd_ctdef, const ObIArray &old_row, const ObIArray &new_row, @@ -1281,14 +1354,9 @@ int ObTableDmlCgService::generate_das_upd_ctdef(ObTableCtx &ctx, if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_upd_ctdef))) { LOG_WARN("fail to generate das dml ctdef", K(ret)); - } else if (OB_FAIL(generate_updated_column_ids(ctx, - assign_exprs, - das_upd_ctdef.column_ids_, - das_upd_ctdef.updated_column_ids_))) { + } else if (OB_FAIL(generate_updated_column_ids(ctx, das_upd_ctdef.updated_column_ids_))) { LOG_WARN("fail to add updated column ids", K(ret)); - } else if (OB_FAIL(generate_column_ids(ctx, - ctx.get_old_row_exprs(), - dml_column_ids))) { + } else if (OB_FAIL(generate_column_ids(ctx, dml_column_ids))) { LOG_WARN("fail to generate dml column ids", K(ret)); } else if (OB_FAIL(generate_projector(dml_column_ids, // new row and old row's columns id das_upd_ctdef.column_ids_, // schmea column ids for given index_tid @@ -1303,34 +1371,22 @@ int ObTableDmlCgService::generate_das_upd_ctdef(ObTableCtx &ctx, } int ObTableDmlCgService::generate_updated_column_ids(ObTableCtx &ctx, - const ObIArray &assign_exprs, - const ObIArray &column_ids, ObIArray &updated_column_ids) { int ret = OB_SUCCESS; + ObIArray &assigns = ctx.get_assignments(); updated_column_ids.reset(); - if (!assign_exprs.empty()) { - if (OB_FAIL(updated_column_ids.reserve(assign_exprs.count()))) { - LOG_WARN("fail to init updated column ids array", K(ret), K(assign_exprs.count())); - } else { - ObColumnRefRawExpr *col_expr = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < assign_exprs.count(); i++) { - if (assign_exprs.at(i)->get_expr_type() == T_FUN_COLUMN_CONV && ctx.has_auto_inc()) { // 兼容自增场景下的列转换表达式 - if (OB_FAIL(updated_column_ids.push_back(ctx.get_auto_inc_column_id()))) { - LOG_WARN("fail to add updated column id", K(ret), K(ctx.get_auto_inc_column_id())); - } - } else { - if (!assign_exprs.at(i)->is_column_ref_expr()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected expr type", K(ret), K(assign_exprs.at(i)->get_expr_type())); - } else if (FALSE_IT(col_expr = static_cast(assign_exprs.at(i)))) { - } else if (!has_exist_in_array(column_ids, col_expr->get_column_id())) { - //not found in column ids, ignore it - } else if (OB_FAIL(updated_column_ids.push_back(col_expr->get_column_id()))) { - LOG_WARN("fail to add updated column id", K(ret), K(col_expr->get_column_id())); - } - } + if (OB_FAIL(updated_column_ids.reserve(assigns.count()))) { + LOG_WARN("fail to init updated column ids array", K(ret), K(assigns.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + const ObTableAssignment &assign = assigns.at(i); + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (OB_FAIL(updated_column_ids.push_back(assign.column_item_->column_id_))) { + LOG_WARN("fail to add updated column id", K(ret), K(assign)); } } } @@ -1340,56 +1396,38 @@ int ObTableDmlCgService::generate_updated_column_ids(ObTableCtx &ctx, int ObTableDmlCgService::generate_upd_assign_infos(ObTableCtx &ctx, ObIAllocator &allocator, - const ObIArray &assign_exprs, ObTableUpdCtDef &udp_ctdef) { int ret = OB_SUCCESS; - int64_t assign_cnt = assign_exprs.count(); + ObIArray &assigns = ctx.get_assignments(); + int64_t assign_cnt = assigns.count(); ColContentFixedArray &assign_infos = udp_ctdef.assign_columns_; if (OB_FAIL(assign_infos.init(assign_cnt))) { LOG_WARN("fail to init assign info array", K(ret), K(assign_cnt)); } for (int64_t i = 0; OB_SUCC(ret) && i < assign_cnt; ++i) { + const ObTableAssignment &assign = assigns.at(i); ColumnContent column_content; int64_t idx = 0; - if (assign_exprs.at(i)->get_expr_type() == T_FUN_COLUMN_CONV && ctx.has_auto_inc()) { // 兼容自增场景下的列转换表达式 - column_content.auto_filled_timestamp_ = false; - column_content.is_nullable_ = false; - column_content.is_predicate_column_ = false; - column_content.is_implicit_ = false; - if (OB_FAIL(ob_write_string(allocator, - ctx.get_auto_inc_column_name(), - column_content.column_name_))) { - LOG_WARN("fail to copy column name", K(ret), K(ctx.get_auto_inc_column_name())); - } else if (!has_exist_in_array(udp_ctdef.das_ctdef_.column_ids_, ctx.get_auto_inc_column_id(), &idx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("assign column not exists in old column", K(ret), K(ctx.get_auto_inc_column_name())); - } else if (FALSE_IT(column_content.projector_index_ = static_cast(idx))) { - //do nothing - } else if (OB_FAIL(assign_infos.push_back(column_content))) { - LOG_WARN("fail to store colum content to assign infos", K(ret), K(column_content)); - } - } else { - ObColumnRefRawExpr *col = static_cast(assign_exprs.at(i)); - column_content.auto_filled_timestamp_ = col->get_result_type().has_result_flag(ON_UPDATE_NOW_FLAG); - column_content.is_nullable_ = !col->get_result_type().is_not_null_for_write(); - column_content.is_predicate_column_ = false; - column_content.is_implicit_ = false; - if (OB_FAIL(ob_write_string(allocator, - col->get_column_name(), - column_content.column_name_))) { - LOG_WARN("fail to copy column name", K(ret), K(col->get_column_name())); - } else if (!has_exist_in_array(udp_ctdef.das_ctdef_.column_ids_, col->get_column_id(), &idx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("assign column not exists in old column", K(ret), KPC(col)); - } else if (FALSE_IT(column_content.projector_index_ = static_cast(idx))) { - //do nothing - } else if (OB_FAIL(assign_infos.push_back(column_content))) { - LOG_WARN("fail to store colum content to assign infos", K(ret), K(column_content)); - } + column_content.auto_filled_timestamp_ = assign.column_expr_->get_result_type().has_result_flag(ON_UPDATE_NOW_FLAG); + column_content.is_nullable_ = !assign.column_expr_->get_result_type().is_not_null_for_write(); + column_content.is_predicate_column_ = false; + column_content.is_implicit_ = false; + if (OB_FAIL(ob_write_string(allocator, + assign.column_expr_->get_column_name(), + column_content.column_name_))) { + LOG_WARN("fail to copy column name", K(ret), K(assign.column_expr_->get_column_name())); + } else if (!has_exist_in_array(udp_ctdef.das_ctdef_.column_ids_, assign.column_expr_->get_column_id(), &idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column not exists in old column", K(ret), KPC(assign.column_expr_)); + } else if (FALSE_IT(column_content.projector_index_ = static_cast(idx))) { + //do nothing + } else if (OB_FAIL(assign_infos.push_back(column_content))) { + LOG_WARN("fail to store colum content to assign infos", K(ret), K(column_content)); } } + return ret; } @@ -1400,17 +1438,13 @@ int ObTableDmlCgService::generate_delete_ctdef(ObTableCtx &ctx, int ret = OB_SUCCESS; ObSEArray old_row; ObSEArray new_row; - ObSEArray tmp_exprs; - const ObIArray &exprs = ctx.get_all_exprs().get_expr_array(); + ObSEArray table_column_exprs; - if (OB_FAIL(replace_exprs_with_dependant(exprs, tmp_exprs))) { + if (OB_FAIL(replace_exprs_with_dependant(ctx, table_column_exprs))) { LOG_WARN("fail to replace exprs with dependant", K(ret)); - } else if (OB_FAIL(old_row.assign(tmp_exprs))) { + } else if (OB_FAIL(old_row.assign(table_column_exprs))) { LOG_WARN("fail to assign old row expr", K(ret)); - } else if (OB_FAIL(generate_base_ctdef(ctx, - del_ctdef, - old_row, - new_row))) { + } else if (OB_FAIL(generate_base_ctdef(ctx, del_ctdef, old_row, new_row))) { LOG_WARN("fail to generate dml base ctdef", K(ret)); } else if (OB_FAIL(generate_das_del_ctdef(ctx, ctx.get_ref_table_id(), @@ -1438,9 +1472,7 @@ int ObTableDmlCgService::generate_das_del_ctdef(ObTableCtx &ctx, if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_del_ctdef))) { LOG_WARN("fail to generate das dml ctdef", K(ret)); - } else if (OB_FAIL(generate_column_ids(ctx, - ctx.get_all_exprs().get_expr_array(), - dml_column_ids))) { + } else if (OB_FAIL(generate_column_ids(ctx, dml_column_ids))) { LOG_WARN("fail to generate dml column ids", K(ret)); } else if (OB_FAIL(generate_projector(dml_column_ids, das_del_ctdef.column_ids_, @@ -1482,6 +1514,12 @@ int ObTableDmlCgService::generate_related_del_ctdef(ObTableCtx &ctx, return ret; } +/* + generate replace ctdef which consists of insert ctdef and delete ctdef. + - generate insert ctdef. + - generate rowkey info which use for fetch duplicated rowkey. + - generate delete ctdef. +*/ int ObTableDmlCgService::generate_replace_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, ObTableReplaceCtDef &replace_ctdef) @@ -1503,6 +1541,7 @@ int ObTableDmlCgService::generate_table_rowkey_info(ObTableCtx &ctx, ObTableInsCtDef &ins_ctdef) { int ret = OB_SUCCESS; + ObIArray &items = ctx.get_column_items(); ObDASInsCtDef &das_ins_ctdef = ins_ctdef.das_ctdef_; ObSEArray rowkey_column_ids; ObSEArray rowkey_exprs; @@ -1510,32 +1549,23 @@ int ObTableDmlCgService::generate_table_rowkey_info(ObTableCtx &ctx, if (OB_FAIL(get_rowkey_exprs(ctx, rowkey_exprs))) { LOG_WARN("fail to get rowkey exprs", K(ret), K(ctx)); + } else if (items.count() < rowkey_exprs.count()) { + ret = OB_ERR_UNDEFINED; + LOG_WARN("invalid column item count", K(ret), K(items), K(rowkey_exprs)); } for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_exprs.count(); ++i) { + const ObTableColumnItem &item = items.at(i); ObRawExpr *expr = rowkey_exprs.at(i); - if (expr->get_expr_type() == T_FUN_COLUMN_CONV && ctx.has_auto_inc()) { // 兼容自增场景下的列转换表达式 - if (OB_FAIL(rowkey_column_ids.push_back(ctx.get_auto_inc_column_id()))) { - LOG_WARN("fail to push base column id", K(ret), K(ctx.get_auto_inc_column_id())); - } else if (OB_FAIL(rowkey_column_types.push_back(expr->get_result_type()))) { - LOG_WARN("fail to push column type", K(ret), KPC(expr)); - } - } else { - if (!expr->is_column_ref_expr()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expr type is not column_ref expr", K(ret), KPC(expr)); - } else { - ObColumnRefRawExpr *col_expr = static_cast(expr); - if (OB_FAIL(rowkey_column_ids.push_back(col_expr->get_column_id()))) { - LOG_WARN("fail to push base column id", K(ret), K(col_expr->get_column_id())); - } else if (OB_FAIL(rowkey_column_types.push_back(col_expr->get_result_type()))) { - LOG_WARN("fail to push column type", K(ret), KPC(col_expr)); - } - } + if (OB_FAIL(rowkey_column_ids.push_back(item.column_id_))) { + LOG_WARN("fail to push base column id", K(ret), K(item)); + } else if (OB_FAIL(rowkey_column_types.push_back(item.expr_->get_result_type()))) { + LOG_WARN("fail to push column type", K(ret), K(item)); } } - if (FAILEDx(das_ins_ctdef.table_rowkey_cids_.init(rowkey_column_ids.count()))) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(das_ins_ctdef.table_rowkey_cids_.init(rowkey_column_ids.count()))) { LOG_WARN("fail to init table rowkey column ids", K(ret), K(rowkey_column_ids.count())); } else if (OB_FAIL(append(das_ins_ctdef.table_rowkey_cids_, rowkey_column_ids))) { LOG_WARN("fail to append table rowkey column id", K(ret), K(rowkey_column_ids)); @@ -1548,8 +1578,7 @@ int ObTableDmlCgService::generate_table_rowkey_info(ObTableCtx &ctx, return ret; } -int ObTableDmlCgService::get_rowkey_exprs(ObTableCtx &ctx, - ObIArray &rowkey_exprs) +int ObTableDmlCgService::get_rowkey_exprs(ObTableCtx &ctx, ObIArray &rowkey_exprs) { int ret = OB_SUCCESS; const ObTableSchema *table_schema = ctx.get_table_schema(); @@ -1559,8 +1588,7 @@ int ObTableDmlCgService::get_rowkey_exprs(ObTableCtx &ctx, LOG_WARN("table schema is null", K(ret)); } else { const int rowkey_cnt = table_schema->get_rowkey_column_num(); - const ObIArray &all_exprs = ctx.is_for_insertup() ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); + const ObIArray &all_exprs = ctx.get_all_exprs_array(); for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; i++) { if (OB_FAIL(rowkey_exprs.push_back(all_exprs.at(i)))) { LOG_WARN("fail to push back rowkey expr", K(ret), K(i)); @@ -1572,53 +1600,36 @@ int ObTableDmlCgService::get_rowkey_exprs(ObTableCtx &ctx, } int ObTableDmlCgService::generate_tsc_ctdef(ObTableCtx &ctx, + ObIArray &access_exprs, ObDASScanCtDef &tsc_ctdef) { int ret = OB_SUCCESS; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); - const ObIArray &all_exprs = ctx.is_for_insertup() ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); - ObSEArray access_exprs; - ObIArray &filter_exprs = ctx.get_filter_exprs(); - const ObTableSchema *table_schema = ctx.get_table_schema(); tsc_ctdef.ref_table_id_ = ctx.get_index_table_id(); - const uint64_t tenant_id = MTL_ID(); + const uint64_t tenant_id = ctx.get_tenant_id(); + ObSEArray column_ids; if (OB_FAIL(ctx.get_schema_guard().get_schema_version(TABLE_SCHEMA, tenant_id, tsc_ctdef.ref_table_id_, tsc_ctdef.schema_version_))) { LOG_WARN("fail to get schema version", K(ret), K(tenant_id), K(tsc_ctdef.ref_table_id_)); - } else if (OB_FAIL(access_exprs.assign(all_exprs))) { - LOG_WARN("fail to assign access exprs", K(ret)); - } else if (OB_FAIL(cg.generate_rt_exprs(access_exprs, - tsc_ctdef.pd_expr_spec_.access_exprs_))) { - LOG_WARN("fail to generate rt exprs ", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(access_exprs, tsc_ctdef.pd_expr_spec_.access_exprs_))) { + LOG_WARN("fail to generate rt exprs ", K(ret), K(access_exprs)); } else if (OB_FAIL(tsc_ctdef.access_column_ids_.init(access_exprs.count()))) { LOG_WARN("fail to init access_column_ids_ ", K(ret)); - } else { - ARRAY_FOREACH(access_exprs, i) { - if (access_exprs.at(i)->get_expr_type() == T_FUN_COLUMN_CONV && ctx.has_auto_inc()) { // 兼容自增场景下的列转换表达式 - if (OB_FAIL(tsc_ctdef.access_column_ids_.push_back(ctx.get_auto_inc_column_id()))) { - LOG_WARN("fail to add column id", K(ret), K(ctx.get_auto_inc_column_id())); - } - } else { - ObColumnRefRawExpr *ref_col_expr = static_cast(access_exprs.at(i)); - if (OB_ISNULL(ref_col_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null column ref expr", K(ret), K(i)); - } else if (OB_FAIL(tsc_ctdef.access_column_ids_.push_back(ref_col_expr->get_column_id()))) { - LOG_WARN("fail to add column id", K(ret), K(ref_col_expr->get_column_id())); - } - } - } + } else if (OB_FAIL(ctx.get_table_schema()->get_column_ids(column_ids))) { + LOG_WARN("fail to get column ids", K(ret)); + } else if (OB_FAIL(tsc_ctdef.access_column_ids_.assign(column_ids))) { + LOG_WARN("fail to assign column ids", K(ret), K(column_ids)); } + if (OB_SUCC(ret)) { + const ObTableSchema *table_schema = ctx.get_table_schema(); tsc_ctdef.table_param_.get_enable_lob_locator_v2() = (ctx.get_cur_cluster_version() >= CLUSTER_VERSION_4_1_0_0); if (OB_FAIL(tsc_ctdef.table_param_.convert(*table_schema, tsc_ctdef.access_column_ids_))) { LOG_WARN("fail to convert table param", K(ret)); - } else if (OB_FAIL(ObTableTscCgService::generate_das_result_output(tsc_ctdef, - tsc_ctdef.access_column_ids_))) { + } else if (OB_FAIL(ObTableTscCgService::generate_das_result_output(tsc_ctdef, tsc_ctdef.access_column_ids_))) { LOG_WARN("generate das result output failed", K(ret)); } } @@ -1632,7 +1643,6 @@ int ObTableDmlCgService::generate_single_constraint_info(ObTableCtx &ctx, ObUniqueConstraintInfo &constraint_info) { int ret = OB_SUCCESS; - constraint_info.table_id_ = table_id; constraint_info.index_tid_ = index_schema.get_table_id(); if (!index_schema.is_index_table()) { @@ -1643,25 +1653,18 @@ int ObTableDmlCgService::generate_single_constraint_info(ObTableCtx &ctx, if (OB_SUCC(ret)) { uint64_t rowkey_column_id = OB_INVALID_ID; - const ObIArray &all_exprs = ctx.is_for_insertup() ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); ObIArray &column_exprs = constraint_info.constraint_columns_; const ObRowkeyInfo &rowkey_info = index_schema.get_rowkey_info(); for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_info.get_size(); ++i) { + const ObTableColumnItem *item = nullptr; if (OB_FAIL(rowkey_info.get_column_id(i, rowkey_column_id))) { LOG_WARN("fail to get rowkey column id", K(ret)); - } else { - const ObIArray &all_column_exprs = ctx.get_all_column_ref_exprs(); - for (int64_t j = 0; OB_SUCC(ret) && j < all_column_exprs.count(); j++) { - ObColumnRefRawExpr *ref_expr = all_column_exprs.at(j); - if (OB_ISNULL(ref_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ref_expr is null", K(ret), K(j)); - } else if (ref_expr->get_column_id() == rowkey_column_id - && OB_FAIL(column_exprs.push_back(ref_expr))) { - LOG_WARN("fail to push back column ref expr to constraint columns", K(ret), K(j)); - } - } + } else if (OB_FAIL(ctx.get_column_item_by_column_id(rowkey_column_id, item))) { + LOG_WARN("fail to get column item", K(ret), K(ctx), K(rowkey_column_id)); + } else if (OB_ISNULL(item)) { + // do nothing, not found + } else if (OB_FAIL(column_exprs.push_back(item->expr_))) { + LOG_WARN("fail to push back column expr", K(ret), K(item)); } } } @@ -1731,6 +1734,7 @@ int ObTableDmlCgService::generate_constraint_ctdefs(ObTableCtx &ctx, ObSEArray cst_infos; ObRowkeyCstCtdef *rowkey_cst_ctdef = nullptr; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); + ObIArray &all_exprs = ctx.get_all_exprs_array(); if (OB_FAIL(generate_constraint_infos(ctx, cst_infos))) { LOG_WARN("fail to generate constraint infos", K(ret), K(ctx)); @@ -1751,8 +1755,15 @@ int ObTableDmlCgService::generate_constraint_ctdefs(ObTableCtx &ctx, LOG_WARN("fail to init rowkey", K(ret), K(cst_columns.count())); } else { for (int64_t j = 0; OB_SUCC(ret) && j < cst_columns.count(); ++j) { + const ObTableColumnItem *item = nullptr; + ObColumnRefRawExpr *ref_expr = cst_columns.at(j); ObExpr *expr = nullptr; - if (OB_FAIL(cg.generate_rt_expr(*cst_columns.at(j), expr))) { + if (OB_FAIL(ctx.get_column_item_by_expr(ref_expr, item))) { + LOG_WARN("fail to column item by expr", K(ret), K(*ref_expr)); + } else if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item is null", K(ret), K(ctx)); + } else if (OB_FAIL(cg.generate_rt_expr(*item->raw_expr_, expr))) { LOG_WARN("fail to generate rt expr", K(ret)); } else if (OB_FAIL(rowkey_cst_ctdef->rowkey_expr_.push_back(expr))) { LOG_WARN("fail to push back rt expr", K(ret)); @@ -1773,25 +1784,22 @@ int ObTableDmlCgService::generate_conflict_checker_ctdef(ObTableCtx &ctx, ObConflictCheckerCtdef &conflict_checker_ctdef) { int ret = OB_SUCCESS; - const ObIArray &exprs = ctx.is_for_insertup() ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); - ObSEArray table_column_exprs; ObSEArray rowkey_exprs; + ObSEArray table_column_exprs; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); if (OB_FAIL(get_rowkey_exprs(ctx, rowkey_exprs))) { LOG_WARN("fail to get table rowkey exprs", K(ret), K(ctx)); - } else if (OB_FAIL(generate_tsc_ctdef(ctx, conflict_checker_ctdef.das_scan_ctdef_))) { - LOG_WARN("fail to generate das_scan_ctdef", K(ret)); - } else if (OB_FAIL(generate_constraint_ctdefs(ctx, - allocator, - conflict_checker_ctdef.cst_ctdefs_))) { + } else if (OB_FAIL(replace_exprs_with_dependant(ctx, table_column_exprs))) { + LOG_WARN("fail to replace exprs with dependant", K(ret)); + } else if (OB_FAIL(generate_tsc_ctdef(ctx, table_column_exprs, conflict_checker_ctdef.das_scan_ctdef_))) { + LOG_WARN("fail to generate das_scan_ctdef", K(ret), K(table_column_exprs)); + } else if (OB_FAIL(generate_constraint_ctdefs(ctx, allocator, conflict_checker_ctdef.cst_ctdefs_))) { LOG_WARN("fail to generate constraint infos", K(ret), K(ctx)); - } else if (OB_FAIL(cg.generate_rt_exprs(rowkey_exprs, - conflict_checker_ctdef.data_table_rowkey_expr_))) { + } else if (OB_FAIL(cg.generate_rt_exprs(rowkey_exprs, conflict_checker_ctdef.data_table_rowkey_expr_))) { LOG_WARN("fail to generate data table rowkey expr", K(ret), K(rowkey_exprs)); - } else if (OB_FAIL(cg.generate_rt_exprs(exprs, conflict_checker_ctdef.table_column_exprs_))) { - LOG_WARN("fail to generate table columns rt exprs ", K(ret)); + } else if (OB_FAIL(cg.generate_rt_exprs(table_column_exprs, conflict_checker_ctdef.table_column_exprs_))) { + LOG_WARN("fail to generate table columns rt exprs ", K(ret), K(table_column_exprs)); } else { conflict_checker_ctdef.rowkey_count_ = ctx.get_table_schema()->get_rowkey_column_num(); } @@ -1801,7 +1809,6 @@ int ObTableDmlCgService::generate_conflict_checker_ctdef(ObTableCtx &ctx, int ObTableDmlCgService::generate_insert_up_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, - const ObTableCtx::ObAssignIds &assign_ids, ObTableInsUpdCtDef &ins_up_ctdef) { int ret = OB_SUCCESS; @@ -1810,10 +1817,7 @@ int ObTableDmlCgService::generate_insert_up_ctdef(ObTableCtx &ctx, LOG_WARN("fail to generate insert ctdef", K(ret), K(ctx)); } else if (OB_FAIL(generate_table_rowkey_info(ctx, ins_up_ctdef.ins_ctdef_))) { LOG_WARN("fail to generate table rowkey info", K(ret), K(ctx)); - } else if (OB_FAIL(generate_update_ctdef(ctx, - allocator, - assign_ids, - ins_up_ctdef.upd_ctdef_))) { + } else if (OB_FAIL(generate_update_ctdef(ctx, allocator, ins_up_ctdef.upd_ctdef_))) { LOG_WARN("fail to generate update ctdef", K(ret), K(ctx)); } @@ -1834,7 +1838,7 @@ int ObTableDmlCgService::generate_ttl_ctdef(ObTableCtx &ctx, LOG_WARN("fail to generate table rowkey info", K(ret), K(ctx)); } else if (OB_FAIL(generate_delete_ctdef(ctx, allocator, ttl_ctdef.del_ctdef_))) { LOG_WARN("fail to generate delete ctdef", K(ret), K(ctx)); - } else if (OB_FAIL(generate_update_ctdef(ctx, allocator, ctx.get_assign_ids(), ttl_ctdef.upd_ctdef_))) { + } else if (OB_FAIL(generate_update_ctdef(ctx, allocator, ttl_ctdef.upd_ctdef_))) { LOG_WARN("fail to generate update ctdef", K(ret), K(ctx)); } else if (filter_exprs.count() != 1) { ret = OB_ERR_UNEXPECTED; @@ -1852,15 +1856,13 @@ int ObTableDmlCgService::generate_lock_ctdef(ObTableCtx &ctx, int ret = OB_SUCCESS; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); ObArray old_row; - const ObIArray &exprs = ctx.get_all_exprs().get_expr_array(); ObSEArray tmp_exprs; - if (OB_FAIL(replace_exprs_with_dependant(exprs, tmp_exprs))) { + if (OB_FAIL(replace_exprs_with_dependant(ctx, tmp_exprs))) { LOG_WARN("fail to replace exprs with dependant", K(ret)); } else if (OB_FAIL(old_row.assign(tmp_exprs))) { LOG_WARN("fail to assign old row expr", K(ret)); - } else if (OB_FAIL(cg.generate_rt_exprs(old_row, - lock_ctdef.old_row_))) { + } else if (OB_FAIL(cg.generate_rt_exprs(old_row, lock_ctdef.old_row_))) { LOG_WARN("fail to generate lock rt exprs", K(ret), K(old_row)); } else if (OB_FAIL(generate_das_lock_ctdef(ctx, ctx.get_ref_table_id(), @@ -1883,9 +1885,7 @@ int ObTableDmlCgService::generate_das_lock_ctdef(ObTableCtx &ctx, if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_lock_ctdef))) { LOG_WARN("fail to generate das dml ctdef", K(ret)); - } else if (OB_FAIL(generate_column_ids(ctx, - ctx.get_all_exprs().get_expr_array(), - dml_column_ids))) { + } else if (OB_FAIL(generate_column_ids(ctx, dml_column_ids))) { LOG_WARN("fail to generate dml column ids", K(ret)); } else if (OB_FAIL(generate_projector(dml_column_ids, das_lock_ctdef.column_ids_, @@ -1906,15 +1906,13 @@ int ObTableDmlCgService::generate_base_ctdef(ObTableCtx &ctx, { int ret = OB_SUCCESS; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); - const ObIArray &exprs = (ctx.is_for_update() || ctx.is_for_insertup()) ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); - if (OB_FAIL(generate_column_ids(ctx, exprs, base_ctdef.column_ids_))) { + if (OB_FAIL(generate_column_ids(ctx, base_ctdef.column_ids_))) { LOG_WARN("fail to generate dml column ids", K(ret)); } else if (OB_FAIL(cg.generate_rt_exprs(old_row, base_ctdef.old_row_))) { - LOG_WARN("fail to generate old row exprs", K(ret)); + LOG_WARN("fail to generate old row exprs", K(ret), K(old_row)); } else if (OB_FAIL(cg.generate_rt_exprs(new_row, base_ctdef.new_row_))) { - LOG_WARN("fail to generate new row exprs", K(ret)); + LOG_WARN("fail to generate new row exprs", K(ret), K(new_row)); } return ret; @@ -1928,14 +1926,10 @@ int ObTableDmlCgService::generate_das_ins_ctdef(ObTableCtx &ctx, int ret = OB_SUCCESS; ObArray dml_column_ids; ObArray empty_old_row; - const ObIArray &exprs = ctx.is_for_insertup() ? - ctx.get_old_row_exprs() : ctx.get_all_exprs().get_expr_array(); if (OB_FAIL(generate_das_base_ctdef(index_tid, ctx, das_ins_ctdef))) { LOG_WARN("fail to generate das dml ctdef", K(ret)); - } else if (OB_FAIL(generate_column_ids(ctx, - exprs, - dml_column_ids))) { + } else if (OB_FAIL(generate_column_ids(ctx, dml_column_ids))) { LOG_WARN("fail to generate dml column ids", K(ret)); } else if (OB_FAIL(generate_projector(dml_column_ids, // new row and old row's columns id das_ins_ctdef.column_ids_, // schmea column ids for given index_tid @@ -2083,32 +2077,20 @@ int ObTableDmlCgService::convert_table_param(ObTableCtx &ctx, return ret; } -int ObTableDmlCgService::generate_column_ids(ObTableCtx &ctx, - const ObIArray &exprs, - ObIArray &column_ids) +int ObTableDmlCgService::generate_column_ids(ObTableCtx &ctx, ObIArray &column_ids) { int ret = OB_SUCCESS; + ObIArray &items = ctx.get_column_items(); column_ids.reset(); - if (!exprs.empty()) { - if (OB_FAIL(column_ids.reserve(exprs.count()))) { - LOG_WARN("fail to reserve column ids capacity", K(ret), K(exprs.count())); - } else { - ARRAY_FOREACH(exprs, i) { - if (exprs.at(i)->get_expr_type() == T_FUN_COLUMN_CONV && (ctx.get_opertion_type() != ObTableOperationType::DEL) // 特判下，若为delete/update/get/scan操作 - && (ctx.get_opertion_type() != ObTableOperationType::UPDATE) // 则走原有的列引用表达式 - && (ctx.get_opertion_type() != ObTableOperationType::GET) - && (ctx.get_opertion_type() != ObTableOperationType::SCAN)) { - if (OB_FAIL(column_ids.push_back(ctx.get_auto_inc_column_id()))) { - LOG_WARN("fail to push back column id", K(ret), K(ctx.get_auto_inc_column_id())); - } - } else { - ObColumnRefRawExpr *col_expr = static_cast(exprs.at(i)); - if (OB_FAIL(column_ids.push_back(col_expr->get_column_id()))) { - LOG_WARN("fail to push back column id", K(ret), K(col_expr->get_column_id())); - } - } - } + if (OB_FAIL(column_ids.reserve(items.count()))) { + LOG_WARN("fail to reserve column ids capacity", K(ret), K(items.count())); + } + + for (int64_t i= 0; OB_SUCC(ret) && i < items.count(); i++) { + const ObTableColumnItem &item = items.at(i); + if (OB_FAIL(column_ids.push_back(item.column_id_))) { + LOG_WARN("fail to push back column id", K(ret)); } } @@ -2216,7 +2198,6 @@ int ObTableDmlCgService::generate_related_ins_ctdef(ObTableCtx &ctx, int ObTableDmlCgService::generate_related_upd_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, - const ObIArray &assign_exprs, const ObIArray &old_row, const ObIArray &new_row, const ObIArray &full_row, @@ -2234,7 +2215,6 @@ int ObTableDmlCgService::generate_related_upd_ctdef(ObTableCtx &ctx, LOG_WARN("fail to allocate update related das ctdef", K(ret)); } else if (OB_FAIL(generate_das_upd_ctdef(ctx, related_index_tids.at(i), - assign_exprs, *related_das_ctdef, old_row, new_row, @@ -2294,12 +2274,11 @@ int ObTableTscCgService::replace_gen_col_exprs(const ObTableCtx &ctx, if (!ctx.get_table_schema()->has_generated_column()) { // do nothing } else { - ObArray res_access_expr; - const ObIArray &pairs = ctx.get_gen_dependants_pairs(); + ObSEArray res_access_expr; + const ObIArray &items = ctx.get_column_items(); ObColumnRefRawExpr *ref_expr = nullptr; - const int64_t N = access_exprs.count(); - for (int64_t i = 0; i < N && OB_SUCC(ret); i++) { + for (int64_t i = 0; i < access_exprs.count() && OB_SUCC(ret); i++) { ObRawExpr *expr = access_exprs.at(i); if (!expr->is_column_ref_expr()) { ret = OB_ERR_UNEXPECTED; @@ -2310,11 +2289,11 @@ int ObTableTscCgService::replace_gen_col_exprs(const ObTableCtx &ctx, LOG_WARN("fail to push back expr", K(ret)); } } else { - for (int j = 0; j < pairs.count() && OB_SUCC(ret); j++) { - if (ref_expr == pairs.at(j).first) { - const ObIArray &ref_exprs = pairs.at(j).second; - if (OB_FAIL(append_array_no_dup(res_access_expr, ref_exprs))) { - LOG_WARN("fail to append array no dup", K(ret), K(res_access_expr), K(ref_exprs)); + for (int j = 0; j < items.count() && OB_SUCC(ret); j++) { + const ObTableColumnItem &item = items.at(j); + if (item.expr_ == expr) { + if (OB_FAIL(append_array_no_dup(res_access_expr, item.dependant_exprs_))) { + LOG_WARN("fail to append array no dup", K(ret), K(res_access_expr), K(item)); } } } @@ -2340,7 +2319,7 @@ int ObTableTscCgService::generate_access_ctdef(const ObTableCtx &ctx, ObDASScanCtDef &das_tsc_ctdef) { int ret = OB_SUCCESS; - ObArray access_exprs; + ObSEArray access_exprs; const ObIArray &select_exprs = ctx.get_select_exprs(); const ObIArray &rowkey_exprs = ctx.get_rowkey_exprs(); const ObIArray &index_exprs = ctx.get_index_exprs(); @@ -2378,18 +2357,30 @@ int ObTableTscCgService::generate_access_ctdef(const ObTableCtx &ctx, if (!is_index_table && OB_FAIL(replace_gen_col_exprs(ctx, access_exprs))) { LOG_WARN("fail to replace generate exprs", K(ret)); } else if (OB_FAIL(generate_rt_exprs(ctx, allocator, access_exprs, das_tsc_ctdef.pd_expr_spec_.access_exprs_))) { - LOG_WARN("fail to generate access rt exprs", K(ret)); + LOG_WARN("fail to generate access rt exprs", K(ret), K(access_exprs)); } else if (OB_FAIL(das_tsc_ctdef.access_column_ids_.init(access_exprs.count()))) { LOG_WARN("fail to init access column ids", K(ret), K(access_exprs.count())); } else { - ObColumnRefRawExpr *col_expr = NULL; for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); i++) { - if (!access_exprs.at(i)->is_column_ref_expr()) { + ObRawExpr *raw_expr = access_exprs.at(i); + if (OB_ISNULL(raw_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected expr type", K(ret), K(access_exprs.at(i)->get_expr_type())); - } else if (FALSE_IT(col_expr = static_cast(access_exprs.at(i)))) { - } else if (OB_FAIL(das_tsc_ctdef.access_column_ids_.push_back(col_expr->get_column_id()))) { - LOG_WARN("fail to push back column id", K(ret)); + LOG_WARN("expr is null", K(ret), K(access_exprs)); + } else if (raw_expr->is_column_ref_expr()) { + ObColumnRefRawExpr *col_ref_expr = static_cast(raw_expr); + if (OB_FAIL(das_tsc_ctdef.access_column_ids_.push_back(col_ref_expr->get_column_id()))) { + LOG_WARN("fail to push back column id", K(ret)); + } + } else { // calculate expr, find in column items + const ObTableColumnItem *item = nullptr; + if (OB_FAIL(ctx.get_column_item_by_expr(raw_expr, item))) { + LOG_WARN("fail to get column item", K(ret), K(*raw_expr)); + } else if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item not found", K(ret), K(ctx)); + } else if (OB_FAIL(das_tsc_ctdef.access_column_ids_.push_back(item->column_id_))) { + LOG_WARN("fail to push back column id", K(ret), K(*item)); + } } } } @@ -2433,40 +2424,39 @@ int ObTableTscCgService::generate_table_param(const ObTableCtx &ctx, ObDASScanCtDef &das_tsc_ctdef) { int ret = OB_SUCCESS; - ObArray tsc_out_cols; + ObSEArray tsc_out_cols; const ObTableSchema *table_schema = nullptr; const ObIArray &select_exprs = ctx.get_select_exprs(); if (!ctx.is_index_scan() // 非索引扫描 || (ctx.is_index_scan() && das_tsc_ctdef.ref_table_id_ == ctx.get_ref_table_id()) // 索引扫描回表 || (ctx.is_index_scan() && !ctx.is_index_back())) { //索引扫描不需要回表 - ObColumnRefRawExpr *col_expr = NULL; if (ctx.is_index_scan() && !ctx.is_index_back()) { table_schema = ctx.get_index_schema(); } else { table_schema = ctx.get_table_schema(); } for (int64_t i = 0; OB_SUCC(ret) && i < select_exprs.count(); i++) { - ObRawExpr *expr = select_exprs.at(i); - if (!expr->is_column_ref_expr()) { + ObRawExpr *raw_expr = select_exprs.at(i); + if (OB_ISNULL(raw_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected expr type", K(ret), K(*expr)); - } else if (FALSE_IT(col_expr = static_cast(expr))) { - } else if (!col_expr->is_virtual_generated_column()) { - if (OB_FAIL(tsc_out_cols.push_back(col_expr->get_column_id()))) { - LOG_WARN("fail to push back column id", K(ret)); - } + LOG_WARN("expr is null", K(ret)); } else { - // output虚拟生成列转换为ouput其依赖的列 - const ObIArray &pairs = ctx.get_gen_dependants_pairs(); - for (int64_t j = 0; j < pairs.count() && OB_SUCC(ret); j++) { - if (col_expr == pairs.at(j).first) { - const ObIArray &dependant_exprs = pairs.at(j).second; - for (int64_t k = 0; k < dependant_exprs.count() && OB_SUCC(ret); k++) { - ObColumnRefRawExpr *dep_col_expr = static_cast(dependant_exprs.at(k)); - if (OB_FAIL(add_var_to_array_no_dup(tsc_out_cols, dep_col_expr->get_column_id()))) { - LOG_WARN("fail to add column id", K(ret), K(tsc_out_cols), K(*dep_col_expr)); - } + const ObTableColumnItem *item = nullptr; + if (OB_FAIL(ctx.get_column_item_by_expr(raw_expr, item))) { + LOG_WARN("fail to get column item", K(ret), K(*raw_expr)); + } else if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item not found", K(ret), K(ctx)); + } else if (!item->is_generated_column_) { + if (OB_FAIL(tsc_out_cols.push_back(item->column_id_))) { + LOG_WARN("fail to push back column id", K(ret), K(tsc_out_cols), K(*item)); + } + } else { // generate column. push dependent column ids + for (int64_t j = 0; j < item->dependant_exprs_.count() && OB_SUCC(ret); j++) { + ObColumnRefRawExpr *dep_col_expr = static_cast(item->dependant_exprs_.at(j)); + if (OB_FAIL(add_var_to_array_no_dup(tsc_out_cols, dep_col_expr->get_column_id()))) { + LOG_WARN("fail to add column id", K(ret), K(tsc_out_cols), K(*dep_col_expr)); } } } @@ -2517,16 +2507,24 @@ int ObTableTscCgService::generate_output_exprs(const ObTableCtx &ctx, { int ret = OB_SUCCESS; ObStaticEngineCG cg(ctx.get_cur_cluster_version()); + const ObIArray &items = ctx.get_column_items(); const ObIArray &select_exprs = ctx.get_select_exprs(); - const int64_t N = select_exprs.count(); - for (int64_t i = 0; i < N && OB_SUCC(ret); i++) { + + for (int64_t i = 0; i < select_exprs.count() && OB_SUCC(ret); i++) { ObExpr *rt_expr = nullptr; - ObRawExpr *expr = select_exprs.at(i); - ObColumnRefRawExpr *col_ref_expr = static_cast(expr); - if (col_ref_expr->is_virtual_generated_column()) { - expr = col_ref_expr->get_dependant_expr(); + ObRawExpr *raw_expr = select_exprs.at(i); + const ObTableColumnItem *item = nullptr; + if (OB_FAIL(ctx.get_column_item_by_expr(raw_expr, item))) { + LOG_WARN("fail to get column item", K(ret), K(*raw_expr)); + } else if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item not found", K(ret), K(ctx)); + } else if (item->is_virtual_generated_column_) { // output dependant expr when virtual expr + raw_expr = item->expr_->get_dependant_expr(); } - if (OB_FAIL(cg.generate_rt_expr(*expr, rt_expr))) { + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(cg.generate_rt_expr(*raw_expr, rt_expr))) { LOG_WARN("fail to generate rt expr", K(ret)); } else if (OB_FAIL(output_exprs.push_back(rt_expr))) { LOG_WARN("fail to push back rt expr", K(ret), K(output_exprs)); @@ -2595,29 +2593,14 @@ int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, ObTableCtx &ctx, ObTableApiUpdateSpec &spec) { - int ret = OB_SUCCESS; - - if (OB_FAIL(ObTableDmlCgService::generate_update_ctdef(ctx, - alloc, - ctx.get_assign_ids(), - spec.get_ctdef()))) { - LOG_WARN("fail to generate update ctdef", K(ret)); - } - - return ret; + return ObTableDmlCgService::generate_update_ctdef(ctx, alloc, spec.get_ctdef()); } int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, ObTableCtx &ctx, ObTableApiDelSpec &spec) { - int ret = OB_SUCCESS; - - if (OB_FAIL(ObTableDmlCgService::generate_delete_ctdef(ctx, alloc, spec.get_ctdef()))) { - LOG_WARN("fail to generate delete ctdef", K(ret)); - } - - return ret; + return ObTableDmlCgService::generate_delete_ctdef(ctx, alloc, spec.get_ctdef()); } int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, @@ -2625,9 +2608,8 @@ int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, ObTableApiReplaceSpec &spec) { int ret = OB_SUCCESS; - ObTableReplaceCtDef &ctdef = spec.get_ctdef(); - if (OB_FAIL(ObTableDmlCgService::generate_replace_ctdef(ctx, alloc, ctdef))) { + if (OB_FAIL(ObTableDmlCgService::generate_replace_ctdef(ctx, alloc, spec.get_ctdef()))) { LOG_WARN("fail to generate replace ctdef", K(ret)); } else if (OB_FAIL(ObTableDmlCgService::generate_conflict_checker_ctdef(ctx, alloc, @@ -2643,23 +2625,13 @@ int ObTableSpecCgService::generate_spec(ObIAllocator &alloc, ObTableApiInsertUpSpec &spec) { int ret = OB_SUCCESS; - ObTableInsUpdCtDef &ctdef = spec.get_ctdef(); - if (OB_FAIL(ObTableDmlCgService::generate_insert_up_ctdef(ctx, - alloc, - ctx.get_assign_ids(), - ctdef))) { + if (OB_FAIL(ObTableDmlCgService::generate_insert_up_ctdef(ctx, alloc, spec.get_ctdef()))) { LOG_WARN("fail to generate insert up ctdef", K(ret)); - } else { - const ObIArray &exprs = ctx.get_old_row_exprs(); - ObStaticEngineCG cg(ctx.get_cur_cluster_version()); - if (OB_FAIL(ObTableDmlCgService::generate_conflict_checker_ctdef(ctx, - alloc, - spec.get_conflict_checker_ctdef()))) { - LOG_WARN("fail to generate conflict checker ctdef", K(ret)); - } else if (OB_FAIL(cg.generate_rt_exprs(exprs, spec.get_all_saved_exprs()))) { - LOG_WARN("fail to generate rt exprs ", K(ret)); - } + } else if (OB_FAIL(ObTableDmlCgService::generate_conflict_checker_ctdef(ctx, + alloc, + spec.get_conflict_checker_ctdef()))) { + LOG_WARN("fail to generate conflict checker ctdef", K(ret)); } return ret; @@ -2669,42 +2641,7 @@ int ObTableSpecCgService::generate_spec(common::ObIAllocator &alloc, ObTableCtx &ctx, ObTableApiLockSpec &spec) { - int ret = OB_SUCCESS; - - if (OB_FAIL(ObTableDmlCgService::generate_lock_ctdef(ctx, spec.get_ctdef()))) { - LOG_WARN("fail to generate lock ctdef", K(ret)); - } - - return ret; -} - -int ObTableExprCgService::generate_autoinc_nextval_expr(ObTableCtx &ctx, - ObRawExpr *&expr, - const ObColumnSchemaV2 &col_schema) -{ - int ret = OB_SUCCESS; - - ObSysFunRawExpr *func_expr = NULL; - if (OB_FAIL(ctx.get_expr_factory().create_raw_expr(T_FUN_SYS_AUTOINC_NEXTVAL, func_expr))) { - LOG_WARN("fail to create nextval expr", K(ret)); - } else { - func_expr->set_func_name(ObString::make_string(N_AUTOINC_NEXTVAL)); - if (OB_NOT_NULL(expr) && OB_FAIL(func_expr->add_param_expr(expr))) { // 将最底部的列转换表达式挂到自增列表达式之下 - LOG_WARN("fail to add collumn conv expr to function param", K(ret)); - } else if (OB_FAIL(func_expr->formalize(&ctx.get_session_info()))) { - LOG_WARN("fail to extract info", K(ret)); - } else if (OB_FAIL(ObAutoincNextvalExtra::init_autoinc_nextval_extra(&ctx.get_allocator(), - reinterpret_cast(func_expr), - ctx.get_table_id(), - col_schema.get_column_id(), - ctx.get_table_name(), - col_schema.get_column_name()))) { - LOG_WARN("fail to init autoinc_nextval_extra", K(ret), K(ctx.get_table_name()), K(col_schema)); - } else { - expr = func_expr; - } - } - return ret; + return ObTableDmlCgService::generate_lock_ctdef(ctx, spec.get_ctdef()); } int ObTableSpecCgService::generate_spec(common::ObIAllocator &alloc, diff --git a/src/observer/table/ob_table_cg_service.h b/src/observer/table/ob_table_cg_service.h index 67582ef09c..46a2b2c906 100644 --- a/src/observer/table/ob_table_cg_service.h +++ b/src/observer/table/ob_table_cg_service.h @@ -36,10 +36,6 @@ public: static int generate_exprs(ObTableCtx &ctx, common::ObIAllocator &allocator, ObExprFrameInfo &expr_frame_info); - // 基于table schema构造全表列原生列引用表达式和生成列表达式 - static int generate_column_raw_exprs(ObTableCtx &ctx); - // 构造更新需要的表达式 - static int generate_update_raw_exprs(ObTableCtx &ctx); // 基于原生表达式生成表达式内存布局 static int generate_expr_frame_info(ObTableCtx &ctx, common::ObIAllocator &allocator, @@ -60,26 +56,13 @@ public: const common::ObIArray &delta_exprs, const ObTableEntity &entity); static int refresh_update_exprs_frame(ObTableCtx &ctx, - const common::ObIArray &old_row, const common::ObIArray &new_row, - const common::ObIArray &full_assign_row, - const ObTableCtx::ObAssignIds &assign_ids, const ObTableEntity &entity); static int refresh_insert_up_exprs_frame(ObTableCtx &ctx, const common::ObIArray &ins_new_row, - const common::ObIArray &delta_exprs, + const common::ObIArray &delta_row, const ObTableEntity &entity); - static int refresh_generated_column_related_frame(ObTableCtx &ctx, - const common::ObIArray &old_row, - const common::ObIArray &full_assign_row, - const ObTableCtx::ObAssignIds &assign_ids, - const ObColumnSchemaV2 &col_schema); - static int generate_assign_exprs(ObTableCtx &ctx, - const ObTableCtx::ObAssignIds &assign_ids, - common::ObIArray &assign_exprs); private: - static int init_datum_param_store(ObTableCtx &ctx, - int64_t capacity); static int refresh_exprs_frame(ObTableCtx &ctx, const common::ObIArray &exprs, const ObTableEntity &entity); @@ -90,42 +73,52 @@ private: const common::ObIArray &exprs, const ObTableEntity &entity); static int refresh_assign_exprs_frame(ObTableCtx &ctx, - const common::ObIArray &old_rows, - const common::ObIArray &new_rows, - const common::ObIArray &full_assign_rows, - const ObTableCtx::ObAssignIds &assign_ids, + const common::ObIArray &new_row, const ObTableEntity &entity); static int refresh_delta_exprs_frame(ObTableCtx &ctx, - const common::ObIArray &delta_exprs, + const common::ObIArray &delta_row, const ObTableEntity &entity); - static int generate_full_assign_raw_exprs(ObTableCtx &ctx); - static int genreate_filter_exprs(ObTableCtx &ctx); -private: - // 通过column_name在表达式数组获取列引用表达式 - static ObRawExpr* get_ref_raw_expr(const common::ObIArray &all_exprs, - const common::ObString &col_name); - // 构造生成列表达式 - static int build_generated_column_expr(ObTableCtx &ctx, - sql::ObColumnRefRawExpr &col_expr, - const common::ObString &expr_str, - const common::ObIArray &exprs); - // 处理生成列表达式 - static int resolve_generated_column_expr(ObTableCtx &ctx); - // 构造列引用原生表达式 - static int generate_column_ref_raw_expr(ObTableCtx &ctx, - const ObColumnSchemaV2 &col_schema, - sql::ObRawExpr *&expr); - // 构建列自增表达式 - static int generate_autoinc_nextval_expr(ObTableCtx &ctx, - ObRawExpr *&expr, - const ObColumnSchemaV2 &col_schema); - static int build_expire_expr(ObTableCtx &ctx, sql::ObRawExpr *&expire_expr); + static int generate_assignments(ObTableCtx &ctx); + + static int generate_filter_exprs(ObTableCtx &ctx); + + static int generate_delta_expr(ObTableCtx &ctx, ObTableAssignment &assign); + + static int generate_assign_expr(ObTableCtx &ctx, ObTableAssignment &assign); + + static int build_generated_column_expr(ObTableCtx &ctx, + ObTableColumnItem &item, + const ObString &expr_str, + sql::ObRawExpr *&expr, + sql::ObRawExpr *delta_expr = nullptr); + + static int generate_autoinc_nextval_expr(ObTableCtx &ctx, + const ObTableColumnItem &item, + sql::ObRawExpr *&expr); + + static int generate_expire_expr(ObTableCtx &ctx, sql::ObRawExpr *&expr); + + static int generate_current_timestamp_expr(ObTableCtx &ctx, + const ObTableColumnItem &item, + sql::ObRawExpr *&expr); + + static int generate_all_column_exprs(ObTableCtx &ctx); + + static int resolve_exprs(ObTableCtx &ctx); + + static int add_extra_column_exprs(ObTableCtx &ctx); + static int write_datum(ObTableCtx &ctx, common::ObIAllocator &allocator, const sql::ObExpr &expr, sql::ObEvalCtx &eval_ctx, const ObObj &obj); + + static int write_autoinc_datum(ObTableCtx &ctx, + const sql::ObExpr &expr, + sql::ObEvalCtx &eval_ctx, + const ObObj &obj); private: DISALLOW_COPY_AND_ASSIGN(ObTableExprCgService); }; @@ -148,7 +141,6 @@ public: ObTableInsCtDef &ins_ctdef); static int generate_update_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, - const ObTableCtx::ObAssignIds &assign_ids, ObTableUpdCtDef &upd_ctdef); static int generate_delete_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, @@ -158,7 +150,6 @@ public: ObTableReplaceCtDef &replace_ctdef); static int generate_insert_up_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, - const ObTableCtx::ObAssignIds &assign_ids, ObTableInsUpdCtDef &ins_up_ctdef); static int generate_lock_ctdef(ObTableCtx &ctx, ObTableLockCtDef &lock_ctdef); @@ -173,16 +164,13 @@ private: ObTableDmlBaseCtDef &base_ctdef, common::ObIArray &old_row, common::ObIArray &new_row); - static int generate_column_ids(ObTableCtx &ctx, - const common::ObIArray &exprs, - common::ObIArray &column_ids); + static int generate_column_ids(ObTableCtx &ctx, common::ObIArray &column_ids); static int generate_das_ins_ctdef(ObTableCtx &ctx, uint64_t index_tid, sql::ObDASInsCtDef &das_ins_ctdef, const common::ObIArray &new_row); static int generate_das_upd_ctdef(ObTableCtx &ctx, uint64_t index_tid, - const common::ObIArray &assign_exprs, sql::ObDASUpdCtDef &das_upd_ctdef, const common::ObIArray &old_row, const common::ObIArray &new_row, @@ -196,12 +184,9 @@ private: sql::ObDASLockCtDef &das_lock_ctdef, const common::ObIArray &old_row); static int generate_updated_column_ids(ObTableCtx &ctx, - const common::ObIArray &assign_exprs, - const common::ObIArray &column_ids, common::ObIArray &updated_column_ids); static int generate_upd_assign_infos(ObTableCtx &ctx, ObIAllocator &allocator, - const common::ObIArray &assign_exprs, ObTableUpdCtDef &udp_ctdef); static int generate_das_base_ctdef(uint64_t index_tid, ObTableCtx &ctx, @@ -223,7 +208,6 @@ private: sql::DASInsCtDefArray &ins_ctdefs); static int generate_related_upd_ctdef(ObTableCtx &ctx, ObIAllocator &allocator, - const common::ObIArray &assign_exprs, const common::ObIArray &old_row, const common::ObIArray &new_row, const common::ObIArray &full_row, @@ -232,11 +216,13 @@ private: ObIAllocator &allocator, const common::ObIArray &old_row, sql::DASDelCtDefArray &del_ctdefs); - static int get_rowkey_exprs(ObTableCtx &ctx, - common::ObIArray &rowkey_exprs); + + static int get_rowkey_exprs(ObTableCtx &ctx, common::ObIArray &rowkey_exprs); + static int generate_table_rowkey_info(ObTableCtx &ctx, ObTableInsCtDef &ins_ctdef); static int generate_tsc_ctdef(ObTableCtx &ctx, + common::ObIArray &access_exprs, sql::ObDASScanCtDef &tsc_ctdef); static int generate_single_constraint_info(ObTableCtx &ctx, const share::schema::ObTableSchema &index_schema, @@ -247,7 +233,7 @@ private: static int generate_constraint_ctdefs(ObTableCtx &ctx, ObIAllocator &allocator, sql::ObRowkeyCstCtdefArray &cst_ctdefs); - static int replace_exprs_with_dependant(const common::ObIArray &src_exprs, + static int replace_exprs_with_dependant(ObTableCtx &ctx, common::ObIArray &dst_exprs); private: DISALLOW_COPY_AND_ASSIGN(ObTableDmlCgService); diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index 6fa7948aa5..3084082ddd 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -55,6 +55,11 @@ int ObTableCtx::get_tablet_by_rowkey(const ObRowkey &rowkey, return ret; } +/* + init session info + 1. expr/das/translation need session. + 2. we use the session pool for authentication because the authentication information is stored in the session pool. +*/ int ObTableCtx::init_sess_info(ObTableApiCredential &credential) { int ret = OB_SUCCESS; @@ -94,6 +99,206 @@ int ObTableCtx::init_common(ObTableApiCredential &credential, return ret; } +/* + 1. ObTableColumnItem record some column info, such as column id、column name、and so on. + 2. we record some specific elements like is_stored_generated_column_ + is_auto_increment_、auto_filled_timestamp_ for specific function. + 3. cascaded_column_ids_ is for update stored generate column. + such as: + - create table t( + `c1` int primary key, + `c2` varchar(20), + `c3` varchar(20) generated always as (substring(`c2`, 1, 4) stored)); + - insert into t(`c1`, `c2`) values(1, 'hello'); + - update t set `c2`='world' where `c1`=1; + `c3` should be updated as well. +*/ +int ObTableCtx::construct_column_items() +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(table_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table schema is null", K(ret)); + } else { + const ObColumnSchemaV2 *col_schema = nullptr; + ObTableSchema::const_column_iterator iter = table_schema_->column_begin(); + ObTableSchema::const_column_iterator end = table_schema_->column_end(); + for (int64_t i = 0; OB_SUCC(ret) && iter != end; ++iter, i++) { + col_schema = *iter; + if (OB_ISNULL(col_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is NULL", K(ret)); + } else { + ObTableColumnItem item; + item.col_idx_ = i; + item.column_id_ = col_schema->get_column_id(); + item.table_id_ = col_schema->get_table_id(); + item.column_name_ = col_schema->get_column_name_str(); + item.default_value_ = col_schema->get_cur_default_value(); + item.is_generated_column_ = col_schema->is_generated_column(); + item.is_stored_generated_column_ = col_schema->is_stored_generated_column(); + item.is_virtual_generated_column_ = col_schema->is_virtual_generated_column(); + item.is_auto_increment_ = col_schema->is_autoincrement(); + item.generated_expr_str_ = item.default_value_.get_string(); + item.auto_filled_timestamp_ = col_schema->is_on_update_current_timestamp(); + if (item.is_auto_increment_ && OB_FAIL(add_auto_inc_param(*col_schema))) { + LOG_WARN("fail to add auto inc param", K(ret), K(item)); + } else if (item.is_generated_column_ + && OB_FAIL(col_schema->get_cascaded_column_ids(item.cascaded_column_ids_))) { + LOG_WARN("fail to get cascaded column ids", K(ret), K(item), K(*col_schema)); + } else if (OB_FAIL(column_items_.push_back(item))) { + LOG_WARN("fail to push back column item", K(ret), K_(column_items), K(item)); + } + } + } + } + + return ret; +} + +int ObTableCtx::get_column_item_by_column_id(uint64_t column_id, const ObTableColumnItem *&item) const +{ + int ret = OB_SUCCESS; + + if (column_items_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column items is empty", K(ret)); + } else { + bool found = false; + for (int64_t i = 0; i < column_items_.count() && !found; i++) { + const ObTableColumnItem &tmp_item = column_items_.at(i); + if (tmp_item.column_id_ == column_id) { + found = true; + item = &tmp_item; + } + } + } + + return ret; +} + +int ObTableCtx::get_column_item_by_expr(ObRawExpr *raw_expr, const ObTableColumnItem *&item) const +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(raw_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("raw_expr is null", K(ret)); + } else if (column_items_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column items is empty", K(ret)); + } else { + bool found = false; + for (int64_t i = 0; i < column_items_.count() && !found; i++) { + const ObTableColumnItem &tmp_item = column_items_.at(i); + if (tmp_item.raw_expr_ == raw_expr) { + found = true; + item = &tmp_item; + } + } + } + + return ret; +} + +int ObTableCtx::get_column_item_by_expr(ObColumnRefRawExpr *expr, const ObTableColumnItem *&item) const +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("raw_expr is null", K(ret)); + } else if (column_items_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column items is empty", K(ret)); + } else { + bool found = false; + for (int64_t i = 0; i < column_items_.count() && !found; i++) { + const ObTableColumnItem &tmp_item = column_items_.at(i); + if (tmp_item.expr_ == expr) { + found = true; + item = &tmp_item; + } + } + } + + return ret; +} + +int ObTableCtx::get_expr_from_column_items(const ObString &col_name, ObRawExpr *&expr) const +{ + int ret = OB_SUCCESS; + + if (column_items_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column items is empty", K(ret)); + } else { + bool found = false; + for (int64_t i = 0; OB_SUCC(ret) && i < column_items_.count() && !found; i++) { + const ObTableColumnItem &item = column_items_.at(i); + if (0 == item.column_name_.case_compare(col_name)) { + if (OB_ISNULL(item.expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("item expr is null", K(ret), K(item)); + } else { + found = true; + expr = item.expr_; + } + } + } + } + + return ret; +} + +int ObTableCtx::get_expr_from_assignments(const ObString &col_name, ObRawExpr *&expr) const +{ + int ret = OB_SUCCESS; + + bool found = false; + for (int64_t i = 0; OB_SUCC(ret) && i < assigns_.count() && !found; i++) { + const ObTableAssignment &assign = assigns_.at(i); + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret)); + } else if (0 == assign.column_item_->column_name_.case_compare(col_name)) { + found = true; + expr = assign.expr_; + } + } + + return ret; +} + +/* + 1. ObConflictChecker need ObPhysicalPlanCtx. + 2. now() expr need ObPhysicalPlanCtx.cur_time_. + 3. das need ObPhysicalPlanCtx to check task should retry or not. +*/ +int ObTableCtx::init_physical_plan_ctx(int64_t timeout_ts, int64_t tenant_schema_version) +{ + int ret = OB_SUCCESS; + void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); + } else { + ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); + phy_plan_ctx->set_timeout_timestamp(timeout_ts); // ObConflictChecker::init_das_scan_rtdef 需要 + phy_plan_ctx->set_tenant_schema_version(tenant_schema_version); + phy_plan_ctx->set_cur_time(ObTimeUtility::current_time()); + exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); + } + return ret; +} + +/* + init table context common param, such as tenant_id_, database_id_，and so on. we also do extra things: + - get session info + - construct column items + - adjust entity +*/ int ObTableCtx::init_common(ObTableApiCredential &credential, const common::ObTabletID &arg_tablet_id, const common::ObString &arg_table_name, @@ -110,7 +315,7 @@ int ObTableCtx::init_common(ObTableApiCredential &credential, arg_table_name, false, /* is_index */ table_schema_))) { - LOG_WARN("fail to get table schema", K(ret), K(tenant_id), "database_id", credential.database_id_, K(arg_table_name)); + LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(database_id), K(arg_table_name)); } else if (OB_FAIL(inner_init_common(credential, arg_tablet_id, arg_table_name, timeout_ts))) { LOG_WARN("fail to inner init common", KR(ret), K(credential), K(arg_tablet_id), K(arg_table_name), K(timeout_ts)); @@ -127,19 +332,22 @@ int ObTableCtx::inner_init_common(ObTableApiCredential &credential, int ret = OB_SUCCESS; bool is_cache_hit = false; const ObTenantSchema *tenant_schema = nullptr; - const uint64_t tenant_id = credential.tenant_id_; - const uint64_t database_id = credential.database_id_; + tenant_id_ = credential.tenant_id_; + database_id_ = credential.database_id_; ObTabletID tablet_id = arg_tablet_id; + if (OB_ISNULL(table_schema_)) { ret = OB_ERR_UNKNOWN_TABLE; - LOG_WARN("table schema is null", K(ret), K(tenant_id), K(table_name)); - } else if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id, tenant_schema))) { - LOG_WARN("fail to get tenant schema", K(ret), K(tenant_id)); + LOG_WARN("table schema is null", K(ret), K(table_name)); + } else if (OB_FAIL(schema_guard_.get_tenant_info(tenant_id_, tenant_schema))) { + LOG_WARN("fail to get tenant schema", K(ret), K_(tenant_id)); } else if (OB_ISNULL(tenant_schema)) { ret = OB_SCHEMA_ERROR; LOG_WARN("tenant schema is null", K(ret)); } else if (OB_FAIL(init_sess_info(credential))) { LOG_WARN("fail to init session info", K(ret), K(credential)); + } else if (OB_FAIL(init_physical_plan_ctx(timeout_ts, tenant_schema->get_schema_version()))) { + LOG_WARN("fail to init physical plan ctx", K(ret)); } else if (!arg_tablet_id.is_valid()) { if (is_scan_) { // 扫描场景使用table_schema上的tablet id,客户端已经做了路由分发 if (table_schema_->is_partitioned_table()) { // 不支持分区表 @@ -161,17 +369,17 @@ int ObTableCtx::inner_init_common(ObTableApiCredential &credential, } if (OB_FAIL(ret)) { - } else if (OB_FAIL(GCTX.location_service_->get(tenant_id, + } else if (OB_FAIL(GCTX.location_service_->get(tenant_id_, tablet_id, 0, /* expire_renew_time */ is_cache_hit, ls_id_))) { LOG_WARN("fail to get ls id", K(ret), K(tablet_id), K(table_name)); + } else if (OB_FAIL(construct_column_items())) { + LOG_WARN("fail to construct column items", K(ret)); } else if (!is_scan_ && OB_FAIL(adjust_entity())) { LOG_WARN("fail to adjust entity", K(ret)); } else { - tenant_id_ = tenant_id; - database_id_ = database_id; table_name_ = table_name; ref_table_id_ = table_schema_->get_table_id(); index_table_id_ = ref_table_id_; @@ -182,14 +390,10 @@ int ObTableCtx::inner_init_common(ObTableApiCredential &credential, is_ttl_table_ = !table_schema_->get_ttl_definition().empty(); } - if (OB_SUCC(ret) && OB_FAIL(init_phy_plan_ctx())) { - LOG_WARN("fail to init physical plan ctx", KR(ret)); - } - return ret; } -// 获取rowkey或者二级索引的columns type +// get columns type from index_schema or primary table schema int ObTableCtx::generate_columns_type(ObIArray &columns_type) { int ret = OB_SUCCESS; @@ -284,6 +488,13 @@ int ObTableCtx::read_real_lob(ObIAllocator &allocator, ObObj &obj) return ret; } +/* + check column type + - nullable + - check data type mismatch or not + - check collation for string type and convert obj type to the column type (char, varchar or text) + - check accuracy +*/ int ObTableCtx::adjust_column_type(const ObExprResType &column_type, ObObj &obj) { @@ -339,6 +550,13 @@ int ObTableCtx::adjust_column_type(const ObExprResType &column_type, return ret; } +/* + check user obj is valid or not, we check: + - column type + - collation type + - nullable + - accuracy. +*/ int ObTableCtx::adjust_column(const ObColumnSchemaV2 &col_schema, ObObj &obj) { int ret = OB_SUCCESS; @@ -353,6 +571,11 @@ int ObTableCtx::adjust_column(const ObColumnSchemaV2 &col_schema, ObObj &obj) return ret; } +/* + check user rowkey is valid or not. + 1. rowkey count should equal schema rowkey count, except for auto increment. + 2. rowkey value should be valid. +*/ int ObTableCtx::adjust_rowkey() { int ret = OB_SUCCESS; @@ -365,16 +588,11 @@ int ObTableCtx::adjust_rowkey() const int64_t schema_rowkey_cnt = table_schema_->get_rowkey_column_num(); const int64_t entity_rowkey_cnt = rowkey.get_obj_cnt(); bool has_auto_inc = false; // only one auto increment column in a table - bool is_full_filled = entity_rowkey_cnt == schema_rowkey_cnt; // 是否把主键值填全，存在自增列场景下可以不填全 + bool is_full_filled = entity_rowkey_cnt == schema_rowkey_cnt; // allow not full filled when rowkey has auto_increment; const ObRowkeyInfo &rowkey_info = table_schema_->get_rowkey_info(); const ObColumnSchemaV2 *col_schema = nullptr; uint64_t column_id = OB_INVALID_ID; ObObj *obj_ptr = rowkey.get_obj_ptr(); - // 不存在自增列的情况下，全部校验 - // 存在自增列情况下，用户填了值，全部校验 - // 存在自增列情况下，用户没有填值，自增列不校验，其他列校验 - // idx 是为了在主键中存在自增列是为了解决使用i访问obj_ptr会出现访问越界的问题，因为在主键存在自增列场景下，用户可以 - // 不填自增列值，这时rowkey中的obj数量少于schema上的rowkey数量 for (int64_t i = 0, idx = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { bool need_check = true; if (OB_FAIL(rowkey_info.get_column_id(i, column_id))) { @@ -387,7 +605,7 @@ int ObTableCtx::adjust_rowkey() if (col_schema->is_part_key_column()) { ret = OB_NOT_SUPPORTED; LOG_WARN("auto increment column could not be partition column", K(ret), K(*col_schema)); - } else if (!is_full_filled) { // 当前列是自增列，但是用户没有填值，不需要校验 + } else if (!is_full_filled) { // curr column is auto_increment and user not fill，no need to check need_check = false; } } @@ -416,11 +634,16 @@ int ObTableCtx::adjust_rowkey() return ret; } +/* + check user properties is valid or not. + 1. rowkey column should not appear in properties, except for get operation. + 2. we do not check column when do get operation, cause property value is empty. +*/ int ObTableCtx::adjust_properties() { int ret = OB_SUCCESS; - bool is_get = (ObTableOperationType::Type::GET == operation_type_); + if (OB_ISNULL(table_schema_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table schema is null", K(ret)); @@ -449,6 +672,10 @@ int ObTableCtx::adjust_properties() return ret; } +/* + check the legality of entity. + - we do not check htable's rowkey, cause htable's rowkey is in properties. +*/ int ObTableCtx::adjust_entity() { int ret = OB_SUCCESS; @@ -482,6 +709,11 @@ bool ObTableCtx::has_exist_in_columns(const ObIArray &columns, return exist; } +/* + genreate key range when do scan. + 1. check the legality of column obj in query range. + 2. fill primary key object when do index scan when user not filled them. +*/ int ObTableCtx::generate_key_range(const ObIArray &scan_ranges) { int ret = OB_SUCCESS; @@ -580,6 +812,14 @@ int ObTableCtx::generate_key_range(const ObIArray &scan_ranges) return ret; } +/* + init scan parameters. + 1. init some common scan parameters such as is_weak_read_、scan_order_、limit_、is_index_back_ and so on. + 2. genreate key range。 + 3. check is_index_back_ when do index scan, need index back if the query column is not in the index table. + 4. select_col_ids_ is same order with schema, + query_col_ids_ and query_col_names_ is same with user query column order. +*/ int ObTableCtx::init_scan(const ObTableQuery &query, const bool &is_wead_read) { @@ -624,7 +864,7 @@ int ObTableCtx::init_scan(const ObTableQuery &query, } else if (query.is_aggregate_query() && OB_FAIL(init_agg_cell_proj(query.get_aggregations().count()))) { LOG_WARN("fail to init agg cell proj", K(ret)); } else { - // select_col_ids用schema序 + // select_col_ids_ is same order with schema int64_t cell_idx = 0; for (ObTableSchema::const_column_iterator iter = table_schema_->column_begin(); OB_SUCC(ret) && iter != table_schema_->column_end(); ++iter, cell_idx++) { @@ -645,14 +885,14 @@ int ObTableCtx::init_scan(const ObTableQuery &query, LOG_WARN("failed to add aggregate projector", K(ret), K(cell_idx), K(column_name)); } else if (is_index_scan_ && !is_index_back_ && OB_ISNULL(column_schema = index_schema_->get_column_schema(column_name))) { - is_index_back_ = true; // 判断是否需要回表,查询的列不在索引表上即需要回表 + is_index_back_ = true; } } else if (has_exist_in_columns(select_columns, column_schema->get_column_name_str())) { if (OB_FAIL(select_col_ids_.push_back(column_schema->get_column_id()))) { LOG_WARN("fail to add column id", K(ret)); } else if (is_index_scan_ && !is_index_back_ && OB_ISNULL(column_schema = index_schema_->get_column_schema(column_name))) { - is_index_back_ = true; // 判断是否需要回表,查询的列不在索引表上即需要回表 + is_index_back_ = true; } } } @@ -662,7 +902,7 @@ int ObTableCtx::init_scan(const ObTableQuery &query, LOG_WARN("select_col_ids or select_metas count is not equal to select_columns", K(select_columns), K(select_col_ids_)); } else if (!select_all_columns) { - // query_col_ids_是用户查询序 + // query_col_ids_ is user query order for (int64_t i = 0; OB_SUCC(ret) && i < select_columns.count(); i++) { if (OB_ISNULL(column_schema = table_schema_->get_column_schema(select_columns.at(i)))) { ret = OB_SCHEMA_ERROR; @@ -682,56 +922,94 @@ int ObTableCtx::init_scan(const ObTableQuery &query, } int ObTableCtx::init_insert() +{ + return init_dml_related_tid(); +} + +/* + stored generate column should be updated when reference column was updated. + such as: + - create table t( + `c1` int primary key, + `c2` varchar(20), + `c3` varchar(20) generated always as (substring(`c2`, 1, 4) stored)); + - insert into t(`c1`, `c2`) values(1, 'hello'); + - update t set `c2`='world' where `c1`=1; + c3 should be update. + 1. match stored generated column by column_id, generated column record cascaded_column_ids_. + c1-16 c2-17 c3-18 + cascaded_column_ids_-(17) + 2. alloc ObTableColumnItem for new ObTableAssignment. +*/ +int ObTableCtx::add_stored_generated_column_assignment(const ObTableAssignment &assign) { int ret = OB_SUCCESS; - ObPhysicalPlanCtx *phy_plan_ctx = get_physical_plan_ctx(); - if (OB_ISNULL(phy_plan_ctx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("phy_plan_ctx is null", K(ret)); - } else if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { - LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); + + for (int64_t i = 0; OB_SUCC(ret) && i < column_items_.count(); i++) { + ObTableColumnItem &item = column_items_.at(i); + if (item.is_stored_generated_column_) { + bool match = false; + for (int64_t j = 0; j < item.cascaded_column_ids_.count() && !match; j++) { + const uint64_t column_id = item.cascaded_column_ids_.at(j); + if (column_id == assign.column_item_->column_id_) { + match = true; + } + } + if (match) { + void *buf = ctx_allocator_.alloc(sizeof(ObTableColumnItem)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObTableColumnItem", K(ret), K(sizeof(ObTableColumnItem))); + } else { + ObTableColumnItem *tmp_item = new(buf) ObTableColumnItem(); + *tmp_item = item; + ObTableAssignment tmp_assign(tmp_item); + if (OB_FAIL(assigns_.push_back(tmp_assign))) { + ctx_allocator_.free(buf); + LOG_WARN("fail to push back assignment", K(ret), K_(assigns), K(tmp_assign)); + } + } + } + } } return init_dml_related_tid(); } -int ObTableCtx::init_assign_ids(ObAssignIds &assign_ids, - const ObTableEntity &entity) +/* + init assignments when do update or insertUp. + 1. user assignment should be add. + 2. stored generate column should be add when reference column is updated. + such as: + - create table t( + `c1` int primary key, + `c2` varchar(20), + `c3` varchar(20) generated always as (substring(`c2`, 1, 4) stored)); + - insert into t(`c1`, `c2`) values(1, 'hello'); + - update t set `c2`='world' where `c1`=1; + c3 should be update. + 3. on update current timestamp should be add. +*/ +int ObTableCtx::init_assignments(const ObTableEntity &entity) { int ret = OB_SUCCESS; - ObArray storage_col_ids; + ObObj prop_obj; - if (OB_ISNULL(table_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret)); - } else if (OB_FAIL(table_schema_->get_column_ids(storage_col_ids))) { - LOG_WARN("fail to get column ids", K(ret)); - } else if (OB_FAIL(assign_ids.init(storage_col_ids.count()))) { - LOG_WARN("fail to init assign_ids capacity", K(ret), K(storage_col_ids.count())); - } else { - ObObj prop_value; // useless - const share::schema::ObColumnSchemaV2 *col_schema = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < storage_col_ids.count(); i++) { - ObString cname; - ObAssignId assign_id; - if (OB_ISNULL(col_schema = table_schema_->get_column_schema(storage_col_ids.at(i)))) { - ret = OB_ERR_COLUMN_NOT_FOUND; - LOG_WARN("column not exists", K(ret), K(cname)); - } else if (col_schema->is_stored_generated_column()) { - assign_id.column_id_ = storage_col_ids.at(i); - assign_id.idx_ = i; - if (OB_FAIL(assign_ids.push_back(assign_id))) { - LOG_WARN("fail to push back assign id", K(ret), K(assign_id)); - } - } else if (FALSE_IT(cname = col_schema->get_column_name_str())) { - // do nothing - } else if (OB_SUCCESS != entity.get_property(cname, prop_value)) { - // 这一列没有被更新，跳过 - } else { - assign_id.column_id_ = storage_col_ids.at(i); - assign_id.idx_ = i; - if (OB_FAIL(assign_ids.push_back(assign_id))) { - LOG_WARN("fail to push back assign id", K(ret), K(assign_id)); - } + for (int64_t i = 0; OB_SUCC(ret) && i < column_items_.count(); i++) { + ObTableColumnItem &item = column_items_.at(i); + if (OB_SUCCESS == entity.get_property(item.column_name_, prop_obj)) { + ObTableAssignment assign(&item); + assign.assign_value_ = prop_obj; // shadow copy when prop_obj is string type + assign.is_assigned_ = true; + if (OB_FAIL(assigns_.push_back(assign))) { + LOG_WARN("fail to push back assignment", K(ret), K_(assigns), K(assign)); + } else if (table_schema_->has_generated_column() + && OB_FAIL(add_stored_generated_column_assignment(assign))) { + LOG_WARN("fail to add soterd generated column assignment", K(ret), K(assign)); + } + } else if (item.auto_filled_timestamp_) { // on update current timestamp + ObTableAssignment assign(&item); + if (OB_FAIL(assigns_.push_back(assign))) { + LOG_WARN("fail to push back assignment", K(ret), K_(assigns), K(assign)); } } } @@ -739,38 +1017,51 @@ int ObTableCtx::init_assign_ids(ObAssignIds &assign_ids, return ret; } +/* + init scan parameters. + 1. init assignments + 2. init scan parameters for child executor(scan executor) of update executor. + 3. init related index table ids, cause we need to write index table as well. +*/ int ObTableCtx::init_update() { int ret = OB_SUCCESS; is_for_update_ = true; - const bool is_batch = OB_ISNULL(batch_op_) ? false : true; - const ObTableEntity *entity = nullptr; - ObAssignIds assign_ids(allocator_); - // 1. init assign_ids_ - if (OB_FAIL(init_assign_ids(assign_ids_, static_cast(*entity_)))) { - LOG_WARN("fail to init assign ids", K(ret), K(*entity_)); - } - // 2. init scan - if (OB_SUCC(ret)) { + // 1. init assignments + if (OB_FAIL(init_assignments(static_cast(*entity_)))) { + LOG_WARN("fail to init assignments", K(ret), K(*entity_)); + } else { + // 2. init scan index_table_id_ = ref_table_id_; is_index_scan_ = false; is_index_back_ = false; is_get_ = true; scan_order_ = ObQueryFlag::Forward; - if (OB_FAIL(table_schema_->get_column_ids(select_col_ids_))) { // init select_col_ids_ + if (OB_FAIL(table_schema_->get_column_ids(select_col_ids_))) { LOG_WARN("fail to get column ids", K(ret)); + } else if (OB_FAIL(init_dml_related_tid())) { // 3. init related index table id + LOG_WARN("fail to init dml related table ids", K(ret)); } } - // 3. init related index table id - if (OB_SUCC(ret) && OB_FAIL(init_dml_related_tid())) { - LOG_WARN("fail to init dml related table ids", K(ret)); - } - return ret; } +/* + init delete parameters. + 1. init scan parameters for child executor(scan executor) of delete executor. + - in htable, we should not query stored generate column. + create table if not exists htable1_cf1 ( + `K` varbinary(1024), + `Q` varbinary(256), + `T` bigint, + `V` varbinary(1024), + `K_PREFIX` varbinary(1024) GENERATED ALWAYS AS (substr(`K`,1,32)) STORED, primary key(`K`, `Q`, `T`)); + `K_PREFIX` should not be query. + - htable use query and delete, is_get_ should not be set. + 2. init related index table ids, cause we need to delete index table as well. +*/ int ObTableCtx::init_delete() { int ret = OB_SUCCESS; @@ -788,7 +1079,7 @@ int ObTableCtx::init_delete() if (OB_ISNULL(column_schema = table_schema_->get_column_schema_by_idx(i))) { ret = OB_SCHEMA_ERROR; LOG_WARN("fail to get column schema bu idx", K(ret), K(i)); - } else if (column_schema->is_generated_column()) { + } else if (column_schema->is_virtual_generated_column()) { // skip } else if (OB_FAIL(select_col_ids_.push_back(column_schema->get_column_id()))) { LOG_WARN("fail to push back column id", K(ret), K(column_schema->get_column_id())); @@ -805,42 +1096,6 @@ int ObTableCtx::init_delete() return ret; } -int ObTableCtx::init_phy_plan_ctx() -{ - int ret = OB_SUCCESS; - void *buf = allocator_.alloc(sizeof(ObPhysicalPlanCtx)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObPhysicalPlanCtx", K(ret), K(sizeof(ObPhysicalPlanCtx))); - } else { - ObPhysicalPlanCtx *phy_plan_ctx = new(buf) ObPhysicalPlanCtx(allocator_); - phy_plan_ctx->set_timeout_timestamp(timeout_ts_); // ObConflictChecker::init_das_scan_rtdef 需要 - phy_plan_ctx->set_tenant_schema_version(tenant_schema_version_); - phy_plan_ctx->set_cur_time(ObTimeUtility::current_time()); - exec_ctx_.set_physical_plan_ctx(phy_plan_ctx); - } - return ret; -} - -int ObTableCtx::init_replace() -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(init_dml_related_tid())) { - LOG_WARN("fail to init dml related tids", K(ret)); - } else { - ObPhysicalPlanCtx *phy_plan_ctx = get_physical_plan_ctx(); - if (OB_ISNULL(phy_plan_ctx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("phy_plan_ctx is null", K(ret)); - } else if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { - LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); - } - } - - return ret; -} - int ObTableCtx::init_ttl_delete(ObRowkey &start_key) { int ret = OB_SUCCESS; @@ -870,6 +1125,20 @@ int ObTableCtx::init_ttl_delete(ObRowkey &start_key) return ret; } +/* + init replace parameters. + - init related index table ids, cause we need to replace index table as well. +*/ +int ObTableCtx::init_replace() +{ + return init_dml_related_tid(); +} + +/* + init insertUp parameters. + 1. init update + 2. reset for is_for_update_ flag, cause init_update() had set is_for_update_=true +*/ int ObTableCtx::init_insert_up() { int ret = OB_SUCCESS; @@ -877,14 +1146,6 @@ int ObTableCtx::init_insert_up() if (OB_FAIL(init_update())) { LOG_WARN("fail to init update", K(ret)); - } else { - ObPhysicalPlanCtx *phy_plan_ctx = get_physical_plan_ctx(); - if (OB_ISNULL(phy_plan_ctx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("phy_plan_ctx is null", K(ret)); - } else if (OB_FAIL(add_auto_inc_param(*phy_plan_ctx))) { - LOG_WARN("fail to add auto inc param", K(ret), K(phy_plan_ctx)); - } } // reset for update flag @@ -892,6 +1153,11 @@ int ObTableCtx::init_insert_up() return ret; } +/* + init get parameters, get operation use scan executor. + 1. init scan parameters + 2. get all column when user not filled select columns. +*/ int ObTableCtx::init_get() { int ret = OB_SUCCESS; @@ -927,143 +1193,160 @@ int ObTableCtx::init_get() return ret; } +/* + init append parameters, append operation use insertUp executor. + 1. init return_affected_entity_ and return_rowkey_ + 2. init insertUp, cause append operation use insertUp executor. + 3. construct generated_expr_str, cause append column use stored to generate columns for functionality. + - expr string: concat_ws('', `%s`, `%s`), `%s` mean column name + 3.1 no record in database + do: append(c1, "abc") + expr: concat_ws('', `c1`, `c1-delta`) + result: "abc", cause c1 is null + 3.1 "abc" in database + do: append(c1, "efg") + expr: concat_ws('', `c1`, `c1-delta`) + result: "abcefg" +*/ int ObTableCtx::init_append(bool return_affected_entity, bool return_rowkey) { int ret = OB_SUCCESS; return_affected_entity_ = return_affected_entity; return_rowkey_ = return_rowkey; - ObSEArray, 8> properties; if (OB_FAIL(init_insert_up())) { LOG_WARN("fail to init insert up", K(ret)); - } else if (OB_FAIL(entity_->get_properties(properties))) { - LOG_WARN("fail to get properties", K(ret)); } else { - const int64_t N = properties.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - ObObj &delta = properties.at(i).second; - if (delta.is_null()) { + for (int64_t i = 0; OB_SUCC(ret) && i < assigns_.count(); i++) { + ObTableAssignment &assign = assigns_.at(i); + const ObObj &delta = assign.assign_value_; + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item is null", K(ret), K(assign)); + } else if (assign.column_item_->auto_filled_timestamp_) { + // do nothing + } else if (delta.is_null()) { ret = OB_OBJ_TYPE_ERROR; LOG_WARN("append NULL is illegal", K(ret), K(delta)); } else if (OB_UNLIKELY(!ob_is_string_type(delta.get_type()))) { ret = OB_OBJ_TYPE_ERROR; LOG_WARN("can only append string type", K(ret), K(delta)); - } - } - } - - if (OB_SUCC(ret)) { - // 构造生成列表达式字符串"concat_ws('', `column_name`, `column_name`)" - const share::schema::ObColumnSchemaV2 *col_schema = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids_.count(); i++) { - uint64_t column_id = assign_ids_.at(i).column_id_; - ObString column_name; - if (OB_ISNULL(col_schema = table_schema_->get_column_schema(column_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get column schema", K(ret), K(column_id)); - } else if (FALSE_IT(column_name = col_schema->get_column_name_str())) { } else { - const int64_t total_len = 22 + column_name.length() + column_name.length(); + const ObString &column_name = assign.column_item_->column_name_; + const int64_t total_len = strlen("concat_ws('', ``, ``)") + 1 + + column_name.length() + column_name.length(); int64_t actual_len = -1; char *buf = NULL; if (OB_ISNULL(buf = static_cast(allocator_.alloc(total_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("fail to alloc memory", K(ret), K(total_len)); + LOG_WARN("fail to alloc memory", K(ret), K(total_len)); } else if ((actual_len = snprintf(buf, total_len, "concat_ws('', `%s`, `%s`)", column_name.ptr(), column_name.ptr())) < 0) { ret = OB_SIZE_OVERFLOW; - LOG_WARN("fail to construct concat_ws expr string", K(ret), K(total_len)); + LOG_WARN("fail to construct concat_ws expr string", K(ret), K(total_len), K(delta)); } else { - ObString expr_str(actual_len, buf); - if (OB_FAIL(expr_strs_.push_back(expr_str))) { - LOG_WARN("fail to push back expr str", K(ret)); - } + ObString generated_expr_str(actual_len, buf); + assign.column_item_->generated_expr_str_ = generated_expr_str; + assign.is_inc_or_append_ = true; } } } } + return ret; } +/* + init increment parameters, append operation use insertUp executor. + 1. init return_affected_entity_ and return_rowkey_ + 2. init insertUp, cause increment operation use insertUp executor. + 3. construct generated_expr_str, cause increment column use stored to generate columns for functionality. + - expr string: IFNULL(`%s`, 0) + `%s`, `%s` mean column name + 3.1 no record in database + do: increment(c1, 1) + expr: IFNULL(`c1`, 0) + `c1_delta` + result: 1, cause c1 is null + 3.1 1 in database + do: increment(c1, 1) + expr: IFNULL(`c1`, 0) + `c1_delta` + result: 2, cause c1 is null +*/ int ObTableCtx::init_increment(bool return_affected_entity, bool return_rowkey) { int ret = OB_SUCCESS; return_affected_entity_ = return_affected_entity; return_rowkey_ = return_rowkey; - ObSEArray, 8> properties; if (OB_FAIL(init_insert_up())) { LOG_WARN("fail to init insert up", K(ret)); - } else if (OB_FAIL(entity_->get_properties(properties))) { - LOG_WARN("fail to get properties", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < properties.count(); i++) { - ObObj &delta = properties.at(i).second; - if (!ob_is_int_tc(delta.get_type())) { + for (int64_t i = 0; OB_SUCC(ret) && i < assigns_.count(); i++) { + ObTableAssignment &assign = assigns_.at(i); + const ObObj &delta = assign.assign_value_; + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column item is null", K(ret), K(assign)); + } else if (assign.column_item_->is_auto_increment_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support increment auto increment column", K(ret), K(assign)); + } else if (assign.column_item_->auto_filled_timestamp_) { + // do nothing + } else if (!ob_is_int_tc(delta.get_type())) { ret = OB_OBJ_TYPE_ERROR; LOG_WARN("delta should only be signed integer type", K(ret), K(delta)); - } - } - } - - if (OB_SUCC(ret)) { - // 构造生成列表达式字符串"ifnull(C1, 0) + C1" - const share::schema::ObColumnSchemaV2 *col_schema = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < assign_ids_.count(); i++) { - uint64_t column_id = assign_ids_.at(i).column_id_; - ObString column_name; - if (OB_ISNULL(col_schema = table_schema_->get_column_schema(column_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get column schema", K(ret), K(column_id)); - } else if (FALSE_IT(column_name = col_schema->get_column_name_str())) { } else { - const int64_t total_len = column_name.length() + column_name.length() + 16; + const ObString &column_name = assign.column_item_->column_name_; + const int64_t total_len = strlen("IFNULL(``, 0) + ``") + 1 + + column_name.length() + column_name.length(); int64_t actual_len = -1; char *buf = NULL; if (OB_ISNULL(buf = static_cast(allocator_.alloc(total_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("fail to alloc memory", K(ret), K(total_len)); - } else if ((actual_len = snprintf(buf, total_len, "IFNULL(%s, 0) + %s", + LOG_WARN("fail to alloc memory", K(ret), K(total_len)); + } else if ((actual_len = snprintf(buf, total_len, "IFNULL(`%s`, 0) + `%s`", column_name.ptr(), column_name.ptr())) < 0) { ret = OB_SIZE_OVERFLOW; - LOG_WARN("fail to construct increment expr string", K(ret), K(total_len)); + LOG_WARN("fail to construct increment expr string", K(ret), K(total_len), K(delta)); } else { - ObString expr_str(actual_len, buf); - if (OB_FAIL(expr_strs_.push_back(expr_str))) { - LOG_WARN("fail to push back expr str", K(ret)); - } + ObString generated_expr_str(actual_len, buf); + assign.column_item_->generated_expr_str_ = generated_expr_str; + assign.is_inc_or_append_ = true; } } } } + return ret; } +/* + classify scan expr for get/delete/update/scan operation +*/ int ObTableCtx::classify_scan_exprs() { int ret = OB_SUCCESS; - const ObIArray *exprs = is_for_update_ ? &old_row_exprs_ : &all_exprs_.get_expr_array(); - int64_t exprs_cnt = exprs->count(); + const ObIArray &exprs = all_exprs_.get_expr_array(); - if (0 == exprs_cnt) { + if (0 == exprs.count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("exprs is empty", K(ret)); + } else if (column_items_.count() > exprs.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unecpect column_items_ count", K(ret), K(column_items_), K(exprs.count())); } else if (!select_exprs_.empty()) { // had classify, do nothing } else { - for (int64_t i = 0; OB_SUCC(ret) && i < exprs_cnt; i++) { - if (exprs->at(i)->is_column_ref_expr()) { - ObColumnRefRawExpr *expr = static_cast(exprs->at(i)); - if (expr->is_rowkey_column() && OB_FAIL(rowkey_exprs_.push_back(expr))) { - LOG_WARN("fail to push back rowkey expr", K(ret)); - } else if (has_exist_in_array(select_col_ids_, expr->get_column_id()) - && OB_FAIL(select_exprs_.push_back(expr))) { - LOG_WARN("fail to push back select expr", K(ret)); - } else if (is_index_scan_ - && has_exist_in_array(index_col_ids_, expr->get_column_id()) - && OB_FAIL(index_exprs_.push_back(expr))) { - LOG_WARN("fail to push back index column expr", K(ret)); - } + for (int64_t i = 0; OB_SUCC(ret) && i < column_items_.count(); i++) { + const ObTableColumnItem &tmp_item = column_items_.at(i); + if (tmp_item.expr_->is_rowkey_column() && OB_FAIL(rowkey_exprs_.push_back(exprs.at(i)))) { + LOG_WARN("fail to push back rowkey expr", K(ret)); + } else if (has_exist_in_array(select_col_ids_, tmp_item.column_id_) + && OB_FAIL(select_exprs_.push_back(exprs.at(i)))) { + LOG_WARN("fail to push back select expr", K(ret)); + } else if (is_index_scan_ + && has_exist_in_array(index_col_ids_, tmp_item.column_id_) + && OB_FAIL(index_exprs_.push_back(exprs.at(i)))) { + LOG_WARN("fail to push back index column expr", K(ret)); } } } @@ -1084,7 +1367,8 @@ int ObTableCtx::init_exec_ctx() return ret; } -// 初始化 das context 中的 table_loc 和 tablet_loc + +// get primary tablet location and all index tablet location, add them to das context. int ObTableCtx::init_das_context(ObDASCtx &das_ctx) { int ret = OB_SUCCESS; @@ -1230,7 +1514,21 @@ int ObTableCtx::init_dml_related_tid() ret = OB_SCHEMA_ERROR; LOG_WARN("null index schema", K(ret)); } else if (index_schema->is_index_local_storage()) { - if (OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { + if (ObTableOperationType::Type::UPDATE == operation_type_) { // check index column is updated or not + bool found = false; + for (int64_t j = 0; OB_SUCC(ret) && j < assigns_.count() && !found; j++) { + const ObTableAssignment &assign = assigns_.at(j); + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (OB_NOT_NULL(index_schema->get_column_schema(assign.column_item_->column_id_))) { + found = true; + } + } + if (found && OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { + LOG_WARN("fail to add related index ids", K(ret), K(index_schema->get_table_id())); + } + } else if (OB_FAIL(related_index_ids_.push_back(index_schema->get_table_id()))) { LOG_WARN("fail to add related index ids", K(ret), K(index_schema->get_table_id())); } } @@ -1273,57 +1571,59 @@ int ObTableCtx::add_aggregate_proj(int64_t cell_idx, return ret; } -int ObTableCtx::add_auto_inc_param(ObPhysicalPlanCtx &phy_plan_ctx) +/* + add auto increment parameters + - each auto increment column contains a param + - only support one auto increment column in mysql mode +*/ +int ObTableCtx::add_auto_inc_param(const ObColumnSchemaV2 &column_schema) { int ret = OB_SUCCESS; - ObIArray &auto_params = phy_plan_ctx.get_autoinc_params(); - for (ObTableSchema::const_column_iterator iter = table_schema_->column_begin(); - OB_SUCC(ret) && iter != table_schema_->column_end() && !has_auto_inc_; ++iter) { - const ObColumnSchemaV2 *col_schema = *iter; - if (OB_ISNULL(col_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("column schema is NULL", K(ret)); - } else if (col_schema->is_autoincrement()) { + ObIArray &auto_params = exec_ctx_.get_physical_plan_ctx()->get_autoinc_params(); + int64_t auto_increment_cache_size = -1; + + if (!auto_params.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected auto params count, should be empty", K(ret)); + } else if (OB_FAIL(get_session_info().get_auto_increment_cache_size(auto_increment_cache_size))) { + LOG_WARN("fail to get increment factor", K(ret)); + } else { + AutoincParam param; + param.tenant_id_ = tenant_id_; + param.autoinc_table_id_ = table_schema_->get_table_id(); + param.autoinc_first_part_num_ = table_schema_->get_first_part_num(); + param.autoinc_table_part_num_ = table_schema_->get_all_part_num(); + param.autoinc_col_id_ = column_schema.get_column_id(); + param.auto_increment_cache_size_ = auto_increment_cache_size; + param.part_level_ = table_schema_->get_part_level(); + ObObjType column_type = column_schema.get_data_type(); + param.autoinc_col_type_ = column_type; + param.autoinc_desired_count_ = 0; + param.autoinc_mode_is_order_ = table_schema_->is_order_auto_increment_mode(); + param.autoinc_version_ = table_schema_->get_truncate_version(); + param.total_value_count_ = 1; + param.autoinc_increment_ = 1; + param.autoinc_offset_ = 1; + param.part_value_no_order_ = true; + if (column_schema.is_tbl_part_key_column()) { + // don't keep intra-partition value asc order when partkey column is auto inc + param.part_value_no_order_ = true; + } + if (OB_FAIL(auto_params.prepare_allocate(1))) { // only one auto inc in a table + LOG_WARN("fail to init auto params", K(ret)); + } else { + auto_params.at(0) = param; has_auto_inc_ = true; - int64_t auto_increment_cache_size = -1; - if (OB_FAIL(get_session_info().get_auto_increment_cache_size(auto_increment_cache_size))) { - LOG_WARN("fail to get increment factor", K(ret)); - } else { - AutoincParam ¶m = get_auto_inc_param(); - param.tenant_id_ = tenant_id_; - param.autoinc_table_id_ = ref_table_id_; - param.autoinc_first_part_num_ = table_schema_->get_first_part_num(); - param.autoinc_table_part_num_ = table_schema_->get_all_part_num(); - param.autoinc_col_id_ = col_schema->get_column_id(); - param.auto_increment_cache_size_ = auto_increment_cache_size; - param.part_level_ = table_schema_->get_part_level(); - ObObjType column_type = col_schema->get_data_type(); - param.autoinc_col_type_ = column_type; - param.autoinc_desired_count_ = 0; - param.autoinc_mode_is_order_ = table_schema_->is_order_auto_increment_mode(); - param.autoinc_version_ = table_schema_->get_truncate_version(); - param.total_value_count_ = 1; - param.autoinc_increment_ = 1; - param.autoinc_offset_ = 1; - param.part_value_no_order_ = true; - // for generate column id when use auto inc - set_auto_inc_column_id(col_schema->get_column_id()); - set_auto_inc_column_name(col_schema->get_column_name_str()); - if (col_schema->is_tbl_part_key_column()) { - // don't keep intra-partition value asc order when partkey column is auto inc - param.part_value_no_order_ = true; - } - if (OB_FAIL(auto_params.prepare_allocate(1))) { // only one auto inc in a table - LOG_WARN("fail to init auto params", K(ret)); - } else { - auto_params.at(0) = param; - } - } } } + return ret; } +/* + update auto increment local and gobal value. + - we need to update local and gobal value after user set specific value. +*/ int ObTableCtx::update_auto_inc_value() { int ret = OB_SUCCESS; @@ -1339,7 +1639,7 @@ int ObTableCtx::update_auto_inc_value() return ret; } -// 获取索引表的tablet_id +// get index table's tablet id int ObTableCtx::get_related_tablet_id(const share::schema::ObTableSchema &index_schema, ObTabletID &related_tablet_id) { diff --git a/src/observer/table/ob_table_context.h b/src/observer/table/ob_table_context.h index f0f35f5493..2196c2d18e 100644 --- a/src/observer/table/ob_table_context.h +++ b/src/observer/table/ob_table_context.h @@ -17,6 +17,7 @@ #include "sql/resolver/expr/ob_raw_expr_util.h" #include "sql/engine/dml/ob_dml_ctx_define.h" #include "sql/das/ob_das_scan_op.h" // for ObDASScanRtDef +#include "sql/resolver/dml/ob_dml_stmt.h" #include "share/table/ob_table.h" #include "ob_table_session_pool.h" @@ -24,23 +25,64 @@ namespace oceanbase { namespace table { - -// 用于存放ctx下自增列的信息 -struct ObTableAutoInc +struct ObTableColumnItem : public sql::ColumnItem { -public: - ObTableAutoInc() - : param_(), - auto_inc_column_id_(OB_INVALID_ID), - auto_inc_column_name_() - { - } - TO_STRING_KV(K_(param), - K_(auto_inc_column_id), - K_(auto_inc_column_name)); - AutoincParam param_; - uint64_t auto_inc_column_id_; - ObString auto_inc_column_name_; + ObTableColumnItem() + : sql::ColumnItem(), + raw_expr_(nullptr), + is_generated_column_(false), + is_stored_generated_column_(false), + is_virtual_generated_column_(false), + is_auto_increment_(false) + {} + TO_STRING_KV("ColumnItem", static_cast(*this), + KPC_(raw_expr), + K_(is_generated_column), + K_(is_stored_generated_column), + K_(is_virtual_generated_column), + K_(cascaded_column_ids), + K_(generated_expr_str), + K_(dependant_exprs), + K_(is_auto_increment)); + sql::ObRawExpr *raw_expr_; // column ref expr or calculate expr + bool is_generated_column_; + bool is_stored_generated_column_; + bool is_virtual_generated_column_; + common::ObSEArray cascaded_column_ids_; + // default equal item.default_value_.get_string() + // specific value in append and increment operation + common::ObString generated_expr_str_; + common::ObSEArray dependant_exprs_; + bool is_auto_increment_; +}; + +struct ObTableAssignment : public sql::ObAssignment +{ + ObTableAssignment() + : sql::ObAssignment(), + column_item_(nullptr), + is_inc_or_append_(false), + delta_expr_(nullptr), + is_assigned_(false) + {} + ObTableAssignment(ObTableColumnItem *item) + : sql::ObAssignment(), + column_item_(item), + is_inc_or_append_(false), + delta_expr_(nullptr), + is_assigned_(false) + {} + TO_STRING_KV("ObAssignment", static_cast(*this), + KPC_(column_item), + K_(is_inc_or_append), + KPC_(delta_expr), + K_(assign_value), + K_(is_assigned)); + ObTableColumnItem *column_item_; + bool is_inc_or_append_; // for append/increment + sql::ObColumnRefRawExpr *delta_expr_; // for append/increment + common::ObObj assign_value_; + bool is_assigned_; // did user assign specific value or not }; enum ObTableExecutorType @@ -62,19 +104,6 @@ enum ObTableExecutorType // 2.在try_process()中进行初始化 class ObTableCtx { -public: - struct ObAssignId { - ObAssignId() - : idx_(OB_INVALID_ID), - column_id_(OB_INVALID_ID) - {} - TO_STRING_KV("index", idx_, - "column_id", column_id_); - uint64_t idx_; - uint64_t column_id_; - }; - typedef common::ObFixedArray ObAssignIds; - typedef std::pair> ObGenDenpendantsPair; public: explicit ObTableCtx(common::ObIAllocator &allocator) : allocator_(allocator), @@ -84,11 +113,8 @@ public: expr_factory_(allocator_), all_exprs_(false), loc_meta_(allocator_), - assign_ids_(allocator_), agg_cell_proj_(allocator_), - auto_inc_param_(), - has_auto_inc_(false), - all_column_ref_exprs_() + has_auto_inc_(false) { // common is_init_ = false; @@ -134,6 +160,8 @@ public: K_(index_tablet_id), K_(ls_id), K_(tenant_schema_version), + K_(column_items), + K_(assigns), // scan to string K_(is_scan), K_(is_index_scan), @@ -172,6 +200,9 @@ public: OB_INLINE sql::ObExecContext& get_exec_ctx() { return exec_ctx_; } OB_INLINE sql::ObRawExprFactory& get_expr_factory() { return expr_factory_; } OB_INLINE sql::ObRawExprUniqueSet& get_all_exprs() { return all_exprs_; } + OB_INLINE ObIArray& get_all_exprs_array() { + return const_cast &>(all_exprs_.get_expr_array()); + } OB_INLINE sql::ObSQLSessionInfo& get_session_info() { return sess_guard_.get_sess_info();} OB_INLINE const sql::ObSQLSessionInfo& get_session_info() const @@ -179,6 +210,10 @@ public: OB_INLINE int64_t get_tenant_schema_version() const { return tenant_schema_version_; } OB_INLINE ObTableOperationType::Type get_opertion_type() const { return operation_type_; } OB_INLINE bool is_init() const { return is_init_; } + OB_INLINE const ObIArray& get_column_items() const { return column_items_; } + OB_INLINE ObIArray& get_column_items() { return column_items_; } + OB_INLINE const ObIArray& get_assignments() const { return assigns_; } + OB_INLINE ObIArray& get_assignments() { return assigns_; } // for scan OB_INLINE bool is_scan() const { return is_scan_; } OB_INLINE bool is_index_scan() const { return is_index_scan_; } @@ -202,16 +237,15 @@ public: OB_INLINE const common::ObIArray& get_query_col_names() const { return query_col_names_; } // for update OB_INLINE bool is_for_update() const { return is_for_update_; } - OB_INLINE const common::ObIArray& get_expr_strs() const { return expr_strs_; } OB_INLINE bool is_inc_or_append() const { return ObTableOperationType::Type::APPEND == operation_type_ || ObTableOperationType::Type::INCREMENT == operation_type_; } - OB_INLINE ObIArray& get_old_row_exprs() { return old_row_exprs_; } - OB_INLINE ObIArray& get_full_assign_exprs() { return full_assign_exprs_; } - OB_INLINE ObIArray& get_delta_exprs() { return delta_exprs_; } - OB_INLINE const ObAssignIds& get_assign_ids() const { return assign_ids_; } + OB_INLINE bool is_dml() const + { + return ObTableOperationType::Type::GET != operation_type_ && !is_scan_; + } // for dml OB_INLINE const ObIArray& get_related_index_ids() const { return related_index_ids_; } OB_INLINE bool is_for_insertup() const { return is_for_insertup_; } @@ -228,24 +262,11 @@ public: OB_INLINE bool return_affected_entity() const { return return_affected_entity_;} OB_INLINE bool return_rowkey() const { return return_rowkey_;} OB_INLINE uint64_t get_cur_cluster_version() const { return cur_cluster_version_;} - OB_INLINE common::ObIArray& get_gen_dependants_pairs() - { - return gen_dependants_pairs_; - } - OB_INLINE const common::ObIArray& get_gen_dependants_pairs() const - { - return gen_dependants_pairs_; - } OB_INLINE bool has_generated_column() const { return table_schema_->has_generated_column(); } // for aggregate OB_INLINE const common::ObIArray &get_agg_projs() const { return agg_cell_proj_; } - // for auto inc - OB_INLINE uint64_t get_auto_inc_column_id() { return auto_inc_param_.auto_inc_column_id_; } - OB_INLINE ObString get_auto_inc_column_name() { return auto_inc_param_.auto_inc_column_name_; } OB_INLINE ObPhysicalPlanCtx *get_physical_plan_ctx() { return exec_ctx_.get_physical_plan_ctx(); } OB_INLINE bool has_auto_inc() { return has_auto_inc_; } - // for rowkey constraint info - OB_INLINE common::ObIArray &get_all_column_ref_exprs() { return all_column_ref_exprs_; } //////////////////////////////////////// setter //////////////////////////////////////////////// // for common OB_INLINE void set_init_flag(bool is_init) { is_init_ = is_init; } @@ -261,8 +282,6 @@ public: // for htable OB_INLINE void set_batch_operation(const ObTableBatchOperation *batch_op) { batch_op_ = batch_op; } // for auto inc - OB_INLINE void set_auto_inc_column_id(const uint64_t &auto_inc_column_id) { auto_inc_param_.auto_inc_column_id_ = auto_inc_column_id; } - OB_INLINE void set_auto_inc_column_name(const ObString &auto_inc_column_name) { auto_inc_param_.auto_inc_column_name_ = auto_inc_column_name; } OB_INLINE bool need_auto_inc_expr() { // delete/update/get/scan操作只需要生成列引用表达式 @@ -311,14 +330,19 @@ public: int init_trans(transaction::ObTxDesc *trans_desc, const transaction::ObTxReadSnapshot &tx_snapshot); int init_das_context(ObDASCtx &das_ctx); + int init_physical_plan_ctx(int64_t timeout_ts, int64_t tenant_schema_version); // 更新全局自增值 int update_auto_inc_value(); // init table context for ttl operation bool is_ttl_table() const { return is_ttl_table_; } void set_is_ttl_table(bool is_ttl_table) { is_ttl_table_ = is_ttl_table; } - int init_phy_plan_ctx(); int init_ttl_delete(ObRowkey &start_key); + int get_column_item_by_column_id(uint64_t column_id, const ObTableColumnItem *&item) const; + int get_column_item_by_expr(sql::ObRawExpr *raw_expr, const ObTableColumnItem *&item) const; + int get_column_item_by_expr(sql::ObColumnRefRawExpr *expr, const ObTableColumnItem *&item) const; + int get_expr_from_column_items(const common::ObString &col_name, sql::ObRawExpr *&expr) const; + int get_expr_from_assignments(const common::ObString &col_name, sql::ObRawExpr *&expr) const; public: // convert lob的allocator需要保证obj写入表达式后才能析构 static int convert_lob(common::ObIAllocator &allocator, ObObj &obj); @@ -336,8 +360,8 @@ private: // for dml int init_dml_related_tid(); // for update - int init_assign_ids(ObAssignIds &assign_ids, - const ObTableEntity &entity); + int init_assignments(const ObTableEntity &entity); + int add_stored_generated_column_assignment(const ObTableAssignment &assign); // Init size of aggregation project array. // // @param [in] size The agg size @@ -350,17 +374,12 @@ private: // @return Returns OB_SUCCESS on success, error code otherwise. int add_aggregate_proj(int64_t cell_idx, const common::ObString &column_name, const ObIArray &aggregations); - AutoincParam &get_auto_inc_param() { return auto_inc_param_.param_; } - - // Add auto inc param to phy_plan_ctx. - // - // @param [in] phy_plan_ctx The phy_plan_ctx. - // @return Returns OB_SUCCESS on success, error code otherwise. - int add_auto_inc_param(ObPhysicalPlanCtx &phy_plan_ctx); + int add_auto_inc_param(const share::schema::ObColumnSchemaV2 &column_schema); private: + int construct_column_items(); int cons_column_type(const share::schema::ObColumnSchemaV2 &column_schema, - sql::ObExprResType &column_type); + sql::ObExprResType &column_type); int adjust_column_type(const ObExprResType &column_type, ObObj &obj); int adjust_column(const ObColumnSchemaV2 &col_schema, ObObj &obj); int adjust_rowkey(); @@ -401,6 +420,8 @@ private: ObTableApiSessGuard sess_guard_; sql::ObDASTableLocMeta loc_meta_; int64_t tenant_schema_version_; + common::ObSEArray column_items_; + common::ObSEArray assigns_; // for scan bool is_scan_; bool is_index_scan_; @@ -409,38 +430,30 @@ private: bool is_get_; bool read_latest_; // default true, false in single get and multi get common::ObQueryFlag::ScanOrder scan_order_; - common::ObArray select_exprs_; - common::ObArray rowkey_exprs_; - common::ObArray index_exprs_; - common::ObArray filter_exprs_; - common::ObArray select_col_ids_; // 基于schema序的select column id - common::ObArray query_col_ids_; // 用户查询的select column id - common::ObArray query_col_names_; // 用户查询的select column name，引用的是schema上的列名 - common::ObArray index_col_ids_; + common::ObSEArray select_exprs_; + common::ObSEArray rowkey_exprs_; + common::ObSEArray index_exprs_; + common::ObSEArray filter_exprs_; + common::ObSEArray select_col_ids_; // 基于schema序的select column id + common::ObSEArray query_col_ids_; // 用户查询的select column id + common::ObSEArray query_col_names_; // 用户查询的select column name，引用的是schema上的列名 + common::ObSEArray index_col_ids_; const share::schema::ObTableSchema *index_schema_; int64_t offset_; int64_t limit_; common::ObSEArray key_ranges_; - // for generate column - common::ObArray gen_dependants_pairs_; // 生成列及其依赖列数组 // for update bool is_for_update_; ObTableOperationType::Type operation_type_; - common::ObArray old_row_exprs_; - common::ObArray full_assign_exprs_; - ObAssignIds assign_ids_; // agg cell index in schema common::ObFixedArray agg_cell_proj_; // for auto inc - ObTableAutoInc auto_inc_param_; bool has_auto_inc_; // for increment/append - common::ObSEArray expr_strs_; - common::ObArray delta_exprs_; // for increment/append bool return_affected_entity_; bool return_rowkey_; // for dml - common::ObSEArray related_index_ids_; + common::ObSEArray related_index_ids_; bool is_for_insertup_; ObTableEntityType entity_type_; const ObITableEntity *entity_; @@ -448,8 +461,6 @@ private: const ObTableBatchOperation *batch_op_; // for lob adapt uint64_t cur_cluster_version_; - // for rowkey constraint info - common::ObSEArray all_column_ref_exprs_; bool is_ttl_table_; private: DISALLOW_COPY_AND_ASSIGN(ObTableCtx); @@ -563,8 +574,7 @@ public: ObTableUpdCtDef(common::ObIAllocator &alloc) : ObTableDmlBaseCtDef(alloc), full_row_(alloc), - full_assign_row_(alloc), - delta_exprs_(alloc), + delta_row_(alloc), das_ctdef_(alloc), assign_columns_(alloc), related_ctdefs_(alloc), @@ -576,14 +586,12 @@ public: { } TO_STRING_KV(K_(full_row), - K_(full_assign_row), - K_(delta_exprs), + K_(delta_row), K_(das_ctdef), K_(assign_columns), K_(related_ctdefs)); ExprFixedArray full_row_; - ExprFixedArray full_assign_row_; - ExprFixedArray delta_exprs_; // for increment/append + ExprFixedArray delta_row_; ObDASUpdCtDef das_ctdef_; ColContentFixedArray assign_columns_; DASUpdCtDefArray related_ctdefs_; diff --git a/src/observer/table/ob_table_end_trans_cb.h b/src/observer/table/ob_table_end_trans_cb.h index d226c2895d..f6d707ed3b 100644 --- a/src/observer/table/ob_table_end_trans_cb.h +++ b/src/observer/table/ob_table_end_trans_cb.h @@ -12,7 +12,7 @@ #ifndef _OB_TABLE_END_TRANS_CB_H #define _OB_TABLE_END_TRANS_CB_H 1 -#include "ob_rpc_async_response.h" +#include "ob_table_rpc_response_sender.h" #include "sql/ob_end_trans_callback.h" #include "share/table/ob_table.h" #include "ob_htable_lock_mgr.h" @@ -61,14 +61,15 @@ private: ObTableEntity result_entity_; common::ObArenaAllocator allocator_; ObTableOperationResult result_; - obrpc::ObRpcAsyncResponse response_sender_; + obrpc::ObTableRpcResponseSender response_sender_; }; class ObTableBatchExecuteEndTransCb: public ObTableAPITransCb { public: ObTableBatchExecuteEndTransCb(rpc::ObRequest *req, ObTableOperationType::Type table_operation_type) - :response_sender_(req, result_), + : entity_factory_("TableBatchCbEntFac", MTL_ID()), + response_sender_(req, result_), table_operation_type_(table_operation_type) { } @@ -87,7 +88,7 @@ private: common::ObArenaAllocator allocator_; table::ObTableEntityFactory entity_factory_; ObTableBatchOperationResult result_; - obrpc::ObRpcAsyncResponse response_sender_; + obrpc::ObTableRpcResponseSender response_sender_; ObTableOperationType::Type table_operation_type_; }; diff --git a/src/observer/table/ob_table_execute_processor.cpp b/src/observer/table/ob_table_execute_processor.cpp index bd74682b27..ef395d9941 100644 --- a/src/observer/table/ob_table_execute_processor.cpp +++ b/src/observer/table/ob_table_execute_processor.cpp @@ -20,6 +20,7 @@ #include "ob_table_scan_executor.h" #include "ob_table_cg_service.h" #include "observer/ob_req_time_service.h" +#include "ob_table_move_response.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -55,6 +56,7 @@ ObTableApiExecuteP::ObTableApiExecuteP(const ObGlobalContext &gctx) :ObTableRpcProcessor(gctx), allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), tb_ctx_(allocator_), + default_entity_factory_("TableExecuteEncFac", MTL_ID()), need_rollback_trans_(false), query_timeout_ts_(0) { @@ -205,10 +207,12 @@ int ObTableApiExecuteP::try_process() uint64_t table_id = arg_.table_id_; bool is_index_supported = true; const ObTableOperation &table_operation = arg_.table_operation_; - if (ObTableOperationType::GET != table_operation.type()) { - if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { - LOG_WARN("failed to get table id", K(ret)); - } else if (OB_FAIL(check_table_index_supported(table_id, is_index_supported))) { + if (OB_FAIL(get_table_id(arg_.table_name_, arg_.table_id_, table_id))) { + LOG_WARN("failed to get table id", K(ret)); + } else if (FALSE_IT(table_id_ = arg_.table_id_)) { + } else if (FALSE_IT(tablet_id_ = arg_.tablet_id_)) { + } else if (ObTableOperationType::GET != table_operation.type()) { + if (OB_FAIL(check_table_index_supported(table_id, is_index_supported))) { LOG_WARN("fail to check index supported", K(ret), K(table_id)); } } @@ -323,12 +327,26 @@ uint64_t ObTableApiExecuteP::get_request_checksum() int ObTableApiExecuteP::response(const int retcode) { int ret = OB_SUCCESS; - if (!need_retry_in_queue_ && !did_async_end_trans()) { + if (!need_retry_in_queue_ && !had_do_response()) { if (OB_SUCC(ret) && ObTableEntityType::ET_HKV == arg_.entity_type_) { // @note modify the value of timestamp to be positive ret = ObTableRpcProcessorUtil::negate_htable_timestamp(result_entity_); } - if (OB_SUCC(ret)) { + + // return the package even if negate_htable_timestamp fails + const ObRpcPacket *rpc_pkt = &reinterpret_cast(req_->get_packet()); + if (is_require_rerouting_err(retcode) && rpc_pkt->require_rerouting()) { + // response rerouting packet + ObTableMoveResponseSender sender(req_, retcode); + if (OB_FAIL(sender.init(arg_.table_id_, arg_.tablet_id_, *gctx_.schema_service_))) { + LOG_WARN("fail to init move response sender", K(ret), K_(arg)); + } else if (OB_FAIL(sender.response())) { + LOG_WARN("fail to do move response", K(ret)); + } + if (OB_FAIL(ret)) { + ret = ObRpcProcessor::response(retcode); // do common response when do move response failed + } + } else { ret = ObRpcProcessor::response(retcode); } } @@ -371,8 +389,6 @@ int ObTableApiExecuteP::process_get() ObNewRow *row = nullptr; if (OB_FAIL(check_arg2())) { LOG_WARN("fail to check arg", K(ret)); - } else if (OB_FAIL(init_tb_ctx())) { - LOG_WARN("fail to init table ctx", K(ret)); } else if (OB_FAIL(init_read_trans(arg_.consistency_level_, tb_ctx_.get_ls_id(), tb_ctx_.get_timeout_ts()))) { @@ -433,7 +449,7 @@ ObTableAPITransCb *ObTableApiExecuteP::new_callback(rpc::ObRequest *req) int ObTableApiExecuteP::before_response(int error_code) { // NOTE: when check_timeout failed, the result.entity_ is null, and serialize result cause coredump - if (!did_async_end_trans() && OB_ISNULL(result_.get_entity())) { + if (!had_do_response() && OB_ISNULL(result_.get_entity())) { result_.set_entity(result_entity_); } return ObTableRpcProcessor::before_response(error_code); diff --git a/src/observer/table/ob_table_executor.cpp b/src/observer/table/ob_table_executor.cpp index ed308b7801..ddea72352f 100644 --- a/src/observer/table/ob_table_executor.cpp +++ b/src/observer/table/ob_table_executor.cpp @@ -72,18 +72,9 @@ void ObTableApiExecutor::clear_evaluated_flag() ObExprFrameInfo *expr_info = const_cast(tb_ctx_.get_expr_frame_info()); if (OB_NOT_NULL(expr_info)) { for (int64_t i = 0; i < expr_info->rt_exprs_.count(); i++) { - if (!tb_ctx_.has_auto_inc()) { // 如果是inc/append场景下进行了自增操作，则不应该清自增列转换的flag，特判 + const ObExpr &expr = expr_info->rt_exprs_.at(i); + if (expr.type_ != T_FUN_SYS_AUTOINC_NEXTVAL) { expr_info->rt_exprs_.at(i).clear_evaluated_flag(eval_ctx_); - } else { - if (expr_info->rt_exprs_.at(i).type_ == T_FUN_COLUMN_CONV) { - if (expr_info->rt_exprs_.at(i).args_[4]->type_ == T_FUN_SYS_AUTOINC_NEXTVAL) { - // do nothing - } else { - expr_info->rt_exprs_.at(i).clear_evaluated_flag(eval_ctx_); - } - } else { - expr_info->rt_exprs_.at(i).clear_evaluated_flag(eval_ctx_); - } } } } diff --git a/src/observer/table/ob_table_insert_up_executor.cpp b/src/observer/table/ob_table_insert_up_executor.cpp index 16ffe80c7e..2164d62282 100644 --- a/src/observer/table/ob_table_insert_up_executor.cpp +++ b/src/observer/table/ob_table_insert_up_executor.cpp @@ -87,13 +87,12 @@ int ObTableApiInsertUpExecutor::refresh_exprs_frame(const ObTableEntity *entity) const ObTableInsCtDef &ins_ctdef = insert_up_spec_.get_ctdef().ins_ctdef_; const ObTableUpdCtDef &upd_ctdef = insert_up_spec_.get_ctdef().upd_ctdef_; - clear_evaluated_flag(); if (OB_ISNULL(entity)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("entity is null", K(ret)); } else if (OB_FAIL(ObTableExprCgService::refresh_insert_up_exprs_frame(tb_ctx_, ins_ctdef.new_row_, - upd_ctdef.delta_exprs_, + upd_ctdef.delta_row_, *entity))) { LOG_WARN("fail to refresh insert up exprs frame", K(ret), K(*entity)); } @@ -108,8 +107,11 @@ int ObTableApiInsertUpExecutor::get_next_row_from_child() if (cur_idx_ >= 1) { ret = OB_ITER_END; - } else if (OB_FAIL(refresh_exprs_frame(entity))) { - LOG_WARN("fail to refresh exprs frame", K(ret)); + } else { + clear_evaluated_flag(); + if (OB_FAIL(refresh_exprs_frame(entity))) { + LOG_WARN("fail to refresh exprs frame", K(ret)); + } } return ret; @@ -158,6 +160,21 @@ int ObTableApiInsertUpExecutor::try_update_row() return ret; } +int ObTableApiInsertUpExecutor::cache_insert_row() +{ + int ret = OB_SUCCESS; + const ObExprPtrIArray &new_row_exprs = get_primary_table_new_row(); + + if (OB_FAIL(ObChunkDatumStore::StoredRow::build(insert_row_, new_row_exprs, eval_ctx_, allocator_))) { + LOG_WARN("fail to build stored row", K(ret), K(new_row_exprs)); + } else if (OB_ISNULL(insert_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache insert row is null", K(ret)); + } + + return ret; +} + // 通过主键在conflict_checker_中找到冲突旧行，执行更新 // 注意，这里更新后还可能出现二级索引冲突，eg: // create table t (C1 int, C2 varchar(10), primary key(C1), UNIQUE KEY idx_c2 (C2)); @@ -170,45 +187,32 @@ int ObTableApiInsertUpExecutor::do_insert_up_cache() { int ret = OB_SUCCESS; ObSEArray constraint_values; - bool is_skipped = false; - ObChunkDatumStore::StoredRow *insert_row = NULL; ObTableUpdRtDef &upd_rtdef = insert_up_rtdef_.upd_rtdef_; - const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); - const ObExprPtrIArray &new_row_exprs = get_primary_table_insert_row(); - // new_row_exprs因为冲突已经被conflict_checker_刷为冲突行，因此需要重新刷一遍 - if (OB_FAIL(refresh_exprs_frame(entity))) { - LOG_WARN("fail to refresh exprs frame", K(ret)); - } else if (OB_FAIL(ObChunkDatumStore::StoredRow::build(insert_row, new_row_exprs, eval_ctx_, allocator_))) { - LOG_WARN("fail to build stored row", K(ret), K(new_row_exprs)); + if (OB_ISNULL(insert_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert row is NULL", K(ret)); + } else if (OB_FAIL(conflict_checker_.check_duplicate_rowkey(insert_row_, constraint_values, true))) { + LOG_WARN("fail to check duplicated key", K(ret), KPC_(insert_row)); } else { - if (OB_ISNULL(insert_row)) { + upd_rtdef.found_rows_++; + const ObChunkDatumStore::StoredRow *upd_new_row = insert_row_; + const ObChunkDatumStore::StoredRow *upd_old_row = constraint_values.at(0).current_datum_row_; + if (OB_ISNULL(upd_old_row)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("insert row is NULL", K(ret)); - } else if (OB_FAIL(conflict_checker_.check_duplicate_rowkey(insert_row, - constraint_values, - true))) { - LOG_WARN("fail to check duplicated key", K(ret), KPC(insert_row)); - } else { - upd_rtdef.found_rows_++; - const ObChunkDatumStore::StoredRow *upd_new_row = insert_row; - const ObChunkDatumStore::StoredRow *upd_old_row = constraint_values.at(0).current_datum_row_; - if (OB_ISNULL(upd_old_row)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("upd_old_row is NULL", K(ret)); - } else if (OB_FAIL(check_whether_row_change(*upd_old_row, - *upd_new_row, - insert_up_spec_.get_ctdef().upd_ctdef_, - is_row_changed_))) { - LOG_WARN("fail to check whether row change", K(ret)); - } else if (is_row_changed_) { - // do update - clear_evaluated_flag(); - if (OB_FAIL(conflict_checker_.update_row(upd_new_row, upd_old_row))) { - LOG_WARN("fail to update row in conflict_checker", K(ret), KPC(upd_new_row), KPC(upd_old_row)); - } else { - upd_changed_rows_++; - } + LOG_WARN("upd_old_row is NULL", K(ret)); + } else if (OB_FAIL(check_whether_row_change(*upd_old_row, + *upd_new_row, + insert_up_spec_.get_ctdef().upd_ctdef_, + is_row_changed_))) { + LOG_WARN("fail to check whether row change", K(ret)); + } else if (is_row_changed_) { + // do update + clear_evaluated_flag(); + if (OB_FAIL(conflict_checker_.update_row(upd_new_row, upd_old_row))) { + LOG_WARN("fail to update row in conflict_checker", K(ret), KPC(upd_new_row), KPC(upd_old_row)); + } else { + upd_changed_rows_++; } } } @@ -258,7 +262,6 @@ int ObTableApiInsertUpExecutor::do_update(const ObRowkey &constraint_rowkey, insert_up_rtdef_.upd_rtdef_, upd_rtctx_)); OZ(to_expr_skip_old(*constraint_value.current_datum_row_, - constraint_rowkey, insert_up_spec_.get_ctdef().upd_ctdef_)); clear_evaluated_flag(); OZ(insert_upd_new_row_to_das(insert_up_spec_.get_ctdef().upd_ctdef_, @@ -267,7 +270,6 @@ int ObTableApiInsertUpExecutor::do_update(const ObRowkey &constraint_rowkey, } else if (NULL == constraint_value.baseline_datum_row_ && NULL != constraint_value.current_datum_row_) { // 单单是唯一索引冲突的时候，会走这个分支 OZ(to_expr_skip_old(*constraint_value.current_datum_row_, - constraint_rowkey, insert_up_spec_.get_ctdef().upd_ctdef_)); OZ(insert_upd_new_row_to_das(insert_up_spec_.get_ctdef().upd_ctdef_, insert_up_rtdef_.upd_rtdef_, @@ -295,6 +297,8 @@ int ObTableApiInsertUpExecutor::get_next_row() } else if (!is_duplicated()) { insert_rows_ = 1; LOG_TRACE("try insert is not duplicated", K(ret), K(insert_rows_)); + } else if (OB_FAIL(cache_insert_row())) { + LOG_WARN("fail to cache insert row", K(ret)); } else if (OB_FAIL(fetch_conflict_rowkey(conflict_checker_))) { LOG_WARN("fail to fetch conflict row", K(ret)); } else if (OB_FAIL(reset_das_env(insert_up_rtdef_.ins_rtdef_))) { diff --git a/src/observer/table/ob_table_insert_up_executor.h b/src/observer/table/ob_table_insert_up_executor.h index d4d0865d05..50196d536d 100644 --- a/src/observer/table/ob_table_insert_up_executor.h +++ b/src/observer/table/ob_table_insert_up_executor.h @@ -25,8 +25,7 @@ public: ObTableApiInsertUpSpec(common::ObIAllocator &alloc, const ObTableExecutorType type) : ObTableApiModifySpec(alloc, type), insert_up_ctdef_(alloc), - conflict_checker_ctdef_(alloc), - all_saved_exprs_(alloc) + conflict_checker_ctdef_(alloc) { } public: @@ -34,12 +33,9 @@ public: OB_INLINE ObTableInsUpdCtDef& get_ctdef() { return insert_up_ctdef_; } OB_INLINE const sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() const { return conflict_checker_ctdef_; } OB_INLINE sql::ObConflictCheckerCtdef& get_conflict_checker_ctdef() { return conflict_checker_ctdef_; } - OB_INLINE const common::ObIArray& get_all_saved_exprs() const { return all_saved_exprs_; } - OB_INLINE common::ObIArray& get_all_saved_exprs() { return all_saved_exprs_; } private: ObTableInsUpdCtDef insert_up_ctdef_; sql::ObConflictCheckerCtdef conflict_checker_ctdef_; - sql::ExprFixedArray all_saved_exprs_; private: DISALLOW_COPY_AND_ASSIGN(ObTableApiInsertUpSpec); }; @@ -52,6 +48,7 @@ public: allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), insert_up_spec_(spec), insert_up_rtdef_(), + insert_row_(nullptr), insert_rows_(0), upd_changed_rows_(0), upd_rtctx_(eval_ctx_, exec_ctx_, get_fake_modify_op()), @@ -110,6 +107,7 @@ private: int try_insert_row(); int try_update_row(); int do_insert_up_cache(); + int cache_insert_row(); int prepare_final_insert_up_task(); int do_update(const ObRowkey &constraint_rowkey, const sql::ObConflictValue &constraint_value); @@ -119,6 +117,7 @@ private: common::ObArenaAllocator allocator_; const ObTableApiInsertUpSpec &insert_up_spec_; ObTableInsUpdRtDef insert_up_rtdef_; + ObChunkDatumStore::StoredRow *insert_row_; int64_t insert_rows_; int64_t upd_changed_rows_; sql::ObDMLRtCtx upd_rtctx_; diff --git a/src/observer/table/ob_table_modify_executor.cpp b/src/observer/table/ob_table_modify_executor.cpp index 4359e9ed0e..1a55101c75 100644 --- a/src/observer/table/ob_table_modify_executor.cpp +++ b/src/observer/table/ob_table_modify_executor.cpp @@ -410,7 +410,6 @@ int ObTableApiModifyExecutor::check_whether_row_change(const ObChunkDatumStore:: } int ObTableApiModifyExecutor::to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, - const ObRowkey &constraint_rowkey, const ObTableUpdCtDef &upd_ctdef) { int ret = OB_SUCCESS; @@ -430,37 +429,35 @@ int ObTableApiModifyExecutor::to_expr_skip_old(const ObChunkDatumStore::StoredRo } // 2. refresh assign column expr datum - const ObTableCtx::ObAssignIds &assign_ids = tb_ctx_.get_assign_ids(); - const int64_t N = assign_ids.count(); - for (uint64_t i = 0; OB_SUCC(ret) && i < N; ++i) { - uint64_t assign_id = assign_ids.at(i).idx_; - const ObColumnSchemaV2 *col_schema = nullptr; - if (OB_ISNULL(col_schema = table_schema->get_column_schema_by_idx(assign_id))) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("fail to get column schema", K(ret), K(assign_id), K(*table_schema)); - } else if (assign_id >= store_row.cnt_) { - ret = OB_ERROR_OUT_OF_RANGE; - LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(store_row.cnt_)); - } else if (assign_id >= new_row.count()) { - ret = OB_ERROR_OUT_OF_RANGE; - LOG_WARN("assign idx out of range", K(ret), K(assign_id), K(new_row.count())); - } else if (col_schema->is_virtual_generated_column()) { + const ObIArray &assigns = tb_ctx_.get_assignments(); + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + const ObTableAssignment &assign = assigns.at(i); + if (OB_ISNULL(assign.column_item_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("should not have virtual generated expr", K(ret)); - } else if (col_schema->is_stored_generated_column()) { - ObTableCtx &ctx = const_cast(tb_ctx_); - if (OB_FAIL(ObTableExprCgService::refresh_generated_column_related_frame(ctx, - upd_ctdef.old_row_, - upd_ctdef.full_assign_row_, - assign_ids, - *col_schema))) { - LOG_WARN("fail to refresh generated column related frame", K(ret), K(ctx), K(*col_schema)); - } + LOG_WARN("assign column item is null", K(ret), K(assign)); + } else if (new_row.count() < assign.column_item_->col_idx_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assign projector_index_", K(ret), K(new_row), K(assign.column_item_)); + } else if (assign.column_item_->is_virtual_generated_column_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("virtual generated column not support to update", K(ret), K(assign)); } else { - const ObExpr *expr = new_row.at(assign_id); - expr->locate_expr_datum(eval_ctx_) = store_row.cells()[assign_id]; - expr->get_eval_info(eval_ctx_).evaluated_ = true; - expr->get_eval_info(eval_ctx_).projected_ = true; + ObExpr *expr = new_row.at(assign.column_item_->col_idx_); + if (OB_ISNULL(expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("expr is null", K(ret)); + } else if (assign.column_item_->is_stored_generated_column_) { + // do nothing, stored generated column not need to fill + } else if (assign.column_item_->auto_filled_timestamp_ && !assign.is_assigned_) { + ObDatum *tmp_datum = nullptr; + if (OB_FAIL(expr->eval(eval_ctx_, tmp_datum))) { + LOG_WARN("fail to eval current timestamp expr", K(ret)); + } + } else { + expr->locate_expr_datum(eval_ctx_) = store_row.cells()[assign.column_item_->col_idx_]; + expr->get_eval_info(eval_ctx_).evaluated_ = true; + expr->get_eval_info(eval_ctx_).projected_ = true; + } } } } diff --git a/src/observer/table/ob_table_modify_executor.h b/src/observer/table/ob_table_modify_executor.h index 1d4de7542f..df92faa077 100644 --- a/src/observer/table/ob_table_modify_executor.h +++ b/src/observer/table/ob_table_modify_executor.h @@ -108,7 +108,6 @@ protected: const ObTableUpdCtDef &upd_ctdef, bool &is_row_changed); int to_expr_skip_old(const ObChunkDatumStore::StoredRow &store_row, - const ObRowkey &constraint_rowkey, const ObTableUpdCtDef &upd_ctdef); int generate_del_rtdef_for_update(const ObTableUpdCtDef &upd_ctdef, ObTableUpdRtDef &upd_rtdef); diff --git a/src/observer/table/ob_table_move_response.cpp b/src/observer/table/ob_table_move_response.cpp new file mode 100644 index 0000000000..c011b413a8 --- /dev/null +++ b/src/observer/table/ob_table_move_response.cpp @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SERVER +#include "ob_table_move_response.h" +#include "share/schema/ob_schema_getter_guard.h" +#include "observer/ob_server_struct.h" +#include "share/partition_table/ob_partition_location.h" + +using namespace oceanbase::observer; +using namespace oceanbase::common; +using namespace oceanbase::table; +//////////////////////////////////////////////////////////////// + +int ObTableMoveResponseSender::get_replica(const uint64_t table_id, + const common::ObTabletID &tablet_id, + table::ObTableMoveReplicaInfo &replica) +{ + int ret = OB_SUCCESS; + bool is_cache_hit = false; + int64_t expire_renew_time = INT64_MAX; //对于get接口，需要传一个最大值，表示需要拿最新的location cache，并让老的失效掉 + share::ObLSID ls_id; + share::ObLSLocation ls_loc; + share::ObLSReplicaLocation replica_loc; + + if (OB_FAIL(GCTX.location_service_->get(MTL_ID(), tablet_id, expire_renew_time, is_cache_hit, ls_id))) { + LOG_WARN("fail to get partition", K(ret), K(table_id), K(tablet_id)); + } else if (OB_FAIL(GCTX.location_service_->get(GCONF.cluster_id, MTL_ID(), ls_id, expire_renew_time, is_cache_hit, ls_loc))) { + LOG_WARN("fail get partition", K(ret), K(table_id), K(tablet_id)); + } else if (OB_FAIL(ls_loc.get_leader(replica_loc))) { + LOG_WARN("fail to get strong leader replica", K(ret)); + } else { + replica.server_ = replica_loc.get_server(); + replica.role_ = replica_loc.get_role(); + replica.replica_type_ = replica_loc.get_replica_type(); + replica.part_renew_time_ = ls_loc.get_renew_time(); + } + + return ret; +} + +int ObTableMoveResponseSender::init(const uint64_t table_id, + const common::ObTabletID &tablet_id, + share::schema::ObMultiVersionSchemaService &schema_service) +{ + int ret = OB_SUCCESS; + + ObTableMoveReplicaInfo &replica = result_.get_replica_info(); + if (OB_FAIL(get_replica(table_id, tablet_id, replica))) { + LOG_WARN("fail to get partition info", K(ret), K(table_id), K(tablet_id)); + } else { + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTableSchema *table_schema = nullptr; + if (OB_FAIL(schema_service.get_tenant_schema_guard(MTL_ID(), schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(MTL_ID(), table_id, table_schema))) { + LOG_WARN("fail to get table schema", K(table_id), K(ret)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("NULL ptr", K(ret), K(table_id)); + } else { + replica.set_table_id(table_id); + replica.set_schema_version(table_schema->get_schema_version()); + replica.set_tablet_id(tablet_id); + + // set move pcode + response_sender_.set_pcode(obrpc::OB_TABLE_API_MOVE); + LOG_DEBUG("move response init successfully", K(replica)); + } + } + + return ret; +} diff --git a/src/observer/table/ob_table_move_response.h b/src/observer/table/ob_table_move_response.h new file mode 100644 index 0000000000..937a444fb4 --- /dev/null +++ b/src/observer/table/ob_table_move_response.h @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_TABLE_MOVE_RESPONSE_H +#define _OB_TABLE_MOVE_RESPONSE_H 1 +#include "ob_table_rpc_response_sender.h" +#include "share/table/ob_table.h" + +namespace oceanbase +{ +namespace observer +{ +class ObTableMoveResponseSender +{ +public: + ObTableMoveResponseSender(rpc::ObRequest *req, const int ret_code) + :response_sender_(req, result_, ret_code) + { + } + virtual ~ObTableMoveResponseSender() = default; + OB_INLINE table::ObTableMoveResult& get_result() { return result_; } + int init(const uint64_t table_id, + const common::ObTabletID &tablet_id, + share::schema::ObMultiVersionSchemaService &schema_service); + int response() { return response_sender_.response(common::OB_SUCCESS); }; +private: + int get_replica(const uint64_t table_id, + const common::ObTabletID &tablet_id, + table::ObTableMoveReplicaInfo &replica); +private: + table::ObTableMoveResult result_; + obrpc::ObTableRpcResponseSender response_sender_; +private: + DISALLOW_COPY_AND_ASSIGN(ObTableMoveResponseSender); +}; + +} // end namespace server +} // end namespace oceanbase + +#endif /* _OB_TABLE_MOVE_RESPONSE_H */ diff --git a/src/observer/table/ob_table_op_wrapper.cpp b/src/observer/table/ob_table_op_wrapper.cpp index 35cc3c55fc..ac5c88e812 100644 --- a/src/observer/table/ob_table_op_wrapper.cpp +++ b/src/observer/table/ob_table_op_wrapper.cpp @@ -81,47 +81,49 @@ int ObTableOpWrapper::process_affected_entity(ObTableCtx &tb_ctx, } else if (OB_FAIL(op_result.get_entity(result_entity))) { LOG_WARN("fail to get result entity", K(ret), K(result_entity)); } else { - ObIAllocator &allocator = tb_ctx.get_allocator(); - const ObIArray *full_assign_exprs = nullptr; const ObIArray *ins_exprs = nullptr; + const ObIArray *upd_exprs = nullptr; bool use_insert_expr = false; if (TABLE_API_EXEC_INSERT_UP == spec.get_type()) { const ObTableApiInsertUpSpec &ins_up_spec = static_cast(spec); - full_assign_exprs = &ins_up_spec.get_ctdef().upd_ctdef_.full_assign_row_; ins_exprs = &ins_up_spec.get_ctdef().ins_ctdef_.new_row_; + upd_exprs = &ins_up_spec.get_ctdef().upd_ctdef_.new_row_; use_insert_expr = !static_cast(executor).is_insert_duplicated(); } else { ObTableApiTTLExecutor &ttl_executor = static_cast(executor); const ObTableApiTTLSpec &ttl_spec = static_cast(spec); - full_assign_exprs = &ttl_spec.get_ctdef().upd_ctdef_.full_assign_row_; ins_exprs = &ttl_spec.get_ctdef().ins_ctdef_.new_row_; + upd_exprs = &ttl_spec.get_ctdef().upd_ctdef_.new_row_; use_insert_expr = !ttl_executor.is_insert_duplicated() || ttl_executor.is_expired(); } - const ObTableCtx::ObAssignIds &assign_ids = tb_ctx.get_assign_ids(); - const int64_t N = assign_ids.count(); - ObObj *obj_array = static_cast(allocator.alloc(sizeof(ObObj) * N)); + + ObIArray &assigns = tb_ctx.get_assignments(); + ObIAllocator &allocator = tb_ctx.get_allocator(); + ObObj *obj_array = static_cast(allocator.alloc(sizeof(ObObj) * assigns.count())); if (OB_ISNULL(obj_array)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("faild to alloc memory for objs", K(ret)); - } else if (OB_ISNULL(full_assign_exprs)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("full assign exprs is null", K(ret)); + LOG_WARN("faild to alloc memory for objs", K(ret), K(assigns.count())); } else if (OB_ISNULL(ins_exprs)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert exprs is null", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - uint64_t idx = assign_ids.at(i).idx_; - uint64_t column_id = assign_ids.at(i).column_id_; - const ObColumnSchemaV2 *column_schema = nullptr; - if (OB_ISNULL(column_schema = tb_ctx.get_table_schema()->get_column_schema(column_id))) { - ret = OB_ERR_COLUMN_NOT_FOUND; - LOG_WARN("column not exist", K(ret), K(column_id)); + for (int64_t i = 0; OB_SUCC(ret) && i < assigns.count(); i++) { + ObTableAssignment &assign = assigns.at(i); + uint64_t project_idx = OB_INVALID_ID; + if (OB_ISNULL(assign.column_item_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assign column item is nullptr", K(ret), K(assign)); + } else if (FALSE_IT(project_idx = assign.column_item_->col_idx_)) { + } else if (use_insert_expr && ins_exprs->count() <= project_idx) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected insert index", K(ret), K(ins_exprs), K(assign)); + } else if (!use_insert_expr && upd_exprs->count() <= project_idx) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assign index", K(ret), K(upd_exprs), K(assign)); } else { ObObj &obj = obj_array[i]; - ObExpr *rt_expr = use_insert_expr ? ins_exprs->at(idx) : full_assign_exprs->at(idx); + ObExpr *rt_expr = use_insert_expr ? ins_exprs->at(project_idx) : upd_exprs->at(project_idx); ObDatum *datum = nullptr; - const ObString &column_name = column_schema->get_column_name_str(); if (OB_FAIL(rt_expr->eval(executor.get_eval_ctx(), datum))) { LOG_WARN("fail to eval datum", K(ret), K(*rt_expr)); } else if (OB_FAIL(datum->to_obj(obj, rt_expr->obj_meta_))) { @@ -129,8 +131,8 @@ int ObTableOpWrapper::process_affected_entity(ObTableCtx &tb_ctx, } else if (is_lob_storage(obj.get_type()) && OB_FAIL(ObTableCtx::read_real_lob(allocator, obj))) { LOG_WARN("fail to read lob", K(ret), K(obj)); - } else if (OB_FAIL(result_entity->set_property(column_name, obj))) { - LOG_WARN("fail to set property", K(ret), K(column_name), K(obj)); + } else if (OB_FAIL(result_entity->set_property(assign.column_item_->column_name_, obj))) { + LOG_WARN("fail to set property", K(ret), K(assign), K(obj)); } } } diff --git a/src/observer/table/ob_table_query_and_mutate_processor.cpp b/src/observer/table/ob_table_query_and_mutate_processor.cpp index 1dae911880..f9e783e5de 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.cpp +++ b/src/observer/table/ob_table_query_and_mutate_processor.cpp @@ -35,6 +35,7 @@ ObTableQueryAndMutateP::ObTableQueryAndMutateP(const ObGlobalContext &gctx) :ObTableRpcProcessor(gctx), allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), tb_ctx_(allocator_), + default_entity_factory_("QueryAndMutateEntFac", MTL_ID()), end_in_advance_(false) { } @@ -954,19 +955,19 @@ int ObTableQueryAndMutateP::try_process() int64_t affected_rows = 0; const bool is_hkv = (ObTableEntityType::ET_HKV == arg_.entity_type_); ObHTableLockHandle *lock_handle = nullptr; - uint64_t table_id = OB_INVALID_ID; if (OB_FAIL(init_scan_tb_ctx(cache_guard))) { LOG_WARN("fail to init scan table ctx", K(ret)); - } else if (FALSE_IT(table_id = tb_ctx_.get_ref_table_id())) { + } else if (FALSE_IT(table_id_ = arg_.table_id_)) { + } else if (FALSE_IT(tablet_id_ = arg_.tablet_id_)) { } else if (is_hkv && OB_FAIL(HTABLE_LOCK_MGR->acquire_handle(lock_handle))) { LOG_WARN("fail to get htable lock handle", K(ret)); - } else if (is_hkv && OB_FAIL(ObHTableUtils::lock_htable_row(table_id, query, *lock_handle, ObHTableLockMode::EXCLUSIVE))) { - LOG_WARN("fail to lock htable row", K(ret), K(table_id), K(query)); + } else if (is_hkv && OB_FAIL(ObHTableUtils::lock_htable_row(table_id_, query, *lock_handle, ObHTableLockMode::EXCLUSIVE))) { + LOG_WARN("fail to lock htable row", K(ret), K_(table_id), K(query)); } else if (OB_FAIL(start_trans(false, /* is_readonly */ sql::stmt::T_UPDATE, consistency_level, - table_id, + table_id_, tb_ctx_.get_ls_id(), get_timeout_ts()))) { LOG_WARN("fail to start readonly transaction", K(ret)); diff --git a/src/observer/table/ob_table_query_common.cpp b/src/observer/table/ob_table_query_common.cpp index b51e2e2bb7..4137029f86 100644 --- a/src/observer/table/ob_table_query_common.cpp +++ b/src/observer/table/ob_table_query_common.cpp @@ -25,7 +25,7 @@ int ObTableQueryUtils::check_htable_query_args(const ObTableQuery &query, int ret = OB_SUCCESS; const ObIArray &select_columns = tb_ctx.get_query_col_names(); int64_t N = select_columns.count(); - if (N != 4) { + if (N != 4 && N != 5) { // htable maybe has prefix generated column ret = OB_INVALID_ARGUMENT; LOG_WARN("TableQuery with htable_filter should select 4 columns", K(ret), K(N)); } diff --git a/src/observer/table/ob_table_query_processor.cpp b/src/observer/table/ob_table_query_processor.cpp index 3a9ba46dd3..0896494075 100644 --- a/src/observer/table/ob_table_query_processor.cpp +++ b/src/observer/table/ob_table_query_processor.cpp @@ -269,6 +269,8 @@ int ObTableQueryP::try_process() LOG_WARN("fail to get spec from cache", K(ret)); } else if (OB_FAIL(spec->create_executor(tb_ctx_, executor))) { LOG_WARN("fail to generate executor", K(ret), K(tb_ctx_)); + } else if (FALSE_IT(table_id_ = arg_.table_id_)) { + } else if (FALSE_IT(tablet_id_ = arg_.tablet_id_)) { } else if (OB_FAIL(start_trans(true, /* is_readonly */ sql::stmt::T_SELECT, arg_.consistency_level_, diff --git a/src/observer/table/ob_table_query_sync_processor.cpp b/src/observer/table/ob_table_query_sync_processor.cpp index 024b6290f8..efd3464523 100644 --- a/src/observer/table/ob_table_query_sync_processor.cpp +++ b/src/observer/table/ob_table_query_sync_processor.cpp @@ -598,6 +598,8 @@ int ObTableQuerySyncP::try_process() } else if (OB_FAIL(get_query_session(query_session_id_, query_session_))) { LOG_WARN("fail to get query session", K(ret), K(query_session_id_)); } else if (FALSE_IT(timeout_ts_ = get_timeout_ts())) { + } else if (FALSE_IT(table_id_ = arg_.table_id_)) { + } else if (FALSE_IT(tablet_id_ = arg_.tablet_id_)) { } else { if (ObQueryOperationType::QUERY_START == arg_.query_type_) { ret = process_query_start(); diff --git a/src/observer/table/ob_table_query_sync_processor.h b/src/observer/table/ob_table_query_sync_processor.h index f5d816cb76..e39c758e72 100644 --- a/src/observer/table/ob_table_query_sync_processor.h +++ b/src/observer/table/ob_table_query_sync_processor.h @@ -186,7 +186,6 @@ public: explicit ObTableQuerySyncP(const ObGlobalContext &gctx); virtual ~ObTableQuerySyncP() {} virtual int deserialize() override; - protected: virtual int check_arg() override; virtual int try_process() override; @@ -195,6 +194,7 @@ protected: virtual uint64_t get_request_checksum() override; virtual table::ObTableAPITransCb *new_callback(rpc::ObRequest *req) override; + private: int process_query_start(); int process_query_next(); diff --git a/src/observer/table/ob_table_replace_executor.cpp b/src/observer/table/ob_table_replace_executor.cpp index 3d230952ba..5aa29d2132 100644 --- a/src/observer/table/ob_table_replace_executor.cpp +++ b/src/observer/table/ob_table_replace_executor.cpp @@ -208,18 +208,11 @@ int ObTableApiReplaceExecutor::do_insert() const ObTableEntity *entity = static_cast(tb_ctx_.get_entity()); const ObTableReplaceCtDef &ctdef = replace_spec_.get_ctdef(); - if (OB_FAIL(refresh_exprs_frame(entity))) { - LOG_WARN("fail to refresh expr frame", K(ret)); - } else if (tb_ctx_.has_auto_inc()) { - for (int64_t i = 0; i < ctdef.ins_ctdef_.new_row_.count(); i++) { // 在自增的场景下，由于自增列的列引用表达式被用户输入的值覆盖 - if (ctdef.ins_ctdef_.new_row_.at(i)->type_ == T_FUN_COLUMN_CONV) { // 故需要手动清空eval的flag - ctdef.ins_ctdef_.new_row_.at(i)->get_eval_info(eval_ctx_).evaluated_ = false; - } - } - } - - if (OB_FAIL(ret)) { - // do nothing + if (OB_ISNULL(insert_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert row is null", K(ret)); + } else if (OB_FAIL(insert_row_->to_expr(get_primary_table_new_row(), eval_ctx_))) { + LOG_WARN("stored row to expr faild", K(ret)); } else if (OB_FAIL(insert_row_to_das(ctdef.ins_ctdef_, replace_rtdef_.ins_rtdef_))) { LOG_WARN("shuffle insert row failed", K(ret)); } else { @@ -229,6 +222,21 @@ int ObTableApiReplaceExecutor::do_insert() return ret; } +int ObTableApiReplaceExecutor::cache_insert_row() +{ + int ret = OB_SUCCESS; + const ObExprPtrIArray &new_row_exprs = get_primary_table_new_row(); + + if (OB_FAIL(ObChunkDatumStore::StoredRow::build(insert_row_, new_row_exprs, eval_ctx_, allocator_))) { + LOG_WARN("fail to build stored row", K(ret), K(new_row_exprs)); + } else if (OB_ISNULL(insert_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache insert row is null", K(ret)); + } + + return ret; +} + int ObTableApiReplaceExecutor::prepare_final_replace_task() { int ret = OB_SUCCESS; @@ -278,6 +286,8 @@ int ObTableApiReplaceExecutor::get_next_row() LOG_WARN("fail to post all das task", K(ret)); } else if (!is_duplicated()) { LOG_DEBUG("try insert is not duplicated", K(ret)); + } else if (OB_FAIL(cache_insert_row())) { + LOG_WARN("fail to cache insert row", K(ret)); } else if (OB_FAIL(fetch_conflict_rowkey(conflict_checker_))) { LOG_WARN("fail to fetch conflict row", K(ret)); } else if (OB_FAIL(reset_das_env(replace_rtdef_.ins_rtdef_))) { diff --git a/src/observer/table/ob_table_replace_executor.h b/src/observer/table/ob_table_replace_executor.h index d3244ac47b..d5140581c4 100644 --- a/src/observer/table/ob_table_replace_executor.h +++ b/src/observer/table/ob_table_replace_executor.h @@ -48,6 +48,7 @@ public: : ObTableApiModifyExecutor(ctx), allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), replace_spec_(replace_spec), + insert_row_(NULL), insert_rows_(0), delete_rows_(0), conflict_checker_(allocator_, eval_ctx_, replace_spec_.get_conflict_checker_ctdef()), @@ -93,6 +94,7 @@ private: const ObChunkDatumStore::StoredRow *replace_row, const ObChunkDatumStore::StoredRow *delete_row); int prepare_final_replace_task(); + int cache_insert_row(); int do_delete(ObConflictRowMap *primary_map); int do_insert(); int reuse(); @@ -100,6 +102,7 @@ private: common::ObArenaAllocator allocator_; const ObTableApiReplaceSpec &replace_spec_; ObTableReplaceRtDef replace_rtdef_; + ObChunkDatumStore::StoredRow *insert_row_; int64_t insert_rows_; int64_t delete_rows_; sql::ObConflictChecker conflict_checker_; diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index dc8e4fe5c4..c4f05f003c 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -29,6 +29,8 @@ #include "storage/tx/ob_trans_service.h" #include "ob_table_session_pool.h" #include "storage/tx/wrs/ob_weak_read_util.h" +#include "ob_table_move_response.h" +#include "share/table/ob_table_config_util.h" using namespace oceanbase::observer; using namespace oceanbase::common; @@ -89,9 +91,9 @@ int ObTableLoginP::process() } } // whether the client should refresh location cache - if (OB_SUCCESS != ret && is_bad_routing_err(ret)) { - ObRpcProcessor::bad_routing_ = true; - LOG_WARN("[TABLE] login bad routing", K(ret), "bad_routing", ObRpcProcessor::bad_routing_); + if (OB_SUCCESS != ret && is_require_rerouting_err(ret)) { + ObRpcProcessor::require_rerouting_ = true; + LOG_WARN("[TABLE] login require rerouting", K(ret), "require_rerouting", ObRpcProcessor::require_rerouting_); } ObTenantStatEstGuard stat_guard(result_.tenant_id_); #ifndef NDEBUG @@ -223,7 +225,7 @@ ObTableApiProcessorBase::ObTableApiProcessorBase(const ObGlobalContext &gctx) need_retry_in_queue_(false), retry_count_(0), trans_desc_(NULL), - did_async_end_trans_(false) + had_do_response_(false) { need_audit_ = GCONF.enable_sql_audit; trans_state_ptr_ = &trans_state_; @@ -233,7 +235,7 @@ void ObTableApiProcessorBase::reset_ctx() { trans_state_ptr_->reset(); trans_desc_ = NULL; - did_async_end_trans_ = false; + had_do_response_ = false; } int ObTableApiProcessorBase::get_ls_id(const ObTabletID &tablet_id, ObLSID &ls_id) @@ -568,7 +570,7 @@ int ObTableApiProcessorBase::async_commit_trans(rpc::ObRequest *req, int64_t tim callback.callback(ret); } // ignore the return code of end_trans - did_async_end_trans_ = true; // don't send response in this worker thread + had_do_response_ = true; // don't send response in this worker thread // @note the req_ may be freed, req_processor can not be read any more. // The req_has_wokenup_ MUST set to be true, otherwise req_processor will invoke req_->set_process_start_end_diff, cause memory core // @see ObReqProcessor::run() req_->set_process_start_end_diff(ObTimeUtility::current_time()); @@ -880,15 +882,15 @@ int ObTableRpcProcessor::process() if (OB_FAIL(process_with_retry(RpcProcessor::arg_.credential_, get_timeout_ts()))) { if (OB_NOT_NULL(request_string_)) { // request_string_ has been generated if enable sql_audit LOG_WARN("fail to process table_api request", K(ret), K(stat_event_type_), K(request_string_)); - } else if (did_async_end_trans()) { // req_ may be freed + } else if (had_do_response()) { // req_ may be freed LOG_WARN("fail to process table_api request", K(ret), K(stat_event_type_)); } else { LOG_WARN("fail to process table_api request", K(ret), K(stat_event_type_), "request", RpcProcessor::arg_); } - // whether the client should refresh location cache - if (is_bad_routing_err(ret)) { - ObRpcProcessor::bad_routing_ = true; - LOG_WARN("table_api request bad routing", K(ret), "bad_routing", ObRpcProcessor::bad_routing_); + // whether the client should refresh location cache and retry + if (is_require_rerouting_err(ret)) { + ObRpcProcessor::require_rerouting_ = true; + LOG_WARN("table_api request require rerouting", K(ret), "require_rerouting", ObRpcProcessor::require_rerouting_); } } return ret; @@ -910,12 +912,26 @@ int ObTableRpcProcessor::before_response(int error_code) } template -int ObTableRpcProcessor::response(const int retcode) +int ObTableRpcProcessor::response(int error_code) { int ret = OB_SUCCESS; // if it is waiting for retry in queue, the response can NOT be sent. - if (!need_retry_in_queue_) { - ret = RpcProcessor::response(retcode); + if (!need_retry_in_queue_ && !had_do_response()) { + const ObRpcPacket *rpc_pkt = &reinterpret_cast(this->req_->get_packet()); + if (is_require_rerouting_err(error_code) && rpc_pkt->require_rerouting()) { + // response rerouting packet + ObTableMoveResponseSender sender(this->req_, error_code); + if (OB_FAIL(sender.init(ObTableApiProcessorBase::table_id_, ObTableApiProcessorBase::tablet_id_, *gctx_.schema_service_))) { + LOG_WARN("fail to init move response sender", K(ret), K(RpcProcessor::arg_)); + } else if (OB_FAIL(sender.response())) { + LOG_WARN("fail to do move response", K(ret)); + } + if (OB_FAIL(ret)) { + ret = RpcProcessor::response(error_code); // do common response when do move response failed + } + } else { + ret = RpcProcessor::response(error_code); + } } return ret; } @@ -978,17 +994,19 @@ void ObTableRpcProcessor::generate_sql_id() snprintf(audit_record_.sql_id_, (int32_t)sizeof(audit_record_.sql_id_), "TABLEAPI0x%04Xvv%016lX", RpcProcessor::PCODE, checksum); } -bool oceanbase::observer::is_bad_routing_err(const int err) +bool oceanbase::observer::is_require_rerouting_err(const int err) { - // bad routing check : whether client should refresh location cache + // rerouting: whether client should refresh location cache and retry // Now, following the same logic as in ../mysql/ob_query_retry_ctrl.cpp - return (is_master_changed_error(err) - || is_server_down_error(err) - || is_partition_change_error(err) - || is_server_status_error(err) - || is_unit_migrate(err) - || is_transaction_rpc_timeout_err(err) - || is_has_no_readable_replica_err(err) - || is_select_dup_follow_replic_err(err) - || is_trans_stmt_need_retry_error(err)); + bool is_err = is_master_changed_error(err) + || is_server_down_error(err) + || is_partition_change_error(err) + || is_server_status_error(err) + || is_unit_migrate(err) + || is_transaction_rpc_timeout_err(err) + || is_has_no_readable_replica_err(err) + || is_select_dup_follow_replic_err(err) + || is_trans_stmt_need_retry_error(err); + + return is_err && ObKVFeatureModeUitl::is_rerouting_enable(); } diff --git a/src/observer/table/ob_table_rpc_processor.h b/src/observer/table/ob_table_rpc_processor.h index 8c586a6acd..eeb37b87d6 100644 --- a/src/observer/table/ob_table_rpc_processor.h +++ b/src/observer/table/ob_table_rpc_processor.h @@ -134,11 +134,11 @@ public: const share::ObLSID &ls_id, int64_t timeout_ts); void release_read_trans(); - inline bool did_async_end_trans() const { return did_async_end_trans_; } inline transaction::ObTxDesc *get_trans_desc() { return trans_desc_; } int get_tablet_by_rowkey(uint64_t table_id, const ObIArray &rowkeys, ObIArray &tablet_ids); inline transaction::ObTxReadSnapshot &get_tx_snapshot() { return tx_snapshot_; } + inline bool had_do_response() const { return had_do_response_; } int get_table_id(const ObString &table_name, const uint64_t arg_table_id, uint64_t &real_table_id) const; protected: virtual int check_arg() = 0; @@ -181,11 +181,13 @@ protected: ObTableRetryPolicy retry_policy_; bool need_retry_in_queue_; int32_t retry_count_; + uint64_t table_id_; + ObTabletID tablet_id_; protected: // trans control sql::TransState trans_state_; transaction::ObTxDesc *trans_desc_; - bool did_async_end_trans_; + bool had_do_response_; // asynchronous transactions return packet in advance sql::TransState *trans_state_ptr_; transaction::ObTxReadSnapshot tx_snapshot_; }; diff --git a/src/observer/table/ob_table_rpc_processor_util.h b/src/observer/table/ob_table_rpc_processor_util.h index e44d69473d..350771664c 100644 --- a/src/observer/table/ob_table_rpc_processor_util.h +++ b/src/observer/table/ob_table_rpc_processor_util.h @@ -276,7 +276,7 @@ private: ~ObTableRpcProcessorUtil() = delete; }; -bool is_bad_routing_err(const int err); +bool is_require_rerouting_err(const int err); } // end namespace observer } // end namespace oceanbase diff --git a/src/observer/table/ob_rpc_async_response.h b/src/observer/table/ob_table_rpc_response_sender.h similarity index 77% rename from src/observer/table/ob_rpc_async_response.h rename to src/observer/table/ob_table_rpc_response_sender.h index b5b1aee043..e687f6419f 100644 --- a/src/observer/table/ob_rpc_async_response.h +++ b/src/observer/table/ob_table_rpc_response_sender.h @@ -10,8 +10,8 @@ * See the Mulan PubL v2 for more details. */ -#ifndef _OB_RPC_ASYNC_RESPONSE_H -#define _OB_RPC_ASYNC_RESPONSE_H 1 +#ifndef _OB_TABLE_RPC_RESPONSE_SENDER_H +#define _OB_TABLE_RPC_RESPONSE_SENDER_H 1 #include "rpc/ob_request.h" #include "rpc/obrpc/ob_rpc_packet.h" #include "rpc/frame/ob_req_processor.h" @@ -25,30 +25,40 @@ namespace obrpc { // this class is copied from ObRpcProcessor template -class ObRpcAsyncResponse +class ObTableRpcResponseSender { public: - ObRpcAsyncResponse(rpc::ObRequest *req, T &result) + ObTableRpcResponseSender(rpc::ObRequest *req, T &result, const int exec_ret_code = common::OB_SUCCESS) :req_(req), result_(result), + exec_ret_code_(exec_ret_code), + pcode_(ObRpcPacketCode::OB_INVALID_RPC_CODE), using_buffer_(NULL) - {} - virtual ~ObRpcAsyncResponse() = default; - int response(const int retcode); + { + if (OB_NOT_NULL(req_)) { + const ObRpcPacket *rpc_pkt = &reinterpret_cast(req_->get_packet()); + pcode_ = rpc_pkt->get_pcode(); + } + } + virtual ~ObTableRpcResponseSender() = default; + int response(const int cb_param); + OB_INLINE void set_pcode(ObRpcPacketCode pcode) { pcode_ = pcode; } private: int serialize(); - int do_response(ObRpcPacket *response_pkt, bool bad_routing); + int do_response(ObRpcPacket *response_pkt, bool require_rerouting); char *easy_alloc(int64_t size) const; // disallow copy - DISALLOW_COPY_AND_ASSIGN(ObRpcAsyncResponse); + DISALLOW_COPY_AND_ASSIGN(ObTableRpcResponseSender); private: rpc::ObRequest *req_; T &result_; + const int exec_ret_code_; // processor执行的返回码 + ObRpcPacketCode pcode_; common::ObDataBuffer *using_buffer_; }; template -char *ObRpcAsyncResponse::easy_alloc(int64_t size) const +char *ObTableRpcResponseSender::easy_alloc(int64_t size) const { void *buf = NULL; if (OB_ISNULL(req_)) { @@ -60,7 +70,7 @@ char *ObRpcAsyncResponse::easy_alloc(int64_t size) const } template -int ObRpcAsyncResponse::serialize() +int ObTableRpcResponseSender::serialize() { int ret = common::OB_SUCCESS; if (OB_ISNULL(using_buffer_)) { @@ -77,12 +87,15 @@ int ObRpcAsyncResponse::serialize() } template -int ObRpcAsyncResponse::do_response(ObRpcPacket *response_pkt, bool bad_routing) +int ObTableRpcResponseSender::do_response(ObRpcPacket *response_pkt, bool require_rerouting) { int ret = common::OB_SUCCESS; if (OB_ISNULL(req_)) { ret = common::OB_ERR_NULL_VALUE; RPC_OBRPC_LOG(WARN, "req is NULL", K(ret)); + } else if (ObRpcPacketCode::OB_INVALID_RPC_CODE == pcode_) { + ret = common::OB_ERR_UNEXPECTED; + RPC_OBRPC_LOG(WARN, "pcode is invalid", K(ret), K_(pcode), KPC_(req)); } else { const ObRpcPacket *rpc_pkt = &reinterpret_cast(req_->get_packet()); // TODO: fufeng, make force_destroy_second as a configure item @@ -94,12 +107,11 @@ int ObRpcAsyncResponse::do_response(ObRpcPacket *response_pkt, bool bad_routi // _OB_LOG(ERROR, "pkt process too long time: pkt_receive_ts=%ld, pkt_code=%d", rts, pcode); // } //copy packet into req buffer - ObRpcPacketCode pcode = rpc_pkt->get_pcode(); ObRpcPacket *packet = NULL; req_->set_trace_point(rpc::ObRequest::OB_EASY_REQUEST_RPC_ASYNC_RSP); if (OB_SUCC(ret)) { packet = response_pkt; - packet->set_pcode(pcode); + packet->set_pcode(pcode_); packet->set_chid(rpc_pkt->get_chid()); packet->set_session_id(0); // not stream packet->set_trace_id(rpc_pkt->get_trace_id()); @@ -115,8 +127,8 @@ int ObRpcAsyncResponse::do_response(ObRpcPacket *response_pkt, bool bad_routi packet->set_pop_process_start_diff(req_->get_pop_process_start_diff()); packet->set_process_start_end_diff(req_->get_process_start_end_diff()); packet->set_process_end_response_diff(req_->get_process_end_response_diff()); - if (bad_routing) { - packet->set_bad_routing(); + if (require_rerouting) { + packet->set_require_rerouting(); } packet->calc_checksum(); } @@ -127,9 +139,10 @@ int ObRpcAsyncResponse::do_response(ObRpcPacket *response_pkt, bool bad_routi } template -int ObRpcAsyncResponse::response(const int retcode) +int ObTableRpcResponseSender::response(const int cb_param) { int ret = common::OB_SUCCESS; + int retcode = (cb_param == OB_SUCCESS ? exec_ret_code_ : cb_param); if (OB_ISNULL(req_)) { ret = common::OB_INVALID_ARGUMENT; RPC_OBRPC_LOG(WARN, "invalid req, maybe stream rpc timeout", K(ret), K(retcode), @@ -202,21 +215,22 @@ int ObRpcAsyncResponse::response(const int retcode) using_buffer_->get_position()))) { RPC_OBRPC_LOG(WARN, "serialize result code fail", K(ret)); } else { - // also send result if process successfully. - if (common::OB_SUCCESS == retcode) { + // 1. send result if process successfully. + // 2. send result if require rerouting + if (common::OB_SUCCESS == retcode || observer::is_require_rerouting_err(retcode)) { if (OB_FAIL(serialize())) { RPC_OBRPC_LOG(WARN, "serialize result fail", K(ret)); } } } - // routing check : whether client should refresh location cache and retry + // rerouting: whether client should refresh location cache and retry // Now, following the same logic as in ../mysql/ob_query_retry_ctrl.cpp - bool bad_routing = false; + bool require_rerouting = false; if (OB_SUCC(ret)) { - if (common::OB_SUCCESS != retcode && observer::is_bad_routing_err(retcode)) { - bad_routing = true; - RPC_OBRPC_LOG(WARN, "bad routing", K(retcode), K(bad_routing)); + if (common::OB_SUCCESS != retcode && observer::is_require_rerouting_err(retcode)) { + require_rerouting = true; + RPC_OBRPC_LOG(INFO, "require rerouting", K(retcode), K(require_rerouting)); } } @@ -224,8 +238,8 @@ int ObRpcAsyncResponse::response(const int retcode) ObRpcPacket *pkt = new (pkt_buf) ObRpcPacket(); //Response rsp(sessid, is_stream_, is_last, pkt); pkt->set_content(using_buffer_->get_data(), using_buffer_->get_position()); - if (OB_FAIL(do_response(pkt, bad_routing))) { - RPC_OBRPC_LOG(WARN, "response data fail", K(ret)); + if (OB_FAIL(do_response(pkt, require_rerouting))) { + RPC_OBRPC_LOG(WARN, "response data fail", K(ret), K(retcode)); } } @@ -236,4 +250,4 @@ int ObRpcAsyncResponse::response(const int retcode) } // end namespace obrpc } // end namespace oceanbase -#endif /* _OB_RPC_ASYNC_RESPONSE_H */ +#endif /* _OB_TABLE_RPC_RESPONSE_SENDER_H */ diff --git a/src/observer/table/ob_table_scan_executor.cpp b/src/observer/table/ob_table_scan_executor.cpp index 2ce3e5318e..4c2d1baf95 100644 --- a/src/observer/table/ob_table_scan_executor.cpp +++ b/src/observer/table/ob_table_scan_executor.cpp @@ -118,7 +118,7 @@ int ObTableApiScanExecutor::prepare_das_task() scan_op = static_cast(task_op); scan_op->set_scan_ctdef(&scan_spec_.get_ctdef().scan_ctdef_); scan_op->set_scan_rtdef(&tsc_rtdef_.scan_rtdef_); - scan_op->set_can_part_retry(nullptr == tsc_rtdef_.scan_rtdef_.sample_info_); + scan_op->set_can_part_retry(false); tsc_rtdef_.scan_rtdef_.table_loc_->is_reading_ = true; if (scan_spec_.get_ctdef().lookup_ctdef_ != nullptr) { //is local index lookup, need to set the lookup ctdef to the das scan op diff --git a/src/observer/table/ob_table_update_executor.cpp b/src/observer/table/ob_table_update_executor.cpp index fff876ed46..a2dadb59b7 100644 --- a/src/observer/table/ob_table_update_executor.cpp +++ b/src/observer/table/ob_table_update_executor.cpp @@ -21,8 +21,7 @@ namespace oceanbase { namespace table { -int ObTableApiUpdateExecutor::process_single_operation(const ObTableEntity *entity, - const ObTableCtx::ObAssignIds &assign_ids) +int ObTableApiUpdateExecutor::process_single_operation(const ObTableEntity *entity) { int ret = OB_SUCCESS; common::ObIArray &key_ranges = tb_ctx_.get_key_ranges(); @@ -49,10 +48,7 @@ int ObTableApiUpdateExecutor::process_single_operation(const ObTableEntity *enti LOG_WARN("fail to get next row", K(ret)); } } else if (OB_FAIL(ObTableExprCgService::refresh_update_exprs_frame(tb_ctx_, - upd_ctdef->old_row_, upd_ctdef->new_row_, - upd_ctdef->full_assign_row_, - assign_ids, *entity))) { LOG_WARN("fail to refresh update exprs frame", K(ret), K(*entity), K(cur_idx_)); } @@ -69,7 +65,7 @@ int ObTableApiUpdateExecutor::get_next_row_from_child() if (cur_idx_ >= 1) { ret = OB_ITER_END; - } else if (OB_FAIL(process_single_operation(entity, tb_ctx_.get_assign_ids()))) { + } else if (OB_FAIL(process_single_operation(entity))) { if (OB_ITER_END != ret) { LOG_WARN("fail to process single update operation", K(ret)); } diff --git a/src/observer/table/ob_table_update_executor.h b/src/observer/table/ob_table_update_executor.h index 47a0784b4b..862b3389b1 100644 --- a/src/observer/table/ob_table_update_executor.h +++ b/src/observer/table/ob_table_update_executor.h @@ -53,8 +53,7 @@ private: int get_next_row_from_child(); int update_row_to_das(); int upd_rows_post_proc(); - int process_single_operation(const ObTableEntity *entity, - const ObTableCtx::ObAssignIds &assign_ids); + int process_single_operation(const ObTableEntity *entity); private: const ObTableApiUpdateSpec &upd_spec_; ObTableUpdRtDef upd_rtdef_; diff --git a/src/observer/table/ttl/ob_table_ttl_executor.cpp b/src/observer/table/ttl/ob_table_ttl_executor.cpp index 5aeb9f2766..7bdc697f09 100644 --- a/src/observer/table/ttl/ob_table_ttl_executor.cpp +++ b/src/observer/table/ttl/ob_table_ttl_executor.cpp @@ -93,7 +93,7 @@ int ObTableApiTTLExecutor::refresh_exprs_frame(const ObTableEntity *entity) LOG_WARN("entity is null", K(ret)); } else if (OB_FAIL(ObTableExprCgService::refresh_ttl_exprs_frame(tb_ctx_, ins_ctdef.new_row_, - upd_ctdef.delta_exprs_, + upd_ctdef.delta_row_, *entity))) { LOG_WARN("fail to refresh ttl exprs frame", K(ret), K(*entity)); } @@ -277,7 +277,6 @@ int ObTableApiTTLExecutor::update_row_to_das() ttl_rtdef_.upd_rtdef_, upd_rtctx_)); OZ(to_expr_skip_old(*constraint_value.current_datum_row_, - constraint_rowkey, ttl_spec_.get_ctdef().upd_ctdef_)); clear_evaluated_flag(); OZ(insert_upd_new_row_to_das(ttl_spec_.get_ctdef().upd_ctdef_, @@ -286,7 +285,6 @@ int ObTableApiTTLExecutor::update_row_to_das() } else if (NULL == constraint_value.baseline_datum_row_ && NULL != constraint_value.current_datum_row_) { // 单单是唯一索引冲突的时候，会走这个分支 OZ(to_expr_skip_old(*constraint_value.current_datum_row_, - constraint_rowkey, ttl_spec_.get_ctdef().upd_ctdef_)); OZ(insert_upd_new_row_to_das(ttl_spec_.get_ctdef().upd_ctdef_, ttl_rtdef_.upd_rtdef_, diff --git a/src/observer/table/ttl/ob_table_ttl_task.cpp b/src/observer/table/ttl/ob_table_ttl_task.cpp index 7051f6cdab..898e3ab17d 100644 --- a/src/observer/table/ttl/ob_table_ttl_task.cpp +++ b/src/observer/table/ttl/ob_table_ttl_task.cpp @@ -37,10 +37,10 @@ ObTableTTLDeleteTask::ObTableTTLDeleteTask(): is_inited_(false), param_(), info_(NULL), - allocator_(ObMemAttr(MTL_ID(), "TTLDeleteTask")), + allocator_(ObMemAttr(MTL_ID(), "TTLDelTaskCtx")), rowkey_(), ttl_tablet_mgr_(NULL), - default_entity_factory_("TTLEntityFac") + rowkey_allocator_(ObMemAttr(MTL_ID(), "TTLDelTaskRKey")) { } @@ -133,12 +133,12 @@ int ObTableTTLDeleteTask::process() ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else { - lib::ContextParam param; - param.set_mem_attr(MTL_ID(), "TTLDeleteTask", ObCtxIds::DEFAULT_CTX_ID) - .set_properties(lib::USE_TL_PAGE_OPTIONAL); - CREATE_WITH_TEMP_CONTEXT(param) { - bool need_stop = false; - while(!need_stop) { + bool need_stop = false; + while(!need_stop) { + lib::ContextParam param; + param.set_mem_attr(MTL_ID(), "TTLDeleteMemCtx", ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + CREATE_WITH_TEMP_CONTEXT(param) { if (OB_FAIL(process_one())) { if (OB_ITER_END != ret) { LOG_WARN("fail to process one", KR(ret)); @@ -147,6 +147,7 @@ int ObTableTTLDeleteTask::process() if (OB_FAIL(ttl_tablet_mgr_->report_task_status(const_cast(*info_), param_, need_stop))) { LOG_WARN("fail to report ttl task status", KR(ret)); } + allocator_.reuse(); } } } @@ -169,8 +170,8 @@ int ObTableTTLDeleteTask::process_one() param_, PER_TASK_DEL_ROWS, rowkey_); - SMART_VAR(ObTableCtx, tb_ctx, allocator_) { - if (OB_FAIL(init_scan_tb_ctx(tb_ctx, cache_guard))) { + SMART_VAR(ObTableCtx, scan_ctx, allocator_) { + if (OB_FAIL(init_scan_tb_ctx(scan_ctx, cache_guard))) { LOG_WARN("fail to init tb ctx", KR(ret)); } else if (OB_FAIL(ObTableApiProcessorBase::start_trans_( false, @@ -178,20 +179,20 @@ int ObTableTTLDeleteTask::process_one() tx_snapshot, ObTableConsistencyLevel::STRONG, &trans_state, - tb_ctx.get_table_id(), - tb_ctx.get_ls_id(), + scan_ctx.get_table_id(), + scan_ctx.get_ls_id(), get_timeout_ts()))) { LOG_WARN("fail to start trans", KR(ret)); - } else if (OB_FAIL(tb_ctx.init_trans(trans_desc, tx_snapshot))) { + } else if (OB_FAIL(scan_ctx.init_trans(trans_desc, tx_snapshot))) { LOG_WARN("fail to init trans", KR(ret)); - } else if (OB_FAIL(cache_guard.get_spec(&tb_ctx, scan_spec))) { + } else if (OB_FAIL(cache_guard.get_spec(&scan_ctx, scan_spec))) { LOG_WARN("fail to get scan spec from cache", KR(ret)); } else { ObTableTTLDeleteRowIterator row_iter; ObTableApiExecutor *executor = nullptr; - if (OB_FAIL(scan_spec->create_executor(tb_ctx, executor))) { - LOG_WARN("fail to generate executor", KR(ret), K(tb_ctx)); - } else if (OB_FAIL(row_iter.init(*tb_ctx.get_table_schema(), ttl_operation))){ + if (OB_FAIL(scan_spec->create_executor(scan_ctx, executor))) { + LOG_WARN("fail to generate executor", KR(ret), K(scan_ctx)); + } else if (OB_FAIL(row_iter.init(*scan_ctx.get_table_schema(), ttl_operation))){ LOG_WARN("fail to init ttl row iterator", KR(ret)); } else if (OB_FAIL(row_iter.open(static_cast(executor)))) { LOG_WARN("fail to open scan row iterator", KR(ret)); @@ -207,7 +208,7 @@ int ObTableTTLDeleteTask::process_one() if (OB_NOT_NULL(scan_spec)) { scan_spec->destroy_executor(executor); - tb_ctx.set_expr_info(nullptr); + scan_ctx.set_expr_info(nullptr); } } } @@ -240,7 +241,6 @@ int ObTableTTLDeleteTask::init_scan_tb_ctx(ObTableCtx &tb_ctx, ObTableApiCacheGu int ret = OB_SUCCESS; ObExprFrameInfo *expr_frame_info = nullptr; tb_ctx.set_scan(true); - tb_ctx.set_operation_type(ObTableOperationType::DEL); if (tb_ctx.is_init()) { LOG_INFO("tb ctx has been inited", K(tb_ctx)); } else if (OB_FAIL(tb_ctx.init_common(credential_, @@ -273,16 +273,16 @@ int ObTableTTLDeleteTask::process_ttl_delete(const ObITableEntity &new_entity, transaction::ObTxReadSnapshot &snapshot) { int ret = OB_SUCCESS; - SMART_VAR(ObTableCtx, tb_ctx, allocator_) { - ObTableApiSpec *spec = nullptr; - ObTableApiExecutor *executor = nullptr; - ObTableOperationResult op_result; - if (OB_FAIL(init_tb_ctx(new_entity, tb_ctx))) { + ObTableApiSpec *spec = nullptr; + ObTableApiExecutor *executor = nullptr; + ObTableOperationResult op_result; + SMART_VAR(ObTableCtx, delete_ctx, allocator_) { + if (OB_FAIL(init_tb_ctx(new_entity, delete_ctx))) { LOG_WARN("fail to init table ctx", K(ret), K(new_entity)); - } else if (OB_FAIL(tb_ctx.init_trans(trans_desc, snapshot))) { - LOG_WARN("fail to init trans", K(ret), K(tb_ctx)); - } else if (OB_FAIL(ObTableOpWrapper::process_op(tb_ctx, op_result))) { - LOG_WARN("fail to process insert op", K(ret)); + } else if (OB_FAIL(delete_ctx.init_trans(trans_desc, snapshot))) { + LOG_WARN("fail to init trans", K(ret), K(delete_ctx)); + } else if (OB_FAIL(ObTableOpWrapper::process_op(delete_ctx, op_result))) { + LOG_WARN("fail to process delete op", K(ret)); } else { affected_rows = op_result.get_affected_rows(); } @@ -555,31 +555,28 @@ int ObTableTTLDeleteTask::execute_ttl_delete(ObTableTTLDeleteRowIterator &ttl_ro int64_t affected_rows = 0; while (OB_SUCC(ret)) { ObNewRow *row = nullptr; - ObITableEntity *new_entity = nullptr; if (OB_FAIL(ttl_row_iter.get_next_row(row))) { if (OB_ITER_END != ret) { LOG_WARN("fail to get next row", K(ret)); } - } else if (OB_ISNULL(new_entity = default_entity_factory_.alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc entity", K(ret)); } else { int64_t rowkey_cnt = ttl_row_iter.rowkey_cell_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; i++) { - if (OB_FAIL(new_entity->add_rowkey_value(row->get_cell(ttl_row_iter.rowkey_cell_ids_[i])))) { + if (OB_FAIL(delete_entity_.add_rowkey_value(row->get_cell(ttl_row_iter.rowkey_cell_ids_[i])))) { LOG_WARN("fail to add rowkey value", K(ret)); } } if (OB_SUCC(ret)) { int64_t tmp_affect_rows = 0; - if (OB_FAIL(process_ttl_delete(*new_entity, tmp_affect_rows, trans_desc, snapshot))) { + if (OB_FAIL(process_ttl_delete(delete_entity_, tmp_affect_rows, trans_desc, snapshot))) { LOG_WARN("fail to execute table delete", K(ret)); } else { affected_rows += tmp_affect_rows; } } } + delete_entity_.reset(); } if (OB_ITER_END == ret) { @@ -606,7 +603,11 @@ int ObTableTTLDeleteTask::execute_ttl_delete(ObTableTTLDeleteRowIterator &ttl_ro ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected rowkey column count", K(ttl_row_iter.last_row_->get_count()), K(ttl_row_iter.get_rowkey_column_cnt())); } else { + rowkey_allocator_.reuse(); row_key.assign(ttl_row_iter.last_row_->cells_, ttl_row_iter.get_rowkey_column_cnt()); + if (OB_FAIL(row_key.deep_copy(rowkey_, rowkey_allocator_))) { + LOG_WARN("fail to deep copy rowkey", KR(ret)); + } } } diff --git a/src/observer/table/ttl/ob_table_ttl_task.h b/src/observer/table/ttl/ob_table_ttl_task.h index 16ab240e36..d79a26eb0f 100644 --- a/src/observer/table/ttl/ob_table_ttl_task.h +++ b/src/observer/table/ttl/ob_table_ttl_task.h @@ -74,7 +74,6 @@ public: int64_t &affected_rows, transaction::ObTxDesc *trans_desc, transaction::ObTxReadSnapshot &snapshot); - table::ObTableEntityFactory &get_entity_factory() { return default_entity_factory_; } common::ObIAllocator &get_allocator() { return allocator_; } int init_credential(const table::ObTTLTaskParam &ttl_param); @@ -108,8 +107,9 @@ private: common::ObRowkey rowkey_; table::ObTenantTabletTTLMgr *ttl_tablet_mgr_; share::ObLSID ls_id_; - table::ObTableEntityFactory default_entity_factory_; + ObTableEntity delete_entity_; table::ObTableApiCredential credential_; + common::ObArenaAllocator rowkey_allocator_; DISALLOW_COPY_AND_ASSIGN(ObTableTTLDeleteTask); }; diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp index d80aae7e99..6bb1ffbef8 100644 --- a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp @@ -20,6 +20,7 @@ #include "storage/tx_storage/ob_tenant_freezer.h" #include "observer/table/ttl/ob_table_ttl_task.h" #include "observer/table/ob_table_service.h" +#include "share/table/ob_table_config_util.h" namespace oceanbase { @@ -395,6 +396,10 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask ret = OB_ERR_UNEXPECTED; LOG_WARN("the tablet task ctx is null", KR(ret)); } else { + if (!ObKVFeatureModeUitl::is_ttl_enable()) { + local_tenant_task_.ttl_continue_ = false; + LOG_DEBUG("local_tenant_task mark continue is false"); + } // lock task ctx for update common::ObSpinLockGuard ctx_guard(ctx->lock_); ctx->last_modify_time_ = ObTimeUtility::current_time(); @@ -406,6 +411,7 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask task_para = ctx->ttl_para_; is_stop = false; } else { + ctx->task_status_ = OB_TTL_TASK_PENDING; LOG_INFO("pending current task", K(local_tenant_task_.state_), K(local_tenant_task_.ttl_continue_)); } } else if (OB_ITER_END == task_info.err_code_) { @@ -426,9 +432,12 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask } } - //schedule task - if (is_stop && OB_FAIL(try_schedule_remaining_tasks(ctx))) { - LOG_WARN("fail to try schedule task", KR(ret)); + // schedule remaining tasks + if (is_stop) { + LOG_INFO("stop current task", K(ret), KPC(ctx), K_(local_tenant_task)); + if (OB_FAIL(try_schedule_remaining_tasks(ctx))) { + LOG_WARN("fail to try schedule task", KR(ret)); + } } return ret; } @@ -599,7 +608,10 @@ void OBTTLTimerPeriodicTask::runTimerTask() { int ret = OB_SUCCESS; ObCurTraceId::init(GCONF.self_addr_); - if (common::ObTTLUtil::check_can_do_work()) { + if (!ObKVFeatureModeUitl::is_ttl_enable()) { + // do nothing + LOG_DEBUG("ttl is disable"); + } else if (common::ObTTLUtil::check_can_do_work()) { if (OB_FAIL(tablet_ttl_mgr_.check_tenant_memory())) { LOG_WARN("fail to check all tenant memory", KR(ret)); } diff --git a/src/observer/table/ttl/ob_tenant_ttl_manager.cpp b/src/observer/table/ttl/ob_tenant_ttl_manager.cpp index 2c3ef5c78e..340b3d1f09 100644 --- a/src/observer/table/ttl/ob_tenant_ttl_manager.cpp +++ b/src/observer/table/ttl/ob_tenant_ttl_manager.cpp @@ -16,6 +16,7 @@ #include "share/ob_max_id_fetcher.h" #include "share/table/ob_ttl_util.h" #include "lib/oblog/ob_log_module.h" +#include "share/table/ob_table_config_util.h" using namespace oceanbase::share; using namespace oceanbase::common; @@ -30,7 +31,10 @@ void ObClearTTLHistoryTask::runTimerTask() { ObCurTraceId::init(GCONF.self_addr_); int ret = OB_SUCCESS; - if (IS_NOT_INIT) { + if (!ObKVFeatureModeUitl::is_ttl_enable()) { + // do nothing + LOG_DEBUG("ttl is disable"); + } else if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ob clear ttl history task is not init", KR(ret)); } else if (ObTTLUtil::check_can_do_work()) { @@ -581,7 +585,10 @@ void ObTTLTaskScheduler::runTimerTask() { int ret = OB_SUCCESS; ObCurTraceId::init(GCONF.self_addr_); - if (IS_NOT_INIT) { + if (!ObKVFeatureModeUitl::is_ttl_enable()) { + // do nothing + LOG_DEBUG("ttl is disable"); + } else if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ttl task mgr not init", KR(ret)); } else if (OB_FAIL(reload_tenant_task())) { diff --git a/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp b/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp index 9a92def9be..63e5e1e57c 100644 --- a/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp @@ -247,6 +247,19 @@ int ObAllVirtualSysStat::update_all_stats_(const int64_t tenant_id, ObStatEventS } else { // it is ok to not have any records } + + { + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + if (tenant_config.is_valid()) { + MTL_SWITCH(tenant_id) { + auto *tenant_base = MTL_CTX(); + int64_t max_sess_num = tenant_base->get_max_session_num(tenant_config->_resource_limit_max_session_num); + stat_events.get(ObStatEventIds::MAX_SESSION_NUM - ObStatEventIds::STAT_EVENT_ADD_END - 1)->stat_value_ + = max_sess_num; + } + } + } + ret = ret_bk; } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_thread.cpp b/src/observer/virtual_table/ob_all_virtual_thread.cpp index 02dd40af35..7338d3f4c5 100644 --- a/src/observer/virtual_table/ob_all_virtual_thread.cpp +++ b/src/observer/virtual_table/ob_all_virtual_thread.cpp @@ -130,7 +130,7 @@ int ObAllVirtualThread::inner_get_next_row(common::ObNewRow *&row) struct iovec remote_iov = {thread_base + rpc_dest_addr_offset, sizeof(ObAddr)}; wait_event_[0] = '\0'; if (0 != join_addr) { - IGNORE_RETURN snprintf(wait_event_, 64, "thread %u", *(uint32_t*)(thread_base + tid_offset)); + IGNORE_RETURN snprintf(wait_event_, 64, "thread %u", *(uint32_t*)(join_addr + tid_offset)); } else if (OB_NOT_NULL(wait_addr)) { uint32_t val = 0; struct iovec local_iov = {&val, sizeof(val)}; diff --git a/src/observer/virtual_table/ob_tenant_virtual_get_object_definition.cpp b/src/observer/virtual_table/ob_tenant_virtual_get_object_definition.cpp index 24a6bef4a0..4049b6500b 100644 --- a/src/observer/virtual_table/ob_tenant_virtual_get_object_definition.cpp +++ b/src/observer/virtual_table/ob_tenant_virtual_get_object_definition.cpp @@ -319,7 +319,7 @@ int ObGetObjectDefinition::get_constraint_definition(ObString &ddl_str, LOG_WARN("get constraint info failed", K(ret), K(tenant_id), K(database_id), K(constraint_name)); } else if (OB_INVALID_ID == (constraint_id = constraint_info.constraint_id_)) { - // The unique constrant is mocked by a unique index. + // The unique constraint is mocked by a unique index. // If other types of constraint is not exist, we will try to find if the uk exists. // bool is_unique_constraint_exist = false; if (OB_FAIL(schema_guard_->get_idx_schema_by_origin_idx_name( diff --git a/src/pl/dblink/ob_pl_dblink_guard.cpp b/src/pl/dblink/ob_pl_dblink_guard.cpp index 59942f89e6..9021e7e20f 100644 --- a/src/pl/dblink/ob_pl_dblink_guard.cpp +++ b/src/pl/dblink/ob_pl_dblink_guard.cpp @@ -56,8 +56,7 @@ int ObPLDbLinkGuard::get_routine_infos_with_synonym(sql::ObSQLSessionInfo &sessi int64_t object_type; OZ (schema_guard.get_dblink_schema(tenant_id, dblink_name, dblink_schema), tenant_id, dblink_name); OV (OB_NOT_NULL(dblink_schema), OB_DBLINK_NOT_EXIST_TO_ACCESS, dblink_name); - OX (link_type = static_cast(dblink_schema->get_driver_proto())); - OZ (ObPLDblinkUtil::init_dblink(dblink_proxy, dblink_conn, session_info, schema_guard, dblink_name)); + OZ (ObPLDblinkUtil::init_dblink(dblink_proxy, dblink_conn, session_info, schema_guard, dblink_name, link_type)); CK (OB_NOT_NULL(dblink_proxy)); CK (OB_NOT_NULL(dblink_conn)); OZ (ObPLDblinkUtil::print_full_name(alloc_, full_name, part1, part2, part3)); @@ -70,13 +69,24 @@ int ObPLDbLinkGuard::get_routine_infos_with_synonym(sql::ObSQLSessionInfo &sessi sub_object_name, object_type, alloc_)); - OZ (get_dblink_routine_infos(session_info, + OZ (get_dblink_routine_infos(dblink_proxy, + dblink_conn, + session_info, schema_guard, dblink_name, schema_name, object_name, sub_object_name, routine_infos)); + if (OB_NOT_NULL(dblink_proxy) && OB_NOT_NULL(dblink_conn)) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = dblink_proxy->release_dblink(link_type, dblink_conn))) { + LOG_WARN("failed to relese connection", K(tmp_ret)); + } + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + } #endif return ret; } @@ -96,7 +106,8 @@ int ObPLDbLinkGuard::get_dblink_type_with_synonym(sql::ObSQLSessionInfo &session #else common::ObDbLinkProxy *dblink_proxy = NULL; common::sqlclient::ObISQLConnection *dblink_conn = NULL; - OZ (ObPLDblinkUtil::init_dblink(dblink_proxy, dblink_conn, session_info, schema_guard, dblink_name)); + common::sqlclient::DblinkDriverProto link_type = DBLINK_UNKNOWN; + OZ (ObPLDblinkUtil::init_dblink(dblink_proxy, dblink_conn, session_info, schema_guard, dblink_name, link_type)); CK (OB_NOT_NULL(dblink_proxy)); CK (OB_NOT_NULL(dblink_conn)); if (OB_SUCC(ret)) { @@ -119,7 +130,9 @@ int ObPLDbLinkGuard::get_dblink_type_with_synonym(sql::ObSQLSessionInfo &session object_type, alloc_)); OV (static_cast(ObObjectType::PACKAGE) == object_type); - OZ (get_dblink_type_by_name(session_info, + OZ (get_dblink_type_by_name(dblink_proxy, + dblink_conn, + session_info, schema_guard, dblink_name, schema_name, @@ -127,11 +140,22 @@ int ObPLDbLinkGuard::get_dblink_type_with_synonym(sql::ObSQLSessionInfo &session sub_object_name, udt)); } + if (OB_NOT_NULL(dblink_proxy) && OB_NOT_NULL(dblink_conn)) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = dblink_proxy->release_dblink(link_type, dblink_conn))) { + LOG_WARN("failed to relese connection", K(tmp_ret)); + } + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + } #endif return ret; } -int ObPLDbLinkGuard::get_dblink_routine_infos(sql::ObSQLSessionInfo &session_info, +int ObPLDbLinkGuard::get_dblink_routine_infos(common::ObDbLinkProxy *dblink_proxy, + common::sqlclient::ObISQLConnection *dblink_conn, + sql::ObSQLSessionInfo &session_info, share::schema::ObSchemaGetterGuard &schema_guard, const ObString &dblink_name, const ObString &db_name, @@ -166,15 +190,17 @@ int ObPLDbLinkGuard::get_dblink_routine_infos(sql::ObSQLSessionInfo &session_inf OZ (dblink_infos_.push_back(dblink_info)); } } - OZ ((const_cast(dblink_info))->get_routine_infos(session_info, - schema_guard, - alloc_, - dblink_name, - db_name, - pkg_name, - routine_name, - routine_infos, - next_link_object_id_)); + OZ ((const_cast(dblink_info))->get_routine_infos(dblink_proxy, + dblink_conn, + session_info, + schema_guard, + alloc_, + dblink_name, + db_name, + pkg_name, + routine_name, + routine_infos, + next_link_object_id_)); if (OB_SUCC(ret)) { bool is_all_func = true; for (int64_t i = 0; OB_SUCC(ret) && i < routine_infos.count(); i++) { @@ -318,20 +344,13 @@ int ObPLDbLinkGuard::dblink_name_resolve(common::ObDbLinkProxy *dblink_proxy, } #undef BIND_BASIC_BY_POS } - if ((NULL != dblink_conn)) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = static_cast(dblink_conn)->free_oci_stmt())) { - LOG_WARN("failed to close oci result", K(tmp_ret)); - if (OB_SUCC(ret)) { - ret = tmp_ret; - } - } - } #endif return ret; } -int ObPLDbLinkGuard::get_dblink_type_by_name(sql::ObSQLSessionInfo &session_info, +int ObPLDbLinkGuard::get_dblink_type_by_name(common::ObDbLinkProxy *dblink_proxy, + common::sqlclient::ObISQLConnection *dblink_conn, + sql::ObSQLSessionInfo &session_info, share::schema::ObSchemaGetterGuard &schema_guard, const common::ObString &dblink_name, const common::ObString &db_name, @@ -365,7 +384,7 @@ int ObPLDbLinkGuard::get_dblink_type_by_name(sql::ObSQLSessionInfo &session_info OZ (dblink_infos_.push_back(dblink_info)); } } - OZ ((const_cast(dblink_info))->get_udt_by_name(session_info, + OZ ((const_cast(dblink_info))->get_udt_by_name(dblink_proxy, dblink_conn, session_info, schema_guard, alloc_, dblink_name, db_name, pkg_name, udt_name, udt, next_link_object_id_)); #endif diff --git a/src/pl/dblink/ob_pl_dblink_guard.h b/src/pl/dblink/ob_pl_dblink_guard.h index 4a33d06926..afa717922a 100644 --- a/src/pl/dblink/ob_pl_dblink_guard.h +++ b/src/pl/dblink/ob_pl_dblink_guard.h @@ -110,7 +110,9 @@ private: int64_t &object_type, ObIAllocator &alloctor); - int get_dblink_routine_infos(sql::ObSQLSessionInfo &session_info, + int get_dblink_routine_infos(common::ObDbLinkProxy *dblink_proxy, + common::sqlclient::ObISQLConnection *dblink_conn, + sql::ObSQLSessionInfo &session_info, share::schema::ObSchemaGetterGuard &schema_guard, const common::ObString &dblink_name, const common::ObString &db_name, @@ -118,7 +120,9 @@ private: const common::ObString &routine_name, common::ObIArray &routine_infos); - int get_dblink_type_by_name(sql::ObSQLSessionInfo &session_info, + int get_dblink_type_by_name(common::ObDbLinkProxy *dblink_proxy, + common::sqlclient::ObISQLConnection *dblink_conn, + sql::ObSQLSessionInfo &session_info, share::schema::ObSchemaGetterGuard &schema_guard, const common::ObString &dblink_name, const common::ObString &db_name, diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index 2a90a2c60d..b509483515 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -1654,28 +1654,33 @@ int ObPL::parameter_anonymous_block(ObExecContext &ctx, trans_ctx.raw_anonymous_off_ = block->pl_str_off_; trans_ctx.params_ = ¶ms; trans_ctx.buf_ = (char *)trans_ctx.allocator_->alloc(sql.length()); - trans_ctx.buf_size_ = sql.length(); - trans_ctx.p_list_ = parse_result.param_nodes_; - CK (T_STMT_LIST == parse_result.result_tree_->type_ && 1 == parse_result.result_tree_->num_child_); - CK (OB_NOT_NULL(block_node = parse_result.result_tree_->children_[0])); - CK (T_SP_ANONYMOUS_BLOCK == block_node->type_); - CK (OB_NOT_NULL(block_node = block_node->children_[0])); - CK (T_SP_BLOCK_CONTENT == block_node->type_ || T_SP_LABELED_BLOCK == block_node->type_); - OZ (transform_tree(trans_ctx, const_cast(block), block_node, ctx, parse_result)); - if (OB_SUCC(ret)) { - if (trans_ctx.buf_size_ < trans_ctx.buf_len_ + trans_ctx.raw_sql_.length() - trans_ctx.copied_idx_ || - trans_ctx.raw_sql_.length() < trans_ctx.copied_idx_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error about trans_ctx.buf", K(ret)); - } else { - MEMCPY(trans_ctx.buf_ + trans_ctx.buf_len_, - trans_ctx.raw_sql_.ptr() + trans_ctx.copied_idx_, - trans_ctx.raw_sql_.length() - trans_ctx.copied_idx_); - trans_ctx.buf_len_ += trans_ctx.raw_sql_.length() - trans_ctx.copied_idx_; + if (OB_ISNULL(trans_ctx.buf_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate failed", K(sql), K(ret)); + } else { + trans_ctx.buf_size_ = sql.length(); + trans_ctx.p_list_ = parse_result.param_nodes_; + CK (T_STMT_LIST == parse_result.result_tree_->type_ && 1 == parse_result.result_tree_->num_child_); + CK (OB_NOT_NULL(block_node = parse_result.result_tree_->children_[0])); + CK (T_SP_ANONYMOUS_BLOCK == block_node->type_); + CK (OB_NOT_NULL(block_node = block_node->children_[0])); + CK (T_SP_BLOCK_CONTENT == block_node->type_ || T_SP_LABELED_BLOCK == block_node->type_); + OZ (transform_tree(trans_ctx, const_cast(block), block_node, ctx, parse_result)); + if (OB_SUCC(ret)) { + if (trans_ctx.buf_size_ < trans_ctx.buf_len_ + trans_ctx.raw_sql_.length() - trans_ctx.copied_idx_ || + trans_ctx.raw_sql_.length() < trans_ctx.copied_idx_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error about trans_ctx.buf", K(ret)); + } else { + MEMCPY(trans_ctx.buf_ + trans_ctx.buf_len_, + trans_ctx.raw_sql_.ptr() + trans_ctx.copied_idx_, + trans_ctx.raw_sql_.length() - trans_ctx.copied_idx_); + trans_ctx.buf_len_ += trans_ctx.raw_sql_.length() - trans_ctx.copied_idx_; + } } + pc_key.assign_ptr(trans_ctx.buf_, trans_ctx.buf_len_); + OZ (get_pl_function(ctx, params, OB_INVALID_ID, pc_key, cacheobj_guard)); } - pc_key.assign_ptr(trans_ctx.buf_, trans_ctx.buf_len_); - OZ (get_pl_function(ctx, params, OB_INVALID_ID, pc_key, cacheobj_guard)); } } return ret; diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 683a525b6c..4f00be01d7 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -11608,6 +11608,7 @@ int ObPLResolver::add_udt_self_argument(const ObIRoutineInfo *routine_info, expr_factory_, current_block_->get_namespace(), self_argument)); + CK (OB_NOT_NULL(self_argument)); OZ (self_argument->formalize(&resolve_ctx_.session_info_)); OX (udf_info.set_is_udf_udt_cons()); OZ (func.add_expr(self_argument)); @@ -13338,7 +13339,9 @@ ObPLMockSelfArg::~ObPLMockSelfArg() int ret = OB_SUCCESS; if (mocked_) { if (mark_only_) { - expr_params_.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM); + if (OB_FAIL(expr_params_.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM))) { + LOG_WARN("failed to clear flag", K(ret)); + } } else { std::rotate(expr_params_.begin(), expr_params_.begin() + 1, expr_params_.end()); if (!expr_params_.at(expr_params_.count() - 1)->has_flag(IS_UDT_UDF_SELF_PARAM)) { @@ -13478,12 +13481,16 @@ int ObPLResolver::check_routine_callable(const ObPLBlockNS &ns, if (expr_params.count() > 0 && expr_params.at(0)->get_result_type().get_udt_id() == access_idxs.at(access_idxs.count() - 1).var_index_) { - expr_params.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM); + if (OB_FAIL(expr_params.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM))) { + LOG_WARN("failed to clear flag", K(ret)); + } } else if (expr_params.count() > 0 && expr_params.at(0)->get_result_type().is_xml_sql_type() && (T_OBJ_XML == access_idxs.at(access_idxs.count() - 1).var_index_)) { // select 'head' || xmlparse(document '123').getclobval() into a from dual; - expr_params.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM); + if (OB_FAIL(expr_params.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM))) { + LOG_WARN("failed to clear flag", K(ret)); + } } /*else if (expr_params.count() > 0 && expr_params.at(0)->get_expr_type() == T_QUESTIONMARK) { // do nothing ... @@ -13593,7 +13600,6 @@ int ObPLResolver::resolve_construct(ObObjAccessIdent &access_ident, const ObUserDefinedType *user_type = NULL; ObObjAccessIdx access_idx; OV (access_ident.is_pl_udf(), OB_ERR_UNEXPECTED, K(access_ident)); - OZ (q_name.access_idents_.push_back(access_ident)); OZ (ns.get_pl_data_type_by_id(user_type_id, user_type)); CK (OB_NOT_NULL(user_type)); OZ (get_names_by_access_ident(access_ident, @@ -13601,6 +13607,16 @@ int ObPLResolver::resolve_construct(ObObjAccessIdent &access_ident, access_ident.udf_info_.udf_database_, access_ident.udf_info_.udf_package_, access_ident.udf_info_.udf_name_)); + + if (OB_SUCC(ret) && + !access_ident.udf_info_.udf_database_.empty() && + access_ident.udf_info_.udf_database_.case_compare(OB_SYS_DATABASE_NAME) != 0) { + OZ (q_name.access_idents_.push_back(access_ident.udf_info_.udf_database_)); + } + if (OB_SUCC(ret) && !access_ident.udf_info_.udf_package_.empty()) { + OZ (q_name.access_idents_.push_back(access_ident.udf_info_.udf_package_)); + } + OZ (q_name.access_idents_.push_back(access_ident)); OZ (resolve_construct(q_name, access_ident.udf_info_, *user_type, expr)); CK (OB_NOT_NULL(expr)); OZ (func.add_expr(expr)); diff --git a/src/pl/ob_pl_stmt.cpp b/src/pl/ob_pl_stmt.cpp index c95800958d..5cb8d015ac 100644 --- a/src/pl/ob_pl_stmt.cpp +++ b/src/pl/ob_pl_stmt.cpp @@ -1882,6 +1882,14 @@ int ObPLExternalNS::resolve_external_symbol(const common::ObString &name, } break; } + if (OB_FAIL(ret) && !resolve_ctx_.is_sql_scope_) { + // only reset in pl + // udf in sql do not reset this error + ObWarningBuffer *buf = common::ob_get_tsi_warning_buffer(); + if (NULL != buf) { + buf->reset(); + } + } CANCLE_LOG_CHECK_MODE(); return ret; } diff --git a/src/pl/ob_pl_type.cpp b/src/pl/ob_pl_type.cpp index fac2f74343..9aa3e56cdc 100644 --- a/src/pl/ob_pl_type.cpp +++ b/src/pl/ob_pl_type.cpp @@ -1990,10 +1990,27 @@ int ObPLCursorInfo::deep_copy(ObPLCursorInfo &src, common::ObIAllocator *allocat } else if (OB_ISNULL(row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("row is null", K(ret)); - } else if (OB_FAIL(dest_cursor->row_store_.add_row(*row))) { - LOG_WARN("failed to add row to row store", K(ret)); } else { - ++cur; + ObNewRow tmp_row = *row; + for (int64_t i = 0; OB_SUCC(ret) && i < tmp_row.get_count(); ++i) { + ObObj& obj = tmp_row.get_cell(i); + ObObj tmp; + if (obj.is_pl_extend()) { + if (OB_FAIL(pl::ObUserDefinedType::deep_copy_obj(*(dest_cursor->allocator_), obj, tmp))) { + LOG_WARN("failed to copy pl extend", K(ret)); + } else { + obj = tmp; + dest_cursor->complex_objs_.push_back(tmp); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(dest_cursor->row_store_.add_row(tmp_row))) { + LOG_WARN("failed to add row to row store", K(ret)); + } else { + ++cur; + } + } } } diff --git a/src/pl/ob_pl_user_type.cpp b/src/pl/ob_pl_user_type.cpp index 9dcaf9d6c6..994259df77 100644 --- a/src/pl/ob_pl_user_type.cpp +++ b/src/pl/ob_pl_user_type.cpp @@ -3142,9 +3142,10 @@ int ObPLComposite::copy_element(const ObObj &src, need_new_allocator, ignore_del_element)); CK (OB_NOT_NULL(dest_composite)); + uint8_t extend_type = src.get_meta().get_extend_type(); if (src.get_ext() == dest.get_ext()) { OX (dest.set_extend(reinterpret_cast(src_composite), - src.get_meta().get_extend_type(), + extend_type, src.get_val_len())); OZ (ObUserDefinedType::destruct_obj(dest, session)); OZ (ObPLComposite::deep_copy(*dest_composite, @@ -3155,13 +3156,13 @@ int ObPLComposite::copy_element(const ObObj &src, need_new_allocator, ignore_del_element)); OX (dest.set_extend(reinterpret_cast(dest_composite), - src.get_meta().get_extend_type(), + extend_type, src.get_val_len())); OZ (ObUserDefinedType::destruct_obj(dest, session)); OX (dest_composite = src_composite); } OX (dest.set_extend(reinterpret_cast(dest_composite), - src.get_meta().get_extend_type(), + extend_type, src.get_val_len())); #ifdef OB_BUILD_ORACLE_PL } diff --git a/src/pl/parser/pl_parser_mysql_mode.y b/src/pl/parser/pl_parser_mysql_mode.y index 3faafd8ac7..a75de86f75 100644 --- a/src/pl/parser/pl_parser_mysql_mode.y +++ b/src/pl/parser/pl_parser_mysql_mode.y @@ -430,7 +430,15 @@ sql_stmt: | SET /*sql stmt tail*/ { //read sql query string直到读到token';'或者END_P - do_parse_sql_stmt($$, parse_ctx, @1.first_column, @1.last_column, 2, ';', END_P); + ParseNode *sql_stmt = NULL; + do_parse_sql_stmt(sql_stmt, parse_ctx, @1.first_column, @1.last_column, 2, ';', END_P); + if (T_SET_PASSWORD == sql_stmt->type_ || + T_SET_NAMES == sql_stmt->type_ || + T_SET_CHARSET == sql_stmt->type_) { + malloc_non_terminal_node($$, parse_ctx->mem_pool_, T_SQL_STMT, 1, sql_stmt); + } else { + $$ = sql_stmt; + } if(T_VARIABLE_SET == $$->type_) { for(int64_t i = 0; i < $$->num_child_; ++i) { if(OB_UNLIKELY(NULL == $$->children_[i] || NULL == $$->children_[i]->children_[1])) { diff --git a/src/rootserver/balance/ob_all_balance_group_builder.cpp b/src/rootserver/balance/ob_all_balance_group_builder.cpp index 73851135d9..c2c1356c1a 100644 --- a/src/rootserver/balance/ob_all_balance_group_builder.cpp +++ b/src/rootserver/balance/ob_all_balance_group_builder.cpp @@ -69,7 +69,7 @@ int ObAllBalanceGroupBuilder::init(const int64_t tenant_id, } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id) || OB_ISNULL(mod)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(mod)); - } else if (OB_FAIL(tablet_to_ls_.init(MAP_BUCKET_NUM, lib::ObLabel("TabletToLS")))) { + } else if (OB_FAIL(tablet_to_ls_.init(MAP_BUCKET_NUM, lib::ObLabel("TabletToLS"), tenant_id))) { LOG_WARN("create map for tablet to LS fail", KR(ret), K(tenant_id)); } else if (OB_FAIL(tablet_data_size_.create(MAP_BUCKET_NUM, lib::ObLabel("TabletSizeMap")))) { LOG_WARN("create map for tablet data size fail", KR(ret), K(tenant_id)); diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index 02471810ba..1f0a7a16d5 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -1087,9 +1087,9 @@ int ObDDLRedefinitionTask::sync_auto_increment_position() } else if (FALSE_IT(param.global_value_to_sync_ = sequence_value - 1)) { // as sequence_value is an avaliable value. sync value will not be avaliable to user } else { - while (OB_SUCC(ret)) { + for (int64_t retry_cnt = 100; OB_SUCC(ret) && retry_cnt > 0; retry_cnt--) { if (OB_FAIL(auto_inc_service.sync_insert_value_global(param))) { - if (DDL_TABLE_RESTORE == task_type_ && OB_TENANT_NOT_IN_SERVER == ret) { + if (DDL_TABLE_RESTORE == task_type_ && share::ObIDDLTask::in_ddl_retry_white_list(ret)) { if (TC_REACH_TIME_INTERVAL(10L * 1000L * 1000L)) { LOG_INFO("set auto increment position failed, retry", K(ret), K(dst_tenant_id_), K(target_object_id_), K(cur_column_id), K(param)); } diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp index cd7922705b..63b3424513 100755 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp @@ -835,7 +835,7 @@ int ObTableRedefinitionTask::take_effect(const ObDDLTaskStatus next_task_status) } else { LOG_WARN("sync auto increment position failed", K(ret), K(object_id_), K(target_object_id_)); } - } else if (OB_FAIL(need_sync_stats && sync_stats_info())) { + } else if (need_sync_stats && OB_FAIL(sync_stats_info())) { LOG_WARN("fail to sync stats info", K(ret), K(object_id_), K(target_object_id_)); } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(dst_tenant_id_, target_object_id_, ddl_rpc_timeout))) { LOG_WARN("get ddl rpc timeout fail", K(ret)); diff --git a/src/rootserver/freeze/ob_major_freeze_helper.cpp b/src/rootserver/freeze/ob_major_freeze_helper.cpp index b8aaa8208e..d37cb698c8 100644 --- a/src/rootserver/freeze/ob_major_freeze_helper.cpp +++ b/src/rootserver/freeze/ob_major_freeze_helper.cpp @@ -112,8 +112,8 @@ int ObMajorFreezeHelper::get_freeze_info( } } // Skip major freeze for standby tenants and thus avoid OB_MAJOR_FREEZE_NOT_ALLOW incurred by - // standby tenants, only when launching major freeze on more than one tenant. - else if (tenant_info.is_standby() && (info_cnt > 1)) { + // standby tenants, only when launching major freeze on more than one tenant or all_user. + else if (tenant_info.is_standby() && ((info_cnt > 1) || param.freeze_all_user_)) { LOG_INFO("skip major freeze for standby tenant", K(tenant_info)); const char *warn_buf = "standby tenant sync freeze info from primary tenant, not allowed to launch major freeze"; int tmp_ret = OB_SUCCESS; diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index cbb62f419f..12161aeb33 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -17593,7 +17593,7 @@ int ObDDLService::inner_drop_and_create_tablet_(const int64_t &schema_version, LOG_WARN("fail to get frozen status for create tablet", KR(ret), K(tenant_id)); } else { ObTableCreator table_creator(tenant_id, frozen_scn, trans); - if (OB_FAIL(table_creator.init(false/*need_check_tablet_cnt*/))) { + if (OB_FAIL(table_creator.init(true/*need_check_tablet_cnt*/))) { LOG_WARN("table_creator init failed", KR(ret), K(tenant_id)); } else if (1 == create_table_count && create_table_schema_ptrs.at(0)->is_global_index_table()) { if (OB_FAIL(table_creator.add_create_tablets_of_table_arg(*create_table_schema_ptrs.at(0), orig_ls_id_array))) { @@ -22325,14 +22325,15 @@ int ObDDLService::standby_create_root_key( LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(arg)); } else { obrpc::RootKeyType key_type = obrpc::RootKeyType::INVALID; - RootKeyValue root_key; - if (OB_FAIL(get_root_key_from_primary(arg, tenant_id, key_type, root_key))) { + common::ObString root_key; + ObArenaAllocator allocator("root_key"); + + if (OB_FAIL(get_root_key_from_primary(arg, tenant_id, key_type, root_key, allocator))) { LOG_WARN("failed to get root key", KR(ret), K(arg), K(tenant_id)); } else { obrpc::ObRootKeyArg root_key_arg; obrpc::ObRootKeyResult dummy_result; - ObString key_value_str(root_key.ptr()); - if (OB_FAIL(root_key_arg.init(tenant_id, key_type, key_value_str))) { + if (OB_FAIL(root_key_arg.init(tenant_id, key_type, root_key))) { LOG_WARN("failed to init root key arg", KR(ret), K(tenant_id), K(key_type), K(root_key)); } else if (OB_FAIL(notify_root_key(*rpc_proxy_, root_key_arg, addrs, dummy_result))) { LOG_WARN("fail to notify root key", K(ret), K(root_key_arg)); @@ -22344,7 +22345,8 @@ int ObDDLService::standby_create_root_key( int ObDDLService::get_root_key_from_primary(const obrpc::ObCreateTenantArg &arg, const uint64_t tenant_id, obrpc::RootKeyType &key_type, - RootKeyValue &key_value) + common::ObString &key_value, + common::ObIAllocator &allocator) { int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat())) { @@ -22368,7 +22370,7 @@ int ObDDLService::get_root_key_from_primary(const obrpc::ObCreateTenantArg &arg, LOG_WARN("failed to init for get", KR(ret), K(primary_tenant_id)); } if (FAILEDx(get_root_key_from_obs(cluster_id, *rpc_proxy_, root_key_arg, - addr_list, key_type, key_value))) { + addr_list, key_type, key_value, allocator))) { LOG_WARN("failed to get root key from obs", KR(ret), K(cluster_id), K(root_key_arg), K(addr_list)); } @@ -22413,7 +22415,8 @@ int ObDDLService::get_root_key_from_obs( const obrpc::ObRootKeyArg &arg, const common::ObIArray &addrs, obrpc::RootKeyType &key_type, - RootKeyValue &key_value) + common::ObString &key_value, + common::ObIAllocator &allocator) { int ret = OB_SUCCESS; key_type = obrpc::RootKeyType::INVALID; @@ -22465,12 +22468,13 @@ int ObDDLService::get_root_key_from_obs( if (OB_UNLIKELY(obrpc::RootKeyType::INVALID != key_type)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("root key type is conflict", KR(ret), K(key_type), KPC(rpc_result)); - } else if (OB_FAIL(key_value.assign(rpc_result->root_key_))) { + } else if (OB_FAIL(deep_copy_ob_string(allocator, rpc_result->root_key_, key_value))) { LOG_WARN("failed to assign result", KR(ret), KPC(rpc_result)); - } else { + } + if (OB_SUCC(ret)) { key_type = rpc_result->key_type_; } - } else if (OB_UNLIKELY(0 != key_value.str().compare(rpc_result->root_key_))) { + } else if (OB_UNLIKELY(0 != key_value.compare(rpc_result->root_key_))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("root key is conflict", KR(ret), K(key_value), KPC(rpc_result)); } @@ -26288,7 +26292,7 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tenant_id", K(ret)); } else if (OB_FAIL(tenant_ids.push_back(tenant_id))) { - LOG_WARN("fail to push back tenant_id", K(ret), K(tenant_id)); + LOG_WARN("fail to push back tenant_id", KR(ret), K(tenant_id)); } while (!stopped_) { common::ObTimeoutCtx ctx; @@ -26304,7 +26308,7 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver } else { int tmp_ret = OB_SUCCESS; bool is_dropped = false; - if (OB_TMP_FAIL(schema_service_->check_if_tenant_has_been_dropped(tenant_id, is_dropped))) { + if (OB_TMP_FAIL(check_tenant_has_been_dropped_(tenant_id, is_dropped))) { LOG_WARN("fail to check if tenant has been dropped", KR(ret), K(tmp_ret), K(tenant_id)); } else if (is_dropped) { LOG_WARN("tenant has been dropped, just exit", KR(ret), K(tenant_id)); @@ -26324,7 +26328,7 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver int64_t schema_version = OB_INVALID_VERSION; if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version( tenant_id, schema_version))) { - LOG_WARN("fail to get tenant refreshed schema version", K(ret), K(tenant_id)); + LOG_WARN("fail to get tenant refreshed schema version", KR(ret), K(tenant_id)); } else { ObSchemaService *schema_service = schema_service_->get_schema_service(); ObRefreshSchemaInfo schema_info; @@ -26336,7 +26340,7 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver } else if (OB_FAIL(schema_service->inc_sequence_id())) { LOG_WARN("increase sequence_id failed", K(ret)); } else if (OB_FAIL(schema_service->set_refresh_schema_info(schema_info))) { - LOG_WARN("fail to set refresh schema info", K(ret), K(schema_info)); + LOG_WARN("fail to set refresh schema info", KR(ret), K(schema_info)); } else if (OB_NOT_NULL(publish_schema_version)) { *publish_schema_version = schema_version; } @@ -26344,7 +26348,7 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver } if (OB_FAIL(ret) && stopped_) { ret = OB_CANCELED; - LOG_WARN("rs is stopped"); + LOG_WARN("rs is stopped", KR(ret), K(tenant_id)); } THIS_WORKER.set_timeout_ts(original_timeout_us); } @@ -26352,6 +26356,46 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver return ret; } +int ObDDLService::check_tenant_has_been_dropped_( + const uint64_t tenant_id, + bool &is_dropped) +{ + int ret = OB_SUCCESS; + is_dropped = false; + ObSchemaGetterGuard guard; + const ObSimpleTenantSchema *tenant = NULL; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("ddl_service is not init", KR(ret)); + } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(guard.check_if_tenant_has_been_dropped(tenant_id, is_dropped))) { + LOG_WARN("fail to check if tenant has been dropped", KR(ret), K(tenant_id)); + } else if (is_dropped) { + LOG_WARN("tenant has been dropped, just exit", KR(ret), K(tenant_id)); + } else if (OB_FAIL(guard.get_tenant_info(tenant_id, tenant))) { + LOG_WARN("fail to get tenant info", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(tenant)) { + LOG_WARN("tenant not exist, maybe schema is fall behind", KR(ret), K(tenant_id)); + } else if (tenant->is_dropping()) { + ObLSStatusOperator ls_status; + ObLSStatusInfo sys_ls_info; + if (OB_FAIL(ls_status.get_ls_status_info(tenant_id, SYS_LS, sys_ls_info, *sql_proxy_))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_WARN("sys ls not exist, consider that tenant has been dropped", KR(ret), K(tenant_id)); + ret = OB_SUCCESS; + is_dropped = true; + } else { + LOG_WARN("fail to get ls status info", KR(ret), K(tenant_id)); + } + } else if (sys_ls_info.ls_is_tenant_dropping() + || sys_ls_info.ls_is_wait_offline()) { + is_dropped = true; + LOG_INFO("sys ls is not readable, consider that tenant has been dropped", KR(ret), K(tenant_id)); + } + } + return ret; +} + int ObDDLService::notify_refresh_schema(const ObAddrIArray &addrs) { int ret = OB_SUCCESS; @@ -32970,10 +33014,11 @@ int ObDDLSQLTransaction::start(ObISQLClient *proxy, int ObDDLSQLTransaction::start( ObISQLClient *proxy, const uint64_t tenant_id, - bool with_snapshot /*= false*/) + bool with_snapshot /*= false*/, + const int32_t group_id /* = 0*/) { int ret = OB_NOT_SUPPORTED; - UNUSEDx(proxy, with_snapshot, tenant_id); + UNUSEDx(proxy, with_snapshot, tenant_id, group_id); return ret; } diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index 050426ac21..5a7a5165ee 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -99,7 +99,6 @@ class ObDDLService { public: typedef std::pair LSTabletID; - typedef ObFixedLengthString RootKeyValue; public: friend class ObTableGroupHelp; friend class ObStandbyClusterSchemaProcessor; @@ -1924,14 +1923,16 @@ public: int check_need_create_root_key(const obrpc::ObCreateTenantArg &arg, bool &need_create); int get_root_key_from_primary(const obrpc::ObCreateTenantArg &arg, const uint64_t tenant_id, obrpc::RootKeyType &key_type, - RootKeyValue &key_value); + common::ObString &key_value, + common::ObIAllocator &allocator); static int get_root_key_from_obs( const uint64_t &cluster_id, obrpc::ObSrvRpcProxy &rpc_proxy, const obrpc::ObRootKeyArg &arg, const common::ObIArray &addrs, obrpc::RootKeyType &key_type, - RootKeyValue &key_value); + common::ObString &key_value, + common::ObIAllocator &allocator); int standby_create_root_key( const uint64_t tenant_id, const obrpc::ObCreateTenantArg &arg, @@ -2242,6 +2243,9 @@ private: virtual int publish_schema(const uint64_t tenant_id, const common::ObAddrIArray &addrs); + + int check_tenant_has_been_dropped_(const uint64_t tenant_id, bool &is_dropped); + int get_zone_region( const common::ObZone &zone, const common::ObIArray &zone_region_list, @@ -2613,7 +2617,8 @@ public: bool with_snapshot = false) override; virtual int start(ObISQLClient *proxy, const uint64_t tenant_id, - bool with_snapshot = false) override; + bool with_snapshot = false, + const int32_t group_id = 0) override; static int lock_all_ddl_operation( common::ObMySQLTransaction &trans, const uint64_t tenant_id); diff --git a/src/rootserver/ob_disaster_recovery_task.cpp b/src/rootserver/ob_disaster_recovery_task.cpp index 38d9f2c0e0..a84a0a9fb8 100644 --- a/src/rootserver/ob_disaster_recovery_task.cpp +++ b/src/rootserver/ob_disaster_recovery_task.cpp @@ -102,6 +102,7 @@ static const char* disaster_recovery_task_ret_comment_strs[] = { "[rs] task can not execute because server is not alive", "[rs] task can not execute because fail to check paxos replica number", "[rs] task can not execute because replica is not in service", + "[rs] task can not execute because server is permanent offline", ""/*default max*/ }; @@ -678,13 +679,13 @@ int ObMigrateLSReplicaTask::check_paxos_number( int ret = OB_SUCCESS; const ObLSReplica *leader = nullptr; if (OB_FAIL(ls_info.find_leader(leader))) { - LOG_WARN("fail to get leader", K(ret)); - } else if (OB_UNLIKELY(nullptr == leader)) { + LOG_WARN("fail to get leader", KR(ret), K(ls_info)); + } else if (OB_ISNULL(leader)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("leader replica is null", KR(ret)); } else if (leader->get_paxos_replica_number() <= 0) { ret = OB_REBALANCE_TASK_CANT_EXEC; - LOG_WARN("paxos replica number not report", K(ret), KPC(leader)); + LOG_WARN("paxos replica number not report", KR(ret), KPC(leader)); } else if (leader->get_paxos_replica_number() != paxos_replica_number_) { ret = OB_REBALANCE_TASK_CANT_EXEC; LOG_WARN("paxos replica number not match", KR(ret), diff --git a/src/rootserver/ob_disaster_recovery_task.h b/src/rootserver/ob_disaster_recovery_task.h index 202fb69a57..5472532eb6 100644 --- a/src/rootserver/ob_disaster_recovery_task.h +++ b/src/rootserver/ob_disaster_recovery_task.h @@ -68,6 +68,7 @@ enum ObDRTaskRetComment CANNOT_EXECUTE_DUE_TO_SERVER_NOT_ALIVE = 6, CANNOT_EXECUTE_DUE_TO_PAXOS_REPLICA_NUMBER = 7, CANNOT_EXECUTE_DUE_TO_REPLICA_NOT_INSERVICE = 8, + CANNOT_EXECUTE_DUE_TO_SERVER_PERMANENT_OFFLINE = 9, MAX }; diff --git a/src/rootserver/ob_disaster_recovery_task_mgr.cpp b/src/rootserver/ob_disaster_recovery_task_mgr.cpp index 787f7af40d..9f957dde6e 100644 --- a/src/rootserver/ob_disaster_recovery_task_mgr.cpp +++ b/src/rootserver/ob_disaster_recovery_task_mgr.cpp @@ -718,12 +718,30 @@ void ObDRTaskMgr::run3() if (OB_FAIL(try_pop_task(allocator, task))) { LOG_WARN("fail to try pop task", KR(ret)); } else if (OB_NOT_NULL(task)) { - tmp_ret = task->log_execute_start(); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("fail to log task start", KR(tmp_ret), KPC(task)); - } - if (OB_FAIL(execute_task(*task))) { - LOG_WARN("fail to send", KR(ret), KPC(task)); + const ObAddr &dst_server = task->get_dst_server(); + ObServerInfoInTable server_info; + if (OB_FAIL(SVR_TRACER.get_server_info(dst_server, server_info))) { + LOG_WARN("fail to get server_info", KR(ret), K(dst_server)); + } else if (server_info.is_permanent_offline()) { + // dest server permanent offline, do not execute this task, just clean it + LOG_INFO("[DRTASK_NOTICE] dest server is permanent offline, task can not execute", K(dst_server), K(server_info)); + ObThreadCondGuard guard(cond_); + if (OB_SUCCESS != (tmp_ret = async_add_cleaning_task_to_updater( + task->get_task_id(), + task->get_task_key(), + OB_REBALANCE_TASK_CANT_EXEC, + false/*need_record_event*/, + ObDRTaskRetComment::CANNOT_EXECUTE_DUE_TO_SERVER_PERMANENT_OFFLINE, + false/*reach_data_copy_concurrency*/))) { + LOG_WARN("fail to do execute over", KR(tmp_ret), KPC(task)); + } + } else { + if (OB_SUCCESS != (tmp_ret = task->log_execute_start())) { + LOG_WARN("fail to log task start", KR(tmp_ret), KPC(task)); + } + if (OB_FAIL(execute_task(*task))) { + LOG_WARN("fail to send", KR(ret), KPC(task)); + } } free_task_(allocator, task); } else { @@ -734,7 +752,7 @@ void ObDRTaskMgr::run3() LOG_WARN("fail to try dump statistic", KR(tmp_ret), K(last_dump_ts)); } if (OB_SUCCESS != (tmp_ret = try_clean_not_in_schedule_task_in_schedule_list_( - last_check_task_in_progress_ts))) { + last_check_task_in_progress_ts))) { LOG_WARN("fail to try check task in progress", KR(tmp_ret), K(last_check_task_in_progress_ts)); } } diff --git a/src/rootserver/ob_disaster_recovery_worker.cpp b/src/rootserver/ob_disaster_recovery_worker.cpp index 411280af1b..6d348dd11f 100755 --- a/src/rootserver/ob_disaster_recovery_worker.cpp +++ b/src/rootserver/ob_disaster_recovery_worker.cpp @@ -719,6 +719,96 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_encryption_log return ret; } +int ObDRWorker::LocalityAlignment::try_generate_type_transform_task_for_readonly_replica_( + ReplicaDescArray &zone_replica_desc_in_locality, + ReplicaStatDesc &replica_stat_desc, + const int64_t index, + bool &task_generated) +{ + int ret = OB_SUCCESS; + task_generated = false; + if (OB_UNLIKELY(0 > index)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(index)); + } else { + for (int64_t i = zone_replica_desc_in_locality.count() - 1; !task_generated && OB_SUCC(ret) && i >= 0; --i) { + ReplicaDesc &replica_desc = zone_replica_desc_in_locality.at(i); + if (REPLICA_TYPE_READONLY == replica_desc.replica_type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica type unexpected", KR(ret), K(dr_ls_info_)); + } else if (REPLICA_TYPE_FULL == replica_desc.replica_type_) { + if (OB_ISNULL(replica_stat_desc.unit_stat_info_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(replica_stat_desc)); + } else { + const share::ObUnitInfo &unit_info = replica_stat_desc.unit_stat_info_->get_unit_info(); + bool server_is_active = false; + if (!unit_info.unit_.is_active_status()) { + FLOG_INFO("unit status is not normal, can not generate type transform task", K(unit_info)); + } else if (OB_FAIL(SVR_TRACER.check_server_active(unit_info.unit_.server_, server_is_active))) { + LOG_WARN("fail to check server is active", KR(ret), K(unit_info)); + } else if (!server_is_active) { + FLOG_INFO("server status is not active, can not generate type transform task", K(unit_info)); + } else if (OB_FAIL(generate_type_transform_task( + replica_stat_desc, + replica_desc.replica_type_, + replica_desc.memstore_percent_))) { + LOG_WARN("fail to generate type transform task", KR(ret), K(replica_stat_desc)); + } else if (OB_FAIL(zone_replica_desc_in_locality.remove(i))) { + LOG_WARN("fail to remove", KR(ret), K(i), K(replica_stat_desc), K(zone_replica_desc_in_locality)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(index), K(replica_stat_desc), K(replica_stat_map_)); + } else { + task_generated = true; + } + } + } + } + } + return ret; +} + +int ObDRWorker::LocalityAlignment::try_generate_remove_readonly_task_for_duplicate_log_stream_( + ReplicaStatDesc &replica_stat_desc, + share::ObLSReplica &replica, + const int64_t index) +{ + int ret = OB_SUCCESS; + ObUnitTableOperator unit_operator; + common::ObArray unit_info_array; + if (OB_ISNULL(GCTX.sql_proxy_) || OB_UNLIKELY(0 > index)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(index)); + } else if (OB_FAIL(unit_operator.init(*GCTX.sql_proxy_))) { + LOG_WARN("unit operator init failed", KR(ret)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant(gen_user_tenant_id(replica.get_tenant_id()), unit_info_array))) { + LOG_WARN("fail to get unit info array", KR(ret), K(replica)); + } else { + bool replica_need_delete = true; + for (int64_t j = 0; OB_SUCC(ret) && j < unit_info_array.count(); ++j) { + if (unit_info_array.at(j).server_ == replica.get_server()) { + replica_need_delete = false; + break; + } + } + if (OB_FAIL(ret)) { + } else if (replica_need_delete) { + // this R-replica need to be deleted + if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { + LOG_WARN("fail to generate remove replica task", KR(ret), K(replica_stat_desc)); + } else if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(index), K(replica), K(replica_stat_map_)); + } + } else { + // delete this R-replica from memory to avoid removing this replca + if (OB_FAIL(replica_stat_map_.remove(index))) { + LOG_WARN("fail to remove", KR(ret), K(replica_stat_map_), K(index)); + } + } + } + return ret; +} + int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_readonly_replica( ReplicaStatDesc &replica_stat_desc, share::ObLSReplica &replica, @@ -733,61 +823,34 @@ int ObDRWorker::LocalityAlignment::do_generate_locality_task_from_readonly_repli ReplicaDescArray *zone_replica_desc = nullptr; int tmp_ret = locality_map_.get_refactored(zone, zone_replica_desc); if (OB_HASH_NOT_EXIST == tmp_ret) { + // zone has been shrinked, generate remove replica task if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { LOG_WARN("fail to generate remove replica task", KR(ret)); } else if (OB_FAIL(replica_stat_map_.remove(index))) { LOG_WARN("fail to remove", KR(ret)); } - } else if (OB_SUCCESS == tmp_ret && nullptr != zone_replica_desc) { - bool found = false; + } else if (OB_SUCCESS == tmp_ret && OB_NOT_NULL(zone_replica_desc)) { + bool task_generated = false; std::sort(zone_replica_desc->begin(), zone_replica_desc->end()); - for (int64_t i = zone_replica_desc->count() - 1; !found && OB_SUCC(ret) && i >= 0; --i) { - ReplicaDesc &replica_desc = zone_replica_desc->at(i); - if (REPLICA_TYPE_READONLY == replica_desc.replica_type_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("replica type unexpected", KR(ret), K(dr_ls_info_)); - } else if (REPLICA_TYPE_FULL == replica_desc.replica_type_) { - if (OB_ISNULL(replica_stat_desc.unit_stat_info_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(replica_stat_desc)); - } else { - const share::ObUnitInfo &unit_info = replica_stat_desc.unit_stat_info_->get_unit_info(); - bool server_is_active = false; - if (!unit_info.unit_.is_active_status()) { - FLOG_INFO("unit status is not normal, can not generate type transform task", K(unit_info)); - } else if (OB_FAIL(SVR_TRACER.check_server_active(unit_info.unit_.server_, server_is_active))) { - LOG_WARN("fail to check server is active", KR(ret), K(unit_info)); - } else if (!server_is_active) { - FLOG_INFO("server status is not active, can not generate type transform task", K(unit_info)); - } else if (OB_FAIL(generate_type_transform_task( - replica_stat_desc, - replica_desc.replica_type_, - replica_desc.memstore_percent_))) { - LOG_WARN("fail to generate type transform task", KR(ret), K(replica_stat_desc)); - } else if (OB_FAIL(zone_replica_desc->remove(i))) { - LOG_WARN("fail to remove", KR(ret), K(i), K(replica), K(zone_replica_desc)); - } else if (OB_FAIL(replica_stat_map_.remove(index))) { - LOG_WARN("fail to remove", KR(ret), K(index), K(replica), K(replica_stat_map_)); - } else { - found = true; - } - } - } - } - // process not found - if (OB_FAIL(ret)) { - // failed - } else if (found) { - // found, bypass + // try to generate type_transform task if needed + if (OB_FAIL(try_generate_type_transform_task_for_readonly_replica_( + *zone_replica_desc, replica_stat_desc, index, task_generated))) { + LOG_WARN("fail to try generate type transform task", KR(ret), + KPC(zone_replica_desc), K(replica_stat_desc), K(index), K(task_generated)); + } else if (task_generated) { + // a type transform task generated, bypass } else if (zone_replica_desc->is_readonly_all_server_) { - if (OB_FAIL(replica_stat_map_.remove(index))) { - LOG_WARN("fail to remove", KR(ret), K(index), K(replica), K(replica_stat_map_)); + // for duplicate log stream, try to remove redudant R-replicas + if (OB_FAIL(try_generate_remove_readonly_task_for_duplicate_log_stream_(replica_stat_desc, replica, index))) { + LOG_WARN("fail to generate remove replica task for duplicate log stream", + KR(ret), K(replica_stat_desc), K(replica), K(index)); } } else { + // for common log stream, just generate remove replica task if (OB_FAIL(generate_remove_replica_task(replica_stat_desc))) { - LOG_WARN("fail to generate remove replica task", KR(ret)); + LOG_WARN("fail to generate remove replica task", KR(ret), K(replica_stat_desc)); } else if (OB_FAIL(replica_stat_map_.remove(index))) { - LOG_WARN("fail to remove", KR(ret)); + LOG_WARN("fail to remove", KR(ret), K(replica_stat_map_), K(index)); } } } else { @@ -4656,7 +4719,10 @@ int ObDRWorker::generate_disaster_recovery_paxos_replica_number( if (locality_paxos_replica_number >= member_list_cnt_after) { new_paxos_replica_number = curr_paxos_replica_number; found = true; - } else {} // new member cnt greater than paxos_replica_number, not good + } else if (locality_paxos_replica_number + 1 == member_list_cnt_after) { + new_paxos_replica_number = curr_paxos_replica_number + 1; + found = true; + } } else if (curr_paxos_replica_number > locality_paxos_replica_number) { if (curr_paxos_replica_number >= member_list_cnt_after) { new_paxos_replica_number = curr_paxos_replica_number; diff --git a/src/rootserver/ob_disaster_recovery_worker.h b/src/rootserver/ob_disaster_recovery_worker.h index 22c8047832..e3a90b33fe 100755 --- a/src/rootserver/ob_disaster_recovery_worker.h +++ b/src/rootserver/ob_disaster_recovery_worker.h @@ -579,10 +579,21 @@ private: ReplicaStatDesc &replica_stat_desc, share::ObLSReplica &replica, const int64_t index); + + int try_generate_type_transform_task_for_readonly_replica_( + ReplicaDescArray &zone_replica_desc_in_locality, + ReplicaStatDesc &replica_stat_desc, + const int64_t index, + bool &task_generated); + int try_generate_remove_readonly_task_for_duplicate_log_stream_( + ReplicaStatDesc &replica_stat_desc, + share::ObLSReplica &replica, + const int64_t index); int do_generate_locality_task_from_readonly_replica( ReplicaStatDesc &replica_stat_desc, share::ObLSReplica &replica, const int64_t index); + int try_generate_locality_task_from_locality_map(); int try_generate_locality_task_from_paxos_replica_number(); void print_locality_information(); diff --git a/src/rootserver/ob_primary_ls_service.cpp b/src/rootserver/ob_primary_ls_service.cpp index c9f6964a5c..d0ad2a03a1 100755 --- a/src/rootserver/ob_primary_ls_service.cpp +++ b/src/rootserver/ob_primary_ls_service.cpp @@ -422,11 +422,13 @@ int ObPrimaryLSService::check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo const int64_t timeout = GCONF.rpc_timeout; obrpc::ObCheckLSCanOfflineArg arg; can_offline = false; + const uint64_t group_id = info.ls_is_tenant_dropping() ? OBCG_DBA_COMMAND : OBCG_DEFAULT; if (OB_FAIL(arg.init(info.tenant_id_, info.ls_id_, info.status_))) { LOG_WARN("failed to init arg", KR(ret), K(arg)); } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(leader) .by(info.tenant_id_) .timeout(timeout) + .group_id(group_id) .check_ls_can_offline(arg))) { can_offline = false; LOG_WARN("failed to check ls can offline", KR(ret), K(arg), K(info), diff --git a/src/rootserver/ob_tenant_balance_service.cpp b/src/rootserver/ob_tenant_balance_service.cpp index 4759f44954..a4dc2fc551 100755 --- a/src/rootserver/ob_tenant_balance_service.cpp +++ b/src/rootserver/ob_tenant_balance_service.cpp @@ -303,9 +303,10 @@ int ObTenantBalanceService::is_ls_balance_finished(const uint64_t &tenant_id, bo } } else { // standby & restore - if (OB_FAIL(is_standby_tenant_ls_balance_finished_(tenant_id, is_finished))) { - LOG_WARN("fail to execute is_standby_tenant_ls_balance_finished_", KR(ret), K(tenant_id)); - } + is_finished = true; + // if (OB_FAIL(is_standby_tenant_ls_balance_finished_(tenant_id, is_finished))) { + // LOG_WARN("fail to execute is_standby_tenant_ls_balance_finished_", KR(ret), K(tenant_id)); + // } } LOG_TRACE("check whether the tenant has balanced ls", K(ret), K(tenant_id), K(is_primary), K(is_finished)); return ret; diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/ob_tenant_role_transition_service.cpp index a6deecd678..f5c0fb131f 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/ob_tenant_role_transition_service.cpp @@ -745,6 +745,7 @@ int ObTenantRoleTransitionService::get_ls_access_mode_( int64_t rpc_count = 0; ObArray return_code_array; int tmp_ret = OB_SUCCESS; + const uint64_t group_id = share::OBCG_DBA_COMMAND; for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { return_code_array.reset(); const ObLSStatusInfo &info = status_info_array.at(i); @@ -755,9 +756,10 @@ int ObTenantRoleTransitionService::get_ls_access_mode_( } else if (OB_FAIL(arg.init(tenant_id_, info.ls_id_))) { LOG_WARN("failed to init arg", KR(ret), K(tenant_id_), K(info)); // use meta rpc process thread - } else if (OB_FAIL(proxy.call(leader, timeout, GCONF.cluster_id, gen_meta_tenant_id(tenant_id_), arg))) { + } else if (OB_FAIL(proxy.call(leader, timeout, GCONF.cluster_id, tenant_id_, group_id, arg))) { //can not ignore error of each ls - LOG_WARN("failed to send rpc", KR(ret), K(leader), K(timeout), K(tenant_id_), K(arg)); + LOG_WARN("failed to send rpc", KR(ret), K(leader), K(timeout), + K(tenant_id_), K(arg), K(group_id)); } else { rpc_count++; } @@ -841,6 +843,7 @@ int ObTenantRoleTransitionService::do_change_ls_access_mode_( ObChangeLSAccessModeProxy proxy(*rpc_proxy_, &obrpc::ObSrvRpcProxy::change_ls_access_mode); ObAddr leader; obrpc::ObLSAccessModeInfo arg; + const uint64_t group_id = share::OBCG_DBA_COMMAND; for (int64_t i = 0; OB_SUCC(ret) && i < ls_access_info.count(); ++i) { const obrpc::ObLSAccessModeInfo &info = ls_access_info.at(i); const int64_t timeout = ctx.get_timeout(); @@ -851,9 +854,10 @@ int ObTenantRoleTransitionService::do_change_ls_access_mode_( } else if (OB_FAIL(arg.init(tenant_id_, info.get_ls_id(), info.get_mode_version(), target_access_mode, ref_scn, sys_ls_sync_scn))) { LOG_WARN("failed to init arg", KR(ret), K(info), K(target_access_mode), K(ref_scn), K(sys_ls_sync_scn)); - } else if (OB_FAIL(proxy.call(leader, timeout, GCONF.cluster_id, gen_meta_tenant_id(tenant_id_), arg))) { + } else if (OB_FAIL(proxy.call(leader, timeout, GCONF.cluster_id, tenant_id_, group_id, arg))) { //can not ignore of each ls - LOG_WARN("failed to send rpc", KR(ret), K(arg), K(timeout), K(tenant_id_), K(info)); + LOG_WARN("failed to send rpc", KR(ret), K(arg), K(timeout), + K(tenant_id_), K(info), K(group_id)); } }//end for //result @@ -1244,6 +1248,7 @@ int ObTenantRoleTransitionService::get_checkpoints_by_rpc(const uint64_t tenant_ *GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::get_ls_sync_scn); obrpc::ObGetLSSyncScnArg arg; int64_t rpc_count = 0; + const uint64_t group_id = share::OBCG_DBA_COMMAND; for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { const ObLSStatusInfo &info = status_info_array.at(i); const int64_t timeout_us = !THIS_WORKER.is_timeout_ts_valid() ? @@ -1254,8 +1259,9 @@ int ObTenantRoleTransitionService::get_checkpoints_by_rpc(const uint64_t tenant_ } else if (OB_FAIL(arg.init(tenant_id, info.ls_id_, check_sync_to_latest))) { LOG_WARN("failed to init arg", KR(ret), K(tenant_id), K(info)); // use meta rpc process thread - } else if (OB_FAIL(proxy.call(leader, timeout_us, gen_meta_tenant_id(tenant_id), arg))) { - LOG_WARN("failed to send rpc", KR(ret), K(leader), K(timeout_us), K(tenant_id), K(arg)); + } else if (OB_FAIL(proxy.call(leader, timeout_us, GCONF.cluster_id, tenant_id, group_id, arg))) { + LOG_WARN("failed to send rpc", KR(ret), K(leader), K(timeout_us), + K(tenant_id), K(arg), K(group_id)); } else { rpc_count++; } @@ -1418,6 +1424,7 @@ void ObTenantRoleTransitionService::broadcast_tenant_info(const char* const log_ } else if (OB_FAIL(unit_operator.get_units_by_tenant(tenant_id_, units))) { LOG_WARN("failed to get tenant unit", KR(ret), K_(tenant_id)); } else { + //no need user special group OBCG_DBA_COMMAND ObRefreshTenantInfoProxy proxy( *GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::refresh_tenant_info); int64_t rpc_count = 0; @@ -1433,7 +1440,7 @@ void ObTenantRoleTransitionService::broadcast_tenant_info(const char* const log_ } else if (OB_FAIL(arg.init(tenant_id_))) { LOG_WARN("failed to init arg", KR(ret), K_(tenant_id)); // use meta rpc process thread - } else if (OB_FAIL(proxy.call(unit.server_, timeout_us, gen_meta_tenant_id(tenant_id_), arg))) { + } else if (OB_FAIL(proxy.call(unit.server_, timeout_us, GCONF.cluster_id, gen_meta_tenant_id(tenant_id_), arg))) { LOG_WARN("failed to send rpc", KR(ret), K(unit), K(timeout_us), K_(tenant_id), K(arg)); } else { rpc_count++; diff --git a/src/rootserver/ob_tenant_transfer_service.cpp b/src/rootserver/ob_tenant_transfer_service.cpp index 53f77d1e68..6b7534b03e 100644 --- a/src/rootserver/ob_tenant_transfer_service.cpp +++ b/src/rootserver/ob_tenant_transfer_service.cpp @@ -177,10 +177,10 @@ int ObTenantTransferService::process_task_(const ObTransferTask::TaskStatus &tas int ObTenantTransferService::process_init_task_(const ObTransferTaskID task_id) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; ObTransferTask task; ObMySQLTransaction trans; - bool member_list_is_same = false; - bool update_comment_to_wait_for_member_list = false; + ObTransferTaskComment result_comment = EMPTY_COMMENT; ObArray tablet_ids; ObTableLockOwnerID lock_owner_id; ObTransferPartList not_exist_part_list; @@ -205,7 +205,8 @@ int ObTenantTransferService::process_init_task_(const ObTransferTaskID task_id) tenant_id_, task_id, true/*for_update*/, - task))) { + task, + 0/*group_id*/))) { LOG_WARN("fail to get task", KR(ret), K_(tenant_id), K(task_id), K(task)); } else if (OB_UNLIKELY(!task.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -218,15 +219,12 @@ int ObTenantTransferService::process_init_task_(const ObTransferTaskID task_id) *sql_proxy_, task.get_src_ls(), task.get_dest_ls(), - member_list_is_same))) { // can't use trans - LOG_WARN("fail to check ls member_list", KR(ret), K(task), K(member_list_is_same)); - } else if (!member_list_is_same) { + result_comment))) { // can't use trans + LOG_WARN("fail to check ls member_list", KR(ret), K(task)); + } else if (EMPTY_COMMENT != result_comment) { ret = OB_NEED_RETRY; - TTS_INFO("member_lists of src_ls and dest_ls are not same, need retry", - KR(ret), K_(tenant_id), K(member_list_is_same), K(task)); - if (task.get_comment() != ObTransferTaskComment::WAIT_FOR_MEMBER_LIST) { - update_comment_to_wait_for_member_list = true; - } + TTS_INFO("member_lists of src_ls and dest_ls are not same or there has inacitve server in member_list, need retry", + KR(ret), K_(tenant_id), K(task), "result_comment", transfer_task_comment_to_str(result_comment)); } else if (OB_FAIL(lock_table_and_part_( trans, task.get_src_ls(), @@ -288,22 +286,10 @@ int ObTenantTransferService::process_init_task_(const ObTransferTaskID task_id) } } - if ((OB_NEED_RETRY == ret && update_comment_to_wait_for_member_list) - || OB_TRANS_TIMEOUT == ret || OB_TIMEOUT == ret) { - ObTimeoutCtx ctx_comment; - int tmp_ret = OB_SUCCESS; - ObTransferTaskComment comment = (OB_NEED_RETRY == ret) - ? ObTransferTaskComment::WAIT_FOR_MEMBER_LIST - : ObTransferTaskComment::TRANSACTION_TIMEOUT; - if (OB_TMP_FAIL(ctx_comment.set_timeout(2000000/*2s*/))) { // overwrite timeout - LOG_WARN("set default timeout ctx failed", KR(tmp_ret), K(ctx_comment), K_(tenant_id), K(task_id)); - } else if (OB_TMP_FAIL(ObTransferTaskOperator::update_comment( - *sql_proxy_, - tenant_id_, - task_id, - comment))) { - LOG_WARN("update comment failed", KR(tmp_ret), K_(tenant_id), K(task_id), K(comment)); - } + // update comments for expected error codes + if (OB_TMP_FAIL(update_comment_for_expected_errors_(ret, task_id, result_comment))) { + LOG_WARN("update comment for expected errors failed", KR(tmp_ret), KR(ret), + K_(tenant_id), K(task_id), "result_comment", transfer_task_comment_to_str(result_comment)); } if (OB_SUCC(ret)) { @@ -321,24 +307,87 @@ int ObTenantTransferService::process_init_task_(const ObTransferTaskID task_id) ERRSIM_POINT_DEF(EN_TENANT_TRANSFER_CHECK_LS_MEMBER_LIST_NOT_SAME); +// 1.check leader member_lists of src_ls and dest_ls are same +// 2.if member_lists are same, check that all servers in member_list are acitve int ObTenantTransferService::check_ls_member_list_( common::ObISQLClient &sql_proxy, const ObLSID &src_ls, const ObLSID &dest_ls, - bool &is_same) + ObTransferTaskComment &result_comment) { int ret = OB_SUCCESS; + result_comment = EMPTY_COMMENT; + bool all_members_are_active = false; + ObLSReplica::MemberList src_ls_member_list; + ObLSReplica::MemberList dest_ls_member_list; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(get_member_lists_by_inner_sql_( + sql_proxy, + src_ls, + dest_ls, + src_ls_member_list, + dest_ls_member_list))) { + LOG_WARN("get member list by inner sql failed", KR(ret), K(src_ls), K(dest_ls)); + } else if (!ObLSReplica::servers_in_member_list_are_same( + src_ls_member_list, + dest_ls_member_list)) { + // result 1: member_lists are not same + result_comment = WAIT_FOR_MEMBER_LIST; + LOG_WARN("member_list of src_ls and dest_ls are not same", KR(ret), K_(tenant_id), K(src_ls), + K(dest_ls), K(src_ls_member_list), K(dest_ls_member_list), K(result_comment)); + } else if (OB_FAIL(ObLSReplica::check_all_servers_in_member_list_are_active( + src_ls_member_list, + all_members_are_active))) { + LOG_WARN("check all servers in member list are active failed", + KR(ret), K(src_ls_member_list), K(all_members_are_active)); + } else if (!all_members_are_active) { + // result 2: member_lists are same, but server in member_list is inactive + result_comment = INACTIVE_SERVER_IN_MEMBER_LIST; + LOG_WARN("member_list has inactive server", KR(ret), K(src_ls), + K(src_ls_member_list), K(all_members_are_active), K(result_comment)); + } else { + // result 3: member_lists are same && all members are active + result_comment = EMPTY_COMMENT; + TTS_INFO("member_lists of src_ls and dest_ls are same and all members are acitve", + KR(ret), K_(tenant_id), K(src_ls), K(dest_ls), K(all_members_are_active), + K(src_ls_member_list), K(dest_ls_member_list), K(result_comment)); + } + // just for debug + if (OB_FAIL(ret)) { + } else if (OB_IN_STOP_STATE == EN_TENANT_TRANSFER_CHECK_LS_MEMBER_LIST_NOT_SAME) { + result_comment = INACTIVE_SERVER_IN_MEMBER_LIST; + TTS_INFO("errsim tenant transfer check ls member list with inactive server", K(result_comment)); + } else if (OB_STATE_NOT_MATCH == EN_TENANT_TRANSFER_CHECK_LS_MEMBER_LIST_NOT_SAME) { + result_comment = WAIT_FOR_MEMBER_LIST; + TTS_INFO("errsim tenant transfer check ls member list not same", K(result_comment)); + } + return ret; +} + +// get ls leader member list of src_ls and dest_ls +int ObTenantTransferService::get_member_lists_by_inner_sql_( + common::ObISQLClient &sql_proxy, + const ObLSID &src_ls, + const ObLSID &dest_ls, + ObLSReplica::MemberList &src_ls_member_list, + ObLSReplica::MemberList &dest_ls_member_list) +{ + int ret = OB_SUCCESS; + src_ls_member_list.reset(); + dest_ls_member_list.reset(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!src_ls.is_valid() || !dest_ls.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(src_ls), K(dest_ls)); } else { - is_same = false; SMART_VAR(ObISQLClient::ReadResult, result) { ObSqlString sql; ObString src_ls_member_list_str; ObString dest_ls_member_list_str; - ObLSReplica::MemberList src_ls_member_list; - ObLSReplica::MemberList dest_ls_member_list; common::sqlclient::ObMySQLResult *res = NULL; if (OB_FAIL(sql.assign_fmt( "SELECT PAXOS_MEMBER_LIST FROM %s WHERE TENANT_ID = %lu AND ROLE = 'LEADER'" @@ -371,12 +420,8 @@ int ObTenantTransferService::check_ls_member_list_( to_cstring(dest_ls_member_list_str), dest_ls_member_list))) { LOG_WARN("text2member_list failed", KR(ret), K_(tenant_id), K(dest_ls), K(dest_ls_member_list_str)); - } else if (ObLSReplica::servers_in_member_list_are_same(src_ls_member_list, dest_ls_member_list)) { - is_same = true; - } else { - is_same = false; } - + // double check sql result if (OB_FAIL(ret)) { if (OB_UNLIKELY(OB_ITER_END == ret)) { // read less than two rows ret = OB_LEADER_NOT_EXIST; @@ -395,22 +440,9 @@ int ObTenantTransferService::check_ls_member_list_( K(sql), K(src_ls_member_list_str), K(dest_ls_member_list_str)); } else { ret = OB_SUCCESS; - if (is_same) { - LOG_INFO("member_list of src_ls and dest_ls are same", KR(ret), K_(tenant_id), K(src_ls), - K(dest_ls), K(is_same), K(src_ls_member_list), K(dest_ls_member_list)); - } else { - LOG_WARN("member_list of src_ls and dest_ls are not same", KR(ret), K_(tenant_id), K(src_ls), - K(dest_ls), K(is_same), K(src_ls_member_list), K(dest_ls_member_list)); - } } } // end SMART_VAR } - if (OB_SUCC(ret)) { - if (EN_TENANT_TRANSFER_CHECK_LS_MEMBER_LIST_NOT_SAME) { - is_same = false; - TTS_INFO("errsim tenant transfer check ls member list not same", K(is_same)); - } - } return ret; } @@ -1104,7 +1136,8 @@ int ObTenantTransferService::try_cancel_transfer_task(const ObTransferTaskID tas tenant_id_, task_id, true/*for_update*/, - task))) { + task, + 0/*group_id*/))) { if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("fail to get task", KR(ret), K_(tenant_id), K(task_id), K(task)); } else { @@ -1564,6 +1597,50 @@ int ObTenantTransferService::set_transaction_timeout_(common::ObTimeoutCtx &ctx) return ret; } +// err --> comment +// OB_TRANS_TIMEOUT TRANSACTION_TIMEOUT +// OB_TIMEOUT TRANSACTION_TIMEOUT +// OB_NEED_RETRY WAIT_FOR_MEMBER_LIST/INACTIVE_SERVER_IN_MEMBER_LIST +int ObTenantTransferService::update_comment_for_expected_errors_( + const int err, + const ObTransferTaskID &task_id, + const ObTransferTaskComment &result_comment) +{ + int ret = OB_SUCCESS; + ObTransferTaskComment actual_comment = EMPTY_COMMENT; + ObTimeoutCtx ctx; + if (IS_NOT_INIT || OB_ISNULL(sql_proxy_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_SUCCESS == err) { + // skip + } else if (OB_UNLIKELY(!task_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid task_id", KR(ret), K(task_id)); + } else if (OB_TRANS_TIMEOUT == err || OB_TIMEOUT == err) { + actual_comment = TRANSACTION_TIMEOUT; + } else if (OB_NEED_RETRY == err) { + if (WAIT_FOR_MEMBER_LIST != result_comment && INACTIVE_SERVER_IN_MEMBER_LIST != result_comment) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected comment with err", KR(ret), K(err), K(result_comment)); + } else { + actual_comment = result_comment; + } + } + if (OB_FAIL(ret) || EMPTY_COMMENT == actual_comment) { + // do nothing + } else if (OB_FAIL(ctx.set_timeout(GCONF.internal_sql_execute_timeout))) { // overwrite timeout + LOG_WARN("set default timeout ctx failed", KR(ret), K(ctx), K_(tenant_id), K(task_id)); + } else if (OB_FAIL(ObTransferTaskOperator::update_comment( + *sql_proxy_, + tenant_id_, + task_id, + actual_comment))) { + LOG_WARN("update comment failed", KR(ret), K_(tenant_id), K(task_id), K(actual_comment)); + } + return ret; +} + #undef TTS_INFO } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/ob_tenant_transfer_service.h b/src/rootserver/ob_tenant_transfer_service.h index 062deec6ac..087bbac064 100644 --- a/src/rootserver/ob_tenant_transfer_service.h +++ b/src/rootserver/ob_tenant_transfer_service.h @@ -16,6 +16,7 @@ #include "rootserver/ob_tenant_thread_helper.h" // ObTenantThreadHelper #include "share/transfer/ob_transfer_info.h" // ObTransferTask #include "share/ob_balance_define.h" // share::ObBalanceTaskID, share::ObTransferTaskID +#include "share/ls/ob_ls_info.h" // MemberList namespace oceanbase { @@ -122,7 +123,13 @@ private: common::ObISQLClient &sql_proxy, const share::ObLSID &src_ls, const share::ObLSID &dest_ls, - bool &is_same); + share::ObTransferTaskComment &result_comment); + int get_member_lists_by_inner_sql_( + common::ObISQLClient &sql_proxy, + const share::ObLSID &src_ls, + const share::ObLSID &dest_ls, + share::ObLSReplica::MemberList &src_ls_member_list, + share::ObLSReplica::MemberList &dest_ls_member_list); int lock_table_and_part_( ObMySQLTransaction &trans, const share::ObLSID &src_ls, @@ -130,7 +137,7 @@ private: share::ObTransferPartList ¬_exist_part_list, share::ObTransferPartList &lock_failed_part_list, share::ObDisplayTabletList &table_lock_tablet_list, - common::ObIArray &tablet_ids, + common::ObIArray &tablet_ids, transaction::tablelock::ObTableLockOwnerID &lock_owner_id); int unlock_table_and_part_( ObMySQLTransaction &trans, @@ -144,19 +151,19 @@ private: int get_tablet_and_partition_idx_by_object_id_( share::schema::ObSimpleTableSchemaV2 &table_schema, const ObObjectID &part_object_id, - ObTabletID &tablet_id, + common::ObTabletID &tablet_id, int64_t &part_idx); int get_tablet_and_partition_idx_by_object_id_( share::schema::ObSimpleTableSchemaV2 &table_schema, const ObObjectID &part_object_id, - ObTabletID &tablet_id, + common::ObTabletID &tablet_id, int64_t &part_idx, int64_t &subpart_idx); int get_tablet_by_partition_idx_( share::schema::ObSimpleTableSchemaV2 &table_schema, const int64_t part_idx, const int64_t subpart_idx, - ObTabletID &tablet_id); + common::ObTabletID &tablet_id); int check_tenant_schema_is_ready_(bool &is_ready); int unlock_and_clear_task_( const share::ObTransferTaskID task_id, @@ -170,7 +177,7 @@ private: const share::ObTransferPartInfo &part_info, common::ObIAllocator &allocator, share::schema::ObSimpleTableSchemaV2 *&table_schema, - ObTabletID &tablet_id, + common::ObTabletID &tablet_id, int64_t &part_idx, int64_t &subpart_idx); int add_table_lock_( @@ -184,14 +191,14 @@ private: const transaction::tablelock::ObTableLockOwnerID &lock_owner_id, share::schema::ObSimpleTableSchemaV2 &table_schema, const share::ObTransferPartInfo &part_info, - const ObTabletID &tablet_id); + const common::ObTabletID &tablet_id); int generate_related_tablet_ids_( share::schema::ObSimpleTableSchemaV2 &table_schema, const int64_t part_idx, const int64_t subpart_idx, - common::ObIArray &tablet_ids); + common::ObIArray &tablet_ids); int generate_tablet_list_( - const ObIArray &tablet_ids, + const common::ObIArray &tablet_ids, share::ObTransferTabletList &tablet_list); int unlock_table_lock_( ObMySQLTransaction &trans, @@ -213,9 +220,13 @@ private: share::schema::ObSimpleTableSchemaV2 *&table_schema); int record_need_move_table_lock_tablet_( share::schema::ObSimpleTableSchemaV2 &table_schema, - const ObTabletID &tablet_id, + const common::ObTabletID &tablet_id, share::ObDisplayTabletList &table_lock_tablet_list); int set_transaction_timeout_(common::ObTimeoutCtx &ctx); + int update_comment_for_expected_errors_( + const int err, + const share::ObTransferTaskID &task_id, + const share::ObTransferTaskComment &result_comment); private: static const int64_t IDLE_TIME_US = 10 * 1000 * 1000L; // 10s static const int64_t BUSY_IDLE_TIME_US = 100 * 1000L; // 100ms diff --git a/src/rootserver/ob_unit_manager.cpp b/src/rootserver/ob_unit_manager.cpp index 3757c59e82..36c113bd8b 100644 --- a/src/rootserver/ob_unit_manager.cpp +++ b/src/rootserver/ob_unit_manager.cpp @@ -1936,7 +1936,7 @@ int ObUnitManager::get_to_be_deleted_unit_group( } } else { ret = OB_OP_NOT_ALLOW; - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "delete unit group which is not belong to this tenant"); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "delete unit group which does not belong to this tenant"); } } } else { diff --git a/src/rootserver/restore/ob_import_table_job_scheduler.cpp b/src/rootserver/restore/ob_import_table_job_scheduler.cpp index f59c168d82..a6fb90e4df 100644 --- a/src/rootserver/restore/ob_import_table_job_scheduler.cpp +++ b/src/rootserver/restore/ob_import_table_job_scheduler.cpp @@ -350,7 +350,12 @@ int ObImportTableJobScheduler::canceling_(share::ObImportTableJob &job) } else if (!is_exist) { } else { LOG_INFO("[IMPORT_TABLE]cancel import table task", K(arg)); - if (OB_FAIL(ObDDLServerClient::abort_redef_table(arg))) { + share::ObTaskId trace_id(*ObCurTraceId::get_trace_id()); + ObImportResult result; + if (OB_FAIL(result.set_result(OB_CANCELED, trace_id, GCONF.self_addr_))) { + LOG_WARN("failed to set result", K(ret)); + } else if (OB_FALSE_IT(task.set_result(result))) { + } else if (OB_FAIL(ObDDLServerClient::abort_redef_table(arg))) { LOG_WARN("failed to abort redef table", K(ret), K(arg)); } else if (OB_FAIL(task_helper_.advance_status(*sql_proxy_, task, next_status))) { LOG_WARN("failed to cancel import task", K(ret), K(task)); @@ -536,8 +541,17 @@ int ObImportTableTaskScheduler::try_advance_status_(const int err_code) int ret = OB_SUCCESS; if (OB_FAIL(err_code) && ObImportTableUtil::can_retrieable_err(err_code)) { // do nothing } else { + share::ObImportTableTaskStatus next_status = import_task_->get_status().get_next_status(err_code); - if (OB_FAIL(helper_.advance_status(*sql_proxy_, *import_task_, next_status))) { + if (import_task_->get_result().is_succeed()) { // avoid to cover comment + share::ObTaskId trace_id(*ObCurTraceId::get_trace_id()); + ObImportResult result; + if (OB_FAIL(result.set_result(err_code, trace_id, GCONF.self_addr_))) { + LOG_WARN("failed to set result", K(ret)); + } else if (OB_FALSE_IT(import_task_->set_result(result))) { + } + } + if (FAILEDx(helper_.advance_status(*sql_proxy_, *import_task_, next_status))) { LOG_WARN("failed to advance status", K(ret), KPC_(import_task), K(next_status)); } } @@ -667,6 +681,7 @@ int ObImportTableTaskScheduler::construct_import_table_schema_( target_table_schema.set_data_table_id(0); target_table_schema.clear_constraint(); target_table_schema.clear_foreign_key_infos(); + target_table_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_NORMAL); uint64_t database_id = OB_INVALID_ID; if (OB_FAIL(target_tenant_guard.get_database_id(import_task_->get_tenant_id(), diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index 41c4e3f068..a519fe3b85 100755 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -261,6 +261,7 @@ ob_set_subtarget(ob_share common_mixed detect/ob_detect_manager_utils.cpp table/ob_table_load_sql_statistics.cpp table/ob_ttl_util.cpp + table/ob_table_config_util.cpp ) ob_set_subtarget(ob_share tablet diff --git a/src/share/config/ob_config.cpp b/src/share/config/ob_config.cpp index 558ba34870..7e05382abe 100644 --- a/src/share/config/ob_config.cpp +++ b/src/share/config/ob_config.cpp @@ -1110,6 +1110,43 @@ int ObConfigLogArchiveOptionsItem::format_option_str(const char *src, int64_t sr } return ret; } +ObConfigModeItem::ObConfigModeItem(ObConfigContainer *container, + Scope::ScopeInfo scope_info, + const char *name, + const char *def, + ObConfigParser* parser, + const char *info, + const ObParameterAttr attr) +:parser_(parser) +{ + MEMSET(value_, 0, MAX_MODE_BYTES); + if (OB_LIKELY(NULL != container)) { + container->set_refactored(ObConfigStringKey(name), this, 1); + } + init(scope_info, name, def, info, attr); +} + +ObConfigModeItem::~ObConfigModeItem() +{ + if (parser_ != NULL) { + delete parser_; + } +} + +bool ObConfigModeItem::set(const char *str) +{ + bool valid = false; + if (str == NULL || parser_ == NULL) { + valid = false; + OB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "str or parser_ is NULL", K(str), K(parser_)); + } else { + valid = parser_->parse(str, value_, MAX_MODE_BYTES); + if (!valid) { + OB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "parse config item fail", K(str), K(parser_)); + } + } + return valid; +} ObConfigVersionItem::ObConfigVersionItem(ObConfigContainer *container, Scope::ScopeInfo scope_info, diff --git a/src/share/config/ob_config.h b/src/share/config/ob_config.h index cb09d7190c..cea1dd29db 100644 --- a/src/share/config/ob_config.h +++ b/src/share/config/ob_config.h @@ -47,6 +47,7 @@ enum ObConfigItemType{ OB_CONF_ITEM_TYPE_CAPACITY = 9, OB_CONF_ITEM_TYPE_LOGARCHIVEOPT = 10, OB_CONF_ITEM_TYPE_VERSION = 11, + OB_CONF_ITEM_TYPE_MODE = 12, }; enum class ObConfigRangeOpts { @@ -1224,6 +1225,57 @@ private: common::ObSArray config_array_; }; +class ObConfigModeItem: public ObConfigItem +{ +public: + ObConfigModeItem(ObConfigContainer *container, + Scope::ScopeInfo scope_info, + const char *name, + const char *def, + ObConfigParser* parser, + const char *info, + const ObParameterAttr attr = ObParameterAttr()); + virtual ~ObConfigModeItem(); + // get_value() return the real-time value + const uint8_t* get_value() const { return value_; } + // get() return the real-time value if it does not need reboot, otherwise it return initial_value + const uint8_t* get() const { return value_; } + operator const uint8_t* () const { return value_; } + + virtual ObConfigItemType get_config_item_type() const { + return ObConfigItemType::OB_CONF_ITEM_TYPE_MODE; + } +protected: + //use current value to do input operation + bool set(const char *str); + const char *value_ptr() const override + { + return value_str_; + } + const char *value_reboot_ptr() const override + { + return value_reboot_str_; + } + uint64_t value_len() const override + { + return sizeof(value_str_); + } + uint64_t value_reboot_len() const override + { + return sizeof(value_reboot_str_); + } +protected: + static const uint64_t VALUE_BUF_SIZE = 65536UL; + static const int64_t MAX_MODE_BYTES = 32; + ObConfigParser *parser_; + char value_str_[VALUE_BUF_SIZE]; + char value_reboot_str_[VALUE_BUF_SIZE]; + // max bits size: 8 * 32 = 256 + uint8_t value_[MAX_MODE_BYTES]; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigModeItem); +}; + } // namespace common } // namespace oceanbase diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index 575c357bad..b0dfa1ab4c 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -29,7 +29,8 @@ #include "share/ob_resource_limit.h" #include "share/table/ob_ttl_util.h" #include "src/observer/ob_server.h" - +#include "share/table/ob_table_config_util.h" +#include "share/config/ob_config_mode_name_def.h" namespace oceanbase { using namespace share; @@ -875,5 +876,147 @@ bool ObConfigSQLTlsVersionChecker::check(const ObConfigItem &t) const 0 == tmp_str.case_compare("TLSV1.3"); } +int ObModeConfigParserUitl::parse_item_to_kv(char *item, ObString &key, ObString &value) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(item)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "item is NULL", K(ret)); + } else { + // key + char *save_ptr = NULL; + char *key_ptr = STRTOK_R(item, "=", &save_ptr); + ObString tmp_key(key_ptr); + key = tmp_key.trim(); + // value + ObString tmp_value(save_ptr); + value = tmp_value.trim(); + if (value.case_compare("on") != 0 && value.case_compare("off") != 0) { + ret = OB_INVALID_CONFIG; + OB_LOG(WARN, "item value is invalid", K(ret), K(value)); + } + } + return ret; +} + +int ObModeConfigParserUitl::format_mode_str(const char *src, int64_t src_len, char *dst, int64_t dst_len) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(src) || OB_UNLIKELY(src_len <=0) + || OB_ISNULL(dst) || dst_len < (3 * src_len)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid arguments", KR(ret), KP(src), KP(dst), K(src_len), K(dst_len)); + } else { + const char *source_str = src; + const char *locate_str = NULL; + int64_t source_left_len = src_len; + int32_t locate = -1; + int64_t pos = 0; + while (OB_SUCC(ret) && (source_left_len > 0) + && (NULL != (locate_str = STRCHR(source_str, ',')))) { + locate = static_cast(locate_str - source_str); + if (OB_FAIL(databuff_printf(dst, dst_len, pos, "%.*s , ", locate, source_str))) { + OB_LOG(WARN, "failed to databuff_print", K(ret), K(dst), K(locate), K(source_str)); + } else { + source_str = locate_str + 1; + source_left_len -= (locate + 1); + } + } + + if (OB_SUCC(ret) && source_left_len > 0) { + if (OB_FAIL(databuff_printf(dst, dst_len, pos, "%s", source_str))) { + OB_LOG(WARN, "failed to databuff_print", KR(ret), K(dst), K(pos)); + } + } + OB_LOG(DEBUG, "format_option_str", K(ret), K(src), K(dst)); + } + return ret; +} + +int ObModeConfigParserUitl::get_kv_list(char *str, ObIArray> &kv_list) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(str)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "item is NULL", K(ret)); + } else { + ObString key; + ObString value; + char *save_ptr = NULL; + char *token = STRTOK_R(str, ",", &save_ptr); + while (OB_SUCC(ret) && OB_NOT_NULL(token)) { + // trim left space + while (*token == ' ') token++; + // trim right space + uint64_t len = strlen(token); + while (len > 0 && token[len - 1] == ' ') token[--len] = '\0'; + // check and set mode + if (OB_FAIL(parse_item_to_kv(token, key, value))) { + OB_LOG(WARN, "fail to check config item", K(ret)); + } else if (OB_FAIL(kv_list.push_back(std::make_pair(key, value)))) { + OB_LOG(WARN, "fail to push back key and value pair", K(ret), K(key), K(value)); + } else { + token = STRTOK_R(NULL, ",", &save_ptr); + } + } + } + return ret; +} + +bool ObKvFeatureModeParser::parse(const char *str, uint8_t *arr, int64_t len) +{ + bool bret = true; + if (str == NULL || arr == NULL) { + bret = false; + OB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "Get mode config item fail, str or value arr is NULL!"); + } else if (strlen(str) == 0) { + bret = true; + OB_LOG_RET(DEBUG, OB_SUCCESS, "strlen is 0"); + } else { + int tmp_ret = OB_SUCCESS; + ObSEArray, 8> kv_list; + int64_t str_len = strlen(str); + const int64_t buf_len = 3 * str_len; // need replace ',' to ' , ' + char buf[buf_len]; + MEMSET(buf, 0, sizeof(buf)); + MEMCPY(buf, str, str_len); + if (OB_SUCCESS != (tmp_ret = ObModeConfigParserUitl::format_mode_str(str, str_len, buf, buf_len))) { + bret = false; + OB_LOG_RET(WARN, tmp_ret, "fail to format mode str", K(str)); + } else if (OB_SUCCESS != (tmp_ret = ObModeConfigParserUitl::get_kv_list(buf, kv_list))) { + bret = false; + OB_LOG_RET(WARN, tmp_ret, "fail to get kv list", K(str)); + } else { + ObKVFeatureMode kv_mode; + for (int64_t i = 0; i < kv_list.count() && bret; i++) { + uint8_t mode = MODE_DEFAULT; + if (kv_list.at(i).second.case_compare(MODE_VAL_ON) == 0) { + mode = MODE_ON; + } else if (kv_list.at(i).second.case_compare(MODE_VAL_OFF) == 0) { + mode = MODE_OFF; + } else { + bret = false; + OB_LOG_RET(WARN, OB_INVALID_CONFIG, "unknown mode type", K(kv_list.at(i).second)); + } + if (!bret) { + } else if (kv_list.at(i).first.case_compare(MODE_NAME_TTL) == 0) { + kv_mode.set_ttl_mode(mode); + } else if (kv_list.at(i).first.case_compare(MODE_NAME_REROUTING) == 0) { + kv_mode.set_rerouting_mode(mode); + } else if (kv_list.at(i).first.case_compare(MODE_NAME_HOTKEY) == 0) { + kv_mode.set_hotkey_mode(mode); + } else { + bret = false; + OB_LOG_RET(WARN, OB_INVALID_CONFIG, "unknown mode name", K(kv_list.at(i).first)); + } + } // end for + if (bret) { + arr[0] = kv_mode.get_value(); + } + } + } + return bret; +} + } // end of namepace common } // end of namespace oceanbase diff --git a/src/share/config/ob_config_helper.h b/src/share/config/ob_config_helper.h index 0957224e12..7590cf726e 100644 --- a/src/share/config/ob_config_helper.h +++ b/src/share/config/ob_config_helper.h @@ -635,6 +635,40 @@ public: private: DISALLOW_COPY_AND_ASSIGN(ObConfigSQLTlsVersionChecker); }; + +class ObModeConfigParserUitl +{ +public: + // parse config item like: "xxx=yyy" + static int parse_item_to_kv(char *item, ObString &key, ObString &value); + static int get_kv_list(char *str, ObIArray> &kv_list); + // format str for split config item + static int format_mode_str(const char *src, int64_t src_len, char *dst, int64_t dst_len); +}; + +class ObConfigParser +{ +public: + ObConfigParser() {} + virtual ~ObConfigParser() {} + virtual bool parse(const char *str, uint8_t *arr, int64_t len) = 0; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigParser); +}; + +class ObKvFeatureModeParser : public ObConfigParser +{ +public: + ObKvFeatureModeParser() {} + virtual ~ObKvFeatureModeParser() {} + virtual bool parse(const char *str, uint8_t *arr, int64_t len) override; +public: + static const int8_t MODE_DEFAULT = 0b00; + static const int8_t MODE_ON = 0b01; + static const int8_t MODE_OFF = 0b10; + DISALLOW_COPY_AND_ASSIGN(ObKvFeatureModeParser); +}; + typedef __ObConfigContainer ObConfigContainer; } // namespace common diff --git a/src/share/config/ob_config_mode_name_def.h b/src/share/config/ob_config_mode_name_def.h new file mode 100644 index 0000000000..1f6dffeeeb --- /dev/null +++ b/src/share/config/ob_config_mode_name_def.h @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SHARE_CONFIG_OB_CONFIG_MODE_NAME_DEF_H_ +#define OCEANBASE_SHARE_CONFIG_OB_CONFIG_MODE_NAME_DEF_H_ +// _obkv_feature_mode +#define MODE_NAME_TTL "ttl" +#define MODE_NAME_REROUTING "rerouting" +#define MODE_NAME_HOTKEY "hotkey" + +#define MODE_DEFAULT_VAL_TTL true +#define MODE_DEFAULT_VAL_REROUTING false +#define MODE_DEFAULT_VAL_HOTKEY false + +// mode value +#define MODE_VAL_ON "on" +#define MODE_VAL_OFF "off" + +#endif // diff --git a/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp b/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp index 368699bcf9..e397dacc38 100644 --- a/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25051_25100.cpp @@ -160,7 +160,7 @@ int ObInnerTableSchema::user_coll_types_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT /*+ USE_MERGE(T, C, D, T1, D1) */ T.TYPE_NAME AS TYPE_NAME, CAST( CASE C.UPPER_BOUND WHEN 0 THEN 'COLLECTION' ELSE 'TABLE' END AS VARCHAR2(10)) AS COLL_TYPE, C.UPPER_BOUND AS UPPER_BOUND, CAST(NULL AS VARCHAR2(7)) AS ELEM_TYPE_MOD, CAST( CASE BITAND(C.PROPERTIES, 15) WHEN 3 THEN NULL ELSE d1.DATABASE_NAME END AS VARCHAR2(128)) AS ELEM_TYPE_OWNER, CAST( CASE BITAND(C.PROPERTIES, 15) WHEN 3 THEN DECODE (C.ELEM_TYPE_ID, 0, 'NULL', 1, 'NUMBER', 2, 'NUMBER', 3, 'NUMBER', 4, 'NUMBER', 5, 'NUMBER', 6, 'NUMBER', 7, 'NUMBER', 8, 'NUMBER', 9, 'NUMBER', 10, 'NUMBER', 11, 'BINARY_FLOAT', 12, 'BINARY_DOUBLE', 13, 'NUMBER', 14, 'NUMBER', 15, 'NUMBER', 16, 'NUMBER', 17, 'DATE', 18, 'TIMESTAMP', 19, 'DATE', 20, 'TIME', 21, 'YEAR', 22, 'VARCHAR2', 23, 'CHAR', 24, 'HEX_STRING', 25, 'EXT', 26, 'UNKNOWN', 27, 'TINYTEXT', 28, 'TEXT', 29, 'MEDIUMTEXT', 30, DECODE(C.COLL_TYPE, 63, 'BLOB', 'CLOB'), 31, 'BIT', 32, 'ENUM', 33, 'SET', 34, 'ENUM_INNER', 35, 'SET_INNER', 36, CONCAT('TIMESTAMP(', CONCAT(C.SCALE, ') WITH TIME ZONE')), 37, CONCAT('TIMESTAMP(', CONCAT(C.SCALE, ') WITH LOCAL TIME ZONE')), 38, CONCAT('TIMESTAMP(', CONCAT(C.SCALE, ')')), 39, 'RAW', 40, CONCAT('INTERVAL YEAR(', CONCAT(C.SCALE, ') TO MONTH')), 41, CONCAT('INTERVAL DAY(', CONCAT(TRUNC(C.SCALE / 10), CONCAT(') TO SECOND(', CONCAT(MOD(C.SCALE, 10), ')')))), 42, 'FLOAT', 43, 'NVARCHAR2', 44, 'NCHAR', 45, CONCAT('UROWID(', CONCAT(C.LENGTH, ')')), 46, '', 'NOT_SUPPORT') ELSE t1.TYPE_NAME END AS VARCHAR2(324)) AS ELEM_TYPE_NAME, C.LENGTH AS LENGTH, C.NUMBER_PRECISION AS NUMBER_PRECISION, C.SCALE AS SCALE, CAST('CHAR_CS' AS CHAR(7)) AS CHARACTER_SET_NAME, CAST('YES' AS CHAR(7)) AS ELEM_STORAGE, CAST('B' AS CHAR(7)) AS NULLS_STORED FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_COLL_TYPE_REAL_AGENT C ON T.TYPE_ID = C.COLL_TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') AND T.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.DATABASE_ID = USERENV('SCHEMAID') LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON T1.TYPE_ID = C.ELEM_TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND D1.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT /*+ USE_MERGE(T, C, D, T1, D1) */ T.TYPE_NAME AS TYPE_NAME, CAST( CASE C.UPPER_BOUND WHEN 0 THEN 'COLLECTION' ELSE 'TABLE' END AS VARCHAR2(10)) AS COLL_TYPE, C.UPPER_BOUND AS UPPER_BOUND, CAST(NULL AS VARCHAR2(7)) AS ELEM_TYPE_MOD, CAST( CASE BITAND(C.PROPERTIES, 15) WHEN 3 THEN NULL ELSE d1.DATABASE_NAME END AS VARCHAR2(128)) AS ELEM_TYPE_OWNER, CAST( CASE BITAND(C.PROPERTIES, 15) WHEN 3 THEN DECODE (C.ELEM_TYPE_ID, 0, 'NULL', 1, 'NUMBER', 2, 'NUMBER', 3, 'NUMBER', 4, 'NUMBER', 5, 'NUMBER', 6, 'NUMBER', 7, 'NUMBER', 8, 'NUMBER', 9, 'NUMBER', 10, 'NUMBER', 11, 'BINARY_FLOAT', 12, 'BINARY_DOUBLE', 13, 'NUMBER', 14, 'NUMBER', 15, 'NUMBER', 16, 'NUMBER', 17, 'DATE', 18, 'TIMESTAMP', 19, 'DATE', 20, 'TIME', 21, 'YEAR', 22, 'VARCHAR2', 23, 'CHAR', 24, 'HEX_STRING', 25, 'EXT', 26, 'UNKNOWN', 27, 'TINYTEXT', 28, 'TEXT', 29, 'MEDIUMTEXT', 30, DECODE(C.COLL_TYPE, 63, 'BLOB', 'CLOB'), 31, 'BIT', 32, 'ENUM', 33, 'SET', 34, 'ENUM_INNER', 35, 'SET_INNER', 36, CONCAT('TIMESTAMP(', CONCAT(C.SCALE, ') WITH TIME ZONE')), 37, CONCAT('TIMESTAMP(', CONCAT(C.SCALE, ') WITH LOCAL TIME ZONE')), 38, CONCAT('TIMESTAMP(', CONCAT(C.SCALE, ')')), 39, 'RAW', 40, CONCAT('INTERVAL YEAR(', CONCAT(C.SCALE, ') TO MONTH')), 41, CONCAT('INTERVAL DAY(', CONCAT(TRUNC(C.SCALE / 10), CONCAT(') TO SECOND(', CONCAT(MOD(C.SCALE, 10), ')')))), 42, 'FLOAT', 43, 'NVARCHAR2', 44, 'NCHAR', 45, CONCAT('UROWID(', CONCAT(C.LENGTH, ')')), 46, DECODE(C.COLL_TYPE, 63, 'BLOB', 'CLOB'), 47, 'JSON', 48, 'SDO_GEOMETRY', 'NOT_SUPPORT') ELSE t1.TYPE_NAME END AS VARCHAR2(324)) AS ELEM_TYPE_NAME, C.LENGTH AS LENGTH, C.NUMBER_PRECISION AS NUMBER_PRECISION, C.SCALE AS SCALE, CAST('CHAR_CS' AS CHAR(7)) AS CHARACTER_SET_NAME, CAST('YES' AS CHAR(7)) AS ELEM_STORAGE, CAST('B' AS CHAR(7)) AS NULLS_STORED FROM SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_COLL_TYPE_REAL_AGENT C ON T.TYPE_ID = C.COLL_TYPE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D ON T.TENANT_ID = SYS_CONTEXT('USERENV', 'CON_ID') AND T.DATABASE_ID = D.DATABASE_ID AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.DATABASE_ID = USERENV('SCHEMAID') LEFT JOIN SYS.ALL_VIRTUAL_TYPE_REAL_AGENT T1 ON T1.TYPE_ID = C.ELEM_TYPE_ID AND T1.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT D1 ON T1.DATABASE_ID = D1.DATABASE_ID AND D1.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index c8c08b9da2..06c9205ffd 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -12961,6 +12961,8 @@ def_table_schema(**gen_iterate_private_virtual_table_def( # 12429: __all_virtual_data_activity_metrics # 12430: __all_virtual_column_group_mapping +# 12431: __all_virtual_column_group_history +# 12432: __all_virtual_column_gorup_mapping_history # # 余留位置 # @@ -13363,6 +13365,9 @@ def_table_schema(**gen_oracle_mapping_virtual_table_def('15414', all_def_keyword # 15417: __all_virtual_column_group_mapping +# 15418: __all_virtual_cgroup_config +# 15419: __all_virutal_column_group_history +# 15420: __all_virutal_column_group_maping_history # 余留位置 ################################################################################ @@ -29851,7 +29856,9 @@ def_table_schema( """.replace("\n", " "), ) -# +#21479 GV$OB_CGROUP_CONFIG +#21480 V$OB_CGROUP_CONFIG + # 余留位置 ################################################################################ @@ -36243,7 +36250,9 @@ def_table_schema( 43, 'NVARCHAR2', 44, 'NCHAR', 45, CONCAT('UROWID(', CONCAT(C.LENGTH, ')')), - 46, '', + 46, DECODE(C.COLL_TYPE, 63, 'BLOB', 'CLOB'), + 47, 'JSON', + 48, 'SDO_GEOMETRY', 'NOT_SUPPORT') ELSE t1.TYPE_NAME END AS VARCHAR2(324)) AS ELEM_TYPE_NAME, C.LENGTH AS LENGTH, @@ -54377,6 +54386,8 @@ def_table_schema( # 28197: V$OB_SESSION # 28198: GV$OB_PL_CACHE_OBJECT # 28199: V$OB_PL_CACHE_OBJECT +# 28200: GV$OB_CGROUP_CONFIG +# 28201: V$OB_CGROUP_CONFIG ################################################################################ # Lob Table (50000, 70000) diff --git a/src/share/io/ob_io_manager.cpp b/src/share/io/ob_io_manager.cpp index 5dc54050d1..9f54349423 100644 --- a/src/share/io/ob_io_manager.cpp +++ b/src/share/io/ob_io_manager.cpp @@ -614,6 +614,9 @@ int ObTenantIOManager::mtl_init(ObTenantIOManager *&io_service) io_service = holder.get_ptr(); } } + if (OB_SUCC(ret)) { + FLOG_INFO("mtl init tenant io manager success", K(tenant_id), KPC(io_service)); + } return ret; } @@ -645,6 +648,7 @@ void ObTenantIOManager::mtl_destroy(ObTenantIOManager *&io_service) io_service->~ObTenantIOManager(); OB_IO_MANAGER.allocator_.free(io_service); io_service = nullptr; + FLOG_INFO("mtl destroy tenant io manager success", K(tenant_id)); } else if (OB_NOT_NULL(io_service) && io_service->get_ref_cnt() != 0) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("ERROR: tenant io manager ref_cnt is not zero", K(ret)); diff --git a/src/share/ls/ob_ls_info.cpp b/src/share/ls/ob_ls_info.cpp index 0aa6d4c3a5..20255d692b 100644 --- a/src/share/ls/ob_ls_info.cpp +++ b/src/share/ls/ob_ls_info.cpp @@ -16,6 +16,7 @@ #include "share/config/ob_server_config.h" // for KR(), common::ob_error_name(x) #include "share/ls/ob_ls_replica_filter.h" // ObLSReplicaFilter #include "share/ob_share_util.h" // ObShareUtils +#include "share/ob_all_server_tracer.h" // SVR_TRACER #include "lib/string/ob_sql_string.h" // ObSqlString #include "lib/utility/utility.h" // split_on() @@ -324,6 +325,7 @@ bool ObLSReplica::learner_list_is_equal(const common::GlobalLearnerList &a, cons return is_equal; } +// both server and timestamp of member need to be equal bool ObLSReplica::member_list_is_equal(const MemberList &a, const MemberList &b) { bool is_equal = true; @@ -377,6 +379,24 @@ bool ObLSReplica::servers_in_member_list_are_same(const MemberList &a, const Mem return is_same; } +int ObLSReplica::check_all_servers_in_member_list_are_active( + const MemberList &member_list, + bool &all_active) +{ + int ret = OB_SUCCESS; + all_active = true; + ARRAY_FOREACH_X(member_list, idx, cnt, OB_SUCC(ret) && all_active) { + const ObAddr &server = member_list.at(idx).get_server(); + if (OB_FAIL(SVR_TRACER.check_server_alive(server, all_active))) { + all_active = false; + LOG_WARN("check server alive failed", KR(ret), K(server), K(all_active), K(member_list)); + } else if (!all_active) { + LOG_WARN("server in member_list is inactive", KR(ret), K(server), K(member_list)); + } + } + return ret; +} + int64_t ObLSReplica::to_string(char *buf, const int64_t buf_len) const { int64_t pos = 0; diff --git a/src/share/ls/ob_ls_info.h b/src/share/ls/ob_ls_info.h index 999f1aa346..dfe0157369 100644 --- a/src/share/ls/ob_ls_info.h +++ b/src/share/ls/ob_ls_info.h @@ -45,7 +45,7 @@ enum ObReplicaStatus const char *ob_replica_status_str(const ObReplicaStatus status); int get_replica_status(const char* str, ObReplicaStatus &status); -int get_replica_status(const ObString &status_str, ObReplicaStatus &status); +int get_replica_status(const common::ObString &status_str, ObReplicaStatus &status); // [class_full_name] SimpleMember // [class_functions] Use this class to build a member_list consists of this simple SimpleMember @@ -83,6 +83,24 @@ class ObLSReplica public: static const int64_t DEFAULT_REPLICA_COUNT = 7; typedef common::ObSEArray MemberList; + /*---------------------- MemberList related functions begin -----------------------*/ + // format-related functions + static int member_list2text(const MemberList &member_list, ObSqlString &text); + static int text2learner_list(const char *text, GlobalLearnerList &learner_list); + static int text2member_list(const char *text, MemberList &member_list); + // transform ObMemberList into MemberList + static int transform_ob_member_list( + const common::ObMemberList &ob_member_list, + MemberList &member_list); + static bool member_list_is_equal(const MemberList &a, const MemberList &b); + static bool server_is_in_member_list( + const MemberList &member_list, + const common::ObAddr &server); + static bool servers_in_member_list_are_same(const MemberList &a, const MemberList &b); + static int check_all_servers_in_member_list_are_active( + const MemberList &member_list, + bool &all_acitve); + /*---------------------- MemberList related functions end -------------------------*/ // initial-related functions ObLSReplica(); @@ -118,19 +136,6 @@ public: || common::REPLICA_TYPE_FULL == replica_type_ || common::REPLICA_TYPE_LOGONLY == replica_type_; } inline bool is_in_restore() const { return !restore_status_.is_restore_none(); } - // format-related functions - static int member_list2text(const MemberList &member_list, ObSqlString &text); - static int text2learner_list(const char *text, GlobalLearnerList &learner_list); - static int text2member_list(const char *text, MemberList &member_list); - // transform ObMemberList into MemberList - static int transform_ob_member_list( - const common::ObMemberList &ob_member_list, - MemberList &member_list); - static bool member_list_is_equal(const MemberList &a, const MemberList &b); - static bool server_is_in_member_list( - const MemberList &member_list, - const common::ObAddr &server); - static bool servers_in_member_list_are_same(const MemberList &a, const MemberList &b); int64_t to_string(char *buf, const int64_t buf_len) const; // operator-related functions int assign(const ObLSReplica &other); diff --git a/src/share/ob_ddl_task_executor.h b/src/share/ob_ddl_task_executor.h index 3da5a1bbf1..8e631e8b45 100644 --- a/src/share/ob_ddl_task_executor.h +++ b/src/share/ob_ddl_task_executor.h @@ -94,7 +94,8 @@ private: } static bool is_stop_state(const int ret_code) { return common::OB_IN_STOP_STATE == ret_code || common::OB_SERVER_IS_INIT == ret_code || common::OB_SERVER_IS_STOPPING == ret_code - || common::OB_RS_SHUTDOWN == ret_code || common::OB_PARTITION_IS_STOPPED == ret_code; + || common::OB_RS_SHUTDOWN == ret_code || common::OB_PARTITION_IS_STOPPED == ret_code + || common::OB_PARTITION_IS_BLOCKED == ret_code; } static bool is_not_master(const int ret_code) { return common::OB_STATE_NOT_MATCH == ret_code || common::OB_NOT_MASTER == ret_code || OB_RS_NOT_MASTER == ret_code diff --git a/src/share/ob_encryption_util_os.cpp b/src/share/ob_encryption_util_os.cpp index 94bb674ac2..93a198300d 100644 --- a/src/share/ob_encryption_util_os.cpp +++ b/src/share/ob_encryption_util_os.cpp @@ -374,13 +374,11 @@ static void ob_free_openssl(void *ptr, const char *, int) int ObEncryptionUtil::init_ssl_malloc() { int ret = OB_SUCCESS; -#ifdef OB_USE_BABASSL int tmp_ret = CRYPTO_set_mem_functions(ob_malloc_openssl, ob_realloc_openssl, ob_free_openssl); if (OB_UNLIKELY(tmp_ret != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to set crypto mem functions", K(tmp_ret), K(ret)); } -#endif return ret; } diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 9e68694b46..8389563631 100755 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -17583,11 +17583,11 @@ static const _error _error_OB_ERR_DBLINK_REMOTE_ECODE = { .error_solution = "Contact OceanBase Support", .mysql_errno = -1, .sqlstate = "HY000", - .str_error = "Unknown errorcode: %d", - .str_user_error = "Unknown errorcode: %d", + .str_error = "\ndblink remote error code: %d,\nremote error msg: %.*s", + .str_user_error = "\ndblink remote error code: %d,\nremote error msg: %.*s", .oracle_errno = 600, - .oracle_str_error = "ORA-00600: internal error code, arguments: -5975, Unknown errorcode: %d", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5975, Unknown errorcode: %d" + .oracle_str_error = "ORA-00600: internal error code, arguments: -5975, \ndblink remote error code: %d,\nremote error msg: %.*s", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5975, \ndblink remote error code: %d,\nremote error msg: %.*s" }; static const _error _error_OB_ERR_DBLINK_NO_LIB = { .error_name = "OB_ERR_DBLINK_NO_LIB", diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index aaa4f3e958..fe33924d64 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -1612,7 +1612,7 @@ DEFINE_ORACLE_ERROR_EXT(OB_ERR_CHECK_OPTION_VIOLATED, -5971, ER_VIEW_CHECK_FAILE DEFINE_ERROR_EXT(OB_ERR_CHECK_OPTION_ON_NONUPDATABLE_VIEW, -5972, ER_VIEW_NONUPD_CHECK, "HY000", "CHECK OPTION on non-updatable view", "CHECK OPTION on non-updatable view '%.*s.%.*s'"); DEFINE_ORACLE_ERROR(OB_ERR_NO_DESC_FOR_POS, -5973, -1, "HY000", "no descriptor for this position", 24334, "no descriptor for this position"); DEFINE_ORACLE_ERROR(OB_ERR_ILL_OBJ_FLAG, -5974, -1, "HY000", "object specified is incompatible with the flag specified", 4047, "object specified is incompatible with the flag specified"); -DEFINE_ERROR_DEP(OB_ERR_DBLINK_REMOTE_ECODE, -5975, -1, "HY000", "Unknown errorcode: %d", "dblink remote ora error code: %d"); +DEFINE_ERROR_DEP(OB_ERR_DBLINK_REMOTE_ECODE, -5975, -1, "HY000", "\ndblink remote error code: %d,\nremote error msg: %.*s", "\ndblink remote error code: %d,\nremote error msg: %.*s"); DEFINE_ERROR_DEP(OB_ERR_DBLINK_NO_LIB, -5976, -1, "HY000", "oci lib not founded"); DEFINE_ORACLE_ERROR(OB_ERR_PARTITION_EXTENDED_ON_VIEW, -5977, -1, "HY000", "partition-extended object names may only be used with tables and editioning views", 14109, "partition-extended object names may only be used with tables and editioning views"); DEFINE_ORACLE_ERROR(OB_ERR_NOT_ALL_VARIABLE_BIND, -5978, -1, "HY000", "not all variables bound", 1008, "not all variables bound"); diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index e9a10d77d3..eab01fc288 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -3251,7 +3251,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_CHECK_OPTION_ON_NONUPDATABLE_VIEW__USER_ERROR_MSG "CHECK OPTION on non-updatable view '%.*s.%.*s'" #define OB_ERR_NO_DESC_FOR_POS__USER_ERROR_MSG "no descriptor for this position" #define OB_ERR_ILL_OBJ_FLAG__USER_ERROR_MSG "object specified is incompatible with the flag specified" -#define OB_ERR_DBLINK_REMOTE_ECODE__USER_ERROR_MSG "Unknown errorcode: %d" +#define OB_ERR_DBLINK_REMOTE_ECODE__USER_ERROR_MSG "\ndblink remote error code: %d,\nremote error msg: %.*s" #define OB_ERR_DBLINK_NO_LIB__USER_ERROR_MSG "oci lib not founded" #define OB_ERR_PARTITION_EXTENDED_ON_VIEW__USER_ERROR_MSG "partition-extended object names may only be used with tables and editioning views" #define OB_ERR_NOT_ALL_VARIABLE_BIND__USER_ERROR_MSG "not all variables bound" @@ -5379,7 +5379,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_CHECK_OPTION_ON_NONUPDATABLE_VIEW__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5972, CHECK OPTION on non-updatable view '%.*s.%.*s'" #define OB_ERR_NO_DESC_FOR_POS__ORA_USER_ERROR_MSG "ORA-24334: no descriptor for this position" #define OB_ERR_ILL_OBJ_FLAG__ORA_USER_ERROR_MSG "ORA-04047: object specified is incompatible with the flag specified" -#define OB_ERR_DBLINK_REMOTE_ECODE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5975, Unknown errorcode: %d" +#define OB_ERR_DBLINK_REMOTE_ECODE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5975, \ndblink remote error code: %d,\nremote error msg: %.*s" #define OB_ERR_DBLINK_NO_LIB__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5976, oci lib not founded" #define OB_ERR_PARTITION_EXTENDED_ON_VIEW__ORA_USER_ERROR_MSG "ORA-14109: partition-extended object names may only be used with tables and editioning views" #define OB_ERR_NOT_ALL_VARIABLE_BIND__ORA_USER_ERROR_MSG "ORA-01008: not all variables bound" diff --git a/src/share/ob_max_id_fetcher.cpp b/src/share/ob_max_id_fetcher.cpp index cfc154c7e8..3247138e06 100755 --- a/src/share/ob_max_id_fetcher.cpp +++ b/src/share/ob_max_id_fetcher.cpp @@ -87,7 +87,14 @@ const char *ObMaxIdFetcher::max_id_name_info_[OB_MAX_ID_TYPE][2] = { lib::ObMutex ObMaxIdFetcher::mutex_bucket_[MAX_TENANT_MUTEX_BUCKET_CNT]; ObMaxIdFetcher::ObMaxIdFetcher(ObMySQLProxy &proxy) - : proxy_(proxy) + : proxy_(proxy), + group_id_(0) +{ +} + +ObMaxIdFetcher::ObMaxIdFetcher(ObMySQLProxy &proxy, const int32_t group_id) + : proxy_(proxy), + group_id_(group_id) { } @@ -432,7 +439,7 @@ int ObMaxIdFetcher::update_max_id(ObISQLClient &sql_client, const uint64_t tenan zone.ptr(), id_name, ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id)))) { LOG_WARN("sql_string append format string failed", K(ret)); - } else if (OB_FAIL(sql_client.write(exec_tenant_id, sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_client.write(exec_tenant_id, sql.ptr(), group_id_, affected_rows))) { LOG_WARN("sql client write fail", K(sql), K(affected_rows), K(ret)); } else if (!is_single_row(affected_rows)) { ret = OB_INNER_STAT_ERROR; @@ -535,7 +542,7 @@ int ObMaxIdFetcher::insert_initial_value(common::ObISQLClient &sql_client, uint6 zone.ptr(), name, obj.get_type(), static_cast(value), info))) { LOG_WARN("sql string assign failed", K(ret)); - } else if (OB_FAIL(sql_client.write(exec_tenant_id, sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_client.write(exec_tenant_id, sql.ptr(), group_id_, affected_rows))) { LOG_WARN("execute sql failed", K(ret)); } return ret; diff --git a/src/share/ob_max_id_fetcher.h b/src/share/ob_max_id_fetcher.h index 33f61e4971..f55fef8310 100755 --- a/src/share/ob_max_id_fetcher.h +++ b/src/share/ob_max_id_fetcher.h @@ -82,6 +82,7 @@ class ObMaxIdFetcher { public: explicit ObMaxIdFetcher(common::ObMySQLProxy &proxy); + explicit ObMaxIdFetcher(common::ObMySQLProxy &proxy, const int32_t group_id); virtual ~ObMaxIdFetcher(); // For generate new object_ids @@ -114,6 +115,7 @@ private: private: common::ObMySQLProxy &proxy_; static lib::ObMutex mutex_bucket_[MAX_TENANT_MUTEX_BUCKET_CNT]; + int32_t group_id_; DISALLOW_COPY_AND_ASSIGN(ObMaxIdFetcher); }; diff --git a/src/share/ob_primary_standby_service.cpp b/src/share/ob_primary_standby_service.cpp index 75e0224029..7b0b006dac 100644 --- a/src/share/ob_primary_standby_service.cpp +++ b/src/share/ob_primary_standby_service.cpp @@ -447,6 +447,7 @@ int ObPrimaryStandbyService::switch_to_standby( { int ret = OB_SUCCESS; ObAllTenantInfo tenant_info; + const int32_t group_id = 0; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); @@ -505,7 +506,7 @@ int ObPrimaryStandbyService::switch_to_standby( } else if (compat_version < DATA_VERSION_4_2_0_0) { //Regardless of the data_version change and switchover concurrency scenario, //if there is concurrency, the member_list lock that has not been released by the operation and maintenance process - } else if (OB_FAIL(ObMemberListLockUtils::unlock_member_list_when_switch_to_standby(tenant_id, *sql_proxy_))) { + } else if (OB_FAIL(ObMemberListLockUtils::unlock_member_list_when_switch_to_standby(tenant_id, group_id, *sql_proxy_))) { LOG_WARN("failed to unlock member list when switch to standby", K(ret), K(tenant_id)); } if (FAILEDx(role_transition_service.switchover_update_tenant_status(tenant_id, diff --git a/src/share/ob_scanner.cpp b/src/share/ob_scanner.cpp index 2c5eeb7512..fc6a391a00 100644 --- a/src/share/ob_scanner.cpp +++ b/src/share/ob_scanner.cpp @@ -48,7 +48,7 @@ ObScanner::ObScanner(const char *label /*= ObModIds::OB_NEW_SCANNER*/, tenant_id), is_result_accurate_(true), implicit_cursors_(inner_allocator_), - datum_store_(), + datum_store_(label), rcode_(), fb_info_() { @@ -78,7 +78,7 @@ ObScanner::ObScanner(ObIAllocator &allocator, tenant_id), is_result_accurate_(true), implicit_cursors_(allocator), - datum_store_(&allocator), + datum_store_(label, &allocator), rcode_(), fb_info_() { diff --git a/src/share/object/ob_obj_cast.h b/src/share/object/ob_obj_cast.h index e5f4bc5bd9..471c3a2aa6 100644 --- a/src/share/object/ob_obj_cast.h +++ b/src/share/object/ob_obj_cast.h @@ -47,6 +47,7 @@ namespace common #define CM_ERROR_ON_SCALE_OVER (1ULL << 10) #define CM_STRICT_JSON (1ULL << 11) +#define CM_ADD_ZEROFILL (1ULL << 47) #define CM_CS_LEVEL_RESERVED1 (1ULL << 48) #define CM_CS_LEVEL_RESERVED2 (1ULL << 49) #define CM_CS_LEVEL_RESERVED3 (1ULL << 50) @@ -137,6 +138,7 @@ typedef uint64_t ObCastMode; #define CM_SET_CS_LEVEL(mode, level) \ ((mode) &= ~(CM_CS_LEVEL_MASK << CM_CS_LEVEL_SHIFT), \ (mode) |= ((level & CM_CS_LEVEL_MASK) << CM_CS_LEVEL_SHIFT)) +#define CM_IS_ADD_ZEROFILL(mode) ((CM_ADD_ZEROFILL & (mode)) != 0) struct ObObjCastParams { // add params when necessary diff --git a/src/share/parameter/ob_parameter_macro.h b/src/share/parameter/ob_parameter_macro.h index b353593f5d..f695f57b93 100644 --- a/src/share/parameter/ob_parameter_macro.h +++ b/src/share/parameter/ob_parameter_macro.h @@ -26,7 +26,8 @@ SCOPE(_DEF_PARAMETER_RANGE_EASY(access_specifier, param, _ ## SCOPE, name, args)) #define _DEF_PARAMETER_SCOPE_CHECKER_EASY(access_specifier, param, name, SCOPE, args...) \ SCOPE(_DEF_PARAMETER_CHECKER_EASY(access_specifier, param, _ ## SCOPE, name, args)) - +#define _DEF_PARAMETER_SCOPE_PARSER_EASY(access_specifier, param, name, SCOPE, args...) \ + SCOPE(_DEF_PARAMETER_PARSER_EASY(access_specifier, param, _ ## SCOPE, name, args)) #define _DEF_PARAMETER_SCOPE_IP_EASY(access_specifier, param, name, SCOPE, def, args...) \ SCOPE(_DEF_PARAMETER_CHECKER_EASY(access_specifier, param, _ ## SCOPE, name, def, \ common::ObConfigIpChecker, args)) @@ -94,6 +95,18 @@ access_specifier: } \ } name; +#define _DEF_PARAMETER_PARSER_EASY(access_specifier, param, scope, name, def, parser, args...) \ +access_specifier: \ + class ObConfig ## param ## Item ## _ ## name \ + : public common::ObConfig ## param ## Item \ + { \ + public: \ + ObConfig ## param ## Item ## _ ## name() \ + : common::ObConfig ## param ## Item( \ + local_container(), scope, #name, def, \ + new (std::nothrow) parser(), args) {} \ + } name; + //////////////////////////////////////////////////////////////////////////////// #define DEF_INT(args...) \ _DEF_PARAMETER_SCOPE_RANGE_EASY(public, Int, args) @@ -140,6 +153,9 @@ access_specifier: #define DEF_STR_LIST(args...) \ _DEF_PARAMETER_SCOPE_EASY(public, StrList, args) +#define DEF_MODE_WITH_PARSER(args...) \ + _DEF_PARAMETER_SCOPE_PARSER_EASY(public, Mode, args) + #define DEF_LOG_ARCHIVE_OPTIONS_WITH_CHECKER(args...) \ _DEF_PARAMETER_SCOPE_CHECKER_EASY(public, LogArchiveOptions, args) #define DEF_LOG_LEVEL(args...) \ diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index b0a50e87de..af556359e7 100755 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1207,8 +1207,8 @@ DEF_STR_WITH_CHECKER(_audit_mode, OB_TENANT_PARAMETER, "NONE", "MYSQL: use mysql audit" "ORACLE: use oracle audit", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_INT(_max_schema_slot_num, OB_CLUSTER_PARAMETER, "128", "[2,256]", - "the max schema slot number for each tenant, " +DEF_INT(_max_schema_slot_num, OB_TENANT_PARAMETER, "128", "[2,256]", + "the max schema slot number for multi-version schema memory management, " "Range: [2, 256] in integer", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_INT_WITH_CHECKER(_ob_query_rate_limit, OB_TENANT_PARAMETER, "-1", @@ -1557,8 +1557,8 @@ DEF_BOOL(_xsolapi_generate_with_clause, OB_TENANT_PARAMETER, "True", DEF_BOOL(_optimizer_group_by_placement, OB_TENANT_PARAMETER, "True", "enable group by placement transform rule", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_TIME(_wait_interval_after_truncate, OB_CLUSTER_PARAMETER, "30s", "[0s,)", - "time interval for waiting other servers to refresh schema after truncate", +DEF_TIME(_wait_interval_after_parallel_ddl, OB_TENANT_PARAMETER, "30s", "[0s,)", + "time interval for waiting other servers to refresh schema after parallel ddl is done", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_CAP(_rebuild_replica_log_lag_threshold, OB_TENANT_PARAMETER, "0M", "[0M,)", "size of clog files that a replica lag behind leader to trigger rebuild, 0 means never trigger rebuild on purpose. Range: [0, +∞)", @@ -1628,3 +1628,7 @@ DEF_STR_WITH_CHECKER(sql_protocol_min_tls_version, OB_CLUSTER_PARAMETER, "none", "SQL SSL control options, used to specify the minimum SSL/TLS version number. " "values: none, TLSv1, TLSv1.1, TLSv1.2, TLSv1.3", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +// obkv +DEF_MODE_WITH_PARSER(_obkv_feature_mode, OB_CLUSTER_PARAMETER, "", common::ObKvFeatureModeParser, + "_obkv_feature_mode is a option list to control specified OBKV features on/off.", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/share/resource_manager/ob_group_list.h b/src/share/resource_manager/ob_group_list.h index 1059233792..0c3edf5ac2 100644 --- a/src/share/resource_manager/ob_group_list.h +++ b/src/share/resource_manager/ob_group_list.h @@ -16,4 +16,7 @@ CGID_DEF(OBCG_MYSQL_LOGIN, 12) CGID_DEF(OBCG_CDCSERVICE, 13) CGID_DEF(OBCG_DIAG_TENANT, 14) CGID_DEF(OBCG_WR, 15) +CGID_DEF(OBCG_STORAGE_HA_LEVEL1, 16) +CGID_DEF(OBCG_STORAGE_HA_LEVEL2, 17) +CGID_DEF(OBCG_DBA_COMMAND, 18, 1) CGID_DEF(OBCG_LQ, 100) diff --git a/src/share/restore/ob_import_table_struct.cpp b/src/share/restore/ob_import_table_struct.cpp index 1461c2fe29..1bd756111b 100644 --- a/src/share/restore/ob_import_table_struct.cpp +++ b/src/share/restore/ob_import_table_struct.cpp @@ -363,11 +363,6 @@ int ObImportTableTask::parse_from(common::sqlclient::ObMySQLResult &result) bool is_succeed = true; if (0 == STRCMP("FAILED", result_buf)) { is_succeed = false; - } else if (0 == STRCMP("SUCCEESS", result_buf)) { - is_succeed = true; - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid result str", K(ret)); } result_.set_result(is_succeed, comment); } diff --git a/src/share/schema/ob_dependency_info.cpp b/src/share/schema/ob_dependency_info.cpp index 126c7191b6..12e01fcd43 100644 --- a/src/share/schema/ob_dependency_info.cpp +++ b/src/share/schema/ob_dependency_info.cpp @@ -1148,6 +1148,8 @@ int ObReferenceObjTable::process_reference_obj_table(const uint64_t tenant_id, if (OB_INVALID_ID != dep_obj_id) { OZ (task_queue.erase_view_id_from_set(dep_obj_id)); } + } else if (task_queue.is_queue_almost_full()) { + ret = OB_SIZE_OVERFLOW; } else if (OB_FAIL(task_queue.push(task))) { if (OB_UNLIKELY(OB_SIZE_OVERFLOW != ret)) { LOG_WARN("push task failed", K(ret)); diff --git a/src/share/schema/ob_multi_version_schema_service.cpp b/src/share/schema/ob_multi_version_schema_service.cpp index 44a5b258fa..04e469d49b 100644 --- a/src/share/schema/ob_multi_version_schema_service.cpp +++ b/src/share/schema/ob_multi_version_schema_service.cpp @@ -2207,7 +2207,12 @@ int ObMultiVersionSchemaService::add_schema( // try switch allocator if (OB_SUCC(ret)) { bool can_switch = false; - int64_t switch_cnt = ObSchemaService::g_liboblog_mode_ ? init_version_cnt_ : GCONF._max_schema_slot_num; + int64_t max_schema_slot_num = OB_MAX_VERSION_COUNT; + omt::ObTenantConfigGuard tenant_config(OTC_MGR.get_tenant_config_with_lock(tenant_id)); + if (tenant_config.is_valid()) { + max_schema_slot_num = tenant_config->_max_schema_slot_num; + } + const int64_t switch_cnt = ObSchemaService::g_liboblog_mode_ ? init_version_cnt_ : max_schema_slot_num; if (OB_FAIL(mem_mgr->check_can_switch_allocator(switch_cnt, can_switch))) { LOG_WARN("fail to check can switch allocator", KR(ret)); } else if (can_switch) { @@ -2384,8 +2389,9 @@ int ObMultiVersionSchemaService::async_refresh_schema( // do nothing } else { int64_t retry_cnt = 0; - int64_t MAX_RETRY_CNT = 1000; - const __useconds_t RETRY_IDLE_TIME = 100 * 1000L; // 100ms + const __useconds_t RETRY_IDLE_TIME = 10 * 1000L; // 10ms + const int64_t MAX_RETRY_CNT = 100 * 1000 * 1000L / RETRY_IDLE_TIME; // 100s at most + const int64_t SUBMIT_TASK_FREQUENCE = 2 * 1000 * 1000L / RETRY_IDLE_TIME; // each 2s while (OB_SUCC(ret)) { if (THIS_WORKER.is_timeout() || (INT64_MAX == THIS_WORKER.get_timeout_ts() && retry_cnt >= MAX_RETRY_CNT)) { @@ -2400,21 +2406,13 @@ int ObMultiVersionSchemaService::async_refresh_schema( // success break; } else { - if (0 == retry_cnt % 20) { - // try refresh schema each 2s - { - bool is_dropped = false; - ObSchemaGetterGuard guard; - if (OB_FAIL(get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(guard.check_if_tenant_has_been_dropped(tenant_id, is_dropped))) { - LOG_WARN("fail to check if tenant has been dropped", KR(ret), K(tenant_id)); - } else if (is_dropped) { - ret = OB_TENANT_HAS_BEEN_DROPPED; - LOG_WARN("tenant has been dropped", KR(ret), K(tenant_id)); - } - } - if (OB_FAIL(ret)) { + if (0 == retry_cnt % SUBMIT_TASK_FREQUENCE) { + bool is_dropped = false; + if (OB_FAIL(check_if_tenant_has_been_dropped(tenant_id, is_dropped))) { + LOG_WARN("fail to check if tenant has been dropped", KR(ret), K(tenant_id)); + } else if (is_dropped) { + ret = OB_TENANT_HAS_BEEN_DROPPED; + LOG_WARN("tenant has been dropped", KR(ret), K(tenant_id)); } else if (OB_ISNULL(GCTX.ob_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("observice is null", K(ret)); diff --git a/src/share/schema/ob_schema_getter_guard.cpp b/src/share/schema/ob_schema_getter_guard.cpp index a80bd41264..4b26287b2f 100644 --- a/src/share/schema/ob_schema_getter_guard.cpp +++ b/src/share/schema/ob_schema_getter_guard.cpp @@ -3439,10 +3439,49 @@ int ObSchemaGetterGuard::check_single_table_priv(const ObSessionPrivInfo &sessio } } } + if (OB_SUCC(ret) && table_need_priv.is_for_update_) { + if (OB_FAIL(check_single_table_priv_for_update_(session_priv, table_need_priv, priv_mgr))) { + LOG_WARN("failed to check select table for update priv", K(ret)); + } + } } return ret; } +/* select ... from table for update, need select privilege and one of (delete, update lock tables). + * ob donesn't have lock tables yet, then it checks select first and one of (delete、update on table level). + */ +int ObSchemaGetterGuard::check_single_table_priv_for_update_(const ObSessionPrivInfo &session_priv, + const ObNeedPriv &table_need_priv, + const ObPrivMgr &priv_mgr) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = session_priv.tenant_id_; + const uint64_t user_id = session_priv.user_id_; + bool pass = false; + const ObNeedPriv need_priv(table_need_priv.db_, table_need_priv.table_, table_need_priv.priv_level_, + OB_PRIV_UPDATE | OB_PRIV_DELETE, table_need_priv.is_sys_table_, + table_need_priv.is_for_update_); + if (OB_UNLIKELY(!table_need_priv.is_for_update_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should not run this function without for update", K(ret), K(table_need_priv)); + } else if (OB_PRIV_HAS_ANY(session_priv.user_priv_set_, need_priv.priv_set_)) { + /* check ok */ + } else if (OB_FAIL(check_priv_db_or_(session_priv, need_priv, priv_mgr, tenant_id, user_id, pass))) { + LOG_WARN("failed to check priv db or", K(ret)); + } else if (!pass && OB_FAIL(check_priv_table_or_(need_priv, priv_mgr, tenant_id, user_id, pass))) { + LOG_WARN("fail to check priv table or", K(ret)); + } else if (!pass) { + ret = OB_ERR_NO_TABLE_PRIVILEGE; + const char *priv_name = "SELECT with locking clause"; + LOG_USER_ERROR(OB_ERR_NO_TABLE_PRIVILEGE, (int)strlen(priv_name), priv_name, + session_priv.user_name_.length(), session_priv.user_name_.ptr(), + session_priv.host_name_.length(), session_priv.host_name_.ptr(), + table_need_priv.table_.length(), table_need_priv.table_.ptr()); + } else { /* check ok */ } + return ret; +} + int ObSchemaGetterGuard::check_db_priv(const ObSessionPrivInfo &session_priv, const ObString &db, const ObPrivSet need_priv, diff --git a/src/share/schema/ob_schema_getter_guard.h b/src/share/schema/ob_schema_getter_guard.h index cb9fad185f..2b6936f326 100644 --- a/src/share/schema/ob_schema_getter_guard.h +++ b/src/share/schema/ob_schema_getter_guard.h @@ -1153,6 +1153,9 @@ private: int get_table_schemas_in_tenant_(const uint64_t tenant_id, const bool only_view_schema, common::ObIArray &table_schemas); + int check_single_table_priv_for_update_(const ObSessionPrivInfo &session_priv, + const ObNeedPriv &table_need_priv, + const ObPrivMgr &priv_mgr); private: common::ObArenaAllocator local_allocator_; ObMultiVersionSchemaService *schema_service_; diff --git a/src/share/schema/ob_schema_mgr_cache.cpp b/src/share/schema/ob_schema_mgr_cache.cpp index 2314f36010..667f1f46dc 100755 --- a/src/share/schema/ob_schema_mgr_cache.cpp +++ b/src/share/schema/ob_schema_mgr_cache.cpp @@ -500,15 +500,24 @@ int ObSchemaMgrCache::put(ObSchemaMgr *schema_mgr, LOG_INFO("put schema mgr", "schema version", NULL != schema_mgr ? schema_mgr->get_schema_version() : OB_INVALID_VERSION); - if (!check_inner_stat()) { + if (OB_UNLIKELY(!check_inner_stat())) { ret = OB_INNER_STAT_ERROR; - LOG_WARN("inner stat error", K(ret)); - } else if (NULL == schema_mgr) { + LOG_WARN("inner stat error", KR(ret)); + } else if (OB_ISNULL(schema_mgr)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(schema_mgr)); + LOG_WARN("invalid argument", KR(ret), KP(schema_mgr)); + } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == schema_mgr->get_tenant_id())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id", KR(ret), "tenant_id", schema_mgr->get_tenant_id()); } else { ObSchemaMgrItem *dst_item = NULL; bool is_stop = false; + const uint64_t tenant_id = schema_mgr->get_tenant_id(); + int64_t max_schema_slot_num = max_cached_num_; + omt::ObTenantConfigGuard tenant_config(OTC_MGR.get_tenant_config_with_lock(tenant_id)); + if (tenant_config.is_valid()) { + max_schema_slot_num = tenant_config->_max_schema_slot_num; + } TCWLockGuard guard(lock_); // 1. In order to avoid the repeated adjustment of the configuration item _max_schema_slot_num that may cause problems // that may be caused by the invisible version in the history, max_cached_num_ can only be increased during @@ -520,7 +529,7 @@ int ObSchemaMgrCache::put(ObSchemaMgr *schema_mgr, // and the schema_mgr memory management strategy is different from the schema refresh scenario. // In order to reduce unnecessary memory usage, a fixed number of 16 slots is also used. if (!ObSchemaService::g_liboblog_mode_ && FALLBACK != mode_) { - max_cached_num_ = max(max_cached_num_, GCONF._max_schema_slot_num); + max_cached_num_ = max(max_cached_num_, max_schema_slot_num); } int64_t target_pos = -1; for (int64_t i = 0; i < max_cached_num_ && !is_stop; ++i) { diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index ca61313dc8..9d9a315d91 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -8962,6 +8962,7 @@ int ObNeedPriv::deep_copy(const ObNeedPriv &other, common::ObIAllocator &allocat priv_level_ = other.priv_level_; priv_set_ = other.priv_set_; is_sys_table_ = other.is_sys_table_; + is_for_update_ = other.is_for_update_; if (OB_FAIL(ob_write_string(allocator, other.db_, db_))) { LOG_WARN("Fail to deep copy db", K_(db), K(ret)); } else if (OB_FAIL(ob_write_string(allocator, other.table_, table_))) { diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index 59314d31d0..2bf0773527 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -4638,14 +4638,14 @@ struct ObNeedPriv const common::ObString &table, ObPrivLevel priv_level, ObPrivSet priv_set, - const bool is_sys_table) - : db_(db), table_(table), priv_level_(priv_level), - priv_set_(priv_set), is_sys_table_(is_sys_table) + const bool is_sys_table, + const bool is_for_update = false) + : db_(db), table_(table), priv_level_(priv_level), priv_set_(priv_set), + is_sys_table_(is_sys_table), is_for_update_(is_for_update) { } ObNeedPriv() - : db_(), table_(), - priv_level_(OB_PRIV_INVALID_LEVEL), - priv_set_(0), is_sys_table_(false) + : db_(), table_(), priv_level_(OB_PRIV_INVALID_LEVEL), priv_set_(0), is_sys_table_(false), + is_for_update_(false) { } int deep_copy(const ObNeedPriv &other, common::ObIAllocator &allocator); common::ObString db_; @@ -4653,7 +4653,8 @@ struct ObNeedPriv ObPrivLevel priv_level_; ObPrivSet priv_set_; bool is_sys_table_; // May be used to represent the table of schema metadata - TO_STRING_KV(K_(db), K_(table), K_(priv_set), K_(priv_level), K_(is_sys_table)); + bool is_for_update_; + TO_STRING_KV(K_(db), K_(table), K_(priv_set), K_(priv_level), K_(is_sys_table), K_(is_for_update)); }; struct ObStmtNeedPrivs diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index e6c5f4c6a0..44b939a6a8 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -492,15 +492,20 @@ int ObSchemaUtils::construct_inner_table_schemas( } return ret; } + int ObSchemaUtils::try_check_parallel_ddl_schema_in_sync( const ObTimeoutCtx &ctx, const uint64_t tenant_id, - const int64_t schema_version, - const int64_t consensus_timeout) + const int64_t schema_version) { int ret = OB_SUCCESS; int64_t start_time = ObTimeUtility::current_time(); ObMultiVersionSchemaService *schema_service = NULL; + int64_t consensus_timeout = 30 * 1000 * 1000L; // 30s + omt::ObTenantConfigGuard tenant_config(OTC_MGR.get_tenant_config_with_lock(tenant_id)); + if (tenant_config.is_valid()) { + consensus_timeout = tenant_config->_wait_interval_after_parallel_ddl; + } if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || schema_version <= 0 || consensus_timeout < 0)) { @@ -525,7 +530,7 @@ int ObSchemaUtils::try_check_parallel_ddl_schema_in_sync( break; } else { if (REACH_TIME_INTERVAL(1000 * 1000L)) { // 1s - LOG_WARN("schema version not sync", K(tenant_id), + LOG_WARN("schema version not sync", K(tenant_id), K(consensus_timeout), K(refreshed_schema_version), K(consensus_schema_version), K(schema_version)); } ob_usleep(10 * 1000L); // 10ms diff --git a/src/share/schema/ob_schema_utils.h b/src/share/schema/ob_schema_utils.h index 2731dd78a5..7bc96aca35 100644 --- a/src/share/schema/ob_schema_utils.h +++ b/src/share/schema/ob_schema_utils.h @@ -156,8 +156,7 @@ public: static int try_check_parallel_ddl_schema_in_sync( const ObTimeoutCtx &ctx, const uint64_t tenant_id, - const int64_t schema_version, - const int64_t consensus_timeout); + const int64_t schema_version); private: static int get_tenant_variable(schema::ObSchemaGetterGuard &schema_guard, uint64_t tenant_id, diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 8fa6fbf2e7..870e480b8e 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1869,6 +1869,7 @@ int ObTableSchema::add_column(const ColumnType &column) && OB_FAIL(check_if_oracle_compat_mode(is_oracle_mode))) { // When deserialize column in physical restore, tenant id is wrong. We need to use lib::is_oracle_mode() to do this check. SHARE_SCHEMA_LOG(WARN, "check if_oracle_compat_mode failed", K(ret), K(tenant_id_), K(table_id_)); + is_oracle_mode = lib::is_oracle_mode(); ret = OB_SUCCESS; SHARE_SCHEMA_LOG(WARN, "replace error code to OB_SUCCESS, because tenant_id is invalid in physical restore", K(ret), K(tenant_id_), K(table_id_), K(is_oracle_mode)); diff --git a/src/share/stat/ob_hybrid_hist_estimator.cpp b/src/share/stat/ob_hybrid_hist_estimator.cpp index 440d3e085e..019f6a9f72 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.cpp +++ b/src/share/stat/ob_hybrid_hist_estimator.cpp @@ -598,7 +598,7 @@ int ObHybridHistograms::build_hybrid_hist(ObIArray &bucket_pairs, dynamic_size = true; // first bucket always contain only one values. following code will handle first value is // popular value or not. - if (bucket_pairs.at(0).is_pop_) { + if (bucket_pairs.at(0).is_pop_ || bucket_num == pop_count + 1) { bucket_size = (total_count - pop_freq) / (bucket_num - pop_count); } else { bucket_size = (total_count - pop_freq - bucket_pairs.at(0).ep_count_) @@ -678,7 +678,7 @@ int ObHybridHistograms::build_hybrid_hist(ObAggregateProcessor::HybridHistExtraR // first bucket always contain only one values. following code will handle first value is // popular value or not. BucketDesc *desc = reinterpret_cast(row->get_extra_payload()); - if (desc->is_pop_) { + if (desc->is_pop_ || bucket_num == pop_count + 1) { bucket_size = (total_count - pop_freq) / (bucket_num - pop_count); } else { bucket_size = (total_count - pop_freq - desc->ep_count_) / (bucket_num - pop_count - 1); diff --git a/src/share/table/ob_table.cpp b/src/share/table/ob_table.cpp index c82cf40296..96212a62c3 100644 --- a/src/share/table/ob_table.cpp +++ b/src/share/table/ob_table.cpp @@ -1716,4 +1716,21 @@ int ObTableAggregation::deep_copy(ObIAllocator &allocator, ObTableAggregation &d OB_SERIALIZE_MEMBER(ObTableAggregation, type_, - column_); \ No newline at end of file + column_); + +//////////////////////////////////////////////////////////////// +OB_SERIALIZE_MEMBER(ObTableMoveReplicaInfo, + table_id_, + schema_version_, + tablet_id_, + server_, + role_, + replica_type_, + part_renew_time_, + reserved_); + + + +OB_SERIALIZE_MEMBER(ObTableMoveResult, + replica_info_, + reserved_); \ No newline at end of file diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index 11fca1385d..e24fd40996 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -21,12 +21,14 @@ #include "lib/container/ob_se_array.h" #include "lib/hash/ob_hashmap.h" #include "lib/list/ob_dlist.h" +#include "lib/net/ob_addr.h" #include "common/ob_common_types.h" #include "common/ob_range.h" #include "rpc/obrpc/ob_poc_rpc_server.h" #include "share/table/ob_table_ttl_common.h" #include "common/rowkey/ob_rowkey.h" +#include "common/ob_role.h" namespace oceanbase { namespace common @@ -144,8 +146,8 @@ template class ObTableEntityFactory: public ObITableEntityFactory { public: - ObTableEntityFactory(const char *label = common::ObModIds::TABLE_PROC) - :alloc_(label) + ObTableEntityFactory(const char *label = common::ObModIds::TABLE_PROC, uint64_t tenant_id = OB_SERVER_TENANT_ID) + :alloc_(label, OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id) {} virtual ~ObTableEntityFactory(); virtual ObITableEntity *alloc() override; @@ -957,6 +959,59 @@ public: common::ObString end_rowkey_; }; +struct ObTableMoveReplicaInfo final +{ + OB_UNIS_VERSION(1); +public: + ObTableMoveReplicaInfo() + : table_id_(common::OB_INVALID_ID), + schema_version_(common::OB_INVALID_VERSION), + tablet_id_(common::ObTabletID::INVALID_TABLET_ID), + role_(common::ObRole::INVALID_ROLE), + replica_type_(common::ObReplicaType::REPLICA_TYPE_MAX), + part_renew_time_(0), + reserved_(0) + {} + virtual ~ObTableMoveReplicaInfo() {} + TO_STRING_KV(K_(table_id), + K_(schema_version), + K_(part_renew_time), + K_(tablet_id), + K_(server), + K_(role), + K_(replica_type), + K_(reserved)); + OB_INLINE void set_table_id(const uint64_t table_id) { table_id_ = table_id; } + OB_INLINE void set_schema_version(const uint64_t schema_version) { schema_version_ = schema_version; } + OB_INLINE void set_tablet_id(const common::ObTabletID &tablet_id) { tablet_id_ = tablet_id; } +public: + uint64_t table_id_; + uint64_t schema_version_; + common::ObTabletID tablet_id_; + common::ObAddr server_; + common::ObRole role_; + common::ObReplicaType replica_type_; + int64_t part_renew_time_; + uint64_t reserved_; +}; + +class ObTableMoveResult final +{ + OB_UNIS_VERSION(1); +public: + ObTableMoveResult() + : reserved_(0) + {} + virtual ~ObTableMoveResult() {} + TO_STRING_KV(K_(replica_info), + K_(reserved)); + + OB_INLINE ObTableMoveReplicaInfo& get_replica_info() { return replica_info_; } +private: + ObTableMoveReplicaInfo replica_info_; + uint64_t reserved_; +}; + } // end namespace table } // end namespace oceanbase diff --git a/src/share/table/ob_table_config_util.cpp b/src/share/table/ob_table_config_util.cpp new file mode 100644 index 0000000000..01fad2d63d --- /dev/null +++ b/src/share/table/ob_table_config_util.cpp @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE +#include "ob_table_config_util.h" +#include "lib/container/ob_se_array.h" +#include "share/config/ob_config_helper.h" +#include "share/config/ob_config_mode_name_def.h" +namespace oceanbase +{ +namespace common +{ +ObKVFeatureMode::ObKVFeatureMode(const uint8_t *values) +{ + if (OB_UNLIKELY(values == NULL)) { + value_ = 0; + is_valid_ = false; + } else { + value_ = values[0]; + is_valid_ = true; + } +} + +void ObKVFeatureMode::set_ttl_mode(uint8_t mode) +{ + is_valid_ = check_mode_valid(mode); + if (is_valid_) { + ttl_mode_ = mode; + } +} + +void ObKVFeatureMode::set_rerouting_mode(uint8_t mode) +{ + is_valid_ = check_mode_valid(mode); + if (is_valid_) { + rerouting_mode_ = mode; + } +} + +void ObKVFeatureMode::set_hotkey_mode(uint8_t mode) +{ + is_valid_ = check_mode_valid(mode); + if (is_valid_) { + hotkey_mode_ = mode; + } +} + +void ObKVFeatureMode::set_value(uint8_t value) +{ + if ((value & 0b11) == 0b11 || ((value >> 2) & 0b11) == 0b11 || ((value >> 4) & 0b11) == 0b11) { + is_valid_ = false; + } else { + is_valid_ = true; + value_ = value; + } +} + + +bool ObKVFeatureMode::is_ttl_enable() { + bool mode = MODE_DEFAULT_VAL_TTL; + if (ttl_mode_ == ObKvFeatureModeParser::MODE_ON) { + mode = true; + } else if (ttl_mode_ == ObKvFeatureModeParser::MODE_OFF) { + mode = false; + } + return mode; +} + +bool ObKVFeatureMode::is_rerouting_enable() { + bool mode = MODE_DEFAULT_VAL_REROUTING; + if (rerouting_mode_ == ObKvFeatureModeParser::MODE_ON) { + mode = true; + } else if (rerouting_mode_ == ObKvFeatureModeParser::MODE_ON) { + mode = false; + } + return mode; +} + +bool ObKVFeatureMode::is_hotkey_enable() { + bool mode = MODE_DEFAULT_VAL_HOTKEY; + if (hotkey_mode_ == ObKvFeatureModeParser::MODE_ON) { + mode = true; + } else if (hotkey_mode_ == ObKvFeatureModeParser::MODE_ON) { + mode = false; + } + return mode; +} + +bool ObKVFeatureModeUitl::is_obkv_feature_enable(ObKVFeatureType feat_type) +{ + bool bret = false; + ObKVFeatureMode cfg(GCONF._obkv_feature_mode); + if (!cfg.is_valid()) { + bret = false; + OB_LOG_RET(WARN, OB_INVALID_ARGUMENT, "unexpected, cfg is invalid"); + } else { + switch (feat_type) { + case ObKVFeatureType::TTL: + bret = cfg.is_ttl_enable(); + break; + case ObKVFeatureType::REROUTING: + bret = cfg.is_rerouting_enable(); + break; + case ObKVFeatureType::HOTKEY: + bret = cfg.is_hotkey_enable(); + break; + default: + OB_LOG_RET(WARN, OB_INVALID_ARGUMENT, "unexpected feature type", K(feat_type)); + break; + } + } + return bret; +} + +bool ObKVFeatureModeUitl::is_ttl_enable() +{ + return is_obkv_feature_enable(ObKVFeatureType::TTL); +} + +bool ObKVFeatureModeUitl::is_rerouting_enable() +{ + return is_obkv_feature_enable(ObKVFeatureType::REROUTING); +} + +bool ObKVFeatureModeUitl::is_hotkey_enable() +{ + return is_obkv_feature_enable(ObKVFeatureType::HOTKEY); +} + +} +} \ No newline at end of file diff --git a/src/share/table/ob_table_config_util.h b/src/share/table/ob_table_config_util.h new file mode 100644 index 0000000000..a41aec3c8f --- /dev/null +++ b/src/share/table/ob_table_config_util.h @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_OB_TABLE_CONFIG_UTIL_H_ +#define OCEANBASE_OB_TABLE_CONFIG_UTIL_H_ +#include "lib/utility/ob_macro_utils.h" +#include "lib/utility/ob_print_utils.h" +#include "share/config/ob_server_config.h" +namespace oceanbase +{ +namespace common +{ +enum ObKVFeatureType { + INVALIDTYPE, + TTL, + REROUTING, + HOTKEY, + MAXTYPE +}; + +class ObKVFeatureMode final +{ +public: + ObKVFeatureMode(): is_valid_(false), value_(0) {} + ObKVFeatureMode(const uint8_t *values); + bool is_valid() { return is_valid_; } + bool check_mode_valid(uint8_t mode) { return mode > 2 ? false : true; } + bool is_ttl_enable(); + bool is_rerouting_enable(); + bool is_hotkey_enable(); + void set_ttl_mode(uint8_t mode); + void set_rerouting_mode(uint8_t mode); + void set_hotkey_mode(uint8_t mode); + void set_value(uint8_t value); + int8_t get_value() const { return value_; } +private: + bool is_valid_; + union { + uint8_t value_; + struct { + uint8_t ttl_mode_ : 2; + uint8_t rerouting_mode_ : 2; + uint8_t hotkey_mode_ : 2; + uint8_t reserver_mode_ :2; + }; + }; +private: + DISALLOW_COPY_AND_ASSIGN(ObKVFeatureMode); +}; + +class ObKVFeatureModeUitl +{ +public: + static bool is_obkv_feature_enable(ObKVFeatureType feat_type); + static bool is_ttl_enable(); + static bool is_rerouting_enable(); + static bool is_hotkey_enable(); +}; + +} +} +#endif \ No newline at end of file diff --git a/src/share/tablet/ob_tablet_to_ls_operator.cpp b/src/share/tablet/ob_tablet_to_ls_operator.cpp index 485f130a9c..50522517a1 100644 --- a/src/share/tablet/ob_tablet_to_ls_operator.cpp +++ b/src/share/tablet/ob_tablet_to_ls_operator.cpp @@ -420,7 +420,8 @@ int ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( const int64_t old_transfer_seq, const ObLSID &old_ls_id, const int64_t new_transfer_seq, - const ObLSID &new_ls_id) + const ObLSID &new_ls_id, + const int32_t group_id) { int ret = OB_SUCCESS; ObSqlString sql; @@ -431,10 +432,11 @@ int ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( || !tablet_id.is_valid() || !old_ls_id.is_valid() || !new_ls_id.is_valid() - || old_ls_id == new_ls_id)) { + || old_ls_id == new_ls_id + || group_id < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), - K(tablet_id), K(old_transfer_seq), K(new_transfer_seq), K(old_ls_id), K(new_ls_id)); + K(tablet_id), K(old_transfer_seq), K(new_transfer_seq), K(old_ls_id), K(new_ls_id), K(group_id)); } else if (OB_FAIL(sql.append_fmt( "UPDATE %s SET transfer_seq = %ld, ls_id = %ld " "WHERE tablet_id = %lu AND transfer_seq = %ld AND ls_id = %ld", @@ -448,7 +450,7 @@ int ObTabletToLSTableOperator::update_ls_id_and_transfer_seq( K(tablet_id), K(old_ls_id), K(new_ls_id), K(old_transfer_seq), K(new_transfer_seq)); } else { int64_t affected_rows = 0; - if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), group_id, affected_rows))) { LOG_WARN("fail to write sql", KR(ret), K(sql), K(tenant_id)); } else if (0 == affected_rows) { ret = OB_ENTRY_NOT_EXIST; diff --git a/src/share/tablet/ob_tablet_to_ls_operator.h b/src/share/tablet/ob_tablet_to_ls_operator.h index 1815549607..d33d4cf7c1 100644 --- a/src/share/tablet/ob_tablet_to_ls_operator.h +++ b/src/share/tablet/ob_tablet_to_ls_operator.h @@ -116,6 +116,7 @@ public: // @param[in] old_ls_id old LS ID // @param[in] new_transfer_seq new Transfer Sequence // @param[in] new_ls_id new LS ID + // @param[in] group_id rpc queue id // // @ret OB_SUCCESS the updation is successful // @ret OB_ENTRY_NOT_EXIST affected rows = 0, @@ -130,7 +131,8 @@ public: const int64_t old_transfer_seq, const ObLSID &old_ls_id, const int64_t new_transfer_seq, - const ObLSID &new_ls_id); + const ObLSID &new_ls_id, + const int32_t group_id); // Get rows from __all_tablet_to_ls according to ObTableIDs // // @param [in] sql_proxy, ObMySQLProxy or ObMySQLTransaction diff --git a/src/share/tablet/ob_tenant_tablet_to_ls_map.h b/src/share/tablet/ob_tenant_tablet_to_ls_map.h index 77b27235a4..a6e6985345 100644 --- a/src/share/tablet/ob_tenant_tablet_to_ls_map.h +++ b/src/share/tablet/ob_tenant_tablet_to_ls_map.h @@ -40,9 +40,10 @@ public: ~ObTenantTabletToLSMap() {} int init(const int64_t bucket_num = 4096, - const lib::ObLabel label = lib::ObLabel("TenantTabletToLSMap")) + const lib::ObLabel label = lib::ObLabel("TenantTabletToLSMap"), + const uint64_t tenant_id = OB_SERVER_TENANT_ID) { - return map_.create(bucket_num, label); + return map_.create(bucket_num, label, ObModIds::OB_HASH_NODE, tenant_id); } void destroy() { map_.destroy(); } diff --git a/src/share/transfer/ob_transfer_info.cpp b/src/share/transfer/ob_transfer_info.cpp index f4292c61a9..c28aa64ba8 100644 --- a/src/share/transfer/ob_transfer_info.cpp +++ b/src/share/transfer/ob_transfer_info.cpp @@ -313,6 +313,7 @@ static const char* TRANSFER_TASK_COMMENT_ARRAY[] = "Task completed as no valid partition", "Task canceled", "Unable to process task due to transaction timeout", + "Unable to process task due to inactive server in member list", "Unknow"/*MAX_COMMENT*/ }; diff --git a/src/share/transfer/ob_transfer_info.h b/src/share/transfer/ob_transfer_info.h index 240b60ea61..454b03551f 100644 --- a/src/share/transfer/ob_transfer_info.h +++ b/src/share/transfer/ob_transfer_info.h @@ -213,6 +213,7 @@ enum ObTransferTaskComment TASK_COMPLETED_AS_NO_VALID_PARTITION = 2, TASK_CANCELED = 3, TRANSACTION_TIMEOUT = 4, + INACTIVE_SERVER_IN_MEMBER_LIST = 5, MAX_COMMENT }; diff --git a/src/share/transfer/ob_transfer_task_operator.cpp b/src/share/transfer/ob_transfer_task_operator.cpp index d30436ed50..4eabd062f8 100644 --- a/src/share/transfer/ob_transfer_task_operator.cpp +++ b/src/share/transfer/ob_transfer_task_operator.cpp @@ -30,19 +30,20 @@ int ObTransferTaskOperator::get( const uint64_t tenant_id, const ObTransferTaskID task_id, const bool for_update, - ObTransferTask &task) + ObTransferTask &task, + const int32_t group_id) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !task_id.is_valid())) { + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !task_id.is_valid() || group_id < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id), K(for_update)); + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id), K(for_update), K(group_id)); } else { ObSqlString sql; SMART_VAR(ObISQLClient::ReadResult, result) { if (OB_FAIL(sql.assign_fmt("SELECT * FROM %s WHERE task_id = %ld%s", OB_ALL_TRANSFER_TASK_TNAME, task_id.id(), for_update ? " FOR UPDATE" : ""))) { LOG_WARN("fail to assign sql", KR(ret), K(task_id), K(for_update)); - } else if (OB_FAIL(sql_proxy.read(result, tenant_id, sql.ptr()))) { + } else if (OB_FAIL(sql_proxy.read(result, tenant_id, sql.ptr(), group_id))) { LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); } else if (OB_ISNULL(result.get_result())) { ret = OB_ERR_UNEXPECTED; @@ -171,14 +172,15 @@ int ObTransferTaskOperator::get_by_src_ls( common::ObISQLClient &sql_proxy, const uint64_t tenant_id, const ObLSID &src_ls, - ObTransferTask &task) + ObTransferTask &task, + const int32_t group_id) { int ret = OB_SUCCESS; const bool is_src_ls = true; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !src_ls.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(src_ls)); - } else if (OB_FAIL(get_by_ls_id_(sql_proxy, tenant_id, src_ls, is_src_ls, task))) { + } else if (OB_FAIL(get_by_ls_id_(sql_proxy, tenant_id, src_ls, is_src_ls, group_id, task))) { LOG_WARN("failed to get by ls id", K(ret), K(tenant_id), K(src_ls)); } return ret; @@ -188,14 +190,15 @@ int ObTransferTaskOperator::get_by_dest_ls( common::ObISQLClient &sql_proxy, const uint64_t tenant_id, const ObLSID &dest_ls, - ObTransferTask &task) + ObTransferTask &task, + const int32_t group_id) { int ret = OB_SUCCESS; const bool is_src_ls = false; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !dest_ls.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(dest_ls)); - } else if (OB_FAIL(get_by_ls_id_(sql_proxy, tenant_id, dest_ls, is_src_ls, task))) { + } else if (OB_FAIL(get_by_ls_id_(sql_proxy, tenant_id, dest_ls, is_src_ls, group_id, task))) { LOG_WARN("failed to get by ls id", K(ret), K(tenant_id), K(dest_ls)); } return ret; @@ -395,7 +398,8 @@ int ObTransferTaskOperator::finish_task( const ObTransferStatus &old_status, const ObTransferStatus &new_status, const int result, - const ObTransferTaskComment &comment) + const ObTransferTaskComment &comment, + const int32_t group_id) { int ret = OB_SUCCESS; bool can_change = false; @@ -432,7 +436,7 @@ int ObTransferTaskOperator::finish_task( LOG_WARN("fail to splice update sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(sql.append_fmt(" AND status='%s'", old_status.str()))) { LOG_WARN("fail to append fmt", KR(ret), K(tenant_id), K(sql), K(old_status)); - } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), group_id, affected_rows))) { LOG_WARN("fail to write sql", KR(ret), K(tenant_id), K(sql), K(affected_rows)); } else if (OB_UNLIKELY(1 != affected_rows)) { ret = OB_STATE_NOT_MATCH; @@ -552,14 +556,16 @@ int ObTransferTaskOperator::update_status_and_result( const ObTransferTaskID task_id, const ObTransferStatus &old_status, const ObTransferStatus &new_status, - const int result) + const int result, + const int32_t group_id) { int ret = OB_SUCCESS; bool can_change = false; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) - || !task_id.is_valid())) { + || !task_id.is_valid() + || group_id < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id)); + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(task_id), K(group_id)); } else if (OB_FAIL(ObTransferStatusHelper::check_can_change_status( old_status, new_status, @@ -584,7 +590,7 @@ int ObTransferTaskOperator::update_status_and_result( LOG_WARN("fail to splice update sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(sql.append_fmt(" AND status='%s'", old_status.str()))) { LOG_WARN("fail to append fmt", KR(ret), K(tenant_id), K(sql), K(old_status)); - } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), group_id, affected_rows))) { LOG_WARN("fail to write sql", KR(ret), K(tenant_id), K(sql), K(affected_rows)); } else if (OB_UNLIKELY(1 != affected_rows && 0 != affected_rows)) { ret = OB_STATE_NOT_MATCH; @@ -603,16 +609,18 @@ int ObTransferTaskOperator::update_start_scn( const uint64_t tenant_id, const ObTransferTaskID task_id, const ObTransferStatus &old_status, - const share::SCN &start_scn) + const share::SCN &start_scn, + const int32_t group_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !task_id.is_valid() || !old_status.is_valid() - || !start_scn.is_valid())) { + || !start_scn.is_valid() + || group_id < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), - K(tenant_id), K(task_id), K(old_status), K(start_scn)); + K(tenant_id), K(task_id), K(old_status), K(start_scn), K(group_id)); } else { ObSqlString sql; ObDMLSqlSplicer dml_splicer; @@ -627,7 +635,7 @@ int ObTransferTaskOperator::update_start_scn( LOG_WARN("fail to splice update sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(sql.append_fmt(" AND status='%s'", old_status.str()))) { LOG_WARN("fail to append fmt", KR(ret), K(tenant_id), K(sql), K(old_status)); - } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), group_id, affected_rows))) { LOG_WARN("fail to write sql", KR(ret), K(tenant_id), K(sql), K(affected_rows)); } else if (OB_UNLIKELY(1 != affected_rows && 0 != affected_rows)) { ret = OB_STATE_NOT_MATCH; @@ -645,16 +653,18 @@ int ObTransferTaskOperator::update_finish_scn( const uint64_t tenant_id, const ObTransferTaskID task_id, const ObTransferStatus &old_status, - const share::SCN &finish_scn) + const share::SCN &finish_scn, + const int32_t group_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !task_id.is_valid() || !old_status.is_valid() - || !finish_scn.is_valid())) { + || !finish_scn.is_valid() + || group_id < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), - K(tenant_id), K(task_id), K(old_status), K(finish_scn)); + K(tenant_id), K(task_id), K(old_status), K(finish_scn), K(group_id)); } else { ObSqlString sql; ObDMLSqlSplicer dml_splicer; @@ -669,7 +679,7 @@ int ObTransferTaskOperator::update_finish_scn( LOG_WARN("fail to splice update sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(sql.append_fmt(" AND status='%s'", old_status.str()))) { LOG_WARN("fail to append fmt", KR(ret), K(tenant_id), K(sql), K(old_status)); - } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_proxy.write(tenant_id, sql.ptr(), group_id, affected_rows))) { LOG_WARN("fail to write sql", KR(ret), K(tenant_id), K(sql), K(affected_rows)); } else if (OB_UNLIKELY(1 != affected_rows)) { ret = OB_STATE_NOT_MATCH; @@ -687,12 +697,13 @@ int ObTransferTaskOperator::get_by_ls_id_( const uint64_t tenant_id, const ObLSID &ls_id, const bool is_src_ls, + const int32_t group_id, ObTransferTask &task) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid())) { + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid() || group_id < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id)); + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(group_id)); } else { ObSqlString sql; SMART_VAR(ObISQLClient::ReadResult, result) { @@ -708,7 +719,7 @@ int ObTransferTaskOperator::get_by_ls_id_( } } - if (FAILEDx(sql_proxy.read(result, tenant_id, sql.ptr()))) { + if (FAILEDx(sql_proxy.read(result, tenant_id, sql.ptr(), group_id))) { LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); } else if (OB_ISNULL(result.get_result())) { ret = OB_ERR_UNEXPECTED; diff --git a/src/share/transfer/ob_transfer_task_operator.h b/src/share/transfer/ob_transfer_task_operator.h index e2a9614e26..de17c19936 100644 --- a/src/share/transfer/ob_transfer_task_operator.h +++ b/src/share/transfer/ob_transfer_task_operator.h @@ -46,6 +46,7 @@ public: * @param [in] task_id: target task_id * @param [in] for_update: select for update * @param [out] task: transfer task + * @param [in] group_id: rpc queue id * @return * - OB_ENTRY_NOT_EXIST: not found * - OB_SUCCESS: successful @@ -56,7 +57,8 @@ public: const uint64_t tenant_id, const ObTransferTaskID task_id, const bool for_update, - ObTransferTask &task); + ObTransferTask &task, + const int32_t group_id); /* * get a transfer task by task_id with create_time and finish_time @@ -123,6 +125,7 @@ public: * @param [in] tenant_id: target tenant_id * @param [in] src_ls: src ls_id * @param [out] task: transfer task + * @param [in] group_id: rpc queue id * @return * - OB_ENTRY_NOT_EXIST: not found * - OB_ERR_UNEXPECTED: more than 1 transfer task on a ls @@ -133,7 +136,8 @@ public: common::ObISQLClient &sql_proxy, const uint64_t tenant_id, const ObLSID &src_ls, - ObTransferTask &task); + ObTransferTask &task, + const int32_t group_id); /* * get transfer task by dest ls (there is no more than 1 transfer task on a ls) @@ -142,6 +146,7 @@ public: * @param [in] tenant_id: target tenant_id * @param [in] dest_ls: destination ls_id * @param [out] task: transfer task + * @param [in] group_id: rpc queue id * @return * - OB_ENTRY_NOT_EXIST: not found * - OB_ERR_UNEXPECTED: more than 1 transfer task on a ls @@ -152,7 +157,8 @@ public: common::ObISQLClient &sql_proxy, const uint64_t tenant_id, const ObLSID &dest_ls, - ObTransferTask &task); + ObTransferTask &task, + const int32_t group_id); /* * insert task @@ -211,6 +217,7 @@ public: * @param [in] new_status: new task status * @param [in] result: return code for the transfer process * @param [in] comment: information for task finish + * @param [in] group_id: rpc queue id * @return * - OB_STATE_NOT_MATCH: task not found or task status mismatch * - OB_SUCCESS: successful @@ -223,7 +230,8 @@ public: const ObTransferStatus &old_status, const ObTransferStatus &new_status, const int result, - const ObTransferTaskComment &comment); + const ObTransferTaskComment &comment, + const int32_t group_id); /* * finish task from INIT status to COMPLETED when part_list is all unreachable @@ -280,6 +288,7 @@ public: * @param [in] old_status: old task status * @param [in] new_status: new task status * @param [in] result: task result + * @param [in] group_id: rpc queue id * @return * - OB_STATE_NOT_MATCH: task not found or task status mismatch * - OB_SUCCESS: successful @@ -291,7 +300,8 @@ public: const ObTransferTaskID task_id, const ObTransferStatus &old_status, const ObTransferStatus &new_status, - const int result); + const int result, + const int32_t group_id); /* @@ -302,6 +312,7 @@ public: * @param [in] task_id: target task_id * @param [in] old_status: old task status * @param [in] start_scn: start scn + * @param [in] group_id: rpc queue id, default is 0 * @return * - OB_STATE_NOT_MATCH: task not found or task status mismatch * - OB_SUCCESS: successful @@ -312,7 +323,8 @@ public: const uint64_t tenant_id, const ObTransferTaskID task_id, const ObTransferStatus &old_status, - const share::SCN &start_scn); + const share::SCN &start_scn, + const int32_t group_id); /* * update finish_scn @@ -322,6 +334,7 @@ public: * @param [in] task_id: target task_id * @param [in] old_status: old task status * @param [in] finish_scn: finish scn + * @param [in] group_id: rpc queue id, default is 0 * @return * - OB_STATE_NOT_MATCH: task not found or task status mismatch * - OB_SUCCESS: successful @@ -332,7 +345,8 @@ public: const uint64_t tenant_id, const ObTransferTaskID, const ObTransferStatus &old_status, - const share::SCN &finish_scn); + const share::SCN &finish_scn, + const int32_t group_id); /* * record transfer task in __all_transfer_task_history @@ -415,6 +429,7 @@ private: const uint64_t tenant_id, const ObLSID &ls_id, const bool is_src_ls, + const int32_t group_id, ObTransferTask &task); static int construct_transfer_tasks_( common::sqlclient::ObMySQLResult &res, diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 95f2a0819f..1d5c9e5bee 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -64,6 +64,7 @@ ob_set_subtarget(ob_sql das das/ob_das_task_result.cpp das/ob_das_spatial_index_lookup_op.cpp das/ob_das_retry_ctrl.cpp + das/ob_das_simple_op.cpp ) ob_set_subtarget(ob_sql dtl diff --git a/src/sql/code_generator/ob_expr_generator_impl.cpp b/src/sql/code_generator/ob_expr_generator_impl.cpp index 5e0a61448c..9b38166e61 100644 --- a/src/sql/code_generator/ob_expr_generator_impl.cpp +++ b/src/sql/code_generator/ob_expr_generator_impl.cpp @@ -676,7 +676,7 @@ inline int ObExprGeneratorImpl::visit_regex_expr(ObOpRawExpr &expr, ObExprRegexp ObIArray ¶m_exprs = expr.get_param_exprs(); if (OB_ISNULL(regexp_op)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("regexpr_op is NULL"); + LOG_WARN("regexp_op is NULL"); } else if (OB_UNLIKELY(2 != param_exprs.count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("regex op should have 2 arguments", K(param_exprs.count())); @@ -779,7 +779,7 @@ inline int ObExprGeneratorImpl::visit_in_expr(ObOpRawExpr &expr, ObExprInOrNotIn } } //for row_type in left_param of EXPR IN - //if min_cluster_version < 3.1, do not check params can use hash optimizition + //if min_cluster_version < 3.1, do not check params can use hash optimization bool param_all_const = true; bool param_all_same_type = true; bool param_all_same_cs_type = true; @@ -875,7 +875,7 @@ inline int ObExprGeneratorImpl::visit_in_expr(ObOpRawExpr &expr, ObExprInOrNotIn ? param_all_same_cs_type : (param_all_same_cs_type &= param_all_same_cs_level)); in_op->set_param_is_ext_type_oracle(param_all_is_ext); - //now only support c1 in (1,2,3,4,5...) to vecotrized + //now only support c1 in (1,2,3,4,5...) to vectorized if (param_all_can_vectorize && expr.get_param_expr(0)->is_vectorize_result()) { in_op->set_param_can_vectorized(); } @@ -893,7 +893,7 @@ int ObExprGeneratorImpl::visit_decode_expr(ObNonTerminalRawExpr &expr, ObExprOra } else { // decode(col, cond1, val1, cond2, val2, ......, condN, valN, def_val) // cmp type of decode is always equal to cond1, or varchar if cond1 is const null. - // res type of decode is always euqal to val1, or varchar if val1 is const null. + // res type of decode is always equal to val1, or varchar if val1 is const null. bool cond_all_same_meta = true; bool val_all_same_meta = true; int64_t param_count = expr.get_param_count(); @@ -1454,7 +1454,7 @@ int ObExprGeneratorImpl::visit_pl_integer_checker_expr(ObOpRawExpr &expr, ObPLIntegerCheckerRawExpr &pl_expr = static_cast(expr); if (OB_ISNULL(checker)) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("pl interger checker operator is null", K(ret)); + LOG_ERROR("pl integer checker operator is null", K(ret)); } else { checker->set_pl_integer_type(pl_expr.get_pl_integer_type()); checker->set_range(pl_expr.get_lower(), pl_expr.get_upper()); @@ -1851,8 +1851,8 @@ int ObExprGeneratorImpl::visit(ObAggFunRawExpr &expr) } // end for FOREACH(e, columnlized_exprs) { - if ((*e)->has_flag(IS_COLUMNLIZED)) { - (*e)->clear_flag(IS_COLUMNLIZED); + if (OB_FAIL((*e)->clear_flag(IS_COLUMNLIZED))) { + LOG_WARN("failed to clear flag", K(ret)); } } } @@ -2107,9 +2107,10 @@ int ObExprGeneratorImpl::generate_expr_operator(ObRawExpr &raw_expr, fetcher.op_ = NULL; ObItemType type = raw_expr.get_expr_type(); if (IS_EXPR_OP(type) && !IS_AGGR_FUN(type)) { - raw_expr.clear_flag(IS_COLUMNLIZED); - // no expr operator for set expr - if (!(type > T_OP_SET && type <= T_OP_EXCEPT)) { + if (OB_FAIL(raw_expr.clear_flag(IS_COLUMNLIZED))) { + LOG_WARN("failed to clear flag", K(ret)); + } else if (!(type > T_OP_SET && type <= T_OP_EXCEPT)) { + // no expr operator for set expr OZ(raw_expr.do_visit(*this)); } } diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 471df021a3..6c617a4ee9 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -479,8 +479,8 @@ int ObStaticEngineCG::clear_all_exprs_specific_flag( if (OB_ISNULL(exprs.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr is null", K(ret), K(i), K(exprs)); - } else { - exprs.at(i)->clear_flag(flag); + } else if (OB_FAIL(exprs.at(i)->clear_flag(flag))) { + LOG_WARN("failed to clear flag", K(ret)); } } @@ -693,7 +693,7 @@ int ObStaticEngineCG::check_vectorize_supported(bool &support, support = false; stop_checking = true; } - LOG_DEBUG("check_vectorie_supported", K(disable_vectorize), K(support), K(stop_checking), + LOG_DEBUG("check_vectorize_supported", K(disable_vectorize), K(support), K(stop_checking), K(op->get_num_of_child())); // continue searching until found an operator with vectorization explicitly disabled for (int64_t i = 0; !stop_checking && OB_SUCC(ret) && i < op->get_num_of_child(); i++) { @@ -709,7 +709,7 @@ int ObStaticEngineCG::check_vectorize_supported(bool &support, // 从raw expr中获取rt_expr，并将raw expr push到cur_op_exprs_中 // -// 设置operater的rt expr, 从raw expr中获取时，均需要通过该接口， +// 设置operator的rt expr, 从raw expr中获取时，均需要通过该接口， // 其中ObStaticEngineExprCG::generate_rt_expr是ObRawExpr的友元函数， 可直接访问ObRawExpr中rt expr， // // 为什么不是ObRawExpr中直接提供访问rt expr的接口给外部使用， 而是用友元函数的方式处理？ @@ -2837,7 +2837,7 @@ int ObStaticEngineCG::generate_spec(ObLogGranuleIterator &op, ObGranuleIteratorS "desc", op.desc_order(), "flags", op.get_gi_flags(), "tsc_ids", spec.pw_dml_tsc_ids_, - "repart_prunint_idx", spec.repart_pruning_tsc_idx_, + "repart_pruning_tsc_idx", spec.repart_pruning_tsc_idx_, K(pwj_gi), K(enable_repart_pruning)); return ret; } @@ -3300,7 +3300,7 @@ int ObStaticEngineCG::generate_spec(ObLogExchange &op, ObPxRepartTransmitSpec &s // repartition_exprs_ only use by null aware anti join // now just support single join key // either repart_keys or repart_sub_keys exists join key - // so we can generate from one of them driectly + // so we can generate from one of them directly if (op.get_repart_keys().count() > 0) { if (OB_FAIL(generate_rt_exprs(op.get_repart_keys(), spec.repartition_exprs_))) { LOG_WARN("failed to generate repart exprs", K(ret)); @@ -3792,7 +3792,7 @@ int ObStaticEngineCG::generate_dist_aggr_group(ObLogGroupBy &op, ObGroupBySpec & const ObDistinctAggrBatch &distinct_batch = op.get_distinct_aggr_batch().at(i); aggr_group_idx += distinct_batch.mocked_aggrs_.count(); if (OB_FAIL(spec.dist_aggr_group_idxes_.push_back(aggr_group_idx))) { - LOG_WARN("failed to push back aggr group aggr inndex", K(ret)); + LOG_WARN("failed to push back aggr group aggr index", K(ret)); } } // end for return ret; @@ -3979,7 +3979,7 @@ int ObStaticEngineCG::generate_normal_tsc(ObLogTableScan &op, ObTableScanSpec &s } root = root->and_next_; } - // TODO @baixian.zr the above optimization is overrided by ObTscCgService::generate_tsc_ctdef before this commit + // TODO @baixian.zr the above optimization is overrode by ObTscCgService::generate_tsc_ctdef before this commit // but after the deep copy of pre_query_range_ is removed in ObTscCgService::generate_tsc_ctdef, // error is returned in such sql 'set global x=y', should fix this; // spec.tsc_ctdef_.pre_query_range_.set_is_equal_and(is_equal_and); @@ -4654,7 +4654,7 @@ int ObStaticEngineCG::generate_join_spec(ObLogJoin &op, ObJoinSpec &spec) for (auto l_output_idx = 0; OB_SUCC(ret) && l_output_idx < nlj.get_left()->output_.count(); l_output_idx++) { - // check if left child expr appears in other_condtion + // check if left child expr appears in other_condition bool appears_in_cond = false; if (OB_FAIL(cond->contain_expr( nlj.get_left()->output_.at(l_output_idx), appears_in_cond))) { @@ -4814,7 +4814,7 @@ int ObStaticEngineCG::generate_join_spec(ObLogJoin &op, ObJoinSpec &spec) //} else if (exec_params.count() == 0) { //// Do nothing //} else if (exec_params.count() != 1) { - //// Only one ? expr for all level expr in connent by clause. + //// Only one ? expr for all level expr in connect by clause. //ret = OB_ERR_UNEXPECTED; //LOG_WARN("unexpected exec params count in connect by", K(exec_params.count()), K(ret)); //} else if (OB_FAIL(nlj_op->init_exec_param_count(exec_params.count()))) { @@ -5052,7 +5052,7 @@ int ObStaticEngineCG::recursive_get_column_expr(const ObColumnRefRawExpr *&colum LOG_WARN("get unexpected null", K(ret)); } else if ((table_item->is_generated_table() || table_item->is_temp_table()) && OB_FAIL(recursive_get_column_expr(inner_column, *table_item))) { - LOG_WARN("faield to recursive get column expr", K(ret)); + LOG_WARN("failed to recursive get column expr", K(ret)); } else { column = inner_column; } @@ -5383,7 +5383,7 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, ObPxMultiPartSSTableInsertS const ObExecContext *exec_ctx = nullptr; ObLogPlan *log_plan = nullptr; if (OB_FAIL(generate_spec(op, static_cast(spec), in_root_job))) { - LOG_WARN("generge multi part sstable insert spec failed", K(ret)); + LOG_WARN("generate multi part sstable insert spec failed", K(ret)); } else if (OB_ISNULL(log_plan = op.get_plan()) || OB_ISNULL(exec_ctx = log_plan->get_optimizer_context().get_exec_ctx())) { ret = OB_INVALID_ARGUMENT; @@ -6621,7 +6621,7 @@ int ObStaticEngineCG::generate_insert_all_with_das(ObLogInsertAll &op, ObTableIn OB_ISNULL(phy_plan_) || OB_UNLIKELY(op.get_table_list().count() != op.get_insert_all_table_info()->count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpeceted error", K(ret), K(phy_plan_), K(op.get_insert_all_table_info())); + LOG_WARN("get unexpected error", K(ret), K(phy_plan_), K(op.get_insert_all_table_info())); } else if (OB_FAIL(spec.ins_ctdefs_.allocate_array(phy_plan_->get_allocator(), op.get_table_list().count()))) { LOG_WARN("allocate insert ctdef array failed", K(ret)); @@ -6723,7 +6723,7 @@ int ObStaticEngineCG::generate_spec(ObLogStatCollector &op, } } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpcted type", K(spec.type_)); + LOG_WARN("unexpected type", K(spec.type_)); } return ret; } @@ -7003,7 +7003,7 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical } if (OB_SUCC(ret) && metas.count() > 0) { if (OB_FAIL(phy_plan.get_encrypt_meta_array().assign(metas))) { - LOG_WARN("fail to assgin encrypt meta", K(ret)); + LOG_WARN("fail to assign encrypt meta", K(ret)); } } } @@ -7546,7 +7546,7 @@ int ObStaticEngineCG::add_output_datum_check_flag(ObOpSpec &spec) } else { // Because the Unpivot will affect the output datum of the SubplanScan, // which is an by designed case, we need to set the SubplanScan operator - // to not check the ouput datum. + // to not check the output datum. spec.get_child(0)->need_check_output_datum_ = false; } } else { diff --git a/src/sql/code_generator/ob_static_engine_expr_cg.cpp b/src/sql/code_generator/ob_static_engine_expr_cg.cpp index bc329f9c7d..c5bac57a29 100644 --- a/src/sql/code_generator/ob_static_engine_expr_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_expr_cg.cpp @@ -229,7 +229,7 @@ int ObStaticEngineExprCG::cg_exprs(const ObIArray &raw_exprs, // cg_expr_parents must be after cg_expr_by_operator, // because cg_expr_by_operator may replace rt_expr.args_ } else if (OB_FAIL(cg_expr_parents(raw_exprs))) { - LOG_WARN("fail to init expr parenets", K(ret), K(raw_exprs)); + LOG_WARN("fail to init expr parents", K(ret), K(raw_exprs)); // init res_buf_len_, frame_idx_, datum_off_, res_buf_off_ } else if (OB_FAIL(cg_all_frame_layout(raw_exprs, expr_info))) { LOG_WARN("fail to init expr data layout", K(ret), K(raw_exprs)); @@ -701,7 +701,7 @@ int ObStaticEngineExprCG::cg_frame_layout_vector_version(const ObIArray &exprs, { int ret = OB_SUCCESS; if (continuous_datum) { - // Layout1: Frame is seperated from meta part and data part. + // Layout1: Frame is separated from meta part and data part. // Meta part(datum header) are allocated continuously. // Reserved data/buf part are allocated continuously // Frame layouts: @@ -987,7 +987,7 @@ int ObStaticEngineExprCG::arrange_datums_data(ObIArray &exprs, CK((total_header_len + expr_data_offset) == frame.frame_size_); } else { // Layout2: Frame is seperated by exprs - // All data(metas + reserved data/buf) within one expr are allocated continously + // All data(metas + reserved data/buf) within one expr are allocated continuously // Frame layouts: // +--------------------------------+ // | Datums in Expr1 | @@ -1271,9 +1271,12 @@ int ObStaticEngineExprCG::alloc_so_check_exprs(const ObIArray &raw_ // stack overflow check expr can not added above T_OP_ROW if (T_OP_ROW == e->type_ && e->parent_cnt_ > 0) { e = e->parents_[0]; - if (T_OP_ROW == e->type_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("T_OP_ROW can not be nested", K(ret)); + if (T_OP_ROW == e->type_ && e->parent_cnt_ > 0) { + e = e->parents_[0]; + if (T_OP_ROW == e->type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("T_OP_ROW can not be nested twice", K(ret)); + } } } if (OB_SUCC(ret) && e->parent_cnt_ > 0) { @@ -1297,7 +1300,7 @@ int ObStaticEngineExprCG::calc_exprs_res_buf_len(const ObIArray &ra rt_expr->res_buf_len_ = min(def_res_len, static_cast(rt_expr->max_length_)); } else { - // max_length may eaual -1 + // max_length may equal -1 rt_expr->res_buf_len_ = def_res_len; } } else { @@ -1336,7 +1339,7 @@ int ObStaticEngineExprCG::create_tmp_frameinfo(const common::ObIArrayinit(UINT64_MAX, mem_attr_.tenant_id_, mem_attr_.ctx_id_, @@ -734,7 +734,7 @@ OB_DEF_DESERIALIZE(ObDASWriteBuffer) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate chunk row store failed", K(ret)); } else { - datum_store_ = new(buffer) ObChunkDatumStore(); + datum_store_ = new(buffer) ObChunkDatumStore(mem_attr_.label_); OB_UNIS_DECODE(*datum_store_); } } diff --git a/src/sql/das/ob_das_factory.cpp b/src/sql/das/ob_das_factory.cpp index e6b539e8a9..7c78fa6968 100644 --- a/src/sql/das/ob_das_factory.cpp +++ b/src/sql/das/ob_das_factory.cpp @@ -18,6 +18,7 @@ #include "sql/das/ob_das_delete_op.h" #include "sql/das/ob_das_update_op.h" #include "sql/das/ob_das_lock_op.h" +#include "sql/das/ob_das_simple_op.h" #include "sql/das/ob_das_extra_data.h" #include "sql/das/ob_das_def_reg.h" #include "sql/das/ob_das_rpc_processor.h" diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 4f4cb23281..fafa417c96 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -711,7 +711,7 @@ OB_SERIALIZE_MEMBER((ObDASScanOp, ObIDASTaskOp), ObDASScanResult::ObDASScanResult() : ObIDASTaskResult(), ObNewRowIterator(), - datum_store_(), + datum_store_("DASScanResult"), result_iter_(), output_exprs_(nullptr), eval_ctx_(nullptr), diff --git a/src/sql/das/ob_das_simple_op.cpp b/src/sql/das/ob_das_simple_op.cpp new file mode 100644 index 0000000000..5ec0210fbf --- /dev/null +++ b/src/sql/das/ob_das_simple_op.cpp @@ -0,0 +1,280 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "sql/das/ob_das_simple_op.h" +#include "sql/das/ob_das_ref.h" +#include "storage/tx_storage/ob_access_service.h" + +namespace oceanbase +{ +namespace sql +{ + +ObDASSimpleOp::ObDASSimpleOp(ObIAllocator &op_alloc) + : ObIDASTaskOp(op_alloc) {} + +int ObDASSimpleOp::release_op() +{ + int ret = OB_SUCCESS; + return ret; +} + +int ObDASSimpleOp::init_task_info(uint32_t row_extend_size) +{ + int ret = OB_SUCCESS; + UNUSED(row_extend_size); + return ret; +} + +int ObDASSimpleOp::swizzling_remote_task(ObDASRemoteInfo *remote_info) +{ + int ret = OB_SUCCESS; + UNUSED(remote_info); + return ret; +} +OB_SERIALIZE_MEMBER((ObDASSimpleOp, ObIDASTaskOp)); + +OB_SERIALIZE_MEMBER(ObDASEmptyCtDef); +OB_SERIALIZE_MEMBER(ObDASEmptyRtDef); + +ObDASSplitRangesOp::ObDASSplitRangesOp(ObIAllocator &op_alloc) + : ObDASSimpleOp(op_alloc), expected_task_count_(0) {} + +int ObDASSplitRangesOp::open_op() +{ + int ret = OB_SUCCESS; + ObAccessService *access_service = MTL(ObAccessService *); + if (OB_FAIL(access_service->split_multi_ranges(ls_id_, + tablet_id_, + ranges_, + expected_task_count_, + op_alloc_, + multi_range_split_array_))) { + LOG_WARN("failed to split multi ranges", K(ret), K_(ls_id), K_(tablet_id)); + } + return ret; +} + +int ObDASSplitRangesOp::fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) +{ + int ret = OB_SUCCESS; + UNUSED(memory_limit); +#if !defined(NDEBUG) + CK(typeid(task_result) == typeid(ObDASSplitRangesResult)); +#endif + if (OB_SUCC(ret)) { + ObDASSplitRangesResult &result = static_cast(task_result); + result.assign(multi_range_split_array_); + has_more = false; + } + return ret; +} + +int ObDASSplitRangesOp::decode_task_result(ObIDASTaskResult *task_result) +{ + int ret = OB_SUCCESS; +#if !defined(NDEBUG) + CK(typeid(*task_result) == typeid(ObDASSplitRangesResult)); + CK(task_id_ == task_result->get_task_id()); +#endif + if (OB_SUCC(ret)) { + ObDASSplitRangesResult *result = static_cast(task_result); + if (OB_FAIL(multi_range_split_array_.assign(result->get_split_array()))) { + LOG_WARN("failed to decode multi_range_split_array", K(ret)); + } + } + return ret; +} + +int ObDASSplitRangesOp::init(const common::ObIArray &ranges, int64_t expected_task_count) +{ + int ret = OB_SUCCESS; + expected_task_count_ = expected_task_count; + if (OB_FAIL(ranges_.assign(ranges))) { + LOG_WARN("failed to assign ranges array", K(ret)); + } + return ret; +} + +OB_SERIALIZE_MEMBER((ObDASSplitRangesOp, ObIDASTaskOp), + ranges_, + expected_task_count_); + +ObDASSplitRangesResult::ObDASSplitRangesResult() + : ObIDASTaskResult() {} + +ObDASSplitRangesResult::~ObDASSplitRangesResult() +{ + multi_range_split_array_.reset(); +} + +int ObDASSplitRangesResult::init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) +{ + int ret = OB_SUCCESS; + UNUSED(op); + UNUSED(alloc); + multi_range_split_array_.reset(); + return ret; +} + +int ObDASSplitRangesResult::reuse() +{ + int ret = OB_SUCCESS; + multi_range_split_array_.reuse(); + return ret; +} + +int ObDASSplitRangesResult::assign(const ObArrayArray &array) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(multi_range_split_array_.assign(array))) { + LOG_WARN("failed to assign multi ranges array", K(ret)); + } + return ret; +} + +OB_SERIALIZE_MEMBER((ObDASSplitRangesResult, ObIDASTaskResult), + multi_range_split_array_); + +ObDASRangesCostOp::ObDASRangesCostOp(common::ObIAllocator &op_alloc) + : ObDASSimpleOp(op_alloc), total_size_(0) {} + +int ObDASRangesCostOp::open_op() +{ + int ret = OB_SUCCESS; + ObAccessService *access_service = MTL(ObAccessService *); + if (OB_FAIL(access_service->get_multi_ranges_cost(ls_id_, + tablet_id_, + ranges_, + total_size_))) { + LOG_WARN("failed to get multi ranges cost", K(ret), K_(ls_id), K_(tablet_id)); + } + return ret; +} + +int ObDASRangesCostOp::fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) +{ + int ret = OB_SUCCESS; + UNUSED(memory_limit); +#if !defined(NDEBUG) + CK(typeid(task_result) == typeid(ObDASSplitRangesResult)); +#endif + if (OB_SUCC(ret)) { + ObDASRangesCostResult &result = static_cast(task_result); + result.set_total_size(total_size_); + has_more = false; + } + return ret; +} + +int ObDASRangesCostOp::decode_task_result(ObIDASTaskResult *task_result) +{ + int ret = OB_SUCCESS; +#if !defined(NDEBUG) + CK(typeid(*task_result) == typeid(ObDASSplitRangesResult)); + CK(task_id_ == task_result->get_task_id()); +#endif + if (OB_SUCC(ret)) { + ObDASRangesCostResult *result = static_cast(task_result); + total_size_ = result->get_total_size(); + } + return ret; +} + +int ObDASRangesCostOp::init(const common::ObIArray &ranges) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ranges_.assign(ranges))) { + LOG_WARN("failed to assign ranges array", K(ret)); + } + return ret; +} + +OB_SERIALIZE_MEMBER((ObDASRangesCostOp, ObIDASTaskOp), + ranges_, + total_size_); + +ObDASRangesCostResult::ObDASRangesCostResult() + : ObIDASTaskResult(), total_size_(0) {} + +int ObDASRangesCostResult::init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) +{ + int ret = OB_SUCCESS; + UNUSED(op); + UNUSED(alloc); + total_size_ = 0; + return ret; +} + +int ObDASRangesCostResult::reuse() +{ + int ret = OB_SUCCESS; + return ret; +} + +OB_SERIALIZE_MEMBER((ObDASRangesCostResult, ObIDASTaskResult), + total_size_); + +int ObDASSimpleUtils::split_multi_ranges(ObExecContext &exec_ctx, + ObDASTabletLoc *tablet_loc, + const common::ObIArray &ranges, + const int64_t expected_task_count, + ObArrayArray &multi_range_split_array) +{ + int ret = OB_SUCCESS; + ObIDASTaskOp *task_op = nullptr; + ObDASSplitRangesOp *split_ranges_op = nullptr; + ObEvalCtx eval_ctx(exec_ctx); + ObDASRef das_ref(eval_ctx, exec_ctx); + if (OB_FAIL(das_ref.create_das_task(tablet_loc, DAS_OP_SPLIT_MULTI_RANGES, task_op))) { + LOG_WARN("prepare das split_multi_ranges task failed", K(ret)); + } else { + split_ranges_op = static_cast(task_op); + if (OB_FAIL(split_ranges_op->init(ranges, expected_task_count))) { + LOG_WARN("failed to init das split ranges op", K(ret)); + } else if (OB_FAIL(das_ref.execute_all_task())) { + LOG_WARN("execute das split_multi_ranges task failed", K(ret)); + } else if (OB_FAIL(multi_range_split_array.assign(split_ranges_op->get_split_array()))) { + LOG_WARN("assgin split multi ranges array failed", K(ret)); + } + } + return ret; +} + +int ObDASSimpleUtils::get_multi_ranges_cost(ObExecContext &exec_ctx, + ObDASTabletLoc *tablet_loc, + const common::ObIArray &ranges, + int64_t &total_size) +{ + int ret = OB_SUCCESS; + ObIDASTaskOp *task_op = nullptr; + ObDASRangesCostOp *ranges_cost_op = nullptr; + ObEvalCtx eval_ctx(exec_ctx); + ObDASRef das_ref(eval_ctx, exec_ctx); + if (OB_FAIL(das_ref.create_das_task(tablet_loc, DAS_OP_GET_RANGES_COST, task_op))) { + LOG_WARN("prepare das get_multi_ranges_cost task failed", K(ret)); + } else { + ranges_cost_op = static_cast(task_op); + if (OB_FAIL(ranges_cost_op->init(ranges))) { + LOG_WARN("failed to init das ranges cost op", K(ret)); + } else if (OB_FAIL(das_ref.execute_all_task())) { + LOG_WARN("execute das get_multi_ranges_cost task failed", K(ret)); + } else { + total_size = ranges_cost_op->get_total_size(); + } + } + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/ob_das_simple_op.h b/src/sql/das/ob_das_simple_op.h new file mode 100644 index 0000000000..d15db053e4 --- /dev/null +++ b/src/sql/das/ob_das_simple_op.h @@ -0,0 +1,147 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_OB_DAS_SIMPLE_OP_H +#define OBDEV_SRC_SQL_DAS_OB_DAS_SIMPLE_OP_H +#include "sql/das/ob_das_task.h" + +namespace oceanbase +{ +namespace common +{ +class ObStoreRange; +} +namespace sql +{ + +class ObDASSimpleOp : public ObIDASTaskOp +{ + OB_UNIS_VERSION(1); +public: + ObDASSimpleOp(common::ObIAllocator &op_alloc); + virtual ~ObDASSimpleOp() = default; + virtual int open_op() = 0; + virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) = 0; + virtual int decode_task_result(ObIDASTaskResult *task_result) = 0; + + virtual int release_op() override; + virtual int init_task_info(uint32_t row_extend_size) override; + virtual int swizzling_remote_task(ObDASRemoteInfo *remote_info) override; +}; + +struct ObDASEmptyCtDef : ObDASBaseCtDef +{ + OB_UNIS_VERSION(1); +public: + ObDASEmptyCtDef(common::ObIAllocator &alloc) + : ObDASBaseCtDef(DAS_OP_INVALID) {} +}; + +struct ObDASEmptyRtDef : ObDASBaseRtDef +{ + OB_UNIS_VERSION(1); +public: + ObDASEmptyRtDef() + : ObDASBaseRtDef(DAS_OP_INVALID) {} +}; + +class ObDASSplitRangesOp : public ObDASSimpleOp +{ + OB_UNIS_VERSION(1); +public: + ObDASSplitRangesOp(common::ObIAllocator &op_alloc); + virtual ~ObDASSplitRangesOp() = default; + virtual int open_op() override; + virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override; + virtual int decode_task_result(ObIDASTaskResult *task_result) override; + int init(const common::ObIArray &ranges, int64_t expected_task_count); + const ObArrayArray &get_split_array() { return multi_range_split_array_; } + INHERIT_TO_STRING_KV("parent", ObDASSimpleOp, + K_(ranges), + K_(expected_task_count), + K_(multi_range_split_array)); +private: + common::ObSEArray ranges_; + int64_t expected_task_count_; + ObArrayArray multi_range_split_array_; +}; + +class ObDASSplitRangesResult : public ObIDASTaskResult +{ + OB_UNIS_VERSION_V(1); +public: + ObDASSplitRangesResult(); + virtual ~ObDASSplitRangesResult(); + virtual int init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) override; + virtual int reuse() override; + const ObArrayArray &get_split_array() const { return multi_range_split_array_; } + ObArrayArray &get_split_array() { return multi_range_split_array_; } + int assign(const ObArrayArray &array); + INHERIT_TO_STRING_KV("parent", ObIDASTaskResult, + K_(multi_range_split_array)); +private: + ObArrayArray multi_range_split_array_; +}; + +class ObDASRangesCostOp : public ObDASSimpleOp +{ + OB_UNIS_VERSION(1); +public: + ObDASRangesCostOp(common::ObIAllocator &op_alloc); + virtual ~ObDASRangesCostOp() = default; + virtual int open_op() override; + virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override; + virtual int decode_task_result(ObIDASTaskResult *task_result) override; + int init(const common::ObIArray &ranges); + int64_t get_total_size() const { return total_size_; } + INHERIT_TO_STRING_KV("parent", ObDASSimpleOp, + K_(ranges), + K_(total_size)); +private: + common::ObSEArray ranges_; + int64_t total_size_; +}; + +class ObDASRangesCostResult : public ObIDASTaskResult +{ + OB_UNIS_VERSION_V(1); +public: + ObDASRangesCostResult(); + virtual ~ObDASRangesCostResult() = default; + virtual int init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) override; + virtual int reuse() override; + int64_t get_total_size() const { return total_size_; } + void set_total_size(int64_t total_size) { total_size_ = total_size; } + INHERIT_TO_STRING_KV("parent", ObIDASTaskResult, + K_(total_size)); +private: + int64_t total_size_; +}; + +class ObDASSimpleUtils +{ +public: + static int split_multi_ranges(ObExecContext &exec_ctx, + ObDASTabletLoc *tablet_loc, + const common::ObIArray &ranges, + const int64_t expected_task_count, + ObArrayArray &multi_range_split_array); + + static int get_multi_ranges_cost(ObExecContext &exec_ctx, + ObDASTabletLoc *tablet_loc, + const common::ObIArray &ranges, + int64_t &total_size); +}; + +} // namespace sql +} // namespace oceanbase +#endif /* OBDEV_SRC_SQL_DAS_OB_DAS_DELETE_OP_H_ */ diff --git a/src/sql/das/ob_das_task.cpp b/src/sql/das/ob_das_task.cpp index 3d00ba95bf..db632fdc8e 100644 --- a/src/sql/das/ob_das_task.cpp +++ b/src/sql/das/ob_das_task.cpp @@ -534,7 +534,7 @@ OB_SERIALIZE_MEMBER(ObDASDataFetchRes, tenant_id_, task_id_, has_more_); ObDASDataFetchRes::ObDASDataFetchRes() - : datum_store_(), + : datum_store_("DASDataFetch"), tenant_id_(0), task_id_(0), has_more_(false) diff --git a/src/sql/das/ob_das_task_result.cpp b/src/sql/das/ob_das_task_result.cpp index 6dd8f934e5..02ca682dfc 100644 --- a/src/sql/das/ob_das_task_result.cpp +++ b/src/sql/das/ob_das_task_result.cpp @@ -36,7 +36,7 @@ ObDASTCB::ObDASTCB() max_batch_size_(0), stored_row_(NULL), stored_row_arr_(NULL), - datum_store_(), + datum_store_("DASTaskResMgr"), result_iter_(), tcb_lock_() { diff --git a/src/sql/dtl/ob_dtl_basic_channel.cpp b/src/sql/dtl/ob_dtl_basic_channel.cpp index 9c115b76b7..02f42ae17c 100644 --- a/src/sql/dtl/ob_dtl_basic_channel.cpp +++ b/src/sql/dtl/ob_dtl_basic_channel.cpp @@ -363,7 +363,7 @@ int ObDtlBasicChannel::mock_eof_buffer(int64_t timeout_ts) int64_t min_buf_size = ObChunkDatumStore::Block::min_buf_size(0); ObDtlLinkedBuffer *buffer = NULL; MTL_SWITCH(tenant_id_) { - ObChunkDatumStore row_store; + ObChunkDatumStore row_store("MockDtlStore"); ObChunkDatumStore::Block* block = NULL; if (OB_ISNULL(buffer = alloc_buf(min_buf_size))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/sql/dtl/ob_dtl_interm_result_manager.cpp b/src/sql/dtl/ob_dtl_interm_result_manager.cpp index f478f17e23..2a8fdd7c03 100644 --- a/src/sql/dtl/ob_dtl_interm_result_manager.cpp +++ b/src/sql/dtl/ob_dtl_interm_result_manager.cpp @@ -241,7 +241,7 @@ int ObDTLIntermResultManager::create_interm_result_info(ObMemAttr &attr, LOG_WARN("fail to alloc ob chunk row store ret", K(ret)); } else { result_info = new(result_info_buf) ObDTLIntermResultInfo(); - result_info->datum_store_ = new(ptr) ObChunkDatumStore(); + result_info->datum_store_ = new(ptr) ObChunkDatumStore("DtlIntermRes"); result_info->is_read_ = false; result_info->trace_id_ = *ObCurTraceId::get_trace_id(); result_info->monitor_info_ = monitor_info; diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index 6afb37dbdc..ece0f22145 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -38,6 +38,7 @@ #include "lib/alloc/malloc_hook.h" #endif #include "pl/ob_pl_user_type.h" +#include "pl/ob_pl.h" namespace oceanbase { @@ -6021,6 +6022,24 @@ int ObAggregateProcessor::get_pl_agg_udf_result(const ObAggrInfo &aggr_info, } else { LOG_TRACE("succeed to get pl agg udf result", K(result_obj), K(result)); } + if (result_obj.is_pl_extend()) { + int tmp_ret = OB_SUCCESS; + if (OB_ISNULL(eval_ctx_.exec_ctx_.get_pl_ctx())) { + tmp_ret = eval_ctx_.exec_ctx_.init_pl_ctx(); + } + if (OB_SUCCESS == tmp_ret && OB_NOT_NULL(eval_ctx_.exec_ctx_.get_pl_ctx())) { + tmp_ret = eval_ctx_.exec_ctx_.get_pl_ctx()->add(result_obj); + } + if (OB_SUCCESS != tmp_ret) { + LOG_ERROR("fail to collect pl collection allocator, may be exist memory issue", K(tmp_ret)); + } + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } + } + + int tmp_ret = OB_SUCCESS; + if ((tmp_ret = pl::ObUserDefinedType::destruct_obj(pl_agg_udf_obj, eval_ctx_.exec_ctx_.get_my_session())) != OB_SUCCESS) { + LOG_WARN("failed to destruct obj, memory may leak", K(ret), K(tmp_ret), K(pl_agg_udf_obj)); } } return ret; diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index 5698dec383..54a620d35a 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -318,7 +318,7 @@ public: { public: explicit GroupConcatExtraResult(common::ObIAllocator &alloc, ObMonitorNode &op_monitor_info) - : ExtraResult(alloc, op_monitor_info), row_count_(0), iter_idx_(0), sort_op_(NULL), separator_datum_(NULL), bool_mark_(alloc) + : ExtraResult(alloc, op_monitor_info), row_count_(0), iter_idx_(0), row_store_(ObModIds::OB_SQL_AGGR_FUN_GROUP_CONCAT), sort_op_(NULL), separator_datum_(NULL), bool_mark_(alloc) { } virtual ~GroupConcatExtraResult(); diff --git a/src/sql/engine/aggregate/ob_hash_groupby_op.h b/src/sql/engine/aggregate/ob_hash_groupby_op.h index 1a7ea116bd..493f5237f6 100644 --- a/src/sql/engine/aggregate/ob_hash_groupby_op.h +++ b/src/sql/engine/aggregate/ob_hash_groupby_op.h @@ -199,7 +199,7 @@ public: { public: DatumStoreLinkPartition(common::ObIAllocator *alloc = nullptr) - : datum_store_(alloc), part_id_(0), part_shift_(0) + : datum_store_(ObModIds::OB_HASH_NODE_GROUP_ROWS, alloc), part_id_(0), part_shift_(0) {} ObChunkDatumStore datum_store_; int64_t part_id_; @@ -227,6 +227,7 @@ public: cur_group_item_idx_(0), cur_group_item_buf_(nullptr), mem_context_(NULL), + group_store_(ObModIds::OB_HASH_NODE_GROUP_ROWS), agged_group_cnt_(0), agged_row_cnt_(0), agged_dumped_cnt_(0), diff --git a/src/sql/engine/basic/ob_chunk_datum_store.cpp b/src/sql/engine/basic/ob_chunk_datum_store.cpp index b1151bccda..c01cc3c01e 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.cpp +++ b/src/sql/engine/basic/ob_chunk_datum_store.cpp @@ -477,8 +477,8 @@ int ObChunkDatumStore::Block::swizzling(int64_t *col_cnt) return ret; } -ObChunkDatumStore::ObChunkDatumStore(common::ObIAllocator *alloc /* = NULL */) - : inited_(false), tenant_id_(0), label_(common::ObModIds::OB_SQL_CHUNK_ROW_STORE), +ObChunkDatumStore::ObChunkDatumStore(const ObLabel &label, common::ObIAllocator *alloc /* = NULL */) + : inited_(false), tenant_id_(0), label_(label), ctx_id_(0), mem_limit_(0), cur_blk_(NULL), cur_blk_buffer_(nullptr), max_blk_size_(0), min_blk_size_(INT64_MAX), default_block_size_(BLOCK_SIZE), @@ -507,7 +507,7 @@ int ObChunkDatumStore::init(int64_t mem_limit, enable_dump_ = enable_dump; tenant_id_ = tenant_id; ctx_id_ = mem_ctx_id; - label_ = label; + UNUSED(label_); if (0 == GCONF._chunk_row_store_mem_limit) { mem_limit_ = mem_limit; } else { @@ -2242,7 +2242,7 @@ OB_DEF_DESERIALIZE(ObChunkDatumStore) } if (!is_inited()) { if (OB_FAIL(init(mem_limit_, tenant_id_, - ctx_id_, "ObChunkRowDE", false/*enable_dump*/))) { + ctx_id_, label_, false/*enable_dump*/))) { LOG_WARN("fail to init chunk row store", K(ret)); } } diff --git a/src/sql/engine/basic/ob_chunk_datum_store.h b/src/sql/engine/basic/ob_chunk_datum_store.h index 151c5ca2ac..5dfa4949a7 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.h +++ b/src/sql/engine/basic/ob_chunk_datum_store.h @@ -919,7 +919,7 @@ public: static const int32_t DATUM_SIZE = sizeof(common::ObDatum); static const int64_t OLD_WORK_AREA_ID = 21; - explicit ObChunkDatumStore(common::ObIAllocator *alloc = NULL); + explicit ObChunkDatumStore(const lib::ObLabel &label, common::ObIAllocator *alloc = NULL); virtual ~ObChunkDatumStore() { reset(); } diff --git a/src/sql/engine/basic/ob_group_join_buffer.cpp b/src/sql/engine/basic/ob_group_join_buffer.cpp index 29612165da..ba8e9df5c9 100644 --- a/src/sql/engine/basic/ob_group_join_buffer.cpp +++ b/src/sql/engine/basic/ob_group_join_buffer.cpp @@ -109,7 +109,7 @@ ObGroupJoinBufffer::ObGroupJoinBufffer() : op_(NULL), spec_(NULL), ctx_(NULL), eval_ctx_(NULL), left_(NULL), right_(NULL), rescan_params_(NULL), left_rescan_params_(NULL), right_rescan_params_(NULL), mem_context_(NULL), - left_store_(), left_store_iter_(), left_store_group_idx_(), + left_store_("JoinBufStore"), left_store_iter_(), left_store_group_idx_(), above_left_group_params_(), above_right_group_params_(), group_params_(), above_group_params_(), last_row_(), last_batch_(), diff --git a/src/sql/engine/basic/ob_hash_partitioning_infrastructure_op.h b/src/sql/engine/basic/ob_hash_partitioning_infrastructure_op.h index 1c22b9f1a7..7b47952bbd 100644 --- a/src/sql/engine/basic/ob_hash_partitioning_infrastructure_op.h +++ b/src/sql/engine/basic/ob_hash_partitioning_infrastructure_op.h @@ -314,7 +314,7 @@ public: { public: ObIntraPartition() : - part_key_(), store_() + part_key_(), store_(ObModIds::OB_SQL_HASH_SET) {} ~ObIntraPartition() { diff --git a/src/sql/engine/basic/ob_material_op_impl.cpp b/src/sql/engine/basic/ob_material_op_impl.cpp index 07e992b39e..90fe6c7e9c 100644 --- a/src/sql/engine/basic/ob_material_op_impl.cpp +++ b/src/sql/engine/basic/ob_material_op_impl.cpp @@ -29,7 +29,7 @@ ObMaterialOpImpl::ObMaterialOpImpl(ObMonitorNode &op_monitor_info, ObSqlWorkArea tenant_id_(OB_INVALID_ID), exec_ctx_(nullptr), mem_context_(nullptr), - datum_store_(), + datum_store_(ObModIds::OB_HASH_NODE_GROUP_ROWS), datum_store_it_(), eval_ctx_(nullptr), profile_(profile), diff --git a/src/sql/engine/basic/ob_topk_op.cpp b/src/sql/engine/basic/ob_topk_op.cpp index 3f5e067e15..0876460f7d 100644 --- a/src/sql/engine/basic/ob_topk_op.cpp +++ b/src/sql/engine/basic/ob_topk_op.cpp @@ -29,8 +29,7 @@ ObTopKSpec::ObTopKSpec(ObIAllocator &alloc, const ObPhyOperatorType type) bool ObTopKSpec::is_valid() const { - return (get_output_count() > 0) && (NULL != org_limit_) && - (NULL != child_) && (child_->get_output_count() > 0); + return OB_NOT_NULL(org_limit_) && OB_NOT_NULL(child_); } OB_SERIALIZE_MEMBER((ObTopKSpec, ObOpSpec), minimum_row_count_, topk_precision_, diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index 56cbac941b..71526789c1 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -568,8 +568,8 @@ int ObCreateTableExecutor::execute(ObExecContext &ctx, ObCreateTableStmt &stmt) LOG_WARN("rpc proxy create table failed", KR(ret), "dst", common_rpc_proxy->get_server()); } else { int64_t refresh_time = ObTimeUtility::current_time(); - if (OB_FAIL(ObSchemaUtils::try_check_parallel_ddl_schema_in_sync(ctx, - tenant_id, res.schema_version_, GCONF._wait_interval_after_truncate))) { + if (OB_FAIL(ObSchemaUtils::try_check_parallel_ddl_schema_in_sync( + ctx, tenant_id, res.schema_version_))) { LOG_WARN("fail to check paralleld ddl schema in sync", KR(ret), K(res)); } int64_t end_time = ObTimeUtility::current_time(); @@ -2208,8 +2208,8 @@ int ObTruncateTableExecutor::execute(ObExecContext &ctx, ObTruncateTableStmt &st } else if (!res.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("truncate invalid ddl_res", KR(ret), K(res)); - } else if (OB_FAIL(ObSchemaUtils::try_check_parallel_ddl_schema_in_sync(ctx, - tenant_id, res.task_id_, GCONF._wait_interval_after_truncate))) { + } else if (OB_FAIL(ObSchemaUtils::try_check_parallel_ddl_schema_in_sync( + ctx, tenant_id, res.task_id_))) { LOG_WARN("fail to check parallel ddl schema in sync", KR(ret), K(res)); } int64_t end_time = ObTimeUtility::current_time(); diff --git a/src/sql/engine/connect_by/ob_cnnt_by_pump.h b/src/sql/engine/connect_by/ob_cnnt_by_pump.h index 10a4021c28..f6f94830d4 100644 --- a/src/sql/engine/connect_by/ob_cnnt_by_pump.h +++ b/src/sql/engine/connect_by/ob_cnnt_by_pump.h @@ -260,7 +260,7 @@ public: : ObConnectByOpPumpBase(), // prior_exprs_result_row_(), pump_stack_() , - datum_store_(), + datum_store_(ObModIds::OB_CONNECT_BY_PUMP), cur_output_exprs_(NULL), hash_filter_rows_(), connect_by_(NULL), diff --git a/src/sql/engine/connect_by/ob_nl_cnnt_by_op.cpp b/src/sql/engine/connect_by/ob_nl_cnnt_by_op.cpp index 54ea7c4b1e..5737f55019 100644 --- a/src/sql/engine/connect_by/ob_nl_cnnt_by_op.cpp +++ b/src/sql/engine/connect_by/ob_nl_cnnt_by_op.cpp @@ -274,7 +274,9 @@ int ObNLConnectByOp::inner_open() K(MY_SPEC.hash_probe_exprs_)); } else { tenant_id = ctx_.get_my_session()->get_effective_tenant_id(); - param.set_mem_attr(tenant_id, ObModIds::OB_CONNECT_BY_PUMP, ObCtxIds::WORK_AREA).set_properties(lib::USE_TL_PAGE_OPTIONAL); + param.set_mem_attr(tenant_id, ObModIds::OB_CONNECT_BY_PUMP, ObCtxIds::WORK_AREA) + .set_properties(lib::USE_TL_PAGE_OPTIONAL) + .set_ablock_size(lib::INTACT_MIDDLE_AOBJECT_SIZE); if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { LOG_WARN("create entity failed", K(ret)); } else if (OB_ISNULL(mem_context_)) { diff --git a/src/sql/engine/dml/ob_table_insert_up_op.h b/src/sql/engine/dml/ob_table_insert_up_op.h index 8db291cb4d..83759f34b9 100644 --- a/src/sql/engine/dml/ob_table_insert_up_op.h +++ b/src/sql/engine/dml/ob_table_insert_up_op.h @@ -82,6 +82,7 @@ public: conflict_checker_(ctx.get_allocator(), eval_ctx_, MY_SPEC.conflict_checker_ctdef_), + insert_up_row_store_("InsertUpRow"), is_ignore_(false) { diff --git a/src/sql/engine/dml/ob_table_replace_op.h b/src/sql/engine/dml/ob_table_replace_op.h index 03825d7e55..dd7b5f7e87 100644 --- a/src/sql/engine/dml/ob_table_replace_op.h +++ b/src/sql/engine/dml/ob_table_replace_op.h @@ -67,7 +67,8 @@ public: delete_rows_(0), conflict_checker_(ctx.get_allocator(), eval_ctx_, - MY_SPEC.conflict_checker_ctdef_) + MY_SPEC.conflict_checker_ctdef_), + replace_row_store_("ReplaceRow") {} virtual ~ObTableReplaceOp() {} diff --git a/src/sql/engine/expr/ob_expr_cast.cpp b/src/sql/engine/expr/ob_expr_cast.cpp index ebe43d424f..30eeca9259 100644 --- a/src/sql/engine/expr/ob_expr_cast.cpp +++ b/src/sql/engine/expr/ob_expr_cast.cpp @@ -441,7 +441,8 @@ int ObExprCast::calc_result_type2(ObExprResType &type, // MySql:cast (1 as decimal(0)) = cast(1 as decimal) // Oracle: cast(1.4 as number) = cast(1.4 as number(-1, -1)) type.set_precision(ObAccuracy::DDL_DEFAULT_ACCURACY2[compatibility_mode][ObNumberType].get_precision()); - } else if (ObIntTC == dst_type.get_type_class() || ObUIntTC == dst_type.get_type_class()) { + } else if ((ObIntTC == dst_type.get_type_class() || ObUIntTC == dst_type.get_type_class()) + && dst_type.get_precision() <= 0) { // for int or uint , the precision = len int32_t len = 0; int16_t length_semantics = LS_BYTE;//unused @@ -471,6 +472,9 @@ int ObExprCast::calc_result_type2(ObExprResType &type, // interval expr need NOT_NULL_FLAG // bug: calc_result_flag2(type, type1, type2); + if (CM_IS_ADD_ZEROFILL(cast_raw_expr->get_extra())) { + type.set_result_flag(ZEROFILL_FLAG); + } } } if (OB_SUCC(ret)) { diff --git a/src/sql/engine/expr/ob_expr_merge_result_type.map b/src/sql/engine/expr/ob_expr_merge_result_type.map index 9868d729f3..23e8f50512 100644 --- a/src/sql/engine/expr/ob_expr_merge_result_type.map +++ b/src/sql/engine/expr/ob_expr_merge_result_type.map @@ -1275,7 +1275,7 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObLongTextType, /* CharType=>ObMediumTextType */ ObLongTextType, /* CharType=>ObLongTextType */ ObVarcharType, /* CharType=>ObBitType */ - ObCharType, /* CharType=>ObEnumType */ + ObVarcharType, /* CharType=>ObEnumType */ ObCharType, /* CharType=>ObSetType */ ObMaxType, /* CharType=>ObEnumInnerType */ ObMaxType, /* CharType=>ObSetInnerType */ @@ -1747,7 +1747,7 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*ObEnumType=>TimeType */ ObVarcharType, /*ObEnumType=>YearType */ ObVarcharType, /*ObEnumType=>VarcharType */ - ObCharType, /*ObEnumType=>CharType */ + ObVarcharType, /*ObEnumType=>CharType */ ObVarcharType, /*ObEnumType=>HexStringType */ ObMaxType, /*ObEnumType=>ExtendType */ ObVarcharType, /*ObEnumType=>UnknownType */ @@ -3956,7 +3956,7 @@ static ObObjType MERGE_RESULT_TYPE_ORACLE[ObMaxType][ObMaxType] = { ObLongTextType, /* CharType=>ObMediumTextType */ ObLongTextType, /* CharType=>ObLongTextType */ ObVarcharType, /* CharType=>ObBitType */ - ObCharType, /* CharType=>ObEnumType */ + ObVarcharType, /* CharType=>ObEnumType */ ObCharType, /* CharType=>ObSetType */ ObMaxType, /* CharType=>ObEnumInnerType */ ObMaxType, /* CharType=>ObSetInnerType */ @@ -4428,7 +4428,7 @@ static ObObjType MERGE_RESULT_TYPE_ORACLE[ObMaxType][ObMaxType] = { ObVarcharType, /*ObEnumType=>TimeType */ ObVarcharType, /*ObEnumType=>YearType */ ObVarcharType, /*ObEnumType=>VarcharType */ - ObCharType, /*ObEnumType=>CharType */ + ObVarcharType, /*ObEnumType=>CharType */ ObVarcharType, /*ObEnumType=>HexStringType */ ObMaxType, /*ObEnumType=>ExtendType */ ObVarcharType, /*ObEnumType=>UnknownType */ diff --git a/src/sql/engine/expr/ob_expr_repeat.cpp b/src/sql/engine/expr/ob_expr_repeat.cpp index 1fda797e82..6e11c31706 100644 --- a/src/sql/engine/expr/ob_expr_repeat.cpp +++ b/src/sql/engine/expr/ob_expr_repeat.cpp @@ -210,6 +210,9 @@ int ObExprRepeat::repeat_text(ObObjType res_type, int64_t tot_length = length * count; int64_t buffer_len = 0; char *buf = NULL; + if (tot_length >= MEM_WARN_THRESHOLD) { + LOG_INFO("once repeat cost over 100M", K(ret), K(tot_length), K(length), K(count)); + } ObTextStringResult result_buffer(res_type, has_lob_header, &allocator); if (OB_FAIL(result_buffer.init(tot_length))) { LOG_WARN("init result failed", K(ret), K(tot_length)); diff --git a/src/sql/engine/expr/ob_expr_repeat.h b/src/sql/engine/expr/ob_expr_repeat.h index 57901bf5e7..51c2de9794 100644 --- a/src/sql/engine/expr/ob_expr_repeat.h +++ b/src/sql/engine/expr/ob_expr_repeat.h @@ -62,6 +62,7 @@ public: ObIAllocator &allocator, const int64_t max_result_size); private: + static const int64_t MEM_WARN_THRESHOLD = 100 * 1024 * 1024; // 100M // disallow copy DISALLOW_COPY_AND_ASSIGN(ObExprRepeat); }; diff --git a/src/sql/engine/expr/ob_expr_validate_password_strength.h b/src/sql/engine/expr/ob_expr_validate_password_strength.h index 658ee4aacd..30071fe860 100644 --- a/src/sql/engine/expr/ob_expr_validate_password_strength.h +++ b/src/sql/engine/expr/ob_expr_validate_password_strength.h @@ -44,7 +44,6 @@ public: const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; static int eval_password_strength(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res_datum); -private: static int calc_password_strength(const common::ObString &password, const ObBasicSessionInfo &session, int &strength); @@ -64,6 +63,7 @@ private: const int64_t password_char_length, const ObBasicSessionInfo &session, bool &passed); +private: static const int64_t VALID_PASSWORD_LENGTH_MIN = 4; static const int64_t PASSWORD_STRENGTH_MULTIPLIER = 25; static const ObValidatePasswordFunc validate_funcs_[STRENGTH_MAX]; diff --git a/src/sql/engine/join/ob_hash_join_basic.h b/src/sql/engine/join/ob_hash_join_basic.h index 116d9abd76..a6b8ca7fcb 100644 --- a/src/sql/engine/join/ob_hash_join_basic.h +++ b/src/sql/engine/join/ob_hash_join_basic.h @@ -162,7 +162,7 @@ public: int32_t part_level, int64_t part_shift, int64_t batchno) - : chunk_row_store_(&alloc), + : chunk_row_store_(common::ObModIds::OB_ARENA_HASH_JOIN, &alloc), inner_callback_(nullptr), part_level_(part_level), part_shift_(part_shift), diff --git a/src/sql/engine/join/ob_merge_join_op.cpp b/src/sql/engine/join/ob_merge_join_op.cpp index a59ff0ec28..ea146bcddb 100644 --- a/src/sql/engine/join/ob_merge_join_op.cpp +++ b/src/sql/engine/join/ob_merge_join_op.cpp @@ -36,6 +36,7 @@ ObMergeJoinOp::ObMergeJoinOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOp : ObJoinOp(exec_ctx, spec, input), state_(JS_JOIN_BEGIN), mem_context_(NULL), + right_cache_(ObModIds::OB_SQL_MERGE_JOIN), stored_row_(NULL), stored_row_idx_(-1), empty_cache_iter_side_(ITER_BOTH), diff --git a/src/sql/engine/join/ob_nested_loop_join_op.cpp b/src/sql/engine/join/ob_nested_loop_join_op.cpp index 869ff38c38..9d6d511f38 100644 --- a/src/sql/engine/join/ob_nested_loop_join_op.cpp +++ b/src/sql/engine/join/ob_nested_loop_join_op.cpp @@ -32,11 +32,11 @@ ObNestedLoopJoinOp::ObNestedLoopJoinOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOpInput *input) : ObBasicNestedLoopJoinOp(exec_ctx, spec, input), - state_(JS_READ_LEFT), mem_context_(nullptr), is_left_end_(false), + state_(JS_READ_LEFT), mem_context_(nullptr), left_store_("NljLStore"), is_left_end_(false), last_store_row_(), save_last_row_(false), defered_right_rescan_(false), batch_rescan_ctl_(), batch_state_(JS_FILL_LEFT), save_last_batch_(false), - batch_mem_ctx_(NULL), stored_rows_(NULL), left_brs_(NULL), left_matched_(NULL), + batch_mem_ctx_(NULL), stored_rows_(NULL), right_store_("NljRStore"), left_brs_(NULL), left_matched_(NULL), need_switch_iter_(false), iter_end_(false), op_max_batch_size_(0), max_group_size_(OB_MAX_BULK_JOIN_ROWS), group_join_buffer_(), diff --git a/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp b/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp index 2829f3992c..99adcb7227 100644 --- a/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp +++ b/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp @@ -138,7 +138,7 @@ int ObPDMLOpBatchRowCache::init_row_store(ObChunkDatumStore *&chunk_row_store) // 进行一个优化： // 1. 如果没有barrier，就不进行的dump // 2. 如果有barrier，就需要进行dump - chunk_row_store = new(buf) ObChunkDatumStore(&allocator); + chunk_row_store = new(buf) ObChunkDatumStore("PDML_ROW_CACHE", &allocator); if (OB_FAIL(chunk_row_store->init(INT64_MAX, // let auto mem mgr take care of mem limit tenant_id_, ObCtxIds::WORK_AREA, diff --git a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp index b4252bdbfd..cb87e89805 100644 --- a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp @@ -396,7 +396,7 @@ int ObPxMultiPartSSTableInsertOp::create_tablet_store(ObTabletID &tablet_id, ObC if (OB_ISNULL(buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate memory for tablet store", K(ret)); - } else if (OB_ISNULL(tablet_store = new (buf) ObChunkDatumStore)) { + } else if (OB_ISNULL(tablet_store = new (buf) ObChunkDatumStore("SSTABLE_INS"))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to new ObChunkDatumStore", K(ret)); } else if (OB_FAIL(tablet_store->init(TABLET_STORE_MEM_LIMIT, diff --git a/src/sql/engine/px/datahub/components/ob_dh_sample.cpp b/src/sql/engine/px/datahub/components/ob_dh_sample.cpp index 22035334b8..0c0e2337c9 100644 --- a/src/sql/engine/px/datahub/components/ob_dh_sample.cpp +++ b/src/sql/engine/px/datahub/components/ob_dh_sample.cpp @@ -97,7 +97,7 @@ OB_DEF_DESERIALIZE(ObDynamicSamplePieceMsg) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); } else { - ObChunkDatumStore *tmp_store = new (tmp_buf) ObChunkDatumStore; + ObChunkDatumStore *tmp_store = new (tmp_buf) ObChunkDatumStore("DYN_SAMPLE_CTX"); if (OB_FAIL(tmp_store->deserialize(buf, data_len, pos))) { LOG_WARN("deserialize datum store failed", K(ret), K(i)); } else if (OB_FAIL(row_stores_.push_back(tmp_store))) { @@ -306,7 +306,7 @@ int ObDynamicSamplePieceMsgCtx::init(const ObIArray &tablet_ids) LOG_WARN("allocate memory failed", K(ret), K(tablet_ids.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { - ObChunkDatumStore *sample_store = new (buf + i * sizeof(ObChunkDatumStore)) ObChunkDatumStore; + ObChunkDatumStore *sample_store = new (buf + i * sizeof(ObChunkDatumStore)) ObChunkDatumStore("DYN_SAMPLE_CTX"); if (OB_FAIL(sample_store->init(0, tenant_id_, ObCtxIds::DEFAULT_CTX_ID, "DYN_SAMPLE_CTX", false/*enable dump*/))) { LOG_WARN("init sample chunk store failed", K(ret), K(i)); diff --git a/src/sql/engine/px/datahub/components/ob_dh_winbuf.h b/src/sql/engine/px/datahub/components/ob_dh_winbuf.h index 55d221db02..215c21d518 100644 --- a/src/sql/engine/px/datahub/components/ob_dh_winbuf.h +++ b/src/sql/engine/px/datahub/components/ob_dh_winbuf.h @@ -76,7 +76,7 @@ public: using WholeMsgProvider = ObWholeMsgProvider; public: ObWinbufWholeMsg() : ready_state_(0), is_empty_(true), is_datum_(false), - row_store_(), datum_store_(), assign_allocator_() + row_store_(), datum_store_("PXDhWinbuf"), assign_allocator_() {} ~ObWinbufWholeMsg() = default; int assign(const ObWinbufWholeMsg &other, common::ObIAllocator *allocator = NULL); diff --git a/src/sql/engine/px/exchange/ob_px_dist_transmit_op.cpp b/src/sql/engine/px/exchange/ob_px_dist_transmit_op.cpp index 2ad5ef609f..38adb14f7e 100644 --- a/src/sql/engine/px/exchange/ob_px_dist_transmit_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_dist_transmit_op.cpp @@ -514,7 +514,7 @@ int ObPxDistTransmitOp::build_row_sample_piece_msg(int64_t expected_range_count, int64_t tenant_id = ctx_.get_my_session()->get_effective_tenant_id(); - ObChunkDatumStore *sample_store = OB_NEWx(ObChunkDatumStore, (&ctx_.get_allocator())); + ObChunkDatumStore *sample_store = OB_NEWx(ObChunkDatumStore, &ctx_.get_allocator(), "DYN_SAMPLE_CTX"); OV(NULL != sample_store, OB_ALLOCATE_MEMORY_FAILED); bool sample_store_dump = false; @@ -577,7 +577,7 @@ int ObPxDistTransmitSpec::register_to_datahub(ObExecContext &ctx) const ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); } else { - ObChunkDatumStore *sample_store = new (chunk_buf) ObChunkDatumStore; + ObChunkDatumStore *sample_store = new (chunk_buf) ObChunkDatumStore("DYN_SAMPLE_CTX"); if (OB_FAIL(sample_store->init(0, ctx.get_my_session()->get_effective_tenant_id(), ObCtxIds::DEFAULT_CTX_ID, "DYN_SAMPLE_CTX", false/*enable dump*/))) { diff --git a/src/sql/engine/px/exchange/ob_px_ms_receive_op.cpp b/src/sql/engine/px/exchange/ob_px_ms_receive_op.cpp index 873049d045..187a976d68 100644 --- a/src/sql/engine/px/exchange/ob_px_ms_receive_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_ms_receive_op.cpp @@ -512,7 +512,7 @@ int ObPxMSReceiveOp::GlobalOrderInput::create_chunk_datum_store( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("create ra row store fail", K(ret)); } else { - row_store = new (buf) ObChunkDatumStore(); + row_store = new (buf) ObChunkDatumStore("PxMSRecvGlobal"); // TODO: llongzhong.wlz 这里应该使用一个参数来控制row_store存储的数据量，或者SQL内存管理自动控制 int64_t mem_limit = 0; row_store->set_allocator(*alloc_); diff --git a/src/sql/engine/px/exchange/ob_px_ms_receive_op.h b/src/sql/engine/px/exchange/ob_px_ms_receive_op.h index 84c382dad0..69dc9074e7 100644 --- a/src/sql/engine/px/exchange/ob_px_ms_receive_op.h +++ b/src/sql/engine/px/exchange/ob_px_ms_receive_op.h @@ -187,7 +187,7 @@ private: public: explicit LocalOrderInput() : MergeSortInput(nullptr, nullptr, false), - datum_store_() + datum_store_("PxMSRecvLocal") { get_row_store_ = &datum_store_; add_row_store_ = &datum_store_; diff --git a/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp b/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp index d79cd93654..fee0935a73 100644 --- a/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_repart_transmit_op.cpp @@ -226,7 +226,7 @@ int ObPxRepartTransmitOp::do_repart_transmit(ObRepartSliceIdxCalc &repart_slice_ { int ret = OB_SUCCESS; // init the ObRepartSliceIdxCalc cache map - if (OB_FAIL(repart_slice_calc.init())) { + if (OB_FAIL(repart_slice_calc.init(ctx_.get_my_session()->get_effective_tenant_id()))) { LOG_WARN("failed to init repart slice calc", K(ret)); } else if (OB_FAIL(send_rows(repart_slice_calc))) { LOG_WARN("failed to send rows", K(ret)); diff --git a/src/sql/engine/px/ob_granule_pump.cpp b/src/sql/engine/px/ob_granule_pump.cpp index ca681c444a..c3c9dc68c7 100644 --- a/src/sql/engine/px/ob_granule_pump.cpp +++ b/src/sql/engine/px/ob_granule_pump.cpp @@ -752,7 +752,8 @@ int ObGranuleSplitter::split_gi_task(ObGranulePumpArgs &args, taskset_ranges, taskset_idxs); } else { - ret = ObGranuleUtil::split_block_ranges(args.ctx_->get_allocator(), + ret = ObGranuleUtil::split_block_ranges(*args.ctx_, + args.ctx_->get_allocator(), tsc, ranges, tablets, @@ -1321,7 +1322,8 @@ int ObPartitionWiseGranuleSplitter::split_insert_gi_task(ObGranulePumpArgs &args LOG_WARN("failed to make whole range", K(ret)); } else if (OB_FAIL(ranges.push_back(each_partition_range))) { LOG_WARN("failed to push partition range to ranges", K(ret)); - } else if (OB_FAIL(ObGranuleUtil::split_block_ranges(args.ctx_->get_allocator(), + } else if (OB_FAIL(ObGranuleUtil::split_block_ranges(*args.ctx_, + args.ctx_->get_allocator(), NULL, ranges, tablets, diff --git a/src/sql/engine/px/ob_granule_util.cpp b/src/sql/engine/px/ob_granule_util.cpp index 20a31c892f..670607f7c0 100644 --- a/src/sql/engine/px/ob_granule_util.cpp +++ b/src/sql/engine/px/ob_granule_util.cpp @@ -27,6 +27,7 @@ #include "share/external_table/ob_external_table_file_mgr.h" #include "share/external_table/ob_external_table_utils.h" #include "sql/engine/table/ob_external_table_access_service.h" +#include "sql/das/ob_das_simple_op.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -132,7 +133,8 @@ int ObGranuleUtil::split_granule_for_external_table(ObIAllocator &allocator, return ret; } -int ObGranuleUtil::split_block_ranges(ObIAllocator &allocator, +int ObGranuleUtil::split_block_ranges(ObExecContext &exec_ctx, + ObIAllocator &allocator, const ObTableScanSpec *tsc,//may be is null, attention use const ObIArray &in_ranges, const ObIArray &tablets, @@ -184,7 +186,8 @@ int ObGranuleUtil::split_block_ranges(ObIAllocator &allocator, } } LOG_TRACE("gi partition granule"); - } else if (OB_FAIL(split_block_granule(allocator, + } else if (OB_FAIL(split_block_granule(exec_ctx, + allocator, tsc, ranges, tablets, @@ -227,7 +230,8 @@ int ObGranuleUtil::remove_empty_range(const common::ObIArray return ret; } -int ObGranuleUtil::split_block_granule(ObIAllocator &allocator, +int ObGranuleUtil::split_block_granule(ObExecContext &exec_ctx, + ObIAllocator &allocator, const ObTableScanSpec *tsc,//may be is null, attention use! const ObIArray &input_ranges, const ObIArray &tablets, @@ -273,10 +277,9 @@ int ObGranuleUtil::split_block_granule(ObIAllocator &allocator, input_store_ranges, need_convert_new_range))) { LOG_WARN("failed to convert new range to store range", K(ret)); - } else if (OB_FAIL(access_service->get_multi_ranges_cost(tablet.ls_id_, - tablet.tablet_id_, - input_store_ranges, - partition_size))) { + } else if (OB_FAIL(ObDASSimpleUtils::get_multi_ranges_cost(exec_ctx, tablets.at(i), + input_store_ranges, + partition_size))) { LOG_WARN("failed to get multi ranges cost", K(ret), K(tablet)); } else { // B to MB @@ -339,7 +342,8 @@ int ObGranuleUtil::split_block_granule(ObIAllocator &allocator, input_store_ranges, need_convert_new_range))) { LOG_WARN("failed to convert new range to store range", K(ret)); - } else if (OB_FAIL(get_tasks_for_partition(allocator, + } else if (OB_FAIL(get_tasks_for_partition(exec_ctx, + allocator, expected_task_cnt, *tablet, input_store_ranges, @@ -470,7 +474,8 @@ int ObGranuleUtil::compute_task_count_each_partition(int64_t total_size, return ret; } -int ObGranuleUtil::get_tasks_for_partition(ObIAllocator &allocator, +int ObGranuleUtil::get_tasks_for_partition(ObExecContext &exec_ctx, + ObIAllocator &allocator, int64_t expected_task_cnt, ObDASTabletLoc &tablet, ObIArray &input_storage_ranges, @@ -504,12 +509,11 @@ int ObGranuleUtil::get_tasks_for_partition(ObIAllocator &allocator, if (!range_independent) { tablet_idx++; } - } else if (OB_FAIL(access_service->split_multi_ranges(tablet.ls_id_, - tablet.tablet_id_, - input_storage_ranges, - expected_task_cnt, - allocator, - multi_range_split_array))) { + } else if (OB_FAIL(ObDASSimpleUtils::split_multi_ranges(exec_ctx, + &tablet, + input_storage_ranges, + expected_task_cnt, + multi_range_split_array))) { LOG_WARN("failed to split multi ranges", K(ret), K(tablet), K(expected_task_cnt)); } else { LOG_TRACE("split multi ranges", diff --git a/src/sql/engine/px/ob_granule_util.h b/src/sql/engine/px/ob_granule_util.h index 469f8448d8..0b3938428d 100644 --- a/src/sql/engine/px/ob_granule_util.h +++ b/src/sql/engine/px/ob_granule_util.h @@ -171,7 +171,8 @@ public: * range_independent IN the random type witch affects the granule_idx * */ - static int split_block_ranges(common::ObIAllocator &allocator, + static int split_block_ranges(ObExecContext &exec_ctx, + common::ObIAllocator &allocator, const ObTableScanSpec *tsc, const common::ObIArray &ranges, const common::ObIArray &tablets, @@ -259,7 +260,8 @@ public: * range_independent IN the random type witch affects the granule_idx * */ - static int split_block_granule(common::ObIAllocator &allocator, + static int split_block_granule(ObExecContext &exec_ctx, + common::ObIAllocator &allocator, const ObTableScanSpec *tsc, const common::ObIArray &input_ranges, const common::ObIArray &tablet_array, @@ -319,7 +321,8 @@ private: * pkey_idx OUT the idx in granule ranges * range_independent IN the random type witch affects the granule_idx */ - static int get_tasks_for_partition(common::ObIAllocator &allocator, + static int get_tasks_for_partition(ObExecContext &exec_ctx, + common::ObIAllocator &allocator, int64_t expected_task_cnt, ObDASTabletLoc &tablet, common::ObIArray &input_storage_ranges, diff --git a/src/sql/engine/sort/ob_sort_op.cpp b/src/sql/engine/sort/ob_sort_op.cpp index 638a10f119..e3bf11131f 100644 --- a/src/sql/engine/sort/ob_sort_op.cpp +++ b/src/sql/engine/sort/ob_sort_op.cpp @@ -281,7 +281,7 @@ int ObSortOp::process_sort_batch() int ObSortOp::scan_all_then_sort() { int ret = OB_SUCCESS; - SMART_VAR(ObChunkDatumStore, cache_store) { + SMART_VAR(ObChunkDatumStore, cache_store, "SORT_CACHE_CTX") { if (OB_FAIL(cache_store.init(2 * 1024 * 1024, ctx_.get_my_session()->get_effective_tenant_id(), ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true/*enable dump*/))) { @@ -338,7 +338,7 @@ int ObSortOp::scan_all_then_sort() int ObSortOp::scan_all_then_sort_batch() { int ret = OB_SUCCESS; - SMART_VAR(ObChunkDatumStore, cache_store) { + SMART_VAR(ObChunkDatumStore, cache_store, "SORT_CACHE_CTX") { if (OB_FAIL(cache_store.init(2 * 1024 * 1024, ctx_.get_my_session()->get_effective_tenant_id(), ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true/*enable dump*/))) { diff --git a/src/sql/engine/sort/ob_sort_op_impl.cpp b/src/sql/engine/sort/ob_sort_op_impl.cpp index ee4bb547f7..abc3f11aba 100644 --- a/src/sql/engine/sort/ob_sort_op_impl.cpp +++ b/src/sql/engine/sort/ob_sort_op_impl.cpp @@ -567,7 +567,7 @@ ObSortOpImpl::ObSortOpImpl(ObMonitorNode &op_monitor_info) : inited_(false), local_merge_sort_(false), need_rewind_(false), got_first_row_(false), sorted_(false), enable_encode_sortkey_(false), mem_context_(NULL), mem_entify_guard_(mem_context_), tenant_id_(OB_INVALID_ID), sort_collations_(nullptr), - sort_cmp_funs_(nullptr), eval_ctx_(nullptr), inmem_row_size_(0), mem_check_interval_mask_(1), + sort_cmp_funs_(nullptr), eval_ctx_(nullptr), datum_store_(ObModIds::OB_SQL_SORT_ROW), inmem_row_size_(0), mem_check_interval_mask_(1), row_idx_(0), heap_iter_begin_(false), imms_heap_(NULL), ems_heap_(NULL), next_stored_row_func_(&ObSortOpImpl::array_next_stored_row), input_rows_(OB_INVALID_ID), input_width_(OB_INVALID_ID), @@ -2338,7 +2338,7 @@ ObPrefixSortImpl::ObPrefixSortImpl(ObMonitorNode &op_monitor_info) : ObSortOpImp base_sort_collations_(), base_sort_cmp_funs_(), prev_row_(nullptr), next_prefix_row_store_(), next_prefix_row_(nullptr), child_(nullptr), self_op_(nullptr), sort_row_count_(nullptr), - selector_(nullptr), selector_size_(0), sort_prefix_rows_(0), + selector_(nullptr), selector_size_(0), sort_prefix_rows_(0), immediate_prefix_store_(ObModIds::OB_SQL_SORT_ROW), immediate_prefix_rows_(nullptr), immediate_prefix_pos_(0), brs_(NULL) { } diff --git a/src/sql/engine/sort/ob_sort_op_impl.h b/src/sql/engine/sort/ob_sort_op_impl.h index 9445927d68..f75574620b 100644 --- a/src/sql/engine/sort/ob_sort_op_impl.h +++ b/src/sql/engine/sort/ob_sort_op_impl.h @@ -26,7 +26,7 @@ namespace sql struct ObSortOpChunk : public common::ObDLinkBase { - explicit ObSortOpChunk(const int64_t level): level_(level), row_(NULL) {} + explicit ObSortOpChunk(const int64_t level): level_(level), datum_store_(ObModIds::OB_SQL_SORT_ROW), row_(NULL) {} int64_t level_; ObChunkDatumStore datum_store_; diff --git a/src/sql/engine/subquery/ob_subplan_filter_op.cpp b/src/sql/engine/subquery/ob_subplan_filter_op.cpp index e9c8ca1126..843f50b87c 100644 --- a/src/sql/engine/subquery/ob_subplan_filter_op.cpp +++ b/src/sql/engine/subquery/ob_subplan_filter_op.cpp @@ -49,6 +49,7 @@ ObSubQueryIterator::ObSubQueryIterator(ObOperator &op) onetime_plan_(false), init_plan_(false), inited_(false), + store_("SpfIter"), mem_entity_(nullptr), id_(-1), parent_(nullptr), @@ -435,7 +436,7 @@ ObSubPlanFilterOp::ObSubPlanFilterOp( max_group_size_(0), current_group_(0), das_batch_params_(), - left_rows_(), + left_rows_("SpfOp"), left_rows_iter_(), last_store_row_(), save_last_row_(false), diff --git a/src/sql/engine/table/ob_table_row_store_op.cpp b/src/sql/engine/table/ob_table_row_store_op.cpp index 299969cbab..ce582de775 100644 --- a/src/sql/engine/table/ob_table_row_store_op.cpp +++ b/src/sql/engine/table/ob_table_row_store_op.cpp @@ -95,7 +95,7 @@ OB_DEF_DESERIALIZE(ObTableRowStoreOpInput) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate row store failed", K(sizeof(ObChunkDatumStore))); } else { - ObChunkDatumStore *datum_store = new(ptr) ObChunkDatumStore(allocator_); + ObChunkDatumStore *datum_store = new(ptr) ObChunkDatumStore("TableRowStoreOp", allocator_); if (OB_FAIL(datum_store->init(UINT64_MAX, common::OB_SERVER_TENANT_ID, ObCtxIds::DEFAULT_CTX_ID, diff --git a/src/sql/engine/window_function/ob_window_function_op.cpp b/src/sql/engine/window_function/ob_window_function_op.cpp index 0348d42c4d..0045edff89 100644 --- a/src/sql/engine/window_function/ob_window_function_op.cpp +++ b/src/sql/engine/window_function/ob_window_function_op.cpp @@ -894,8 +894,8 @@ int ObWindowFunctionOp::NonAggrCellRankLike::eval(RowsReader &row_reader, ObDatum &expr_datum = wf_info_.expr_->locate_datum_for_write(op_.eval_ctx_); wf_info_.expr_->set_evaluated_flag(op_.eval_ctx_); if (T_WIN_FUN_PERCENT_RANK == wf_info_.func_type_) { - if (is_oracle_mode()) { - // result will be zero when only one row within frame + if (ob_is_number_tc(wf_info_.expr_->datum_meta_.type_)) { + //in mysql mode, percent rank may return double if (0 == frame.tail_ - frame.head_) { number::ObNumber res_nmb; res_nmb.set_zero(); @@ -915,7 +915,7 @@ int ObWindowFunctionOp::NonAggrCellRankLike::eval(RowsReader &row_reader, expr_datum.set_number(res_nmb); } } - } else { + } else if (ObDoubleType == wf_info_.expr_->datum_meta_.type_) { if (0 == frame.tail_ - frame.head_) { expr_datum.set_double(0); } else { @@ -923,6 +923,9 @@ int ObWindowFunctionOp::NonAggrCellRankLike::eval(RowsReader &row_reader, double denominator= static_cast(frame.tail_ - frame.head_); expr_datum.set_double(numerator / denominator); } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the result type of window function is unexpected", K(ret), K(wf_info_.expr_->datum_meta_)); } } else if (lib::is_oracle_mode()) { number::ObNumber res_nmb; @@ -1002,7 +1005,7 @@ int ObWindowFunctionOp::NonAggrCellCumeDist::eval(RowsReader &row_reader, } } if (OB_SUCC(ret)) { - if (is_oracle_mode()) { + if (ob_is_number_tc(wf_info_.expr_->datum_meta_.type_)) { // number of row[cur] >= row[:] (whether `>=` or other is depend on ORDER BY) number::ObNumber numerator; // total tuple of current window @@ -1022,7 +1025,7 @@ int ObWindowFunctionOp::NonAggrCellCumeDist::eval(RowsReader &row_reader, expr_datum.set_number(res_nmb); val = static_cast(expr_datum); } - } else { + } else if (ObDoubleType == wf_info_.expr_->datum_meta_.type_) { // number of row[cur] >= row[:] (whether `>=` or other is depend on ORDER BY) double numerator; // total tuple of current window @@ -1033,6 +1036,9 @@ int ObWindowFunctionOp::NonAggrCellCumeDist::eval(RowsReader &row_reader, wf_info_.expr_->set_evaluated_flag(op_.eval_ctx_); expr_datum.set_double(numerator / denominator); val = static_cast(expr_datum); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the result type of window function is unexpected", K(ret), K(wf_info_.expr_->datum_meta_)); } } diff --git a/src/sql/executor/ob_direct_receive_op.cpp b/src/sql/executor/ob_direct_receive_op.cpp index 09b4a11b31..9ead7afd7f 100644 --- a/src/sql/executor/ob_direct_receive_op.cpp +++ b/src/sql/executor/ob_direct_receive_op.cpp @@ -145,12 +145,14 @@ int ObDirectReceiveOp::inner_close() LOG_WARN("session or plan ctx or rpc is NULL", K(ret)); } else { ObQueryRetryInfo retry_info; + const int32_t group_id = OB_INVALID_ID == session->get_expect_group_id() ? 0 : session->get_expect_group_id(); ObExecutorRpcCtx rpc_ctx(session->get_effective_tenant_id(), plan_ctx->get_timeout_timestamp(), ctx_.get_task_exec_ctx().get_min_cluster_version(), &retry_info, session, - plan_ctx->is_plain_select_stmt()); + plan_ctx->is_plain_select_stmt(), + group_id); int tmp_ret = rpc->task_kill(rpc_ctx, resp_handler->get_task_id(), resp_handler->get_dst_addr()); if (OB_SUCCESS != tmp_ret) { LOG_WARN("kill task failed", K(tmp_ret), diff --git a/src/sql/executor/ob_executor_rpc_impl.cpp b/src/sql/executor/ob_executor_rpc_impl.cpp index b8299edabd..0eac7e2789 100644 --- a/src/sql/executor/ob_executor_rpc_impl.cpp +++ b/src/sql/executor/ob_executor_rpc_impl.cpp @@ -54,6 +54,7 @@ int ObExecutorRpcImpl::task_execute(ObExecutorRpcCtx &rpc_ctx, RemoteStreamHandle &real_handler = handler.get_remote_stream_handle(); RemoteStreamHandle::MyHandle &h = real_handler.get_handle(); int64_t timeout_timestamp = rpc_ctx.get_timeout_timestamp(); + const int32_t group_id = rpc_ctx.get_group_id(); if (OB_ISNULL(proxy_) || OB_ISNULL(real_handler.get_result())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("not init", K(ret), K_(proxy), "result", real_handler.get_result()); @@ -71,6 +72,7 @@ int ObExecutorRpcImpl::task_execute(ObExecutorRpcCtx &rpc_ctx, } else if (OB_FAIL(to_proxy .by(tenant_id) .timeout(timeout) + .group_id(group_id) .task_execute(task, *real_handler.get_result(), h))) { LOG_WARN("rpc task_execute fail", K(ret), @@ -121,6 +123,7 @@ int ObExecutorRpcImpl::task_execute_v2(ObExecutorRpcCtx &rpc_ctx, RemoteStreamHandleV2 &real_handler = handler.get_remote_stream_handle_v2(); RemoteStreamHandleV2::MyHandle &h = real_handler.get_handle(); int64_t timeout_timestamp = rpc_ctx.get_timeout_timestamp(); + const int32_t group_id = rpc_ctx.get_group_id(); if (OB_ISNULL(proxy_) || OB_ISNULL(real_handler.get_result())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("not init", K(ret), K_(proxy), "result", real_handler.get_result()); @@ -135,6 +138,7 @@ int ObExecutorRpcImpl::task_execute_v2(ObExecutorRpcCtx &rpc_ctx, } else if (OB_FAIL(to_proxy .by(tenant_id) .timeout(timeout) + .group_id(group_id) .remote_task_execute(task, *real_handler.get_result(), h))) { LOG_WARN("rpc task_execute fail", K(ret), K(tenant_id), K(svr), K(timeout), K(timeout_timestamp)); diff --git a/src/sql/executor/ob_executor_rpc_impl.h b/src/sql/executor/ob_executor_rpc_impl.h index 08146aff97..f730b854a8 100644 --- a/src/sql/executor/ob_executor_rpc_impl.h +++ b/src/sql/executor/ob_executor_rpc_impl.h @@ -233,13 +233,15 @@ public: uint64_t min_cluster_version, ObQueryRetryInfo *retry_info, ObSQLSessionInfo *session, - bool is_plain_select) + bool is_plain_select, + int32_t group_id) : rpc_tenant_id_(rpc_tenant_id), timeout_timestamp_(timeout_timestamp), min_cluster_version_(min_cluster_version), retry_info_(retry_info), session_(session), - is_plain_select_(is_plain_select) + is_plain_select_(is_plain_select), + group_id_(group_id) { } ~ObExecutorRpcCtx() {} @@ -256,11 +258,13 @@ public: inline ObQueryRetryInfo *get_retry_info_for_update() const { return retry_info_; } bool is_retry_for_rpc_timeout() const { return is_plain_select_; } int check_status() const; + int32_t get_group_id() const { return group_id_; } TO_STRING_KV(K_(rpc_tenant_id), K_(timeout_timestamp), K_(min_cluster_version), K_(retry_info), - K_(is_plain_select)); + K_(is_plain_select), + K_(group_id)); private: uint64_t rpc_tenant_id_; int64_t timeout_timestamp_; @@ -269,6 +273,7 @@ private: ObQueryRetryInfo *retry_info_; const ObSQLSessionInfo *session_;//该类中的变量会并发访问，注意session成功并发访问是否正确 bool is_plain_select_;//stmt_type == T_SELECT && not select...for update + int32_t group_id_; private: DISALLOW_COPY_AND_ASSIGN(ObExecutorRpcCtx); }; diff --git a/src/sql/executor/ob_maintain_dependency_info_task.cpp b/src/sql/executor/ob_maintain_dependency_info_task.cpp index 52d785d996..15e0eb0230 100644 --- a/src/sql/executor/ob_maintain_dependency_info_task.cpp +++ b/src/sql/executor/ob_maintain_dependency_info_task.cpp @@ -267,7 +267,10 @@ void ObMaintainDepInfoTaskQueue::run2() if (task->get_retry_times() > 0) { task->set_retry_times(task->get_retry_times() - 1); task->set_last_execute_time(ObTimeUtility::current_time()); - if (OB_FAIL(queue_.push(task))) { + if (is_queue_almost_full()) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("push task to queue failed", K(ret)); + } else if (OB_FAIL(queue_.push(task))) { LOG_WARN("push task to queue failed", K(ret)); } else { rescheduled = true; diff --git a/src/sql/executor/ob_maintain_dependency_info_task.h b/src/sql/executor/ob_maintain_dependency_info_task.h index 914bd1a626..25d09a468b 100644 --- a/src/sql/executor/ob_maintain_dependency_info_task.h +++ b/src/sql/executor/ob_maintain_dependency_info_task.h @@ -90,6 +90,7 @@ class ObMaintainDepInfoTaskQueue: public share::ObAsyncTaskQueue { public: static const int64_t INIT_BKT_SIZE = 512; + constexpr static const double MAX_QUEUE_USAGE_RATIO = 0.8; ObMaintainDepInfoTaskQueue() : last_execute_time_(0) {} virtual ~ObMaintainDepInfoTaskQueue() { @@ -102,6 +103,7 @@ public: { last_execute_time_ = execute_time; } int add_view_id_to_set(const uint64_t view_id) { return view_info_set_.set_refactored(view_id); } int erase_view_id_from_set(const uint64_t view_id) { return view_info_set_.erase_refactored(view_id); } + bool is_queue_almost_full() const { return queue_.size() > queue_.capacity() * MAX_QUEUE_USAGE_RATIO; } private: int64_t last_execute_time_; common::hash::ObHashSet view_info_set_; diff --git a/src/sql/executor/ob_remote_scheduler.cpp b/src/sql/executor/ob_remote_scheduler.cpp index ddca225e56..d24d9d298b 100644 --- a/src/sql/executor/ob_remote_scheduler.cpp +++ b/src/sql/executor/ob_remote_scheduler.cpp @@ -233,12 +233,14 @@ int ObRemoteScheduler::execute_with_sql(ObExecContext &ctx, ObPhysicalPlan *phy_ } if (OB_SUCC(ret)) { ObScanner *scanner = NULL; + const int32_t group_id = OB_INVALID_ID == session->get_expect_group_id() ? 0 : session->get_expect_group_id(); ObExecutorRpcCtx rpc_ctx(session->get_rpc_tenant_id(), plan_ctx->get_timeout_timestamp(), ctx.get_task_exec_ctx().get_min_cluster_version(), retry_info, ctx.get_my_session(), - plan_ctx->is_plain_select_stmt()); + plan_ctx->is_plain_select_stmt(), + group_id); if (OB_FAIL(rpc->task_execute_v2(rpc_ctx, task, task.get_runner_svr(), diff --git a/src/sql/executor/ob_remote_task_executor.cpp b/src/sql/executor/ob_remote_task_executor.cpp index 622028e615..672fc0a302 100644 --- a/src/sql/executor/ob_remote_task_executor.cpp +++ b/src/sql/executor/ob_remote_task_executor.cpp @@ -63,12 +63,14 @@ int ObRemoteTaskExecutor::execute(ObExecContext &query_ctx, ObJob *job, ObTaskIn } else { // 将task_info设成OB_TASK_STATE_RUNNING状态，后面如果重试可能会用到该状态 task_info->set_state(OB_TASK_STATE_RUNNING); + const int32_t group_id = OB_INVALID_ID == session->get_expect_group_id() ? 0 : session->get_expect_group_id(); ObExecutorRpcCtx rpc_ctx(session->get_rpc_tenant_id(), plan_ctx->get_timeout_timestamp(), query_ctx.get_task_exec_ctx().get_min_cluster_version(), retry_info, query_ctx.get_my_session(), - plan_ctx->is_plain_select_stmt()); + plan_ctx->is_plain_select_stmt(), + group_id); if (OB_FAIL(rpc->task_execute(rpc_ctx, task, task_info->get_task_location().get_server(), diff --git a/src/sql/executor/ob_slice_calc.cpp b/src/sql/executor/ob_slice_calc.cpp index ad98e3bdf7..d857c217c7 100644 --- a/src/sql/executor/ob_slice_calc.cpp +++ b/src/sql/executor/ob_slice_calc.cpp @@ -237,10 +237,10 @@ int ObRepartSliceIdxCalc::get_previous_row_tablet_id(ObObj &tablet_id) return ret; } -int ObSlaveMapRepartIdxCalcBase::init() +int ObSlaveMapRepartIdxCalcBase::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - if (OB_FAIL(ObRepartSliceIdxCalc::init())) { + if (OB_FAIL(ObRepartSliceIdxCalc::init(tenant_id))) { LOG_WARN("fail init base", K(ret)); } // 在pkey random情况下，一个partition是可以被其所在的SQC上的所有worker处理的， @@ -254,7 +254,10 @@ int ObSlaveMapRepartIdxCalcBase::init() // p2 : [task4,task5] const ObPxPartChMapTMArray &part_ch_array = part_ch_info_.part_ch_array_; if (OB_SUCC(ret)) { - if (OB_FAIL(part_to_task_array_map_.create(max(1, part_ch_array.count()), ObModIds::OB_SQL_PX))) { + if (OB_FAIL(part_to_task_array_map_.create(max(1, part_ch_array.count()), + ObModIds::OB_SQL_PX, + ObModIds::OB_HASH_NODE, + tenant_id))) { LOG_WARN("fail create part to task array map", "count", part_ch_array.count(), K(ret)); } else { // In ObRepartSliceIdxCalc::init(), the support_vectorized_calc_ has been set to true. @@ -304,9 +307,9 @@ int ObSlaveMapRepartIdxCalcBase::destroy() return ret; } -int ObRepartRandomSliceIdxCalc::init() +int ObRepartRandomSliceIdxCalc::init(uint64_t tenant_id) { - return ObSlaveMapRepartIdxCalcBase::init(); + return ObSlaveMapRepartIdxCalcBase::init(tenant_id); } int ObRepartRandomSliceIdxCalc::destroy() @@ -471,13 +474,13 @@ int ObAffinitizedRepartSliceIdxCalc::get_slice_idx_vec(const ObIArray & return ret; } -int ObRepartSliceIdxCalc::init() +int ObRepartSliceIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; if (px_repart_ch_map_.created()) { ret = OB_INIT_TWICE; LOG_WARN("this map has been init twice", K(ret)); - } else if (OB_FAIL(build_repart_ch_map(px_repart_ch_map_))) { + } else if (OB_FAIL(build_repart_ch_map(px_repart_ch_map_, tenant_id))) { LOG_WARN("failed to build affi hash map", K(ret)); } else if (OB_FAIL(setup_one_side_one_level_info())) { LOG_WARN("fail to build one side on level map", K(ret)); @@ -515,11 +518,14 @@ int ObRepartSliceIdxCalc::setup_one_side_one_level_info() return ret; } -int ObRepartSliceIdxCalc::build_repart_ch_map(ObPxPartChMap &affinity_map) +int ObRepartSliceIdxCalc::build_repart_ch_map(ObPxPartChMap &affinity_map, uint64_t tenant_id) { int ret = OB_SUCCESS; const ObPxPartChMapTMArray &part_ch_array = part_ch_info_.part_ch_array_; - if (OB_FAIL(affinity_map.create(max(1, part_ch_array.count()), ObModIds::OB_SQL_PX))) { + if (OB_FAIL(affinity_map.create(max(1, part_ch_array.count()), + ObModIds::OB_SQL_PX, + ObModIds::OB_HASH_NODE, + tenant_id))) { LOG_WARN("fail create hashmap", "count", part_ch_array.count(), K(ret)); } @@ -795,13 +801,13 @@ ObSlaveMapPkeyRangeIdxCalc::~ObSlaveMapPkeyRangeIdxCalc() destroy(); } -int ObSlaveMapPkeyRangeIdxCalc::init() +int ObSlaveMapPkeyRangeIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(is_inited_)); - } else if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init())) { + } else if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init(tenant_id))) { LOG_WARN("fail init base repart class", K(ret)); } else if (OB_UNLIKELY(nullptr == calc_part_id_expr_ || sort_exprs_.count() <= 0)) { ret = OB_INVALID_ARGUMENT; @@ -1048,10 +1054,10 @@ int ObSlaveMapPkeyRangeIdxCalc::get_slice_idx( } /******************* ObSlaveMapPkeyHashIdxCalc ********************/ -int ObSlaveMapPkeyHashIdxCalc::init() +int ObSlaveMapPkeyHashIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init())) { + if (OB_FAIL(ObSlaveMapRepartIdxCalcBase::init(tenant_id))) { LOG_WARN("fail init base repart class", K(ret)); } else if (affi_hash_map_.created()) { ret = OB_INIT_TWICE; @@ -1339,10 +1345,10 @@ int ObNullAwareHashSliceIdCalc::get_slice_indexes(const ObIArray &exprs return ret; } -int ObNullAwareAffinitizedRepartSliceIdxCalc::init() +int ObNullAwareAffinitizedRepartSliceIdxCalc::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - OZ (ObRepartSliceIdxCalc::init()); + OZ (ObRepartSliceIdxCalc::init(tenant_id)); OX (support_vectorized_calc_ = false); return ret; } diff --git a/src/sql/executor/ob_slice_calc.h b/src/sql/executor/ob_slice_calc.h index 45f1baa17f..81026da4d8 100644 --- a/src/sql/executor/ob_slice_calc.h +++ b/src/sql/executor/ob_slice_calc.h @@ -232,7 +232,7 @@ public: int init_partition_cache_map(); - virtual int init(); + virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID); virtual int destroy() { int ret = OB_SUCCESS; @@ -242,7 +242,7 @@ public: return ret; } - int build_repart_ch_map(ObPxPartChMap &map); + int build_repart_ch_map(ObPxPartChMap &map, uint64_t tenant_id); int build_part2tablet_id_map(); protected: // this is a trick! @@ -320,7 +320,7 @@ protected: {} ~ObSlaveMapRepartIdxCalcBase() = default; protected: - virtual int init() override; + virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override; virtual int destroy() override; protected: // 存储同一个partition所对应的所有task id @@ -390,7 +390,7 @@ public: virtual int get_slice_idx(const ObIArray &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx) override; - virtual int init() override; + virtual int init(uint64_t tenant_id) override; virtual int destroy() override; private: int get_task_idx_by_tablet_id(int64_t tablet_id , int64_t &task_idx); @@ -810,7 +810,7 @@ public: sort_cmp_(sort_cmp_funs, sort_collations) {} virtual ~ObSlaveMapPkeyRangeIdxCalc(); - virtual int init() override; + virtual int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override; virtual int destroy() override; virtual int get_slice_idx( const common::ObIArray &exprs, @@ -932,7 +932,7 @@ public: } ~ObSlaveMapPkeyHashIdxCalc() = default; - int init() override; + int init(uint64_t tenant_id = OB_SERVER_TENANT_ID) override; int destroy() override; // for static engine @@ -1091,7 +1091,7 @@ public: } ~ObNullAwareAffinitizedRepartSliceIdxCalc() = default; - virtual int init() override; + virtual int init(uint64_t tenant_id) override; virtual int get_slice_idx(const ObIArray &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx) { UNUSED(exprs); diff --git a/src/sql/monitor/flt/ob_flt_control_info_mgr.h b/src/sql/monitor/flt/ob_flt_control_info_mgr.h index 00fea19924..36668d46a9 100644 --- a/src/sql/monitor/flt/ob_flt_control_info_mgr.h +++ b/src/sql/monitor/flt/ob_flt_control_info_mgr.h @@ -24,7 +24,7 @@ namespace oceanbase { namespace sql { - // for client identfier + // for client identifier static const char TYPE_I[] = "type_i"; // for mod_act static const char TYPE_MOD_ACT[] = "type_m"; @@ -147,7 +147,7 @@ namespace sql common::ObSEArray mod_infos_; // tenant trace FLTControlInfo tenant_info_; - // allcator + // allocator ObArenaAllocator alloc_; }; diff --git a/src/sql/monitor/ob_exec_stat.h b/src/sql/monitor/ob_exec_stat.h index 5f3020f61e..c5395a1fc6 100644 --- a/src/sql/monitor/ob_exec_stat.h +++ b/src/sql/monitor/ob_exec_stat.h @@ -367,7 +367,7 @@ struct ObAuditRecordData { int64_t plan_id_; int64_t affected_rows_;//delete,update,insert影响的行数,及select选出的行数 int64_t return_rows_; - int64_t partition_cnt_;//该请求涉及的所以parttion个数 + int64_t partition_cnt_;//该请求涉及的所以partition个数 int64_t expected_worker_cnt_; // px 预期分配线程数 int64_t used_worker_cnt_; // px 实际分配线程数 int64_t try_cnt_; //尝试执行次数 diff --git a/src/sql/monitor/ob_exec_stat_collector.cpp b/src/sql/monitor/ob_exec_stat_collector.cpp index 4122e847db..c64f2ec61a 100644 --- a/src/sql/monitor/ob_exec_stat_collector.cpp +++ b/src/sql/monitor/ob_exec_stat_collector.cpp @@ -82,7 +82,7 @@ int ObExecStatCollector::collect_plan_monitor_info(uint64_t job_id, SQL_MONITOR_LOG(WARN, "fail to get operator info by index", K(ret), K(i)); } else if (OB_ISNULL(op_info)) { ret = OB_ERR_UNEXPECTED; - SQL_MONITOR_LOG(WARN, "get invalie op_info", K(ret), K(op_info)); + SQL_MONITOR_LOG(WARN, "get invalid op_info", K(ret), K(op_info)); } else if (OB_FAIL(op_info->set_job_id(job_id))) { SQL_MONITOR_LOG(WARN, "fail to set job id", K(ret), K(job_id)); } else if (OB_FAIL(op_info->set_task_id(task_id))) { @@ -142,7 +142,7 @@ int ObExecStatDispatch::dispatch(bool need_add_monitor, } else if (need_add_monitor && OB_FAIL(monitor_info->add_operator_info(op_info))) { LOG_WARN("fail to add operator info", K(ret), K(op_info)); } else if (need_update_plan && OB_FAIL(plan->op_stats_.add_op_stat(op_info))) { - LOG_WARN("fail to add operatgor info", K(ret), K(op_info)); + LOG_WARN("fail to add operator info", K(ret), K(op_info)); } break; } diff --git a/src/sql/monitor/ob_phy_operator_stats.cpp b/src/sql/monitor/ob_phy_operator_stats.cpp index e047d8bae7..e4b3ee4f1c 100644 --- a/src/sql/monitor/ob_phy_operator_stats.cpp +++ b/src/sql/monitor/ob_phy_operator_stats.cpp @@ -69,7 +69,7 @@ int ObPhyOperatorStats::add_op_stat(ObPhyOperatorMonitorInfo &info) int64_t stat_start_index = copy_start_index + info.get_op_id() * StatId::MAX_STAT; if (stat_start_index < 0 || stat_start_index + StatId::MAX_STAT > array_size_) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invaild array index", K(stat_start_index), K(array_size_)); + LOG_WARN("invalid array index", K(stat_start_index), K(array_size_)); } else { int64_t last_input_rows = 0; int64_t last_output_rows = 0; diff --git a/src/sql/monitor/ob_phy_plan_monitor_info.cpp b/src/sql/monitor/ob_phy_plan_monitor_info.cpp index 2438320786..4d41a10e8f 100644 --- a/src/sql/monitor/ob_phy_plan_monitor_info.cpp +++ b/src/sql/monitor/ob_phy_plan_monitor_info.cpp @@ -41,7 +41,7 @@ int ObPhyPlanMonitorInfo::get_operator_info(int64_t op_id, ObPhyOperatorMonitorI ARRAY_FOREACH_NORET(operator_infos_, idx) { if (operator_infos_.at(idx).get_op_id() == op_id) { if (OB_FAIL(info.assign(operator_infos_.at(idx)))) { - LOG_WARN("fail to assgin to phy_operator info", K(ret)); + LOG_WARN("fail to assign to phy_operator info", K(ret)); } else { ret = OB_SUCCESS; } diff --git a/src/sql/monitor/ob_sql_plan.cpp b/src/sql/monitor/ob_sql_plan.cpp index b64eb45298..988e03bdb4 100644 --- a/src/sql/monitor/ob_sql_plan.cpp +++ b/src/sql/monitor/ob_sql_plan.cpp @@ -13,6 +13,7 @@ #include "sql/resolver/ddl/ob_explain_stmt.h" #include "sql/optimizer/ob_log_values.h" #include "sql/optimizer/ob_log_plan.h" +#include "sql/optimizer/ob_del_upd_log_plan.h" #include "lib/time/Time.h" #include "pl/sys_package/ob_dbms_xplan.h" #include "lib/json/ob_json.h" @@ -208,8 +209,8 @@ int ObSqlPlan::get_plan_outline_info_one_line(PlanText &plan_text, LOG_WARN("failed to get plan tree outline", K(ret)); } else if (OB_FAIL(query_hint.print_transform_hints(plan_text))) { LOG_WARN("failed to print all transform hints", K(ret)); - } else if (OB_FAIL(query_hint.get_global_hint().print_global_hint(plan_text))) { - LOG_WARN("failed to print global hint", K(ret)); + } else if (OB_FAIL(get_global_hint_outline(plan_text, *plan))) { + LOG_WARN("failed to get plan global hint outline", K(ret)); } else { BUF_PRINT_CONST_STR(" END_OUTLINE_DATA*/", plan_text); plan_text.is_outline_data_ = false; @@ -221,6 +222,43 @@ int ObSqlPlan::get_plan_outline_info_one_line(PlanText &plan_text, return ret; } +int ObSqlPlan::get_global_hint_outline(PlanText &plan_text, ObLogPlan &plan) +{ + int ret = OB_SUCCESS; + ObGlobalHint outline_global_hint; + if (OB_FAIL(outline_global_hint.assign(plan.get_optimizer_context().get_global_hint()))) { + LOG_WARN("failed to assign global hint", K(ret)); + } else if (OB_FAIL(construct_outline_global_hint(plan, outline_global_hint))) { + LOG_WARN("failed to construct outline global hint", K(ret)); + } else if (OB_FAIL(outline_global_hint.print_global_hint(plan_text))) { + LOG_WARN("failed to print global hint", K(ret)); + } + return ret; +} + +int ObSqlPlan::construct_outline_global_hint(ObLogPlan &plan, ObGlobalHint &outline_global_hint) +{ + int ret = OB_SUCCESS; + ObDelUpdLogPlan *del_upd_plan = NULL; + outline_global_hint.opt_features_version_ = ObGlobalHint::CURRENT_OUTLINE_ENABLE_VERSION; + outline_global_hint.pdml_option_ = ObPDMLOption::NOT_SPECIFIED; + if (OB_SUCC(ret) && NULL != (del_upd_plan = dynamic_cast(&plan)) + && del_upd_plan->use_pdml()) { + outline_global_hint.pdml_option_ = ObPDMLOption::ENABLE; + } + + if (OB_SUCC(ret)) { + outline_global_hint.parallel_ = ObGlobalHint::UNSET_PARALLEL; + if (plan.get_optimizer_context().is_use_auto_dop()) { + outline_global_hint.merge_parallel_hint(ObGlobalHint::SET_ENABLE_AUTO_DOP); + } else if (plan.get_optimizer_context().get_max_parallel() > ObGlobalHint::DEFAULT_PARALLEL) { + outline_global_hint.merge_parallel_hint(plan.get_optimizer_context().get_max_parallel()); + } + } + + return ret; +} + int ObSqlPlan::inner_store_sql_plan_for_explain(ObExecContext *ctx, const ObString& plan_table, const ObString& statement_id, @@ -475,7 +513,7 @@ int ObSqlPlan::escape_quotes(ObSqlPlanItem &plan_item) } /** - * escape qutotes for string value + * escape quotes for string value * oracle: ' => '' * mysql: ' => \' */ @@ -494,7 +532,7 @@ int ObSqlPlan::inner_escape_quotes(char* &ptr, int64_t &length) int64_t pos = 0; if (OB_ISNULL(buf=(char*)allocator_.alloc(buf_len))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate mempry", K(ret)); + LOG_WARN("failed to allocate memory", K(ret)); } else { for (int64_t i = 0; i < length; ++i) { if (ptr[i] == '\'') { @@ -716,8 +754,8 @@ int ObSqlPlan::get_plan_outline_info(PlanText &plan_text, LOG_WARN("failed to get plan tree outline", K(ret)); } else if (OB_FAIL(query_hint.print_transform_hints(temp_text))) { LOG_WARN("failed to print all transform hints", K(ret)); - } else if (OB_FAIL(query_hint.get_global_hint().print_global_hint(temp_text))) { - LOG_WARN("failed to print global hint", K(ret)); + } else if (OB_FAIL(get_global_hint_outline(temp_text, *plan))) { + LOG_WARN("failed to get plan global hint outline", K(ret)); } else { BUF_PRINT_CONST_STR(NEW_LINE, temp_text); BUF_PRINT_CONST_STR(OUTPUT_PREFIX, temp_text); diff --git a/src/sql/monitor/ob_sql_plan.h b/src/sql/monitor/ob_sql_plan.h index 24158da5b3..e93a45f0f4 100644 --- a/src/sql/monitor/ob_sql_plan.h +++ b/src/sql/monitor/ob_sql_plan.h @@ -188,6 +188,8 @@ private: static int get_plan_tree_outline(PlanText &plan_text, ObLogicalOperator* op); + static int get_global_hint_outline(PlanText &plan_text, ObLogPlan &plan); + static int construct_outline_global_hint(ObLogPlan &plan, ObGlobalHint &outline_global_hint); int get_plan_other_info(PlanText &plan_text, ObLogPlan* plan, diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index 8cf6cc878c..63e7b5c584 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -903,7 +903,7 @@ int ObResultSet::close(int &client_ret) } else if (OB_NOT_NULL(physical_plan_)) { //Because of the async close result we need set the partition_hit flag //to the call back param, than close the result. - //But the das framwork set the patition_hit after result is closed. + //But the das framwork set the partition_hit after result is closed. //So we need to set the partition info at here. if (is_end_trans_async()) { ObCurTraceId::TraceId *cur_trace_id = NULL; @@ -1448,7 +1448,7 @@ int ObResultSet::copy_field_columns(const ObPhysicalPlan &plan) } else if (OB_FAIL(field_columns_.push_back(field))) { LOG_WARN("push back field column failed", K(ret)); } else { - LOG_DEBUG("succs to copy field", K(field)); + LOG_DEBUG("success to copy field", K(field)); } } return ret; @@ -1820,8 +1820,8 @@ int ObRemoteResultSet::copy_field_columns( LOG_WARN("failed to reserve field column array", K(ret), K(N)); } for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { - const ObField &ofield = src_field_columns.at(i); - if (OB_FAIL(field.deep_copy(ofield, &get_mem_pool()))) { + const ObField &obfield = src_field_columns.at(i); + if (OB_FAIL(field.deep_copy(obfield, &get_mem_pool()))) { LOG_WARN("deep copy field failed", K(ret)); } else if (OB_FAIL(field_columns_.push_back(field))) { LOG_WARN("push back field column failed", K(ret)); @@ -1843,7 +1843,7 @@ int ObRemoteResultSet::setup_next_scanner() } else { ObInnerSQLTransmitResult *transmit_result= NULL; - if (!first_response_received_) { /* has not gotten the first scanner responsed */ + if (!first_response_received_) { /* has not gotten the first scanner response */ if (OB_ISNULL(transmit_result = remote_resp_handler_->get_result())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("transmit_result is NULL", K(ret)); @@ -1852,7 +1852,7 @@ int ObRemoteResultSet::setup_next_scanner() } else { scanner_ = &transmit_result->get_scanner(); scanner_iter_ = scanner_->begin(); - first_response_received_ = true; /* has gotten the first scanner responsed already */ + first_response_received_ = true; /* has gotten the first scanner response already */ found_rows_ += scanner_->get_found_rows(); stmt_type_ = transmit_result->get_stmt_type(); const common::ObSArray &src_field_columns = diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index af5157b310..656bd23021 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -7294,7 +7294,8 @@ int ObSPIService::convert_obj(ObPLExecCtx *ctx, LOG_DEBUG("column convert", K(obj.get_meta()), K(result_types[i].get_meta_type()), K(current_type.at(i)), K(result_types[i].get_accuracy())); if (obj.is_pl_extend()/* && pl::PL_RECORD_TYPE == obj.get_meta().get_extend_type()*/ - && result_types[i].get_meta_type().is_ext()) { + && result_types[i].get_meta_type().is_ext() + && obj.get_meta().get_extend_type() != PL_CURSOR_TYPE) { //record嵌object场景，object属性在resolver阶段要求强一致，无需强转 OZ (calc_array.push_back(obj)); } else if (obj.get_meta() == result_types[i].get_meta_type() @@ -7435,6 +7436,7 @@ int ObSPIService::store_result(ObPLExecCtx *ctx, 1 == type_count && 1 == obj_array.count() && obj_array.at(0).is_pl_extend() && + obj_array.at(0).get_meta().get_extend_type() != PL_CURSOR_TYPE && obj_array.at(0).get_meta().get_extend_type() != PL_OPAQUE_TYPE); // xmltypes may need to do cast if (!is_schema_object) { if (OB_SUCC(ret) && type_count != obj_array.count()) { @@ -7766,7 +7768,7 @@ int ObSPIService::store_result(ObPLExecCtx *ctx, } else { OZ (deep_copy_obj(*table->get_allocator(), current_obj, tmp)); } - OZ (store_datum(current_datum, tmp)); + OZ (store_datum(current_datum, tmp, ctx->exec_ctx_->get_my_session())); } } } @@ -7863,7 +7865,7 @@ int ObSPIService::store_datums(ObObj &dest_addr, } for (int64_t i = 0; OB_SUCC(ret) && !is_opaque && i < obj_array.count(); ++i) { - if (OB_FAIL(store_datum(current_datum, obj_array.at(i)))) { + if (OB_FAIL(store_datum(current_datum, obj_array.at(i), session_info))) { LOG_WARN("failed to arrange store", K(dest_addr), K(i), K(obj_array.at(i)), K(obj_array), K(ret)); } } @@ -7871,15 +7873,20 @@ int ObSPIService::store_datums(ObObj &dest_addr, return ret; } -int ObSPIService::store_datum(int64_t ¤t_addr, const ObObj &obj) +int ObSPIService::store_datum(int64_t ¤t_addr, const ObObj &obj, ObSQLSessionInfo *session_info) { int ret = OB_SUCCESS; if (OB_UNLIKELY(0 == current_addr) || obj.is_invalid_type()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Argument passed in is NULL", K(current_addr), K(obj), K(ret)); } else { - new (reinterpret_cast(current_addr))ObObj(obj); - current_addr += sizeof(ObObj); + ObObj *cur_obj = reinterpret_cast(current_addr); + if (OB_FAIL(ObUserDefinedType::destruct_obj(*cur_obj, session_info))) { + LOG_WARN("fail to destruct obj", KPC(cur_obj), K(obj), K(ret)); + } else { + new (cur_obj)ObObj(obj); + current_addr += sizeof(ObObj); + } } return ret; } @@ -8542,11 +8549,14 @@ int ObSPIService::spi_execute_dblink(ObExecContext &exec_ctx, OZ (spi_after_execute_dblink(session, routine_info, allocator, params, exec_params)); } - if (OB_NOT_NULL(dblink_conn)) { + if (OB_NOT_NULL(dblink_proxy) && OB_NOT_NULL(dblink_conn)) { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = dblink_proxy->release_dblink(link_type, dblink_conn))) { LOG_WARN("failed to relese connection", K(tmp_ret)); } + if (OB_SUCC(ret)) { + ret = tmp_ret; + } } return ret; } diff --git a/src/sql/ob_spi.h b/src/sql/ob_spi.h index 015bb9aca2..2f23635c8d 100644 --- a/src/sql/ob_spi.h +++ b/src/sql/ob_spi.h @@ -408,7 +408,7 @@ public: const ObDataType *column_types, int64_t type_count, const bool *exprs_not_null_flag, - const int64_t *pl_integer_rangs, + const int64_t *pl_integer_ranges, bool is_bulk = false, bool is_forall = false, bool is_type_record = false, @@ -424,7 +424,7 @@ public: const ObDataType *column_types, int64_t type_count, const bool *exprs_not_null_flag, - const int64_t *pl_integer_rangs, + const int64_t *pl_integer_ranges, bool is_bulk = false, bool is_returning = false, bool is_type_record = false); @@ -504,7 +504,7 @@ public: const ObDataType *column_types, int64_t type_count, const bool *exprs_not_null_flag, - const int64_t *pl_integer_rangs, + const int64_t *pl_integer_ranges, bool is_bulk, int64_t limit, const ObDataType *return_types, @@ -527,7 +527,7 @@ public: static int spi_extend_collection(pl::ObPLExecCtx *ctx, const ObSqlExpression *collection_expr, - int64_t coluln_count, + int64_t column_count, const ObSqlExpression *n_expr, const ObSqlExpression *i_expr = NULL, uint64_t package_id = OB_INVALID_ID); @@ -791,7 +791,7 @@ private: const ObDataType *column_types, int64_t type_count, const bool *exprs_not_null_flag, - const int64_t *pl_integer_rangs, + const int64_t *pl_integer_ranges, int64_t is_bulk, bool is_forall = false, bool is_type_record = false, @@ -976,7 +976,7 @@ private: static int store_datums(ObObj &dest_addr, ObIArray &result, ObIAllocator *alloc, ObSQLSessionInfo *session_info, bool is_schema_object); - static int store_datum(int64_t ¤t_addr, const ObObj &obj); + static int store_datum(int64_t ¤t_addr, const ObObj &obj, ObSQLSessionInfo *session_info); static const ObPostExprItem &get_last_expr_item(const ObSqlExpression &expr); diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 8704d47024..715575a6b7 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -1030,8 +1030,7 @@ int ObSql::do_real_prepare(const ObString &sql, LOG_WARN("session info is NULL", K(ret)); } else if (OB_FAIL(parser.parse(sql, parse_result, - parse_mode, - false/*is_batched_multi_stmt_split_on*/))) { + parse_mode))) { LOG_WARN("generate syntax tree failed", K(sql), K(ret)); } else if (is_mysql_mode() && ObSQLUtils::is_mysql_ps_not_support_stmt(parse_result)) { @@ -1301,7 +1300,7 @@ int ObSql::handle_pl_prepare(const ObString &sql, LOG_WARN("failed to init result set", K(ret)); } else if (OB_FAIL(ob_write_string(allocator, sess.get_current_query_string(), cur_query))) { LOG_WARN("failed to write string", K(ret)); - } else if (OB_FAIL(sess.store_query_string(trimed_stmt))) { + } else if (OB_FAIL(sess.store_query_string(sql))) { LOG_WARN("store query string fail", K(ret)); } else if (OB_FAIL(parser.parse(sql, parse_result, parse_mode, false, false, true))) { LOG_WARN("generate syntax tree failed", K(sql), K(ret)); @@ -1614,7 +1613,7 @@ int ObSql::handle_ps_prepare(const ObString &stmt, K(ret), K(pctx), K(ps_cache)); } else if (OB_FAIL(ob_write_string(allocator, session.get_current_query_string(), cur_query))) { LOG_WARN("failed to write string", K(ret)); - } else if (OB_FAIL(session.store_query_string(trimed_stmt))) { + } else if (OB_FAIL(session.store_query_string(stmt))) { LOG_WARN("store query string fail", K(ret)); } else { bool need_do_real_prepare = false; @@ -4080,7 +4079,8 @@ int ObSql::parser_and_check(const ObString &outlined_stmt, ObParser parser(allocator, session->get_sql_mode(), session->get_local_collation_connection(), pc_ctx.def_name_ctx_); if (OB_FAIL(parser.parse(outlined_stmt, parse_result, pc_ctx.is_rewrite_sql_ ? UDR_SQL_MODE : STD_MODE, - pc_ctx.sql_ctx_.handle_batched_multi_stmt()))) { + pc_ctx.sql_ctx_.handle_batched_multi_stmt(), + false, lib::is_mysql_mode() && NULL != session->get_pl_context()))) { LOG_WARN("Generate syntax tree failed", K(outlined_stmt), K(ret)); } else if ((PC_PS_MODE == pc_ctx.mode_ || PC_PL_MODE == pc_ctx.mode_) && OB_FAIL(construct_param_store_from_parameterized_params( @@ -4117,6 +4117,7 @@ int ObSql::parser_and_check(const ObString &outlined_stmt, LOG_WARN("parser error number is unexpected, need disconnect", K(ret)); } if (OB_SUCC(ret)) { + stmt::StmtType stmt_type = stmt::T_NONE; if (OB_ISNULL(parse_result.result_tree_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), KP(parse_result.result_tree_)); @@ -4140,6 +4141,8 @@ int ObSql::parser_and_check(const ObString &outlined_stmt, int32_t line_no = 1; LOG_USER_ERROR(OB_ERR_PARSE_SQL, ob_errpkt_strerror(OB_ERR_PARSER_SYNTAX, false), str_len, err_msg, line_no); LOG_WARN("the text query is invalid", K(outlined_stmt), K(children_node->value_), K(ret)); + } else if (OB_FAIL(ObResolverUtils::resolve_stmt_type(parse_result, stmt_type))) { + LOG_WARN("failed to resolve stmt type", K(ret)); } else { ObItemType type = children_node->type_; //如果是非DML语句, 则不进入plan cache @@ -4147,6 +4150,9 @@ int ObSql::parser_and_check(const ObString &outlined_stmt, if (T_SHOW_VARIABLES == type) { is_show_variables = true; } + if (ObStmt::is_ddl_stmt(stmt_type, true)) { + THIS_WORKER.set_timeout_ts(session->get_query_start_time() + GCONF._ob_ddl_timeout); + } if (IS_DML_STMT(type) || is_show_variables) { if (OB_UNLIKELY(NULL == (plan_cache = session->get_plan_cache()))) { ret = OB_ERR_UNEXPECTED; @@ -4437,6 +4443,8 @@ int ObSql::after_get_plan(ObPlanCacheCtx &pc_ctx, param.autoinc_offset_ = session.get_local_auto_increment_offset(); if (pc_ctx.sql_ctx_.is_do_insert_batch_opt()) { param.total_value_count_ = pc_ctx.sql_ctx_.get_insert_batch_row_cnt(); + } else if (phy_plan->is_plain_insert() && pctx->get_array_param_groups().count() == 1) { + param.total_value_count_ = pctx->get_array_param_groups().at(0).row_count_; } } // end for } diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index e81227856f..fa8b8a7df1 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -597,7 +597,7 @@ int ObSQLUtils::calc_calculable_expr(ObSQLSessionInfo *session, SQL_LOG(WARN, "Invalid arguments", K(expr), K(allocator)); } else if (!expr->is_static_scalar_const_expr()) { ret = OB_INVALID_ARGUMENT; - SQL_LOG(WARN, "expr should be calculabe expr", K(*expr), K(ret)); + SQL_LOG(WARN, "expr should be calculable expr", K(*expr), K(ret)); } else if (OB_FAIL(calc_const_expr(session, *expr, result, @@ -691,6 +691,10 @@ int ObSQLUtils::se_calc_const_expr(ObSQLSessionInfo *session, } else { schema_guard = &session->get_cached_schema_guard_info().get_schema_guard(); } + uint64_t effective_tenant_id = session->get_effective_tenant_id(); + if (session->get_ddl_info().is_ddl_check_default_value()) { + effective_tenant_id = OB_SERVER_TENANT_ID; + } SMART_VARS_2((ObExecContext, exec_ctx, tmp_allocator), (ObStaticEngineExprCG, expr_cg, tmp_allocator, session, schema_guard, @@ -699,7 +703,7 @@ int ObSQLUtils::se_calc_const_expr(ObSQLSessionInfo *session, (NULL != out_ctx ? out_ctx->get_min_cluster_version() : GET_MIN_CLUSTER_VERSION()))) { LinkExecCtxGuard link_guard(*session, exec_ctx); exec_ctx.set_my_session(session); - exec_ctx.set_mem_attr(ObMemAttr(session->get_effective_tenant_id(), + exec_ctx.set_mem_attr(ObMemAttr(effective_tenant_id, ObModIds::OB_SQL_EXEC_CONTEXT, ObCtxIds::EXECUTE_CTX_ID)); exec_ctx.set_physical_plan_ctx(&phy_plan_ctx); @@ -1685,6 +1689,26 @@ int ObSQLUtils::get_default_cast_mode(const bool is_explicit_cast, return ret; } +int ObSQLUtils::get_cast_mode_for_replace(const ObRawExpr *expr, + const ObSQLSessionInfo *session, + ObCastMode &cast_mode) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr) || OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", KP(expr), KP(session), K(ret)); + } else if (OB_FAIL(ObSQLUtils::get_default_cast_mode(false,/* explicit_cast */ + 0, /* result_flag */ + session, cast_mode))) { + LOG_WARN("failed to get default cast mode", K(ret)); + } else if (OB_FAIL(ObSQLUtils::set_cs_level_cast_mode(expr->get_collation_level(), cast_mode))) { + LOG_WARN("failed to set cs level cast mode", K(ret)); + } else if (expr->get_result_type().has_result_flag(ZEROFILL_FLAG)) { + cast_mode |= CM_ADD_ZEROFILL; + } + return ret; +} + ObCollationLevel ObSQLUtils::transform_cs_level(const ObCollationLevel cs_level) { // CS_LEVEL_INVALID is not defined as 0, transform the input cs level to make the default 0 @@ -4114,7 +4138,7 @@ int ObSQLUtils::create_encode_sortkey_expr( } else if (OB_FAIL(expr_factory.create_raw_expr(T_FUN_SYS_ENCODE_SORTKEY, encode_expr))) { LOG_WARN("failed to create encode_expr", K(ret)); } else { - // Assamble encode sortkey. + // Assemble encode sortkey. for (int64_t i = start_key; OB_SUCC(ret) && i < order_keys.count(); i++) { ObConstRawExpr *nulls_pos_expr = nullptr; ObConstRawExpr *order_expr = nullptr; @@ -5043,7 +5067,7 @@ int ObSQLUtils::find_synonym_ref_obj(const uint64_t database_id, obj_type = ObObjectType::FUNCTION; schema_version = routine_info->get_schema_version(); } else if (OB_FAIL(guard.get_standalone_procedure_info(tenant_id, database_id, object_name, routine_info))) { - LOG_WARN("failed to get procedore info", K(ret)); + LOG_WARN("failed to get procedure info", K(ret)); } else if (nullptr != routine_info) { exist = true; object_id = routine_info->get_object_id(); @@ -5098,7 +5122,7 @@ int ObSQLUtils::print_identifier_require_quotes(ObCollationType collation_type, require = false; if (OB_ISNULL(info)) { ret = OB_INVALID_ARGUMENT; - OB_LOG(WARN, "arguemnt is invalid", K(ret)); + OB_LOG(WARN, "argument is invalid", K(ret)); } else if (ident.length() > 0 && ident[0] == '$') { require = true; } diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index 1796c43645..52a8b1bca7 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -369,6 +369,9 @@ public: const bool ret_error = false); static void set_insert_update_scope(common::ObCastMode &cast_mode); static bool is_insert_update_scope(common::ObCastMode &cast_mode); + static int get_cast_mode_for_replace(const ObRawExpr *expr, + const ObSQLSessionInfo *session, + ObCastMode &cast_mode); static common::ObCollationLevel transform_cs_level(const common::ObCollationLevel cs_level); static int set_cs_level_cast_mode(const common::ObCollationLevel cs_level, common::ObCastMode &cast_mode); diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index 47bdf39a99..0997ff639b 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -574,7 +574,7 @@ int ObAccessPathEstimation::add_index_info(ObOptimizerContext &ctx, } else if (OB_FAIL(task->paths_.push_back(ap))) { LOG_WARN("failed to push back access path", K(ret)); } else if (OB_ISNULL(index_est_arg = task->arg_.index_params_.alloc_place_holder())) { - ret = OB_ERR_UNEXPECTED; + ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate index argument", K(ret)); } else if (OB_FAIL(get_key_ranges(ctx, allocator, part.tablet_id_, ap, tmp_ranges))) { LOG_WARN("failed to get key ranges", K(ret)); diff --git a/src/sql/optimizer/ob_del_upd_log_plan.cpp b/src/sql/optimizer/ob_del_upd_log_plan.cpp index 637566ac04..d6796c641b 100644 --- a/src/sql/optimizer/ob_del_upd_log_plan.cpp +++ b/src/sql/optimizer/ob_del_upd_log_plan.cpp @@ -1422,7 +1422,7 @@ int ObDelUpdLogPlan::allocate_pdml_insert_as_top(ObLogicalOperator *&top, LOG_WARN("get unexpected null", K(ret)); } if (OB_FAIL(insert_op->compute_property())) { - LOG_WARN("failed to compute propery", K(ret)); + LOG_WARN("failed to compute property", K(ret)); } else { top = insert_op; } diff --git a/src/sql/optimizer/ob_delete_log_plan.cpp b/src/sql/optimizer/ob_delete_log_plan.cpp index 6370f5d369..ece94a3ae0 100644 --- a/src/sql/optimizer/ob_delete_log_plan.cpp +++ b/src/sql/optimizer/ob_delete_log_plan.cpp @@ -275,7 +275,7 @@ int ObDeleteLogPlan::allocate_delete_as_top(ObLogicalOperator *&top, } else if (delete_stmt->is_error_logging() && OB_FAIL(delete_op->extract_err_log_info())) { LOG_WARN("failed to extract error log info", K(ret)); } else if (OB_FAIL(delete_op->compute_property())) { - LOG_WARN("failed to compute propery", K(ret)); + LOG_WARN("failed to compute property", K(ret)); } else { top = delete_op; } diff --git a/src/sql/optimizer/ob_insert_all_log_plan.cpp b/src/sql/optimizer/ob_insert_all_log_plan.cpp index d0ff4436c7..345d181f61 100644 --- a/src/sql/optimizer/ob_insert_all_log_plan.cpp +++ b/src/sql/optimizer/ob_insert_all_log_plan.cpp @@ -127,7 +127,7 @@ int ObInsertAllLogPlan::allocate_insert_values_as_top(ObLogicalOperator *&top) } else if (insert_all_stmt->is_error_logging() && OB_FAIL(values_op->extract_err_log_info())) { LOG_WARN("failed to extract error log exprs", K(ret)); } else if (OB_FAIL(values_op->compute_property())) { - LOG_WARN("failed to compute propery", K(ret)); + LOG_WARN("failed to compute property", K(ret)); } else { if (NULL != top) { values_op->add_child(top); diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 92b404e09f..8f68835106 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -690,9 +690,8 @@ int ObJoinOrder::compute_access_path_parallel(ObIArray &access_pat parallel = ObGlobalHint::UNSET_PARALLEL; ObOptimizerContext *opt_ctx = NULL; ObSQLSessionInfo *session_info = NULL; - int64_t parallel_limit = ObGlobalHint::UNSET_PARALLEL; + int64_t cur_min_parallel = ObGlobalHint::UNSET_PARALLEL; if (OB_ISNULL(get_plan()) || OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) - || OB_UNLIKELY(ObGlobalHint::DEFAULT_PARALLEL > opt_ctx->get_parallel_degree_limit()) || OB_ISNULL(session_info = opt_ctx->get_session_info())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected params", K(ret), K(get_plan()), K(opt_ctx), K(session_info)); @@ -703,37 +702,29 @@ int ObJoinOrder::compute_access_path_parallel(ObIArray &access_pat parallel = ObGlobalHint::DEFAULT_PARALLEL; } else if (OB_FAIL(get_random_parallel(access_paths, parallel))) { LOG_WARN("failed to get random parallel", K(ret)); - } else { - parallel = std::min(parallel, opt_ctx->get_parallel_degree_limit()); } - LOG_TRACE("Auto DOP trace point", K(session_info->is_user_session()), - K(opt_ctx->get_parallel_degree_limit()), K(parallel)); - } else if (OB_FAIL(get_parallel_from_available_access_paths(parallel_limit))) { + LOG_TRACE("Auto DOP trace point", K(session_info->is_user_session()), K(parallel)); + } else if (OB_FAIL(get_parallel_from_available_access_paths(cur_min_parallel))) { LOG_WARN("failed to get parallel from available access paths", K(ret)); } else { - parallel_limit = (ObGlobalHint::DEFAULT_PARALLEL > parallel_limit || - parallel_limit > opt_ctx->get_parallel_degree_limit()) - ? opt_ctx->get_parallel_degree_limit() - : parallel_limit; - const int64_t cost_threshold = opt_ctx->get_parallel_min_scan_time_threshold(); - int64_t calc_parallel = parallel_limit; - int64_t cur_parallel = ObGlobalHint::UNSET_PARALLEL; + int64_t calc_parallel = ObGlobalHint::UNSET_PARALLEL; int64_t das_path_cnt = 0; AccessPath *path = NULL; + bool finish = false; OPT_TRACE_TITLE("begin compute auto dop for table"); - for (int64_t i = 0; calc_parallel > ObGlobalHint::DEFAULT_PARALLEL && OB_SUCC(ret) && i < access_paths.count(); i++) { + for (int64_t i = 0; !finish && OB_SUCC(ret) && i < access_paths.count(); i++) { if (OB_ISNULL(path = access_paths.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(path), K(ret)); } else if (path->use_das_) { ++das_path_cnt; - } else if (OB_FAIL(path->compute_parallel_degree(parallel_limit, cost_threshold, cur_parallel))) { + } else if (OB_FAIL(path->compute_parallel_degree(cur_min_parallel, calc_parallel))) { LOG_WARN("failed to compute parallel degree", K(ret)); } else { - LOG_TRACE("finish compute one path parallel degree", K(i), K(cur_parallel), K(calc_parallel), - K(parallel_limit), K(path->table_id_), K(path->index_id_)); - calc_parallel = std::min(cur_parallel, calc_parallel); - parallel_limit = calc_parallel; + LOG_TRACE("finish compute one path parallel degree", K(i), K(cur_min_parallel), K(calc_parallel), + K(path->table_id_), K(path->index_id_)); + cur_min_parallel = calc_parallel; + finish = ObGlobalHint::DEFAULT_PARALLEL == calc_parallel; } } OPT_TRACE_TITLE("end compute auto dop for table"); @@ -2439,6 +2430,23 @@ int ObJoinOrder::will_use_skip_scan(const uint64_t table_id, use_skip_scan = OptSkipScanState::SS_UNSET; } + if (OB_SUCC(ret) && OptSkipScanState::SS_DISABLE != use_skip_scan) { + // OptColumnMeta for prefix columns may be not added. It's needed to calculate prefix NDV + const ObIArray &column_items = index_info_entry->get_range_info().get_range_columns(); + const int64_t ss_offset = query_range->get_skip_scan_offset(); + const OptSelectivityCtx &ctx = get_plan()->get_selectivity_ctx(); + OptTableMeta *table_meta = NULL; + if (OB_UNLIKELY(column_items.count() < ss_offset) || + OB_ISNULL(table_meta = get_plan()->get_basic_table_metas().get_table_meta_by_table_id(table_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected params", K(ret), K(column_items.count()), K(ss_offset), K(table_meta)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < ss_offset; ++i) { + if (OB_FAIL(table_meta->add_column_meta_no_dup(column_items.at(i).column_id_ , ctx))) { + LOG_WARN("failed to add column meta no duplicate", K(ret)); + } + } + } LOG_TRACE("check use skip scan", K(helper.is_inner_path_), K(hint_force_skip_scan), K(hint_force_no_skip_scan), K(use_skip_scan)); return ret; @@ -4855,8 +4863,7 @@ int AccessPath::assign(const AccessPath &other, common::ObIAllocator *allocator) } // compute auto dop for access path -int AccessPath::compute_parallel_degree(const int64_t parallel_degree_limit, - const int64_t cost_threshold, +int AccessPath::compute_parallel_degree(const int64_t cur_min_parallel_degree, int64_t ¶llel) const { int ret = OB_SUCCESS; @@ -4866,12 +4873,10 @@ int AccessPath::compute_parallel_degree(const int64_t parallel_degree_limit, double cost_threshold_us = 0.0; int64_t cur_parallel_degree_limit = ObGlobalHint::UNSET_PARALLEL; int64_t server_cnt = 0; - if (use_das_ || parallel_degree_limit <= ObGlobalHint::DEFAULT_PARALLEL - || is_virtual_table(est_cost_info_.ref_table_id_) + if (use_das_ || is_virtual_table(est_cost_info_.ref_table_id_) || est_cost_info_.is_unique_) { parallel = ObGlobalHint::DEFAULT_PARALLEL; - } else if (OB_FAIL(check_and_prepare_estimate_parallel_params(parallel_degree_limit, - cost_threshold, + } else if (OB_FAIL(check_and_prepare_estimate_parallel_params(cur_min_parallel_degree, px_part_gi_min_part_per_dop, cost_threshold_us, server_cnt, @@ -4914,21 +4919,20 @@ int AccessPath::compute_parallel_degree(const int64_t parallel_degree_limit, } if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(ObGlobalHint::DEFAULT_PARALLEL > parallel || parallel_degree_limit < parallel)) { + } else if (OB_UNLIKELY(ObGlobalHint::DEFAULT_PARALLEL > parallel || cur_parallel_degree_limit < parallel)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected parallel result", K(ret), K(parallel_degree_limit), K(parallel)); + LOG_WARN("get unexpected parallel result", K(ret), K(cur_parallel_degree_limit), K(parallel)); } else { OPT_TRACE("finish compute one path parallel degree:", parallel); LOG_TRACE("finish compute parallel degree", K(phy_query_range_row_count_), K(query_range_row_count_), - K(parallel), K(parallel_degree_limit), K(cur_parallel_degree_limit), + K(parallel), K(cur_parallel_degree_limit), K(cost_threshold_us), K(px_cost), K(cost), K(pre_cost)); } } return ret; } -int AccessPath::check_and_prepare_estimate_parallel_params(const int64_t input_parallel_degree_limit, - const double cost_threshold_ms, +int AccessPath::check_and_prepare_estimate_parallel_params(const int64_t cur_min_parallel_degree, int64_t &px_part_gi_min_part_per_dop, double &cost_threshold_us, int64_t &server_cnt, @@ -4943,23 +4947,25 @@ int AccessPath::check_and_prepare_estimate_parallel_params(const int64_t input_p ObSQLSessionInfo *session_info = NULL; ObSEArray server_list; if (OB_ISNULL(table_partition_info_) || - OB_UNLIKELY(cost_threshold_ms < 1) || OB_ISNULL(parent_) || OB_ISNULL(parent_->get_plan()) || OB_ISNULL(opt_ctx = &parent_->get_plan()->get_optimizer_context()) || OB_ISNULL(session_info = opt_ctx->get_session_info())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected params", K(ret), K(table_partition_info_), K(cost_threshold_ms), K(parent_), K(session_info)); + LOG_WARN("get unexpected params", K(ret), K(table_partition_info_), K(parent_), K(session_info)); } else if (OB_FAIL(session_info->get_sys_variable(share::SYS_VAR__PX_MIN_GRANULES_PER_SLAVE, px_part_gi_min_part_per_dop))) { LOG_WARN("failed to get sys variable px min granule per slave", K(ret)); } else if (OB_FAIL(table_partition_info_->get_all_servers(server_list))) { LOG_WARN("failed to get all servers", K(ret)); } else { px_part_gi_min_part_per_dop = std::max(1L, px_part_gi_min_part_per_dop); - cost_threshold_us = 1000.0 * cost_threshold_ms; + cost_threshold_us = 1000.0 * std::max(10L, opt_ctx->get_parallel_min_scan_time_threshold()); server_cnt = server_list.count(); - cur_parallel_degree_limit = input_parallel_degree_limit; + cur_parallel_degree_limit = opt_ctx->get_parallel_degree_limit(server_cnt); const int64_t row_parallel_limit = std::floor(phy_query_range_row_count_ / ROW_COUNT_THRESHOLD_PER_DOP); const int64_t ss_scan_parallel_limit = std::floor(est_cost_info_.ss_prefix_ndv_); + if (cur_min_parallel_degree > ObGlobalHint::DEFAULT_PARALLEL && cur_min_parallel_degree < cur_parallel_degree_limit) { + cur_parallel_degree_limit = cur_min_parallel_degree; + } if (row_parallel_limit > ObGlobalHint::DEFAULT_PARALLEL && row_parallel_limit < cur_parallel_degree_limit) { cur_parallel_degree_limit = row_parallel_limit; } diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index af01053599..a068be63cb 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -627,11 +627,9 @@ struct EstimateCostInfo { const ObCostTableScanInfo &get_cost_table_scan_info() const { return est_cost_info_; } ObCostTableScanInfo &get_cost_table_scan_info() { return est_cost_info_; } - int compute_parallel_degree(const int64_t parallel_degree_limit, - const int64_t micro_block_threshold, + int compute_parallel_degree(const int64_t cur_min_parallel_degree, int64_t ¶llel) const; - int check_and_prepare_estimate_parallel_params(const int64_t input_parallel_degree_limit, - const double cost_threshold_ms, + int check_and_prepare_estimate_parallel_params(const int64_t cur_min_parallel_degree, int64_t &px_part_gi_min_part_per_dop, double &cost_threshold_us, int64_t &server_cnt, diff --git a/src/sql/optimizer/ob_log_expr_values.cpp b/src/sql/optimizer/ob_log_expr_values.cpp index a4fc3a91eb..fcfca6c7bb 100644 --- a/src/sql/optimizer/ob_log_expr_values.cpp +++ b/src/sql/optimizer/ob_log_expr_values.cpp @@ -145,7 +145,7 @@ int ObLogExprValues::compute_fd_item_set() if (OB_FAIL(ret)) { /*do nothing*/ } else if (OB_FAIL(deduce_const_exprs_and_ft_item_set(*fd_item_set))) { - LOG_WARN("falied to deduce fd item set", K(ret)); + LOG_WARN("failed to deduce fd item set", K(ret)); } else { set_fd_item_set(fd_item_set); } diff --git a/src/sql/optimizer/ob_log_join.cpp b/src/sql/optimizer/ob_log_join.cpp index a9fc22efe7..95f9a9383d 100644 --- a/src/sql/optimizer/ob_log_join.cpp +++ b/src/sql/optimizer/ob_log_join.cpp @@ -957,7 +957,7 @@ int ObLogJoin::allocate_granule_post(AllocGIContext &ctx) // 并且 nlj 向左支 request 一个 part id // 通知 GI 在迭代 partition granule 时进入 partition pruning 模式 if (OB_FAIL(build_gi_partition_pruning())) { - LOG_WARN("fail deterimine right child partition id", K(ret)); + LOG_WARN("fail determine right child partition id", K(ret)); } } return ret; diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index c890ae18af..2869818ff1 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -4875,28 +4875,20 @@ int ObLogPlan::assign_right_popular_value_to_left(ObExchangeInfo &left_exch_info LOG_WARN("left_exch_info hash_dist_exprs_ is empty or null.", K(ret), K(left_exch_info)); } else { ObObjType expect_type = left_exch_info.hash_dist_exprs_.at(0).expr_->get_result_meta().get_type(); - if (ob_is_string_tc(pv.get_type()) && ob_is_large_text(expect_type)) { - // add lob locator for string - ObString data = pv.get_string(); - ObTextStringResult new_tmp_lob(expect_type, true, &get_allocator()); - if (OB_FAIL(new_tmp_lob.init(data.length()))) { - LOG_WARN("fail to init text string result", K(ret), K(new_tmp_lob), K(data.length())); - } else if (OB_FAIL(new_tmp_lob.append(data))) { - LOG_WARN("fail to append data", K(ret), K(new_tmp_lob), K(data.length())); - } else { - ObString res; - new_tmp_lob.get_result_buffer(res); - ObObj new_pv; - new_pv.set_lob_value(expect_type, res.ptr(), res.length()); - new_pv.set_has_lob_header(); - if (OB_FAIL(left_exch_info.popular_values_.push_back(new_pv))) { - LOG_WARN("failed to push obj to left_exch_info popular_values", K(ret), K(new_pv), K(left_exch_info)); - } + bool need_cast = (!is_lob_storage(pv.get_type()) && is_lob_storage(expect_type)) || + (is_lob_storage(pv.get_type()) && !is_lob_storage(expect_type)); + ObObj new_pv; + if (need_cast) { + ObCastCtx cast_ctx(&get_allocator(), NULL, CM_NONE, pv.get_meta().get_collation_type()); + if (OB_FAIL(ObObjCaster::to_type(expect_type, cast_ctx, pv, new_pv))) { + LOG_WARN("failed to do cast obj", K(ret), K(pv), K(expect_type));; } } else { - if (OB_FAIL(left_exch_info.popular_values_.push_back(pv))) { - LOG_WARN("failed to push obj to left_exch_info popular_values", K(ret), K(pv), K(left_exch_info)); - } + new_pv = pv; + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(left_exch_info.popular_values_.push_back(new_pv))) { + LOG_WARN("failed to push obj to left_exch_info popular_values", K(ret), K(new_pv), K(left_exch_info)); } } } diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index 007db0ce04..486b42772a 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -152,7 +152,7 @@ int ObLogTableScan::get_op_exprs(ObIArray &all_exprs) ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); } else if (OB_FAIL(generate_access_exprs())) { - LOG_WARN("failed to generate acess exprs", K(ret)); + LOG_WARN("failed to generate access exprs", K(ret)); } else if (NULL != limit_count_expr_ && OB_FAIL(all_exprs.push_back(limit_count_expr_))) { LOG_WARN("failed to push back expr", K(ret)); @@ -317,7 +317,7 @@ int ObLogTableScan::copy_filter_before_index_back() } else { bool is_contain_vir_gen_column = false; // ObArray column_exprs; - // scan_pushdown before index back conculde virtual generated column + // scan_pushdown before index back conclude virtual generated column // need copy for avoiding shared expression. if (OB_FAIL(ObRawExprUtils::contain_virtual_generated_column(filters.at(i), is_contain_vir_gen_column))) { LOG_WARN("fail to contain virtual gen column", K(ret)); @@ -1836,7 +1836,7 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text) } else if (!is_skip_scan() && T_INDEX_SS_HINT == hint->get_hint_type()) { /* is not index skip scan but exist index_ss hint */ } else if (OB_FAIL(hint->print_hint(plan_text))) { - LOG_WARN("failed to print indedx hint", K(ret), K(*hint)); + LOG_WARN("failed to print index hint", K(ret), K(*hint)); } } } else {// print all no index @@ -1847,7 +1847,7 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text) ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL", K(ret), K(hint)); } else if (OB_FAIL(hint->print_hint(plan_text))) { - LOG_WARN("failed to print indedx hint", K(ret), K(*hint)); + LOG_WARN("failed to print index hint", K(ret), K(*hint)); } } } diff --git a/src/sql/optimizer/ob_log_topk.cpp b/src/sql/optimizer/ob_log_topk.cpp index 57b33b9d6d..62cb8a501b 100644 --- a/src/sql/optimizer/ob_log_topk.cpp +++ b/src/sql/optimizer/ob_log_topk.cpp @@ -105,7 +105,7 @@ int ObLogTopk::est_cost() if (OB_ISNULL(child = get_child(first_child)) || OB_ISNULL(get_stmt()) || OB_ISNULL(get_plan())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpeced null", K(child), K(get_stmt()), K(get_plan()),K(ret)); + LOG_WARN("get unexpected null", K(child), K(get_stmt()), K(get_plan()),K(ret)); } else if (OB_UNLIKELY((parallel = get_parallel()) < 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected parallel degree", K(ret)); diff --git a/src/sql/optimizer/ob_merge_log_plan.cpp b/src/sql/optimizer/ob_merge_log_plan.cpp index fd238c5335..4b2ae5e7cf 100644 --- a/src/sql/optimizer/ob_merge_log_plan.cpp +++ b/src/sql/optimizer/ob_merge_log_plan.cpp @@ -524,7 +524,7 @@ bool ObMergeLogPlan::match_same_partition(const ObShardingInfo &l_sharding_info, } // When insert_sharding and update_sharding match same partition, -// need add euqal constraints for const params in sharding conditions which equal to part key. +// need add equal constraints for const params in sharding conditions which equal to part key. int ObMergeLogPlan::generate_equal_constraint(ObLogicalOperator &top, ObShardingInfo &insert_sharding, bool &can_gen_cons, @@ -552,7 +552,7 @@ int ObMergeLogPlan::generate_equal_constraint(ObLogicalOperator &top, LOG_WARN("get unexpected null", K(ret), K(target_table_scan)); } else if (OB_FAIL(append(right_conds, target_table_scan->get_range_conditions())) ||OB_FAIL(append(right_conds, target_table_scan->get_filter_exprs()))) { - LOG_WARN("failed to append epxrs", K(ret)); + LOG_WARN("failed to append exprs", K(ret)); } else if (OB_FAIL(insert_sharding.get_all_partition_keys(left_part_keys))) { LOG_WARN("failed to get partition keys", K(ret)); } else if (OB_FAIL(target_table_scan->get_sharding()->get_all_partition_keys(right_part_keys))) { @@ -864,7 +864,7 @@ int ObMergeLogPlan::prepare_table_dml_info_update(const ObMergeTableInfo& merge_ LOG_WARN("failed to get table schema", K(merge_info), K(ret)); } else if (!merge_info.is_link_table_ && OB_FAIL(check_update_part_key(index_schema, table_dml_info))) { - LOG_WARN("faield to check update part key", K(ret)); + LOG_WARN("failed to check update part key", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < table_dml_info->ck_cst_exprs_.count(); ++i) { if (OB_FAIL(ObDMLResolver::copy_schema_expr(optimizer_context_.get_expr_factory(), @@ -1027,7 +1027,7 @@ int ObMergeLogPlan::check_merge_stmt_need_multi_partition_dml(bool &is_multi_par } else if (OB_FAIL(merge_stmt->part_key_has_subquery(has_subquery_part_key))) { LOG_WARN("failed to check part key has subquery", K(ret)); } else if (OB_FAIL(merge_stmt->part_key_has_auto_inc(has_auto_inc_part_key))) { - LOG_WARN("check to check whether part key containts auto inc column", K(ret)); + LOG_WARN("check to check whether part key contains auto inc column", K(ret)); } else if (has_rand_part_key || has_subquery_part_key || has_auto_inc_part_key) { is_multi_part_dml = true; } else { /*do nothing*/ } diff --git a/src/sql/optimizer/ob_opt_est_cost_model.cpp b/src/sql/optimizer/ob_opt_est_cost_model.cpp index f2beaef485..ac2bdb26aa 100644 --- a/src/sql/optimizer/ob_opt_est_cost_model.cpp +++ b/src/sql/optimizer/ob_opt_est_cost_model.cpp @@ -86,7 +86,7 @@ double ObTableMetaInfo::get_micro_block_numbers() const { double ret = 0.0; if (micro_block_count_ <= 0) { - // calculate micore block count use storage statistics + // calculate micro block count use storage statistics ret = 0; } else { // get micro block count from optimizer statistics @@ -1399,7 +1399,7 @@ int ObOptEstCostModel::cost_table_get_one_batch(const ObCostTableScanInfo &est_c OB_FAIL(cost_table_lookup_rpc(index_back_row_count, est_cost_info, network_cost))) { - LOG_WARN("failed to get newwork transform cost for global index", K(ret)); + LOG_WARN("failed to get network transform cost for global index", K(ret)); } else { get_cost = base_cost + ib_cost + network_cost; get_index_back_cost = ib_cost + network_cost; diff --git a/src/sql/optimizer/ob_opt_est_utils.h b/src/sql/optimizer/ob_opt_est_utils.h index 6701cd8bb1..cc36776269 100644 --- a/src/sql/optimizer/ob_opt_est_utils.h +++ b/src/sql/optimizer/ob_opt_est_utils.h @@ -155,7 +155,7 @@ private: uint64_t &convertable_map, int64_t pos); // 1, find common prefix length of strings - // 2, find dynamix base and offset of strings + // 2, find dynamic base and offset of strings // 3, use dynamic base and offset to convert strings to scalars static int convert_strings_to_scalar(const common::ObIArray &origin_strs, common::ObIArray &scalars); diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index 0592f99cfe..0eb02d01e1 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -139,47 +139,74 @@ int OptTableMeta::init(const uint64_t table_id, } //init column ndv for (int64_t i = 0; OB_SUCC(ret) && i < column_ids.count(); ++i) { - column_id = column_ids.at(i); - int64_t global_ndv = 0; - int64_t num_null = 0; - bool is_single_pkey = (1 == pk_ids_.count() && pk_ids_.at(0) == column_id); - ObGlobalColumnStat stat; - if (is_single_pkey) { - global_ndv = rows_; - num_null = 0; - } else if (use_default_stat()) { - global_ndv = std::min(rows, 100L); - num_null = rows * EST_DEF_COL_NULL_RATIO; - } else if (OB_ISNULL(ctx.get_opt_stat_manager()) || OB_ISNULL(ctx.get_session_info())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(ctx.get_opt_stat_manager()), - K(ctx.get_session_info())); - } else if (OB_FAIL(ctx.get_opt_stat_manager()->get_column_stat(ctx.get_session_info()->get_effective_tenant_id(), - ref_table_id_, - all_used_part_id, - column_id, - all_used_global_parts, - rows, - scale_ratio, - stat))) { - LOG_WARN("failed to get column stats", K(ret)); - } else if (0 == stat.ndv_val_ && 0 == stat.null_val_) { - global_ndv = std::min(rows, 100L); - num_null = rows * EST_DEF_COL_NULL_RATIO; - } else if (0 == stat.ndv_val_ && stat.null_val_ > 0) { - global_ndv = 1; - num_null = stat.null_val_; - } else { - global_ndv = stat.ndv_val_; - num_null = stat.null_val_; - } - if (OB_SUCC(ret)) { - column_metas_.at(i).init(column_id, global_ndv, num_null, stat.avglen_val_); + if (OB_FAIL(init_column_meta(ctx, column_ids.at(i), column_metas_.at(i)))) { + LOG_WARN("failed to init column ", K(ret)); } } return ret; } +int OptTableMeta::init_column_meta(const OptSelectivityCtx &ctx, + const uint64_t column_id, + OptColumnMeta &col_meta) +{ + int ret = OB_SUCCESS; + ObGlobalColumnStat stat; + bool is_single_pkey = (1 == pk_ids_.count() && pk_ids_.at(0) == column_id); + int64_t global_ndv = 0; + int64_t num_null = 0; + if (is_single_pkey) { + global_ndv = rows_; + num_null = 0; + } else if (use_default_stat()) { + global_ndv = std::min(rows_, 100.0); + num_null = rows_ * EST_DEF_COL_NULL_RATIO; + } else if (OB_ISNULL(ctx.get_opt_stat_manager()) || OB_ISNULL(ctx.get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx.get_opt_stat_manager()), + K(ctx.get_session_info())); + } else if (OB_FAIL(ctx.get_opt_stat_manager()->get_column_stat(ctx.get_session_info()->get_effective_tenant_id(), + ref_table_id_, + all_used_parts_, + column_id, + all_used_global_parts_, + rows_, + scale_ratio_, + stat))) { + LOG_WARN("failed to get column stats", K(ret)); + } else if (0 == stat.ndv_val_ && 0 == stat.null_val_) { + global_ndv = std::min(rows_, 100.0); + num_null = rows_ * EST_DEF_COL_NULL_RATIO; + } else if (0 == stat.ndv_val_ && stat.null_val_ > 0) { + global_ndv = 1; + num_null = stat.null_val_; + } else { + global_ndv = stat.ndv_val_; + num_null = stat.null_val_; + } + + if (OB_SUCC(ret)) { + col_meta.init(column_id, global_ndv, num_null, stat.avglen_val_); + } + return ret; +} + +int OptTableMeta::add_column_meta_no_dup(const uint64_t column_id, + const OptSelectivityCtx &ctx) +{ + int ret = OB_SUCCESS; + OptColumnMeta *col_meta = NULL; + if (NULL != OptTableMeta::get_column_meta(column_id)) { + /* do nothing */ + } else if (OB_ISNULL(col_meta = column_metas_.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate place holder for column meta", K(ret)); + } else if (OB_FAIL(init_column_meta(ctx, column_id, *col_meta))) { + LOG_WARN("failed to init column meta", K(ret)); + } + return ret; +} + const OptColumnMeta* OptTableMeta::get_column_meta(const uint64_t column_id) const { const OptColumnMeta* column_meta = NULL; diff --git a/src/sql/optimizer/ob_opt_selectivity.h b/src/sql/optimizer/ob_opt_selectivity.h index b451c9a317..f40322d7f3 100644 --- a/src/sql/optimizer/ob_opt_selectivity.h +++ b/src/sql/optimizer/ob_opt_selectivity.h @@ -246,6 +246,12 @@ public: // int update_stat(const double rows, const bool can_reduce, const bool can_enlarge); + int init_column_meta(const OptSelectivityCtx &ctx, + const uint64_t column_id, + OptColumnMeta &col_meta); + + int add_column_meta_no_dup(const uint64_t column_id, const OptSelectivityCtx &ctx); + const OptColumnMeta* get_column_meta(const uint64_t column_id) const; uint64_t get_table_id() const { return table_id_; } diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index a087c6ad04..3ae9c81881 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -432,13 +432,14 @@ int ObOptimizer::check_pdml_enabled(const ObDMLStmt &stmt, // 3. decided by session variable: _enable_parallel_dml is true or _force_parallel_dml_dop > 1; int ret = OB_SUCCESS; ObSqlCtx *sql_ctx = NULL; + ObQueryCtx *query_ctx = NULL; bool can_use_pdml = true; bool session_enable_pdml = false; bool enable_auto_dop = false; uint64_t session_pdml_dop = ObGlobalHint::UNSET_PARALLEL; - if (OB_ISNULL(ctx_.get_exec_ctx())) { + if (OB_ISNULL(ctx_.get_exec_ctx()) || OB_ISNULL(query_ctx = ctx_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(ctx_.get_exec_ctx())); + LOG_WARN("unexpected null", K(ret), K(ctx_.get_exec_ctx()), K(query_ctx)); } else if (OB_ISNULL(sql_ctx = ctx_.get_exec_ctx()->get_sql_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(ctx_.get_exec_ctx())); @@ -464,7 +465,8 @@ int ObOptimizer::check_pdml_enabled(const ObDMLStmt &stmt, // do nothing } else if (ctx_.get_global_hint().get_pdml_option() == ObPDMLOption::ENABLE) { // 1. enable parallel dml by hint - } else if (ctx_.get_global_hint().get_pdml_option() == ObPDMLOption::DISABLE) { + } else if (ctx_.get_global_hint().get_pdml_option() == ObPDMLOption::DISABLE + || query_ctx->get_query_hint().has_outline_data()) { can_use_pdml = false; // 1. disable parallel dml by hint } else if (ctx_.get_global_hint().enable_auto_dop()) { // 2.1 enable parallel dml by auto dop @@ -501,25 +503,10 @@ int ObOptimizer::check_pdml_supported_feature(const ObDelUpdStmt &pdml_stmt, int ret = OB_SUCCESS; share::schema::ObSchemaGetterGuard *schema_guard = ctx_.get_schema_guard(); ObSEArray table_infos; - bool enable_all_pdml_feature = false; // 默认非注入错误情况下，关闭PDML不稳定feature - // 目前通过注入错误的方式来打开PDML不稳定功能，用于PDML全部功能的case回归 - // 对应的event注入任何类型的错误，都会打开PDML非稳定功能 - ret = OB_E(EventTable::EN_ENABLE_PDML_ALL_FEATURE) OB_SUCCESS; - LOG_TRACE("event: check pdml all feature", K(ret)); - if (OB_FAIL(ret)) { - enable_all_pdml_feature = true; - ret = OB_SUCCESS; - ctx_.add_plan_note(PDML_ENABLE_BY_TRACE_EVENT); - } - LOG_TRACE("event: check pdml all feature result", K(ret), K(enable_all_pdml_feature)); - // 检查是否开启全部pdml feature： - // 1. 如果开启，is open = true - // 2. 如果没有开启，需要依次检查被禁止的不稳定的功能，如果存在被禁止的不稳定功能 is open = false + // 依次检查被禁止的不稳定的功能，如果存在被禁止的不稳定功能 is open = false if (OB_ISNULL(schema_guard)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("the schema guard is null", K(ret)); - } else if (enable_all_pdml_feature) { - is_use_pdml = true; } else if (pdml_stmt.is_ignore()) { is_use_pdml = false; ctx_.add_plan_note(PDML_DISABLED_BY_IGNORE); @@ -735,7 +722,10 @@ int ObOptimizer::init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &s int64_t session_force_parallel_dop = ObGlobalHint::UNSET_PARALLEL; bool session_enable_auto_dop = false; bool session_enable_manual_dop = false; - if (ctx_.has_pl_udf()) { + if (OB_ISNULL(ctx_.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("query ctx is nul", K(ret)); + } else if (ctx_.has_pl_udf()) { //following above rule, but if stmt contain pl_udf, force das, parallel should be 1 ctx_.set_parallel_rule(PXParallelRule::PL_UDF_DAS_FORCE_SERIALIZE); } else if (ctx_.has_cursor_expression()) { @@ -749,6 +739,9 @@ int ObOptimizer::init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &s ctx_.set_parallel(ctx_.get_global_hint().get_parallel_degree()); } else if (ctx_.get_global_hint().enable_auto_dop()) { ctx_.set_parallel_rule(PXParallelRule::AUTO_DOP); + } else if (ctx_.get_query_ctx()->get_query_hint().has_outline_data()) { + ctx_.set_parallel_rule(PXParallelRule::MANUAL_HINT); + ctx_.set_parallel(ObGlobalHint::DEFAULT_PARALLEL); } else if (session.is_user_session() && !ctx_.get_global_hint().enable_manual_dop() && OB_FAIL(OB_E(EventTable::EN_ENABLE_AUTO_DOP_FORCE_PARALLEL_PLAN) OB_SUCCESS)) { ret = OB_SUCCESS; @@ -774,7 +767,7 @@ int ObOptimizer::init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &s } else { LOG_TRACE("succeed to init parallel policy", K(session.is_user_session()), K(ctx_.can_use_pdml()), K(ctx_.get_parallel_rule()), K(ctx_.get_parallel()), - K(ctx_.get_parallel_degree_limit()), K(ctx_.get_parallel_min_scan_time_threshold())); + K(ctx_.get_auto_dop_params())); } return ret; } @@ -782,31 +775,38 @@ int ObOptimizer::init_parallel_policy(ObDMLStmt &stmt, const ObSQLSessionInfo &s int ObOptimizer::set_auto_dop_params(const ObSQLSessionInfo &session) { int ret = OB_SUCCESS; - const uint64_t default_parallel_degree_limit = 256; - uint64_t parallel_degree_limit = default_parallel_degree_limit; + uint64_t parallel_degree_limit = 0; uint64_t parallel_min_scan_time_threshold = 1000; - int64_t parallel_servers_target = 0; + AutoDOPParams params; if (!session.is_user_session()) { /* do nothing */ } else if (OB_FAIL(session.get_sys_variable(share::SYS_VAR_PARALLEL_DEGREE_LIMIT, parallel_degree_limit))) { LOG_WARN("failed to get sys variable parallel degree limit", K(ret)); } else if (OB_FAIL(session.get_sys_variable(share::SYS_VAR_PARALLEL_MIN_SCAN_TIME_THRESHOLD, parallel_min_scan_time_threshold))) { LOG_WARN("failed to get sys variable parallel threshold", K(ret)); - } else if (0 != parallel_degree_limit) { - /* do nothing */ - } else if (OB_FAIL(ObSchemaUtils::get_tenant_int_variable(session.get_effective_tenant_id(), - SYS_VAR_PARALLEL_SERVERS_TARGET, - parallel_servers_target))) { - LOG_WARN("fail read tenant variable", K(ret), K(session.get_effective_tenant_id())); - } else if (parallel_servers_target > 0) { - parallel_degree_limit = parallel_servers_target; - } else { - parallel_degree_limit = default_parallel_degree_limit; + } + + if (OB_SUCC(ret) && 0 == parallel_degree_limit) { + const ObTenantBase *tenant = NULL; + int64_t parallel_servers_target = 0; + if (OB_ISNULL(tenant = MTL_CTX())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (session.is_user_session() && + OB_FAIL(ObSchemaUtils::get_tenant_int_variable(session.get_effective_tenant_id(), + SYS_VAR_PARALLEL_SERVERS_TARGET, + parallel_servers_target))) { + LOG_WARN("fail read tenant variable", K(ret), K(session.get_effective_tenant_id())); + } else { + params.unit_min_cpu_ = std::max(tenant->unit_min_cpu(), 0.0); + params.parallel_servers_target_ = std::max(parallel_servers_target, 0L); + } } if (OB_SUCC(ret)) { - ctx_.set_parallel_degree_limit(parallel_degree_limit); - ctx_.set_parallel_min_scan_time_threshold(parallel_min_scan_time_threshold); + params.parallel_min_scan_time_threshold_ = parallel_min_scan_time_threshold; + params.parallel_degree_limit_ = parallel_degree_limit; + ctx_.set_auto_dop_params(params); } return ret; } @@ -1101,7 +1101,7 @@ int ObOptimizer::update_column_usage_infos() LOG_WARN("get unexpected null", K(ret), K(optstat_monitor_mgr)); } else if (OB_FAIL(optstat_monitor_mgr->update_local_cache(ctx_.get_column_usage_infos()))) { LOG_WARN("failed to update local cache", K(ret)); - } else {/*do nothiing*/} + } else {/*do nothing*/} } } return ret; diff --git a/src/sql/optimizer/ob_optimizer.h b/src/sql/optimizer/ob_optimizer.h index b8b9929a14..6ac0ea819f 100644 --- a/src/sql/optimizer/ob_optimizer.h +++ b/src/sql/optimizer/ob_optimizer.h @@ -56,7 +56,7 @@ namespace sql */ PROJECT_PRUNING, /** - * Exchange allocation is also known as parallel optimzation. Though there + * Exchange allocation is also known as parallel optimization. Though there * are multiple possible optimization that can happen in this step, at the * moment we just perform simple check on table location and insert pairs of * exchange nodes at places where data exchange needs to happen(local diff --git a/src/sql/optimizer/ob_optimizer_context.h b/src/sql/optimizer/ob_optimizer_context.h index 491e04cbbb..b524be2a45 100644 --- a/src/sql/optimizer/ob_optimizer_context.h +++ b/src/sql/optimizer/ob_optimizer_context.h @@ -84,6 +84,38 @@ struct TableLocRelInfo common::ObArray table_part_infos_; }; +struct AutoDOPParams { + AutoDOPParams() + : parallel_degree_limit_(0), + parallel_servers_target_(0), + unit_min_cpu_(0), + parallel_min_scan_time_threshold_(1000) + { } + + int64_t get_parallel_degree_limit(const int64_t server_cnt) const { + int64_t limit = 0; + if (0 < parallel_degree_limit_) { + limit = parallel_degree_limit_; + } else if (0 >= parallel_servers_target_ || 0 >= unit_min_cpu_ || 0 >= server_cnt) { + limit = std::max(parallel_servers_target_, server_cnt * unit_min_cpu_); + } else { + limit = std::min(parallel_servers_target_, server_cnt * unit_min_cpu_); + } + return std::max(limit, 1L); + } + bool is_valid() { return parallel_min_scan_time_threshold_ >= 10 + && (parallel_degree_limit_ > 0 || parallel_servers_target_ > 0 + || unit_min_cpu_ > 0); } + TO_STRING_KV(K_(parallel_degree_limit), + K_(parallel_servers_target), + K_(unit_min_cpu), + K_(parallel_min_scan_time_threshold)); + int64_t parallel_degree_limit_; + int64_t parallel_servers_target_; + int64_t unit_min_cpu_; + int64_t parallel_min_scan_time_threshold_; // auto dop threshold for table scan cost +}; + class ObOptimizerContext { @@ -119,8 +151,7 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, px_parallel_rule_(PXParallelRule::USE_PX_DEFAULT), can_use_pdml_(false), max_parallel_(ObGlobalHint::UNSET_PARALLEL), - parallel_degree_limit_(ObGlobalHint::UNSET_PARALLEL), - parallel_min_scan_time_threshold_(-1), + auto_dop_params_(), is_online_ddl_(false), ddl_sample_column_count_(0), is_heap_table_ddl_(false), @@ -237,8 +268,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, void set_serial_set_order(bool force_serial_set_order) { force_serial_set_order_ = force_serial_set_order; } inline int64_t get_parallel() const { return parallel_; } inline int64_t get_max_parallel() const { return max_parallel_; } - inline int64_t get_parallel_degree_limit() const { return parallel_degree_limit_; } - inline int64_t get_parallel_min_scan_time_threshold() const { return parallel_min_scan_time_threshold_; } + inline int64_t get_parallel_degree_limit(const int64_t server_cnt) const { return auto_dop_params_.get_parallel_degree_limit(server_cnt); } + inline int64_t get_parallel_min_scan_time_threshold() const { return auto_dop_params_.parallel_min_scan_time_threshold_; } inline bool force_disable_parallel() const { return px_parallel_rule_ >= PL_UDF_DAS_FORCE_SERIALIZE && px_parallel_rule_ < MAX_OPTION; } inline bool is_use_table_dop() const { return MANUAL_TABLE_DOP == px_parallel_rule_; } @@ -246,8 +277,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, inline bool is_parallel_rule_valid() const { return MAX_OPTION != px_parallel_rule_; } void set_parallel(int64_t parallel) { parallel_ = parallel; } void set_max_parallel(int64_t max_parallel) { max_parallel_ = max_parallel_ < max_parallel ? max_parallel : max_parallel_; } - void set_parallel_degree_limit(int64_t parallel_degree_limit) { parallel_degree_limit_ = parallel_degree_limit; } - void set_parallel_min_scan_time_threshold(int64_t threshold) { parallel_min_scan_time_threshold_ = threshold; } + void set_auto_dop_params(const AutoDOPParams &auto_dop_params) { auto_dop_params_ = auto_dop_params; } + const AutoDOPParams &get_auto_dop_params() { return auto_dop_params_; } void set_can_use_pdml(bool u) { can_use_pdml_ = u; } inline ObFdItemFactory &get_fd_item_factory() { return fd_item_factory_; } void set_is_online_ddl(bool flag) { is_online_ddl_ = flag; } @@ -547,8 +578,7 @@ private: PXParallelRule px_parallel_rule_; bool can_use_pdml_; // can use pdml after check parallel int64_t max_parallel_; - int64_t parallel_degree_limit_; // parallel limit for auto dop - int64_t parallel_min_scan_time_threshold_; // auto dop threshold for table scan cost + AutoDOPParams auto_dop_params_; // parameters to calc dop for Auto DOP bool is_online_ddl_; int64_t ddl_sample_column_count_; bool is_heap_table_ddl_; // we need to treat heap table ddl seperately diff --git a/src/sql/optimizer/ob_update_log_plan.cpp b/src/sql/optimizer/ob_update_log_plan.cpp index 126288324a..d19baad82e 100644 --- a/src/sql/optimizer/ob_update_log_plan.cpp +++ b/src/sql/optimizer/ob_update_log_plan.cpp @@ -157,7 +157,7 @@ int ObUpdateLogPlan::generate_normal_raw_plan() if (OB_FAIL(candi_allocate_sequence())) { LOG_WARN("failed to allocate sequence operator", K(ret)); } else { - LOG_TRACE("succeed to allocate sequance operator", + LOG_TRACE("succeed to allocate sequence operator", K(candidates_.candidate_plans_.count())); } } @@ -212,7 +212,7 @@ int ObUpdateLogPlan::generate_normal_raw_plan() if (OB_FAIL(candi_allocate_update())) { LOG_WARN("failed to allocate update operator", K(ret)); } else { - LOG_TRACE("succeed to allocate normal update opeartor", + LOG_TRACE("succeed to allocate normal update operator", K(candidates_.candidate_plans_.count())); } } @@ -369,7 +369,7 @@ int ObUpdateLogPlan::allocate_update_as_top(ObLogicalOperator *&top, ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate update operator", K(ret)); } else if (OB_FAIL(update_op->assign_dml_infos(index_dml_infos_))) { - LOG_WARN("failed to assgin dml infos", K(ret)); + LOG_WARN("failed to assign dml infos", K(ret)); } else { update_op->set_child(ObLogicalOperator::first_child, top); update_op->set_ignore(update_stmt->is_ignore()); @@ -411,7 +411,7 @@ int ObUpdateLogPlan::candi_allocate_pdml_update() IndexDMLInfo *index_delete_info = nullptr; IndexDMLInfo *index_insert_info = nullptr; // 更新了当前索引的分区键，需要做 row-movement - // 需要为每一个globa index以及primary index分配各自的 update operator，形成如下的计划: + // 需要为每一个global index以及primary index分配各自的 update operator，形成如下的计划: // .... // INSERT INDEX (i3) // DELETE INDEX (i3) @@ -440,7 +440,7 @@ int ObUpdateLogPlan::candi_allocate_pdml_update() } } else { // 在PDML update中，可能包含有多个global index， - // 需要为每一个globa index以及primary index分配各自的 update operator，形成如下的计划: + // 需要为每一个global index以及primary index分配各自的 update operator，形成如下的计划: // .... // UPDATE INDEX (i3) // UPDATE INDEX (i2) @@ -475,7 +475,7 @@ int ObUpdateLogPlan::perform_vector_assign_expr_replacement(ObUpdateStmt *stmt) if (OB_FAIL(replace_alias_ref_expr(value, replace_happened))) { LOG_WARN("failed to replace alias ref expr", K(ret)); } else if (replace_happened && OB_FAIL(value->formalize(session_info))) { - LOG_WARN("failed to foramlize expr", K(ret)); + LOG_WARN("failed to formalize expr", K(ret)); } } } @@ -624,7 +624,7 @@ int ObUpdateLogPlan::prepare_table_dml_info_special(const ObDmlTableInfo& table_ LOG_WARN("failed to get table schema", K(table_info), K(ret)); } else if (!table_info.is_link_table_ && OB_FAIL(check_update_part_key(index_schema, table_dml_info))) { - LOG_WARN("faield to check update part key", K(ret)); + LOG_WARN("failed to check update part key", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < table_dml_info->ck_cst_exprs_.count(); ++i) { if (OB_FAIL(ObDMLResolver::copy_schema_expr(optimizer_context_.get_expr_factory(), @@ -677,7 +677,7 @@ int ObUpdateLogPlan::prepare_table_dml_info_special(const ObDmlTableInfo& table_ LOG_WARN("failed to check update unique key", K(ret)); } else if (!table_info.is_link_table_ && OB_FAIL(check_update_part_key(index_schema, index_dml_info))) { - LOG_WARN("faield to check update part key", K(ret)); + LOG_WARN("failed to check update part key", K(ret)); } else if (OB_FAIL(index_dml_infos.push_back(index_dml_info))) { LOG_WARN("failed to push back index dml info", K(ret)); } diff --git a/src/sql/parser/ob_parser.cpp b/src/sql/parser/ob_parser.cpp index d188e37b12..b466d9442f 100644 --- a/src/sql/parser/ob_parser.cpp +++ b/src/sql/parser/ob_parser.cpp @@ -499,7 +499,7 @@ ObParser::State ObParser::transform_normal( } break; default: { is_not_pl = true; - LOG_WARN_RET(common::OB_ERR_UNEXPECTED, "unexpecte state", K(state)); + LOG_WARN_RET(common::OB_ERR_UNEXPECTED, "unexpected state", K(state)); } break; } return state; @@ -837,14 +837,14 @@ int32_t ObParser::get_well_formed_errlen(const ObCharsetInfo *charset_info, } return res_len; } -// avoid separeting sql by semicolons in quotes or comment. +// avoid separating sql by semicolons in quotes or comment. void ObParser::get_single_sql(const common::ObString &stmt, int64_t offset, int64_t remain, int64_t &str_len) { /* following two flags are used to mark wether we are in comment, if in comment, ';' can't be used to split sql*/ // in -- comment bool comment_flag = false; // in /*! comment */ or /* comment */ bool c_comment_flag = false; - /* follwing three flags are used to mark wether we are in quotes.*/ + /* following three flags are used to mark wether we are in quotes.*/ // in '', single quotes bool sq_flag = false; // in "", double quotes @@ -1240,7 +1240,7 @@ int ObParser::pre_parse(const common::ObString &stmt, PreParseResult &res) { int ret = OB_SUCCESS; - // /*tracd_id=xxx*/ + // /*trace_id=xxx*/ int64_t len = stmt.length(); if (len <= 13) { //do_nothing diff --git a/src/sql/parser/ob_parser.h b/src/sql/parser/ob_parser.h index ce769a14fe..8452003d73 100644 --- a/src/sql/parser/ob_parser.h +++ b/src/sql/parser/ob_parser.h @@ -77,7 +77,7 @@ public: // situation we will try find ';' delimiter to parser part of string in case of save memory, // but this maybe parser error and throw error info. However, we will still try parser remain // string when parse part of string failed, if we throw parse part error info, maybe will let - // someone misunderstand have bug, So, we introduce this mark to decide to throw parser erorr. + // someone misunderstand have bug, So, we introduce this mark to decide to throw parser error. // eg: select '123;' from dual; select '123' from dual; int parse_sql(const common::ObString &stmt, ParseResult &parse_result, diff --git a/src/sql/parser/parse_malloc.cpp b/src/sql/parser/parse_malloc.cpp index bb817a188f..ba334d0a88 100644 --- a/src/sql/parser/parse_malloc.cpp +++ b/src/sql/parser/parse_malloc.cpp @@ -33,7 +33,7 @@ void *malloc_parentheses_info(const size_t nbyte, void *malloc_pool) return ptr; } -// get memory from the thread obStringBuf, and not release untill thread quits +// get memory from the thread obStringBuf, and not release until thread quits void *parse_malloc(const size_t nbyte, void *malloc_pool) { void *ptr = NULL; diff --git a/src/sql/parser/parse_node.c b/src/sql/parser/parse_node.c index 9c167828fa..afb3174df6 100644 --- a/src/sql/parser/parse_node.c +++ b/src/sql/parser/parse_node.c @@ -720,7 +720,7 @@ int64_t get_question_mark(ObQuestionMarkCtx *ctx, void *malloc_pool, const char } else { if (NULL == ctx->name_ && 0 == ctx->capacity_) { ctx->capacity_ = MAX_QUESTION_MARK; - // the errocde will be ignored here. TO BE FIXED. + // the errcode will be ignored here. TO BE FIXED. ctx->name_ = (char **)parse_malloc(sizeof(char*) * MAX_QUESTION_MARK, malloc_pool); } if (ctx->name_ != NULL) { diff --git a/src/sql/parser/parse_node.h b/src/sql/parser/parse_node.h index 3fc9149705..9d84eb2229 100644 --- a/src/sql/parser/parse_node.h +++ b/src/sql/parser/parse_node.h @@ -131,8 +131,8 @@ typedef struct _ParseNode uint32_t is_neg_ : 1;// 记录常量节点的父节点是否为T_OP_NEG节点, 1表示是, 0 表示不是 uint32_t is_hidden_const_ : 1; //1 表示某常量正常parse能识别但fast parse不能识别, 0 表示都能识别。 uint32_t is_tree_not_param_ :1; //1 表示该节点及其子节点常量均不能参数化, 0表示没该限制 - uint32_t length_semantics_ :2; //2 for oralce [char|varbinary] (n b [bytes|char]) - uint32_t is_val_paramed_item_idx_ :1; // T_PROJECT_STRING的vlaues是否是select_item_param_infos数组的下标 + uint32_t length_semantics_ :2; //2 for oracle [char|varbinary] (n b [bytes|char]) + uint32_t is_val_paramed_item_idx_ :1; // T_PROJECT_STRING的values是否是select_item_param_infos数组的下标 uint32_t is_copy_raw_text_ : 1; // 是否回填常量节点的raw_text_，用于select item常量参数化 uint32_t is_column_varchar_ : 1; // 投影列是否是一个常量字符串，用于select item常量参数化 uint32_t is_trans_from_minus_: 1; // 负数常量节点是否是从减号操作转换而来，比如1 - 2，词法阶段会生成一个-2 @@ -174,7 +174,7 @@ typedef struct _ParseNode int64_t text_len_; int64_t pos_; //记录?在带?的sql中的偏移 - struct _ParseNode **children_; /* attributes for non-terninal node, which has children */ + struct _ParseNode **children_; /* attributes for non-terminal node, which has children */ ObStmtLoc stmt_loc_; //临时放在这里，后面要移到parse_stmt_node.h中去 union { int64_t raw_param_idx_; // 常量节点在fp_result.raw_params_中的下标 @@ -243,7 +243,7 @@ typedef struct _ObQuestionMarkCtx // record the minus status while parsing the sql // for example, 'select - -1 from dual' // when parser sees the first '-', pos_ = 7, raw_sql_offset = 7, has_minus_ = true, is_cur_numeric_ = false -// after seeing the second '-', members are reseted, pos_ = 9, raw_sql_offset_ = 9, has_minus_ = true, is_cur_numberic = false +// after seeing the second '-', members are reseted, pos_ = 9, raw_sql_offset_ = 9, has_minus_ = true, is_cur_numeric = false // after seeing '1', is_cur_numeric = true, then param node '-1' is returned typedef struct _ObMinusStatuCtx { diff --git a/src/sql/parser/sql_parser_base.h b/src/sql/parser/sql_parser_base.h index fc39f3424f..ed219780e9 100644 --- a/src/sql/parser/sql_parser_base.h +++ b/src/sql/parser/sql_parser_base.h @@ -240,7 +240,7 @@ do { setup_token_pos_info(node, word_start - 1, word_end - word_start + 1); \ } while (0) -//oralce下生成非保留关键字结点请使用该宏,区别于mysql的是做了大写的转换 +//oracle下生成非保留关键字结点请使用该宏,区别于mysql的是做了大写的转换 #define get_oracle_non_reserved_node(node, malloc_pool, expr_start, expr_end) \ do { \ malloc_terminal_node(node, malloc_pool, T_IDENT); \ diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 42857144d9..f24244c682 100755 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -1511,8 +1511,10 @@ BEGIN(in_c_comment); node->children_[1] = target_node; return REMAP_TABLE_VAL; } + } else { - YY_UNEXPECTED_ERROR("Named binding is only for remap\n"); + yyerror(yylloc, yyextra, "Named binding is only for remap\n"); + return PARSER_SYNTAX_ERROR; } } @@ -1547,7 +1549,8 @@ BEGIN(in_c_comment); return REMAP_TABLE_VAL; } } else { - YY_UNEXPECTED_ERROR("Named binding is only for remap\n"); + yyerror(yylloc, yyextra, "Named binding is only for remap\n"); + return PARSER_SYNTAX_ERROR; } } @@ -1589,7 +1592,8 @@ BEGIN(in_c_comment); return REMAP_TABLE_VAL; } } else { - YY_UNEXPECTED_ERROR("Named binding is only for remap\n"); + yyerror(yylloc, yyextra, "Named binding is only for remap\n"); + return PARSER_SYNTAX_ERROR; } } @@ -1621,7 +1625,8 @@ BEGIN(in_c_comment); return REMAP_TABLE_VAL; } } else { - YY_UNEXPECTED_ERROR("Named binding is only for remap\n"); + yyerror(yylloc, yyextra, "Named binding is only for remap\n"); + return PARSER_SYNTAX_ERROR; } } @@ -1671,7 +1676,8 @@ BEGIN(in_c_comment); return REMAP_TABLE_VAL; } } else { - YY_UNEXPECTED_ERROR("Named binding is only for remap\n"); + yyerror(yylloc, yyextra, "Named binding is only for remap\n"); + return PARSER_SYNTAX_ERROR; } } @@ -1714,7 +1720,8 @@ BEGIN(in_c_comment); return REMAP_TABLE_VAL; } } else { - YY_UNEXPECTED_ERROR("Named binding is only for remap\n"); + yyerror(yylloc, yyextra, "Named binding is only for remap\n"); + return PARSER_SYNTAX_ERROR; } } diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 840932e584..5ae067e52c 100755 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -373,7 +373,7 @@ END_P SET_VAR DELIMITER %type create_tenant_stmt opt_tenant_option_list alter_tenant_stmt drop_tenant_stmt create_standby_tenant_stmt log_restore_source_option %type create_restore_point_stmt drop_restore_point_stmt %type create_resource_stmt drop_resource_stmt alter_resource_stmt -%type cur_timestamp_func cur_time_func cur_date_func now_synonyms_func utc_timestamp_func utc_time_func utc_date_func sys_interval_func sysdate_func +%type cur_timestamp_func cur_time_func cur_date_func now_synonyms_func utc_timestamp_func utc_time_func utc_date_func sys_interval_func sysdate_func cur_user_func %type create_dblink_stmt drop_dblink_stmt dblink tenant opt_cluster opt_dblink %type opt_create_resource_pool_option_list create_resource_pool_option alter_resource_pool_option_list alter_resource_pool_option %type opt_shrink_unit_option id_list opt_shrink_tenant_unit_option @@ -2434,6 +2434,10 @@ MOD '(' expr ',' expr ')' { $$ = $1; } +| cur_user_func +{ + $$ = $1; +} | cur_date_func { $$ = $1; @@ -3077,6 +3081,17 @@ CURTIME '(' ')' } ; +cur_user_func: +CURRENT_USER +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_CURRENT_USER, 1, NULL); +} +| CURRENT_USER '(' ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_CURRENT_USER, 1, NULL); +} +; + cur_date_func: CURDATE '(' ')' { @@ -8322,22 +8337,7 @@ opt_lock_type: ; no_table_select: -select_with_opt_hint opt_query_expression_option_list select_expr_list into_opt %prec LOWER_PARENS -{ - ParseNode *project_list = NULL; - merge_nodes(project_list, result, T_PROJECT_LIST, $3); - - ParseNode *select_node = NULL; - malloc_select_node(select_node, result->malloc_pool_); - select_node->children_[PARSE_SELECT_DISTINCT] = $2; - select_node->children_[PARSE_SELECT_SELECT] = project_list; - select_node->children_[PARSE_SELECT_HINTS] = $1; - select_node->children_[PARSE_SELECT_INTO] = $4; - $$ = select_node; - - setup_token_pos_info(select_node, @1.first_column - 1, 6); -} -| select_with_opt_hint opt_query_expression_option_list select_expr_list into_opt +select_with_opt_hint opt_query_expression_option_list select_expr_list into_opt FROM DUAL opt_where opt_groupby opt_having opt_named_windows { ParseNode *project_list = NULL; @@ -8357,24 +8357,21 @@ FROM DUAL opt_where opt_groupby opt_having opt_named_windows setup_token_pos_info(select_node, @1.first_column - 1, 6); } -| select_with_opt_hint opt_query_expression_option_list select_expr_list into_opt -WHERE opt_hint_value expr opt_groupby opt_having opt_named_windows +| select_with_opt_hint opt_query_expression_option_list select_expr_list into_opt opt_where opt_groupby opt_having opt_named_windows { ParseNode *project_list = NULL; - ParseNode *where_node = NULL; ParseNode *select_node = NULL; merge_nodes(project_list, result, T_PROJECT_LIST, $3); - malloc_non_terminal_node(where_node, result->malloc_pool_, T_WHERE_CLAUSE, 2, $7, $6); setup_token_pos_info($$, @1.first_column - 1, 5); malloc_select_node(select_node, result->malloc_pool_); select_node->children_[PARSE_SELECT_DISTINCT] = $2; select_node->children_[PARSE_SELECT_SELECT] = project_list; - select_node->children_[PARSE_SELECT_WHERE] = where_node; + select_node->children_[PARSE_SELECT_WHERE] = $5; select_node->children_[PARSE_SELECT_HINTS] = $1; select_node->children_[PARSE_SELECT_INTO] = $4; - select_node->children_[PARSE_SELECT_DYNAMIC_GROUP] = $8; - select_node->children_[PARSE_SELECT_DYNAMIC_HAVING] = $9; - select_node->children_[PARSE_SELECT_NAMED_WINDOWS] = $10; + select_node->children_[PARSE_SELECT_DYNAMIC_GROUP] = $6; + select_node->children_[PARSE_SELECT_DYNAMIC_HAVING] = $7; + select_node->children_[PARSE_SELECT_NAMED_WINDOWS] = $8; $$ = select_node; setup_token_pos_info(select_node, @1.first_column - 1, 6); } @@ -11794,7 +11791,12 @@ INTNUM | QUESTIONMARK { $$ = $1; - dup_string($$, result, @1.first_column + 1, @1.last_column); + if (result->pl_parse_info_.is_pl_parse_ || result->pl_parse_info_.is_inner_parse_) { + dup_string($$, result, @1.first_column + 1, @1.last_column); + } else { + yyerror(&@1, result, "question mark as condition arg not allowed\n"); + YYABORT_PARSE_SQL_ERROR; + } } | column_name { @@ -17548,10 +17550,6 @@ NAME_OB { make_name_node($$, result->malloc_pool_, "right"); } -| CURRENT_USER -{ - make_name_node($$, result->malloc_pool_, "current_user"); -} | SYSTEM_USER { make_name_node($$, result->malloc_pool_, "user"); diff --git a/src/sql/plan_cache/ob_pc_ref_handle.h b/src/sql/plan_cache/ob_pc_ref_handle.h index ade8794122..7148f5e026 100644 --- a/src/sql/plan_cache/ob_pc_ref_handle.h +++ b/src/sql/plan_cache/ob_pc_ref_handle.h @@ -190,5 +190,5 @@ private: uint64_t tenant_id_; }; } // end namespace sql -} // end namespace oceanbases +} // end namespace oceanbase #endif // !OCEANBASE_PC_REF_HANDLE_H_ diff --git a/src/sql/plan_cache/ob_plan_cache.cpp b/src/sql/plan_cache/ob_plan_cache.cpp index 88e4ecf127..e7c917019c 100644 --- a/src/sql/plan_cache/ob_plan_cache.cpp +++ b/src/sql/plan_cache/ob_plan_cache.cpp @@ -558,7 +558,7 @@ int ObPlanCache::get_plan(common::ObIAllocator &allocator, if (OB_ISNULL(pc_ctx.sql_ctx_.session_info_) || OB_ISNULL(pc_ctx.sql_ctx_.schema_guard_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguement",K(ret), + LOG_WARN("invalid argument",K(ret), K(pc_ctx.sql_ctx_.schema_guard_), K(pc_ctx.exec_ctx_.get_physical_plan_ctx())); } else if (pc_ctx.sql_ctx_.multi_stmt_item_.is_batched_multi_stmt()) { if (OB_FAIL(construct_multi_stmt_fast_parser_result(allocator, @@ -1906,7 +1906,7 @@ int64_t ObPlanCache::get_label_hold(lib::ObLabel &label) const // } else if (plan->get_mem_size() >= get_mem_high()) { // // plan mem is too big to reach memory highwater, do not add plan // } else if (OB_FAIL(construct_plan_cache_key(pc_ctx, ObLibCacheNameSpace::NS_CRSR))) { -// LOG_WARN("fail to construnct plan cache key", K(ret)); +// LOG_WARN("fail to construct plan cache key", K(ret)); // } else if (OB_FAIL(add_plan_cache(pc_ctx, plan))) { // if (OB_FAIL(deal_add_ps_plan_result(ret, pc_ctx, *plan))) { // LOG_WARN("fail to deal result code", K(ret)); @@ -1946,7 +1946,7 @@ int ObPlanCache::add_ps_plan(T *plan, ObPlanCacheCtx &pc_ctx) } else if (plan->get_mem_size() >= get_mem_high()) { // plan mem is too big to reach memory highwater, do not add plan } else if (OB_FAIL(construct_plan_cache_key(pc_ctx, ObLibCacheNameSpace::NS_CRSR))) { - LOG_WARN("fail to construnct plan cache key", K(ret)); + LOG_WARN("fail to construct plan cache key", K(ret)); } else if (OB_ISNULL(pc_ctx.raw_sql_.ptr())) { ret = OB_ERR_UNEXPECTED; SQL_PC_LOG(WARN, "pc_ctx.raw_sql_.ptr() is NULL, cannot add plan to plan cache by sql", K(ret)); @@ -2022,7 +2022,7 @@ int ObPlanCache::deal_add_ps_plan_result(int add_plan_ret, } } else { pc_ctx.sql_ctx_.self_add_plan_ = true; - LOG_TRACE("Successed to add plan to ObPlanCache", K(cache_object)); + LOG_TRACE("Succeed to add plan to ObPlanCache", K(cache_object)); } return ret; @@ -2186,7 +2186,7 @@ int ObPlanCache::get_ps_plan(ObCacheObjGuard& guard, } else if (FALSE_IT(original_param_cnt = pc_ctx.fp_result_.cache_params_->count())) { // do nothing } else if (OB_FAIL(construct_plan_cache_key(pc_ctx, ObLibCacheNameSpace::NS_CRSR))) { - LOG_WARN("fail to construnct plan cache key", K(ret)); + LOG_WARN("fail to construct plan cache key", K(ret)); } else { ObPsStmtId new_stmt_id = pc_ctx.fp_result_.pc_key_.key_id_; // the remote plan uses key_id is 0 to distinguish, so if key_id is 0, it cannot be set to OB_INVALID_ID diff --git a/src/sql/plan_cache/ob_plan_cache_callback.cpp b/src/sql/plan_cache/ob_plan_cache_callback.cpp index 4d42ff5dfd..efa308f8de 100644 --- a/src/sql/plan_cache/ob_plan_cache_callback.cpp +++ b/src/sql/plan_cache/ob_plan_cache_callback.cpp @@ -57,7 +57,7 @@ from map | dec ref cnt => ref_cnt=0 | | ref plan id x. inc_ref=1 deleting plan x | - | acess plan x --> cause core! + | access plan x --> cause core! */ void ObCacheObjAtomicOp::operator()(ObjKV &entry) diff --git a/src/sql/plan_cache/ob_plan_cache_util.h b/src/sql/plan_cache/ob_plan_cache_util.h index 0ccee2993c..f33d749511 100644 --- a/src/sql/plan_cache/ob_plan_cache_util.h +++ b/src/sql/plan_cache/ob_plan_cache_util.h @@ -368,7 +368,7 @@ struct ObOperatorStat //不再记录last的执行结果 //int64_t last_input_rows_; //上次input rows //int64_t last_rescan_times_; //rescan的次数 - //int64_t last_output_rows_; //output rows in last exectuion + //int64_t last_output_rows_; //output rows in last execution //暂时不支持以下和oracle兼容的统计项 //int64_t last_cr_buffer_gets_; //上次执行逻辑读次数 //int64_t cr_buffer_gets_; //累计逻辑读次数 @@ -539,7 +539,7 @@ struct ObPlanStat int64_t delayed_large_querys_; int64_t delayed_px_querys_; // px query 被丢回队列重试的次数 int64_t expected_worker_count_; // px 预期分配线程数 - int64_t minimal_worker_count_; // minial threads required for query + int64_t minimal_worker_count_; // minimal threads required for query int64_t outline_version_; int64_t outline_id_; bool is_last_exec_succ_; // record whether last execute success @@ -976,14 +976,14 @@ public: * * update cached configs * 1. @Funciton load_influence_plan_config() will load values - * 2. @Function serialize_configs() will serialze config values to strings and plan cache will + * 2. @Function serialize_configs() will serialize config values to strings and plan cache will * compare this string so as to figure out whether configs has changed. * 3. after generate string, @Function should do @Function update_version() * * add configs has influence in execution plan. @see load_influence_plan_config(); * * NOTES: - * to add configs that will influence exectuion plan, please add to following funcs: + * to add configs that will influence execution plan, please add to following funcs: * 1. load_influence_plan_config(); * 2. serialize_configs(); * 3. adds default values to ObBasicSessionInfo::load_default_configs_in_pc() @@ -1011,7 +1011,7 @@ public: } // init tenant_id_ void init(int t_id) {tenant_id_ = t_id;} - // load configs which will influence exectuion plan + // load configs which will influence execution plan int load_influence_plan_config(); // generate config string int serialize_configs(char *buf, int buf_len, int64_t &pos); diff --git a/src/sql/plan_cache/ob_ps_cache.cpp b/src/sql/plan_cache/ob_ps_cache.cpp index 358d5d2f9c..df6f2263e4 100644 --- a/src/sql/plan_cache/ob_ps_cache.cpp +++ b/src/sql/plan_cache/ob_ps_cache.cpp @@ -294,7 +294,7 @@ int ObPsCache::get_or_add_stmt_item(uint64_t db_id, new_item_value->~ObPsStmtItem(); inner_allocator_->free(new_item_value); } else { - LOG_WARN("unexpecte error", K(ret), K(new_stmt_id)); + LOG_WARN("unexpected error", K(ret), K(new_stmt_id)); new_item_value->~ObPsStmtItem(); inner_allocator_->free(new_item_value); } @@ -339,7 +339,7 @@ int ObPsCache::ref_stmt_item(const ObPsSqlKey &ps_sql_key, if (OB_SUCCESS != (callback_ret = op.get_callback_ret())) { ret = callback_ret; if (OB_EAGAIN == ret) { - LOG_INFO("try egain", K(ret), "stmt_id", ps_sql_key, K(retry_cnt)); + LOG_INFO("try again", K(ret), "stmt_id", ps_sql_key, K(retry_cnt)); ob_usleep(static_cast(500)); //sleep 500us } } else if (OB_FAIL(op.get_value(ps_stmt_item))) { @@ -352,7 +352,7 @@ int ObPsCache::ref_stmt_item(const ObPsSqlKey &ps_sql_key, break; } case OB_EAGAIN: { - LOG_WARN("try egain", K(ret), "stmt_id", ps_sql_key, K(retry_cnt)); + LOG_WARN("try again", K(ret), "stmt_id", ps_sql_key, K(retry_cnt)); ob_usleep(static_cast(500)); //sleep 500us break; } @@ -531,7 +531,7 @@ int ObPsCache::fill_ps_stmt_info(const ObResultSet &result, LOG_WARN("invalid argument", K(params), K(columns), K(sql_ctx), K(ret)); } else if (OB_FAIL(ps_stmt_info.reserve_ps_meta_fields(params->count(), columns->count()))) { - LOG_WARN("fail to reserver ps meta field", K(ret)); + LOG_WARN("fail to reserve ps meta field", K(ret)); } for (int i = 0; OB_SUCC(ret) && i < params->count(); ++i) { if (OB_FAIL(ps_stmt_info.add_param_field(params->at(i)))) { @@ -662,7 +662,7 @@ int ObPsCache::ref_stmt_info(const ObPsStmtId stmt_id, ObPsStmtInfo *&ps_stmt_in if (OB_SUCCESS != (callback_ret = op.get_callback_ret())) { ret = callback_ret; if (OB_EAGAIN == ret) { - LOG_INFO("try egain", K(ret), K(stmt_id), K(retry_cnt)); + LOG_INFO("try again", K(ret), K(stmt_id), K(retry_cnt)); ob_usleep(static_cast(500)); //sleep 500us } } else if (OB_FAIL(op.get_value(ps_stmt_info))) { @@ -675,7 +675,7 @@ int ObPsCache::ref_stmt_info(const ObPsStmtId stmt_id, ObPsStmtInfo *&ps_stmt_in break; } case OB_EAGAIN: { - LOG_INFO("try egain", K(ret), K(stmt_id), K(retry_cnt)); + LOG_INFO("try again", K(ret), K(stmt_id), K(retry_cnt)); ob_usleep(static_cast(500)); //sleep 500us break; } diff --git a/src/sql/plan_cache/ob_sql_parameterization.cpp b/src/sql/plan_cache/ob_sql_parameterization.cpp index 377dee8e87..eb5943ff21 100644 --- a/src/sql/plan_cache/ob_sql_parameterization.cpp +++ b/src/sql/plan_cache/ob_sql_parameterization.cpp @@ -133,7 +133,7 @@ TransformTreeCtx::TransformTreeCtx() : } // replace const expr with ? in syntax tree -// seprate params from syntax tree +// separate params from syntax tree int ObSqlParameterization::transform_syntax_tree(ObIAllocator &allocator, const ObSQLSessionInfo &session, const ObIArray *raw_params, @@ -644,7 +644,7 @@ int ObSqlParameterization::transform_tree(TransformTreeCtx &ctx, p_info.raw_text_pos_ = ctx.tree_->sql_str_off_; if (ctx.tree_->sql_str_off_ == -1) { ret = OB_NOT_SUPPORTED; - LOG_WARN("invlid str off", K(lbt()), K(ctx.tree_), + LOG_WARN("invalid str off", K(lbt()), K(ctx.tree_), K(ctx.tree_->raw_param_idx_), K(get_type_name(node_type)), K(session_info.get_current_query_string()), "result_tree_", SJ(ObParserResultPrintWrapper(*ctx.top_node_))); @@ -709,12 +709,12 @@ int ObSqlParameterization::transform_tree(TransformTreeCtx &ctx, // transform `operand - const_num_val` to `operand + (-const_num_val)` if (OB_SUCC(ret) && OB_FAIL(transform_minus_op(*(ctx.allocator_), ctx.tree_, ctx.is_from_pl_))) { - LOG_WARN("failed to transform minums operation", K(ret)); + LOG_WARN("failed to transform minus operation", K(ret)); } else if (lib::is_oracle_mode()) { // in oracle mode, select +-1 from dual is prohibited, but with following orders, it can be executed successfully: - // 1. select +1 from dual; (genereted plan with key: select +? from dual) + // 1. select +1 from dual; (generated plan with key: select +? from dual) // 2. select +-1 from dual; (hit plan before, executed successfully) - // Thus, add a constraint in plan cache: the numeric value following `-` or `+` operators must be posbitive number + // Thus, add a constraint in plan cache: the numeric value following `-` or `+` operators must be positive number if (T_OP_POS == ctx.tree_->type_ || T_OP_NEG == ctx.tree_->type_ ) { if (OB_ISNULL(ctx.tree_->children_) || ctx.tree_->num_child_ != 1 @@ -731,7 +731,7 @@ int ObSqlParameterization::transform_tree(TransformTreeCtx &ctx, } } if (T_LIMIT_CLAUSE == ctx.tree_->type_) { - // limit a offset b, a and b must be postive + // limit a offset b, a and b must be positive // 0 is counted as positive, -0 is counted as negative if (OB_ISNULL(ctx.tree_->children_) || 2 != ctx.tree_->num_child_) { ret = OB_INVALID_ARGUMENT; @@ -745,7 +745,7 @@ int ObSqlParameterization::transform_tree(TransformTreeCtx &ctx, && FALSE_IT(ctx.tree_->children_[1]->is_num_must_be_pos_ = 1)) { } } else if (T_COMMA_LIMIT_CLAUSE == ctx.tree_->type_) { - // limit a, b, a and b must be postive + // limit a, b, a and b must be positive if (OB_ISNULL(ctx.tree_->children_) || 2 != ctx.tree_->num_child_ || OB_ISNULL(ctx.tree_->children_[0]) @@ -890,13 +890,13 @@ int ObSqlParameterization::transform_tree(TransformTreeCtx &ctx, p_info.raw_text_pos_ = root->sql_str_off_; if (root->sql_str_off_ == -1) { ret = OB_NOT_SUPPORTED; - LOG_WARN("invlid str off", K(lbt()), K(ctx.tree_), + LOG_WARN("invalid str off", K(lbt()), K(ctx.tree_), K(root->raw_param_idx_), K(get_type_name(root->type_)), K(session_info.get_current_query_string()), "result_tree_", SJ(ObParserResultPrintWrapper(*ctx.top_node_))); } if (OB_FAIL(ret)) { - // do nithing + // do nothing } else if (OB_FAIL(ctx.sql_info_->parse_infos_.push_back(p_info))) { SQL_PC_LOG(WARN, "fail to push parser info", K(ret)); } @@ -1679,7 +1679,7 @@ int ObSqlParameterization::add_not_param_flag(const ParseNode *node, SqlInfo &sq p_info.raw_text_pos_ = node->sql_str_off_; if (node->sql_str_off_ == -1) { ret = OB_NOT_SUPPORTED; - LOG_WARN("invlid str off", K(lbt()), K(node), + LOG_WARN("invalid str off", K(lbt()), K(node), K(node->raw_param_idx_), K(get_type_name(node->type_))); } if (OB_FAIL(ret)) { @@ -1702,7 +1702,7 @@ int ObSqlParameterization::add_not_param_flag(const ParseNode *node, SqlInfo &sq p_info.raw_text_pos_ = node->sql_str_off_; if (node->sql_str_off_ == -1) { ret = OB_NOT_SUPPORTED; - LOG_WARN("invlid str off", K(lbt()), K(node), + LOG_WARN("invalid str off", K(lbt()), K(node), K(node->raw_param_idx_), K(get_type_name(node->type_))); } if (OB_FAIL(ret)) { @@ -2111,7 +2111,7 @@ int ObSqlParameterization::get_select_item_param_info(const common::ObIArraychildren_[i])) { stack_frames.at(frame_idx).next_child_idx_ = i + 1; } else if (OB_FAIL(stack_frames.push_back(TraverseStackFrame{frame.cur_node_->children_[i], 0}))) { - LOG_WARN("failed to push back eleemnt", K(ret)); + LOG_WARN("failed to push back element", K(ret)); } else { stack_frames.at(frame_idx).next_child_idx_ = i + 1; LOG_DEBUG("after pushing frame", K(stack_frames)); @@ -2162,7 +2162,7 @@ int ObSqlParameterization::resolve_paramed_const(SelectItemTraverseCtx &ctx) SQL_PC_LOG(WARN, "invalid argument", K(ret), K(idx), K(ctx.raw_params_.count())); } else if (OB_ISNULL(ctx.raw_params_.at(idx)) || OB_ISNULL(ctx.raw_params_.at(idx)->node_)) { ret = OB_INVALID_ARGUMENT; - SQL_PC_LOG(WARN, "invalid arguemnt", K(ret)); + SQL_PC_LOG(WARN, "invalid argument", K(ret)); } else { const ParseNode *param_node = ctx.raw_params_.at(idx)->node_; int64_t tmp_len = std::min(ctx.buf_len_ - ctx.param_info_.name_len_, param_node->raw_sql_offset_ - ctx.expr_pos_); diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index 0b32d5ff96..4bd9d737b2 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -924,6 +924,7 @@ int get_dml_stmt_need_privs( need_priv.table_ = table_item->table_name_; need_priv.priv_set_ = priv_set; need_priv.is_sys_table_ = table_item->is_system_table_; + need_priv.is_for_update_ = table_item->for_update_; need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; //no check for information_schema select if (stmt::T_SELECT != dml_stmt->get_stmt_type()) { @@ -1901,6 +1902,7 @@ int get_load_data_stmt_need_privs( need_priv.table_ = load_data_stmt->get_load_arguments().table_name_; need_priv.priv_set_ = OB_PRIV_INSERT; need_priv.is_sys_table_ = false; + need_priv.is_for_update_ = false; need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; ADD_NEED_PRIV(need_priv); } diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 7ee8ef5663..b1bee17192 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -44,6 +44,7 @@ #include "observer/mysql/ob_query_response_time.h" #include "rootserver/ob_rs_job_table_operator.h" //ObRsJobType #include "sql/resolver/cmd/ob_kill_stmt.h" +#include "share/table/ob_table_config_util.h" namespace oceanbase { @@ -4327,6 +4328,10 @@ int ObTableTTLResolver::resolve(const ParseNode& parse_tree) ret = OB_NOT_SUPPORTED; LOG_WARN("TTL command is not supported in data version less than 4.2.1", K(ret), K(tenant_data_version)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "TTL command is not supported in data version less than 4.2.1"); + } else if (!ObKVFeatureModeUitl::is_ttl_enable()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ttl is disable", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "ttl is disable, set by config item _obkv_feature_mode"); } else if (OB_UNLIKELY(T_TABLE_TTL != parse_tree.type_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("type is not T_TABLE_TTL", "type", get_type_name(parse_tree.type_)); @@ -4990,7 +4995,7 @@ int ObRecoverTableResolver::resolve(const ParseNode &parse_tree) } else if (OB_NOT_NULL(parse_tree.children_[8]) && OB_FAIL(Util::resolve_string(parse_tree.children_[8], stmt->get_rpc_arg().restore_tenant_arg_.description_))) { LOG_WARN("failed to resolve desc", K(ret)); -#ifndef OB_BUILD_TDE_SECURITY +#ifdef OB_BUILD_TDE_SECURITY } else if (OB_FAIL(resolve_kms_info_( stmt->get_rpc_arg().restore_tenant_arg_.restore_option_, stmt->get_rpc_arg().restore_tenant_arg_.kms_info_))) { LOG_WARN("failed to resolve kms info", K(ret)); @@ -5322,7 +5327,7 @@ int ObRecoverTableResolver::resolve_backup_set_pwd_(common::ObString &pwd) return ret; } -#ifndef OB_BUILD_TDE_SECURITY +#ifdef OB_BUILD_TDE_SECURITY int ObRecoverTableResolver::resolve_kms_info_(const common::ObString &restore_option, common::ObString &kms_info) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index b83f0fa758..e99d553b38 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -275,7 +275,7 @@ private: const ParseNode *node, share::ObImportRemapArg &remap_arg); int resolve_remap_tablespaces_( const ParseNode *node, share::ObImportRemapArg &remap_arg); -#ifndef OB_BUILD_TDE_SECURITY +#ifdef OB_BUILD_TDE_SECURITY int resolve_kms_info_(const common::ObString &restore_option, common::ObString &kms_info); #endif int resolve_backup_set_pwd_(common::ObString &pwd); diff --git a/src/sql/resolver/cmd/ob_anonymous_block_resolver.cpp b/src/sql/resolver/cmd/ob_anonymous_block_resolver.cpp index 048ea06f7b..1cf31aaca3 100644 --- a/src/sql/resolver/cmd/ob_anonymous_block_resolver.cpp +++ b/src/sql/resolver/cmd/ob_anonymous_block_resolver.cpp @@ -114,7 +114,9 @@ int ObAnonymousBlockResolver::resolve_anonymous_block( false, p_param_list); for (int64_t i = 0; OB_SUCC(ret) && i < params_.query_ctx_->question_marks_count_; ++i) { - OZ (param_list.push_back(ObObjParam(ObObj(ObNullType)))); + ObObjParam param = ObObjParam(ObObj(ObNullType)); + const_cast(param.get_null_meta()).reset(); + OZ (param_list.push_back(param)); } OZ (package_guard.init()); OX (func_ast.set_db_name(params_.session_info_->get_database_name())); diff --git a/src/sql/resolver/cmd/ob_load_data_resolver.cpp b/src/sql/resolver/cmd/ob_load_data_resolver.cpp index c757fa0746..4ac39ca337 100644 --- a/src/sql/resolver/cmd/ob_load_data_resolver.cpp +++ b/src/sql/resolver/cmd/ob_load_data_resolver.cpp @@ -246,6 +246,8 @@ int ObLoadDataResolver::resolve(const ParseNode &parse_tree) ret = OB_NOT_SUPPORTED; LOG_WARN("load data to the view is not supported", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "load data to the view is"); + } else if (OB_FAIL(check_trigger_constraint(tschema))) { + LOG_WARN("check trigger constraint failed", K(ret), KPC(tschema)); } else { load_args.table_id_ = tschema->get_table_id(); load_args.table_name_ = table_name; @@ -269,7 +271,7 @@ int ObLoadDataResolver::resolve(const ParseNode &parse_tree) } else { load_args.combined_name_.assign_ptr(buf, pos); } - LOG_DEBUG("resovle table info result", K(tenant_id), K(database_name), K(table_name)); + LOG_DEBUG("resolve table info result", K(tenant_id), K(database_name), K(table_name)); } } @@ -907,7 +909,7 @@ int ObLoadDataResolver::resolve_column_ref_expr(ObIArray &colum * This function examines subqueries in set clause recursively, * and ensures that no subquery read data from the loaded table. * - * Call deepth has already examined in subquery resolving phase, + * Call depth has already examined in subquery resolving phase, * don't need to do it again */ int recursively_check_subquery_tables(ObSelectStmt *subquery_stmt, uint64_t loaded_table_id) @@ -1180,7 +1182,7 @@ int ObLoadDataResolver::check_if_table_exists(uint64_t tenant_id, LOG_WARN("fail to check table or index exist", K(tenant_id), K(database_id), K(table_name), K(ret)); } else if (!is_table_exist) { - //a alias table is impossiable + //a alias table is impossible //TODO wjh: support synonym tables, return not exist for now //see ObDMLResolver::resolve_table_relation_recursively ret = OB_TABLE_NOT_EXIST; @@ -1192,7 +1194,7 @@ int ObLoadDataResolver::check_if_table_exists(uint64_t tenant_id, cte_table_fisrt, is_hidden, tschema))) { - //it's possiable to get "table not exist" ret here + //it's possible to get "table not exist" ret here LOG_WARN("get table schema failed", K(ret)); } else { table_id = tschema->get_table_id(); @@ -1342,5 +1344,41 @@ int ObLoadDataResolver::resolve_char_node(const ParseNode &node, int32_t &single return ret; } +int ObLoadDataResolver::check_trigger_constraint(const ObTableSchema *table_schema) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema) + || OB_ISNULL(schema_checker_) + || OB_ISNULL(session_info_) + || OB_ISNULL(schema_checker_->get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("object is null", K(ret), K(table_schema), K(schema_checker_), + K(session_info_), K(schema_checker_->get_schema_guard())); + } else { + uint64_t tenant_id = session_info_->get_effective_tenant_id(); + + for (int64_t i = 0; OB_SUCC(ret) && i < table_schema->get_trigger_list().count(); i++) { + const ObTriggerInfo *trg_info = NULL; + share::schema::ObSchemaGetterGuard *schema_guard = schema_checker_->get_schema_guard(); + if (OB_FAIL(schema_guard->get_trigger_info(tenant_id, table_schema->get_trigger_list().at(i), trg_info))) { + LOG_WARN("get trigger info failed", K(ret), K(tenant_id), K(table_schema->get_trigger_list().at(i))); + } else if (OB_ISNULL(trg_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("trigger info is null", K(ret), K(tenant_id), K(table_schema->get_trigger_list().at(i))); + } else if (trg_info->is_enable() + && (trg_info->has_insert_event() || trg_info->has_update_event())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support load data if table has insert or update trigger", K(ret), KPC(trg_info)); + if (lib::is_oracle_mode()) { + LOG_USER_ERROR(OB_NOT_SUPPORTED, "if table has enabled insert or update trigger, load data"); + } else { + LOG_USER_ERROR(OB_NOT_SUPPORTED, "if table has insert or update trigger, load data"); + } + } + } + } + return ret; +} + } // sql } // oceanbase diff --git a/src/sql/resolver/cmd/ob_load_data_resolver.h b/src/sql/resolver/cmd/ob_load_data_resolver.h index 132927f3cb..994bb7206c 100644 --- a/src/sql/resolver/cmd/ob_load_data_resolver.h +++ b/src/sql/resolver/cmd/ob_load_data_resolver.h @@ -57,7 +57,7 @@ public: const common::ObString &table_name, bool cte_table_fisrt, uint64_t& table_id); int validate_stmt(ObLoadDataStmt* stmt); int resolve_hints(const ParseNode &node); - + int check_trigger_constraint(const ObTableSchema *table_schema); private: enum ParameterEnum { ENUM_OPT_LOCAL = 0, diff --git a/src/sql/resolver/dcl/ob_create_user_resolver.cpp b/src/sql/resolver/dcl/ob_create_user_resolver.cpp index 6e91498db9..c3dbb8b232 100644 --- a/src/sql/resolver/dcl/ob_create_user_resolver.cpp +++ b/src/sql/resolver/dcl/ob_create_user_resolver.cpp @@ -120,7 +120,7 @@ int ObCreateUserResolver::resolve(const ParseNode &parse_tree) if (lib::is_oracle_mode() && 0 != host_name.compare(OB_DEFAULT_HOST_NAME)) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "create user with hostname"); - LOG_WARN("create user shoud not use hostname in oracle mode", K(ret)); + LOG_WARN("create user should not use hostname in oracle mode", K(ret)); } ObString password; ObString need_enc_str = ObString::make_string("NO"); @@ -169,7 +169,7 @@ int ObCreateUserResolver::resolve(const ParseNode &parse_tree) } create_user_stmt->set_profile_id(profile_id); //只有oracle模式profile id是有效的 if (OB_SUCC(ret)) { - if (!lib::is_oracle_mode() && OB_FAIL(check_password_strength(password, user_name))) { + if (!lib::is_oracle_mode() && OB_FAIL(check_password_strength(password))) { LOG_WARN("password don't satisfied current policy", K(ret)); } else if (lib::is_oracle_mode() && OB_FAIL(check_oracle_password_strength( params_.session_info_->get_effective_tenant_id(), diff --git a/src/sql/resolver/dcl/ob_dcl_resolver.cpp b/src/sql/resolver/dcl/ob_dcl_resolver.cpp index 8a094d95fc..0b671d08d1 100644 --- a/src/sql/resolver/dcl/ob_dcl_resolver.cpp +++ b/src/sql/resolver/dcl/ob_dcl_resolver.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX SQL_RESV #include "observer/ob_server_struct.h" #include "observer/ob_inner_sql_connection_pool.h" +#include "sql/engine/expr/ob_expr_validate_password_strength.h" #include "sql/resolver/dcl/ob_dcl_resolver.h" #include "sql/session/ob_sql_session_info.h" #include "sql/ob_sql_utils.h" @@ -59,51 +60,46 @@ int ObDCLResolver::check_and_convert_name(ObString &db, ObString &table) return ret; } -int ObDCLResolver::check_password_strength(common::ObString &password, common::ObString &user_name) +int ObDCLResolver::check_password_strength(common::ObString &password) { int ret = OB_SUCCESS; int64_t pw_policy = 0; - uint64_t valid_pw_len = 0; + int64_t check_user_name_flag = 0; + size_t char_len = ObCharset::strlen_char(ObCharset::get_system_collation(), password.ptr(), + static_cast(password.length())); + bool passed = true; if (OB_ISNULL(session_info_)) { ret = OB_NOT_INIT; LOG_WARN("Session info is not inited", K(ret)); - // 0 代表密码政策为low, 1代表密码政策为medium } else if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_POLICY, pw_policy))) { LOG_WARN("fail to get validate_password_policy variable", K(ret)); - } else if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_LENGTH, valid_pw_len))) { - LOG_WARN("fail to get validate_password_length variable", K(ret)); + } else if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_CHECK_USER_NAME, check_user_name_flag))) { + LOG_WARN("fail to get validate_password_check_user_name variable", K(ret)); + } else if (!check_user_name_flag && OB_FAIL(check_user_name(password, session_info_->get_user_name()))) { + LOG_WARN("password cannot be the same with user name", K(ret)); + } else if (OB_FAIL(ObExprValidatePasswordStrength::validate_password_low(password, + char_len, + *session_info_, + passed))) { + LOG_WARN("password len dont satisfied current pw policy", K(ret)); } else if (ObPasswordPolicy::LOW == pw_policy) { - if (OB_FAIL(check_password_len(password, valid_pw_len))) { - LOG_WARN("password len dont satisfied current pw policy", K(ret)); - } + // do nothing } else if (ObPasswordPolicy::MEDIUM == pw_policy) { - uint64_t valid_pw_len = 0; - int64_t check_user_name_flag = 0; - uint64_t mix_case_count = 0; - uint64_t number_count = 0; - uint64_t special_char_count = 0; - if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_CHECK_USER_NAME, check_user_name_flag))) { - LOG_WARN("fail to get validate_password_check_user_name variable", K(ret)); - } else if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_NUMBER_COUNT, number_count))) { - LOG_WARN("fail to get validate_password_number_count variable", K(ret)); - } else if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_SPECIAL_CHAR_COUNT, special_char_count))) { - LOG_WARN("fail to get validate_password_length variable", K(ret)); - } else if (OB_FAIL(session_info_->get_sys_variable(share::SYS_VAR_VALIDATE_PASSWORD_MIXED_CASE_COUNT, mix_case_count))) { - LOG_WARN("fail to get validate_password_mixed_case_count variable", K(ret)); - } else if (OB_FAIL(check_number_count(password, number_count))) { - LOG_WARN("password number count not satisfied current pw policy", K(ret)); - } else if (OB_FAIL(check_special_char_count(password, special_char_count))) { - LOG_WARN("password special char count not satisfied current pw policy", K(ret)); - } else if (OB_FAIL(check_mixed_case_count(password, mix_case_count))) { - LOG_WARN("password mixed case count not satisfied current pw policy", K(ret)); - } else if (!check_user_name_flag && OB_FAIL(check_user_name(password, user_name))) { - LOG_WARN("password cannot be the same with user name", K(ret)); - } else if (OB_FAIL(check_password_len(password, valid_pw_len))) { + if (OB_FAIL(ObExprValidatePasswordStrength::validate_password_medium(password, + char_len, + *session_info_, + passed))) { LOG_WARN("password len dont satisfied current pw policy", K(ret)); } } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("the value of password policy is unexpectd", K(ret)); + LOG_WARN("the value of password policy is unexpected", K(ret)); + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(!passed)) { + ret = OB_ERR_NOT_VALID_PASSWORD; + LOG_WARN("the password is not valid", K(ret)); + } } return ret; } @@ -183,85 +179,7 @@ int ObDCLResolver::check_oracle_password_strength(int64_t tenant_id, return ret; } - -int ObDCLResolver::check_number_count(common::ObString &password, const int64_t &number_count) -{ - int ret = OB_SUCCESS; - int64_t count = 0; - for (int i = 0; OB_SUCC(ret) && i < password.length(); ++i) { - if (password[i] >= '0' && password[i] <= '9') { - count++; - } - if (count >= number_count) { - break; - } - } - if (OB_SUCC(ret)) { - if (number_count > count) { - ret = OB_ERR_NOT_VALID_PASSWORD; - LOG_WARN("the password is not valid", K(ret)); - } - } - return ret; -} - -int ObDCLResolver::check_special_char_count(common::ObString &password, const int64_t &special_char_count) -{ - int ret = OB_SUCCESS; - int64_t count = 0; - for (int i = 0; OB_SUCC(ret) && i < password.length(); ++i) { - if ((password[i] >= '!' && password[i] <= '/')|| - (password[i] >= ':' && password[i] <= '?')) { - count++; - } - if (count >= special_char_count) { - break; - } - } - if (OB_SUCC(ret)) { - if (special_char_count > count) { - ret = OB_ERR_NOT_VALID_PASSWORD; - LOG_WARN("the password is not valid", K(ret)); - } - } - return ret; -} - -int ObDCLResolver::check_mixed_case_count(common::ObString &password, const int64_t &mix_case_count) -{ - int ret = OB_SUCCESS; - int64_t lower_count = 0; - int64_t upper_count = 0; - for (int i = 0; OB_SUCC(ret) && i < password.length(); ++i) { - if (islower(password[i])) { - lower_count++; - } else if (isupper(password[i])) { - upper_count++; - } - if (lower_count >= mix_case_count && upper_count >= mix_case_count) { - break; - } - } - if (OB_SUCC(ret)) { - if (mix_case_count > lower_count || mix_case_count > upper_count) { - ret = OB_ERR_NOT_VALID_PASSWORD; - LOG_WARN("the password is not valid", K(ret)); - } - } - return ret; -} - -int ObDCLResolver::check_password_len(common::ObString &password, const int64_t &password_len) -{ - int ret = OB_SUCCESS; - if (password.length() < password_len) { - ret = OB_ERR_NOT_VALID_PASSWORD; - LOG_WARN("the password is not valid", K(ret)); - } - return ret; -} - -int ObDCLResolver::check_user_name(common::ObString &password, common::ObString &user_name) +int ObDCLResolver::check_user_name(common::ObString &password, const common::ObString &user_name) { int ret = OB_SUCCESS; if (ObCharset::case_insensitive_equal(password, user_name)) { diff --git a/src/sql/resolver/dcl/ob_dcl_resolver.h b/src/sql/resolver/dcl/ob_dcl_resolver.h index 440eec83c7..e75520042c 100644 --- a/src/sql/resolver/dcl/ob_dcl_resolver.h +++ b/src/sql/resolver/dcl/ob_dcl_resolver.h @@ -35,12 +35,8 @@ public: bool skip_enclosed_char = false); protected: int check_and_convert_name(common::ObString &db, common::ObString &table); - int check_password_strength(common::ObString &password, common::ObString &user_name); - int check_number_count(common::ObString &password, const int64_t &number_count); - int check_special_char_count(common::ObString &password, const int64_t &special_char_count); - int check_mixed_case_count(common::ObString &password, const int64_t &mix_case_count); - int check_user_name(common::ObString &password, common::ObString &user_name); - int check_password_len(common::ObString &password, const int64_t &password_len); + int check_password_strength(common::ObString &password); + int check_user_name(common::ObString &password, const common::ObString &user_name); int check_oracle_password_strength(int64_t tenant_id, int64_t profile_id, common::ObString &password, diff --git a/src/sql/resolver/dcl/ob_drop_user_resolver.cpp b/src/sql/resolver/dcl/ob_drop_user_resolver.cpp index 5b898a91c1..fa32b24db8 100644 --- a/src/sql/resolver/dcl/ob_drop_user_resolver.cpp +++ b/src/sql/resolver/dcl/ob_drop_user_resolver.cpp @@ -95,7 +95,7 @@ int ObDropUserResolver::resolve(const ParseNode &parse_tree) SQL_RESV_LOG(WARN, "user_list_node is null", K(ret)); } else { uint64_t tenant_id = params_.session_info_->get_effective_tenant_id(); - // resovle user_list_node + // resolved user_list_node for (int i = 0; i < user_list_node->num_child_ && OB_SUCCESS == ret; ++i) { if (OB_ISNULL(user_list_node->children_[i])) { ret = OB_ERR_PARSE_SQL; diff --git a/src/sql/resolver/dcl/ob_grant_resolver.cpp b/src/sql/resolver/dcl/ob_grant_resolver.cpp index 9425bb3995..6d601dc3ed 100644 --- a/src/sql/resolver/dcl/ob_grant_resolver.cpp +++ b/src/sql/resolver/dcl/ob_grant_resolver.cpp @@ -1337,7 +1337,7 @@ int ObGrantResolver::resolve_mysql(const ParseNode &parse_tree) if (OB_ISNULL(user_node->children_[2])) { ret = OB_ERR_PARSE_SQL; LOG_WARN("The child 2 of user_node should not be NULL", K(ret)); - } else if (OB_FAIL(check_password_strength(pwd, user_name))) { + } else if (OB_FAIL(check_password_strength(pwd))) { LOG_WARN("fail to check password strength", K(ret)); } else if (0 == user_node->children_[2]->value_) { if (!ObSetPasswordResolver::is_valid_mysql41_passwd(pwd)) { diff --git a/src/sql/resolver/dcl/ob_grant_stmt.cpp b/src/sql/resolver/dcl/ob_grant_stmt.cpp index 064407ca53..c0b01bf765 100644 --- a/src/sql/resolver/dcl/ob_grant_stmt.cpp +++ b/src/sql/resolver/dcl/ob_grant_stmt.cpp @@ -117,7 +117,7 @@ int ObGrantStmt::add_role(const common::ObString &role) // Pust the first user_name and host_name into role[0] and role[1] // And the rest of user_names and host_names shall be saved in remain_roles -// The remain_roles is used to keep compatibility with previouse logic +// The remain_roles is used to keep compatibility with previous logic int ObGrantStmt::add_user(const common::ObString &user_name, const common::ObString &host_name) { diff --git a/src/sql/resolver/dcl/ob_set_password_resolver.cpp b/src/sql/resolver/dcl/ob_set_password_resolver.cpp index 3b7d730f72..67d38e1542 100644 --- a/src/sql/resolver/dcl/ob_set_password_resolver.cpp +++ b/src/sql/resolver/dcl/ob_set_password_resolver.cpp @@ -156,7 +156,7 @@ int ObSetPasswordResolver::resolve(const ParseNode &parse_tree) } else { ObString password(static_cast(node->children_[1]->str_len_), node->children_[1]->str_value_); - if (!lib::is_oracle_mode() && OB_FAIL(check_password_strength(password, user_name))) { + if (!lib::is_oracle_mode() && OB_FAIL(check_password_strength(password))) { LOG_WARN("fail to check password strength", K(ret)); } else if (lib::is_oracle_mode() && OB_FAIL( resolve_oracle_password_strength(user_name, host_name, password))) { diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index c7ad995d14..923037eb3e 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -95,7 +95,7 @@ int ObCreateTableResolver::add_primary_key_part(const ObString &column_name, } else if (static_cast(table_id_) > 0 && OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id( session_info_->get_effective_tenant_id(), table_id_, is_oracle_mode))) { - LOG_WARN("fail to check oralce mode", KR(ret), K_(table_id)); + LOG_WARN("fail to check oracle mode", KR(ret), K_(table_id)); } else { ObColumnSchemaV2 *col = NULL; ObTableSchema &table_schema = create_table_stmt->get_create_table_arg().schema_; @@ -767,7 +767,7 @@ int ObCreateTableResolver::resolve(const ParseNode &parse_tree) // Oracle 官方文档关于 references 权限的描述非常少。 // 文档中比较重要的一条是“references 权限不能被授予给角色”， // 再加上测试的结果，所以我们推测references权限进行检查时， - // 检查的不是当前用户是否具有refernces权限，而是去检查子表所在的schema有没有references的权限。 + // 检查的不是当前用户是否具有references权限，而是去检查子表所在的schema有没有references的权限。 // 所以现在的逻辑是 // 1. 当子表和父表相同时，无需检查 // 2. 当子表和父表同属一个schema时，也无需检查，这一点已经在oracle上验证了。 @@ -963,7 +963,7 @@ int ObCreateTableResolver::resolve_partition_option( } if (OB_SUCC(ret) && (OB_NOT_NULL(node) || is_oracle_temp_table_)) { if (OB_FAIL(check_generated_partition_column(table_schema))) { - LOG_WARN("Failed to check generate partiton column", K(ret)); + LOG_WARN("Failed to check generate partition column", K(ret)); } else if (OB_FAIL(table_schema.check_primary_key_cover_partition_column())) { SQL_RESV_LOG(WARN, "fail to check primary key cover partition column", K(ret)); } else if (OB_FAIL(table_schema.check_auto_partition_valid())) { @@ -1202,7 +1202,7 @@ int ObCreateTableResolver::resolve_table_elements(const ParseNode *node, } else if (static_cast(table_id_) > 0 && OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id( session_info_->get_effective_tenant_id(), table_id_, is_oracle_mode))) { - LOG_WARN("fail to check oralce mode", KR(ret), K_(table_id)); + LOG_WARN("fail to check oracle mode", KR(ret), K_(table_id)); } else { ObSEArray gen_col_expr_arr; ParseNode *primary_node = NULL; @@ -1215,7 +1215,7 @@ int ObCreateTableResolver::resolve_table_elements(const ParseNode *node, table_schema.set_tenant_id(tenant_id); bool has_visible_col = false; // 将经过 resolve_column_definition 后的 column schema 存放在 resolved_cols 中 - // 为了支持生成列按照任意顺序定义，在生成全部 column_schema 之后，再按照统一存放到 table_schma 中 + // 为了支持生成列按照任意顺序定义，在生成全部 column_schema 之后，再按照统一存放到 table_schema 中 ObSEArray resolved_cols; //列需要根据租户id区分大小写,这里先把租户id设置进table_schema if (RESOLVE_NON_COL != resolve_rule) { @@ -1674,7 +1674,7 @@ int ObCreateTableResolver::set_nullable_for_cta_column(ObSelectStmt *select_stmt column.set_nullable(!is_not_null); } } else { //mysql mode - if (expr->is_win_func_expr()) {//compatiable with mysql + if (expr->is_win_func_expr()) {//compatible with mysql const ObWinFunRawExpr *win_expr = reinterpret_cast(expr); if (T_WIN_FUN_RANK == win_expr->get_func_type() || T_WIN_FUN_DENSE_RANK == win_expr->get_func_type() || @@ -2510,7 +2510,7 @@ int ObCreateTableResolver::resolve_index_node(const ParseNode *node) } else if (static_cast(table_id_) > 0 && OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id( session_info_->get_effective_tenant_id(), table_id_, is_oracle_mode))) { - LOG_WARN("fail to check oralce mode", KR(ret), K_(table_id)); + LOG_WARN("fail to check oracle mode", KR(ret), K_(table_id)); } else { index_arg_.reset(); ObColumnSortItem sort_item; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.h b/src/sql/resolver/ddl/ob_create_table_resolver.h index f3b2568904..fa9f7a928a 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver.h @@ -107,7 +107,7 @@ private: const ParseNode* node, ObArray &constraint_position_list); int build_partition_key_info(share::schema::ObTableSchema &table_schema, const bool is_subpart); - //resolve partitoin option only used in ObCreateTableResolver now. + //resolve partition option only used in ObCreateTableResolver now. int resolve_partition_option(ParseNode *node, share::schema::ObTableSchema &table_schema, const bool is_partition_option_node_with_opt); @@ -122,7 +122,7 @@ private: int add_new_indexkey_for_oracle_temp_table(const int32_t org_key_len); int add_pk_key_for_oracle_temp_table(ObArray &stats, int64_t &pk_data_length); int set_partition_info_for_oracle_temp_table(share::schema::ObTableSchema &table_schema); - // following four functions should be used only in oralce mode + // following four functions should be used only in oracle mode int generate_primary_key_name_array(const share::schema::ObTableSchema &table_schema, ObIArray &pk_columns_name); int generate_uk_idx_array(const ObIArray &index_arg_list, ObIArray &uk_idx_in_index_arg_list); bool is_pk_uk_duplicate(const ObIArray &pk_columns_name, const ObIArray &index_arg_list, const ObIArray &uk_idx); diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.cpp b/src/sql/resolver/ddl/ob_create_view_resolver.cpp index 5bde72fc60..191e55f229 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_view_resolver.cpp @@ -456,6 +456,7 @@ int ObCreateViewResolver::check_privilege_needed(ObCreateTableStmt &stmt, need_priv.table_ = table_name; need_priv.priv_set_ = OB_PRIV_SELECT; need_priv.is_sys_table_ = table_item->is_system_table_; + need_priv.is_for_update_ = table_item->for_update_; need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; if (OB_FAIL(stmt.add_view_need_priv(need_priv))) { LOG_WARN("Fail to add need_priv", K(ret), K(need_priv)); diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 3577408c0e..4187be25af 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -5332,6 +5332,9 @@ int ObDDLResolver::init_empty_session(const common::ObTimeZoneInfoWrap &tz_info_ } else if (OB_FAIL(table_schema.check_if_oracle_compat_mode(is_oracle_compat_mode))) { LOG_WARN("failed to get table compatibility mode", K(ret)); } else { + ObSessionDDLInfo ddl_info; + ddl_info.set_ddl_check_default_value(true); + empty_session.set_ddl_info(ddl_info); empty_session.set_nls_formats(nls_formats); empty_session.set_compatibility_mode( is_oracle_compat_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE); @@ -5634,6 +5637,8 @@ int ObDDLResolver::calc_default_value(share::schema::ObColumnSchemaV2 &column, uint64_t tenant_id = column.get_tenant_id(); const ObTenantSchema *tenant_schema = NULL; ObSchemaGetterGuard guard; + ObSessionDDLInfo ddl_info; + ddl_info.set_ddl_check_default_value(true); ParamStore empty_param_list( (ObWrapperAllocator(allocator)) ); params.expr_factory_ = &expr_factory; params.allocator_ = &allocator; @@ -5658,6 +5663,8 @@ int ObDDLResolver::calc_default_value(share::schema::ObColumnSchemaV2 &column, lib::is_oracle_mode() ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE))) { } else if (FALSE_IT(empty_session.set_sql_mode( lib::is_oracle_mode() ? DEFAULT_ORACLE_MODE : DEFAULT_MYSQL_MODE))) { + } else if (FALSE_IT(empty_session.set_ddl_info(ddl_info))) { + LOG_WARN("fail to set ddl_info", K(ret)); } else if (OB_FAIL(default_value.get_string(expr_str))) { LOG_WARN("get expr string from default value failed", K(ret), K(default_value)); } else if (OB_FAIL(ObResolverUtils::resolve_default_expr_v2_column_expr(params, expr_str, diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 70d14dde6f..56cde8055e 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -12395,7 +12395,9 @@ int ObDMLResolver::convert_udf_to_agg_expr(ObRawExpr *&expr, } } - ctx.parents_expr_info_.del_member(IS_AGG); + if (OB_SUCC(ret) && OB_FAIL(ctx.parents_expr_info_.del_member(IS_AGG))) { + LOG_WARN("failed to del member", K(ret)); + } } return ret; } @@ -12613,11 +12615,12 @@ int ObDMLResolver::inner_resolve_hints(const ParseNode &node, cur_hints.reuse(); if (OB_ISNULL(hint_node = node.children_[i])) { /* do nothing */ - } else if (T_QB_NAME == hint_node->type_ && !qb_name_conflict) { + } else if (T_QB_NAME == hint_node->type_) { ObString tmp_qb_name; if (OB_FAIL(resolve_qb_name_node(hint_node, tmp_qb_name))) { LOG_WARN("failed to resolve qb name node", K(ret)); - } else if (OB_UNLIKELY(!qb_name.empty() && !tmp_qb_name.empty())) { + } else if (OB_UNLIKELY(qb_name_conflict || (!qb_name.empty() && !tmp_qb_name.empty()))) { + LOG_TRACE("conflict qb_name hint.", K(tmp_qb_name), K(qb_name)); qb_name_conflict = true; qb_name.reset(); } else if (!tmp_qb_name.empty()) { diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 0df8684092..e4036ae95a 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -823,7 +823,7 @@ private: // a synonym may a dblink synonym, while failed to resolving a sysnonym to a table name, // wo should try to resolving to a dblink name. - // in out param: table_name, in: may someting link "remote_schema.test@my_link", out: "test" + // in out param: table_name, in: may something link "remote_schema.test@my_link", out: "test" // out param: dblink_name("my_link"), db_name("remote_schema"), dblink_id(id of my_link) int resolve_dblink_with_synonym(uint64_t tenant_id, ObString &table_name, ObString &dblink_name, ObString &db_name, uint64_t &dblink_id); @@ -853,7 +853,7 @@ private: ObIArray &opt_hints); int resolve_index_hint(const ParseNode &index_node, ObOptHint *&opt_hint); - int resolve_index_hint(const TableItem &table, // resovle mysql mode index hint after table + int resolve_index_hint(const TableItem &table, // resolved mysql mode index hint after table const ParseNode &index_hint_node); int resolve_table_parallel_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_join_order_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); @@ -970,7 +970,7 @@ protected: /* * 在with clause的解析中，我们不能将解析的表添加到影响此plan是否需要进入trans_service - * 的globle_dependency_table中去。 + * 的global_dependency_table中去。 * 例如： * with cte (select * from t1) select 1 from dual; * 如果加入了，上面这句将会进入trans_service，而之后则会报错，因为其实它检测到相关table的 diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index cf68ef7b47..963bd0f38a 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -470,8 +470,6 @@ int ObGlobalHint::print_global_hint(PlanText &plan_text) const if (OB_SUCC(ret) && has_parallel_hint() && !ignore_parallel_for_dblink) { //PARALLEL if (has_parallel_degree()) { PRINT_GLOBAL_HINT_NUM("PARALLEL", parallel_); - } else if (plan_text.is_outline_data_) { - /* do not print parallel policy for outline data */ } else if (enable_auto_dop()) { PRINT_GLOBAL_HINT_STR("PARALLEL( AUTO )"); } else if (enable_manual_dop()) { @@ -502,7 +500,7 @@ int ObGlobalHint::print_global_hint(PlanText &plan_text) const } } // OPTIMIZER_FEATURES_ENABLE - if (OB_SUCC(ret) && (has_valid_opt_features_version() || plan_text.is_outline_data_)) { + if (OB_SUCC(ret) && (has_valid_opt_features_version())) { int64_t cur_pos = 0; const uint64_t version = has_valid_opt_features_version() ? opt_features_version_ : CURRENT_OUTLINE_ENABLE_VERSION; diff --git a/src/sql/resolver/dml/ob_insert_resolver.cpp b/src/sql/resolver/dml/ob_insert_resolver.cpp index f221959c17..6318462c4a 100644 --- a/src/sql/resolver/dml/ob_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_insert_resolver.cpp @@ -200,7 +200,7 @@ int ObInsertResolver::resolve_insert_clause(const ParseNode &node) insert_stmt->get_insert_table_info().ref_table_id_, insert_stmt->get_insert_table_info().table_id_, insert_stmt))) { - LOG_WARN("failed to add new column for oralce label security table", K(ret)); + LOG_WARN("failed to add new column for oracle label security table", K(ret)); } else if (!has_tg && OB_FAIL(generate_autoinc_params(insert_stmt->get_insert_table_info()))) { LOG_WARN("failed to save autoinc params", K(ret)); } else if (OB_FAIL(generate_column_conv_function(insert_stmt->get_insert_table_info()))) { @@ -491,7 +491,7 @@ int ObInsertResolver::resolve_insert_assign(const ParseNode &assign_list) //add column_item if (OB_ISNULL(assign.column_expr_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid assginment variable", K(i), K(j), K(assign)); + LOG_WARN("invalid assignment variable", K(i), K(j), K(assign)); } else if (assign.is_duplicated_) { ret = OB_ERR_FIELD_SPECIFIED_TWICE; LOG_USER_ERROR(OB_ERR_FIELD_SPECIFIED_TWICE, to_cstring(assign.column_expr_->get_column_name())); @@ -1075,7 +1075,7 @@ int ObInsertResolver::try_expand_returning_exprs() int ret = OB_SUCCESS; ObInsertStmt *insert_stmt = NULL; // we do not need expand returning expr in prepare stage because we resolve - // it twice, first in prepare stage, second in actural execution. We can only + // it twice, first in prepare stage, second in actual execution. We can only // do it in second stage // Otherwise if we expand in prepare stage, which will pollute our spell SQL // then got a wrong result @@ -1090,7 +1090,7 @@ int ObInsertResolver::try_expand_returning_exprs() for (int64_t i = 0; OB_SUCC(ret) && i < insert_stmt->get_returning_exprs().count(); i++) { for (int64_t j = 0; OB_SUCC(ret) && j < table_columns.count(); j++) { if (table_columns.at(j)->is_xml_column()) { - // do nothing and will rewrite in trnsform stage + // do nothing and will rewrite in transform stage } else { OZ(ObRawExprUtils::replace_ref_column(insert_stmt->get_returning_exprs().at(i), table_columns.at(j), diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index 89a7ef85f8..e0e8ebf9d9 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -1069,7 +1069,9 @@ int ObSelectResolver::resolve_normal_query(const ParseNode &parse_tree) OZ( resolve_with_clause(parse_tree.children_[PARSE_SELECT_WITH]) ); /* normal select */ - select_stmt->assign_set_op(ObSelectStmt::NONE); + if (OB_SUCC(ret)) { + select_stmt->assign_set_op(ObSelectStmt::NONE); + } OZ( resolve_query_options(parse_tree.children_[PARSE_SELECT_DISTINCT]) ); if (OB_SUCC(ret) && is_only_full_group_by_on(session_info_->get_sql_mode())) { OZ( standard_group_checker_.init() ); diff --git a/src/sql/resolver/dml/ob_select_stmt.h b/src/sql/resolver/dml/ob_select_stmt.h index 74445c7792..dc26c7d7c4 100644 --- a/src/sql/resolver/dml/ob_select_stmt.h +++ b/src/sql/resolver/dml/ob_select_stmt.h @@ -154,7 +154,7 @@ struct ObSelectIntoItem ObItemType into_type_; common::ObObj outfile_name_; common::ObObj filed_str_; // filed terminated str - common::ObObj line_str_; // line teminated str + common::ObObj line_str_; // line terminated str common::ObSEArray user_vars_; // user variables common::ObSEArray pl_vars_; // pl variables char closed_cht_; // all fileds, "123","ab" diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index 3ea280ea45..6a4fc71f85 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -548,7 +548,7 @@ int ObQueryHint::reset_duplicate_qb_name() if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { LOG_WARN("get stmt id from hash map failed", K(ret)); } else if (OB_FAIL(qb_name_map_.set_refactored(qb_names.qb_names_.at(0), i))) { - LOG_WARN("faield to add name map", K(ret)); + LOG_WARN("failed to add name map", K(ret)); } } else if (OB_UNLIKELY(idx < 0 || idx >= i)) { ret = OB_ERR_UNEXPECTED; @@ -704,7 +704,7 @@ int ObQueryHint::print_outline_data(PlanText &plan_text) const bool is_oneline = plan_text.is_oneline_; if (OB_UNLIKELY(1 < stmt_id_hints_.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected stmt id hitns for outline data", K(ret), K(stmt_id_hints_)); + LOG_WARN("unexpected stmt id hints for outline data", K(ret), K(stmt_id_hints_)); } else if (OB_FAIL(BUF_PRINTF("%sBEGIN_OUTLINE_DATA", ObQueryHint::get_outline_indent(is_oneline)))) { } else if (1 == stmt_id_hints_.count() && OB_FAIL(stmt_id_hints_.at(0).print_hints(plan_text, true))) { diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index 2a417df527..e566dc6fc1 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -167,7 +167,7 @@ struct ObQueryHint { int64_t outline_stmt_id_; ObSEArray qb_hints_; // hints with qb name ObSEArray stmt_id_hints_; // hints without qb name, used before transform - ObSEArray trans_list_; // tranform hints from outline data, need keep order + ObSEArray trans_list_; // transform hints from outline data, need keep order ObSEArray outline_trans_hints_; // tranform hints to generate outline data ObSEArray used_trans_hints_; ObSEArray stmt_id_map_; // stmt id -> qb name list, position is stmt id diff --git a/src/sql/resolver/dml/ob_stmt_expr_visitor.cpp b/src/sql/resolver/dml/ob_stmt_expr_visitor.cpp index ed6e6dc17f..fcfe192181 100644 --- a/src/sql/resolver/dml/ob_stmt_expr_visitor.cpp +++ b/src/sql/resolver/dml/ob_stmt_expr_visitor.cpp @@ -133,7 +133,7 @@ int ObSharedExprChecker::init(ObDMLStmt &stmt) } if (OB_SUCC(ret)) { if (OB_FAIL(stmt_expr_set.destroy())) { - LOG_WARN("failed to detroy stmt expr set", K(ret)); + LOG_WARN("failed to destroy stmt expr set", K(ret)); } } return ret; diff --git a/src/sql/resolver/dml/ob_update_resolver.cpp b/src/sql/resolver/dml/ob_update_resolver.cpp index 5386e88f8d..50aa902b25 100644 --- a/src/sql/resolver/dml/ob_update_resolver.cpp +++ b/src/sql/resolver/dml/ob_update_resolver.cpp @@ -172,7 +172,7 @@ int ObUpdateResolver::resolve(const ParseNode &parse_tree) LOG_WARN("failed to resolve error logging", K(ret)); } else if (is_mysql_mode() && !update_stmt->is_ignore() && OB_FAIL(check_join_update_conflict())) { - LOG_WARN("failed to check join update confilct", K(ret)); + LOG_WARN("failed to check join update conflict", K(ret)); } else if (OB_FAIL(update_stmt->formalize_stmt(session_info_))) { LOG_WARN("pull update stmt all expr relation ids failed", K(ret)); } else { /*do nothing*/ } @@ -184,7 +184,7 @@ int ObUpdateResolver::resolve(const ParseNode &parse_tree) } else if (OB_FAIL(update_stmt->check_dml_need_filter_null())) { LOG_WARN("failed to check dml need filter null", K(ret)); } else if (lib::is_mysql_mode() && OB_FAIL(check_safe_update_mode(update_stmt))) { - LOG_WARN("failed to check fullfill safe update mode", K(ret)); + LOG_WARN("failed to check fulfill safe update mode", K(ret)); } else { /*do nothing*/ } } return ret; @@ -195,7 +195,7 @@ int ObUpdateResolver::try_expand_returning_exprs() int ret = OB_SUCCESS; ObUpdateStmt *update_stmt = NULL; // we do not need expand returing expr in prepare stage because we resolve - // it twice, first in prepare stage, second in actural execution. We can only + // it twice, first in prepare stage, second in actual execution. We can only // do it in second stage // Otherwise if we expand in prepare stage, which will pollute our spell SQL // then got a wrong result @@ -227,7 +227,7 @@ int ObUpdateResolver::try_expand_returning_exprs() LOG_WARN("failed to add uncopy exprs", K(ret)); } else if (OB_FAIL(copier.copy_on_replace(update_stmt->get_returning_exprs(), update_stmt->get_returning_exprs()))) { - LOG_WARN("failed to copy on repalce returning exprs", K(ret)); + LOG_WARN("failed to copy on replace returning exprs", K(ret)); } } } diff --git a/src/sql/resolver/dml/ob_update_stmt.h b/src/sql/resolver/dml/ob_update_stmt.h index b254e0441f..21e6abf7cc 100644 --- a/src/sql/resolver/dml/ob_update_stmt.h +++ b/src/sql/resolver/dml/ob_update_stmt.h @@ -21,7 +21,7 @@ namespace oceanbase namespace sql { /** - * UPDATE syntac from MySQL 5.7 + * UPDATE syntax from MySQL 5.7 * * Single-table syntax: UPDATE [LOW_PRIORITY] [IGNORE] table_reference diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 827c63d973..f02219a895 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -431,11 +431,11 @@ int ObRawExpr::pull_relation_id() int ObRawExpr::add_child_flags(const ObExprInfo &flags) { int ret = OB_SUCCESS; - ObExprInfo tmp = flags; - if (INHERIT_MASK_BEGIN < tmp.bit_count()) { - int64_t mask_end = INHERIT_MASK_END < tmp.bit_count() ? - static_cast(INHERIT_MASK_END) : tmp.bit_count() - 1; - if (tmp.do_mask(INHERIT_MASK_BEGIN, mask_end)) { + if (INHERIT_MASK_BEGIN < flags.bit_count()) { + ObExprInfo tmp(flags); + int64_t mask_end = INHERIT_MASK_END < flags.bit_count() ? + static_cast(INHERIT_MASK_END) : flags.bit_count() - 1; + if (OB_FAIL(tmp.do_mask(INHERIT_MASK_BEGIN, mask_end))) { LOG_WARN("failed to do mask", K(ret)); } else if (OB_FAIL(info_.add_members(tmp))) { LOG_WARN("failed to add expr info", K(ret)); @@ -910,7 +910,7 @@ int ObConstRawExpr::inner_deep_copy(ObIRawExprCopier &copier) } else if (OB_FAIL(deep_copy_obj(*inner_alloc_, value_, tmp))) { LOG_WARN("deep copy error", K(value_), K(ret)); } else if (OB_FAIL(ob_write_string(*inner_alloc_, literal_prefix_, literal_prefix_))) { - LOG_WARN("failed to werite string", K(ret)); + LOG_WARN("failed to write string", K(ret)); } else { value_ = tmp; } @@ -3934,7 +3934,7 @@ int ObSysFunRawExpr::get_name_internal(char *buf, const int64_t buf_len, int64_t if (get_param_count() >= 1) { if (OB_ISNULL(get_param_expr(i))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("poram expr is NULL", K(i), K(ret)); + LOG_WARN("param expr is NULL", K(i), K(ret)); } else if (OB_FAIL(get_param_expr(i)->get_name(buf, buf_len, pos, type))) { LOG_WARN("fail to get_name", K(ret)); } else {} @@ -4193,7 +4193,7 @@ int ObNormalDllUdfRawExpr::set_udf_meta(const share::schema::ObUDF &udf) udf_meta_.ret_ = udf.get_ret(); udf_meta_.type_ = udf.get_type(); - /* data from schame, deep copy maybe a better choices */ + /* data from schema, deep copy maybe a better choices */ if (OB_ISNULL(inner_alloc_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("inner allocator or expr factory is NULL", K(inner_alloc_), K(ret)); @@ -4429,7 +4429,7 @@ int ObObjectConstructRawExpr::assign(const ObRawExpr &other) if (OB_FAIL(elem_types_.assign(tmp.elem_types_))) { LOG_WARN("failed to assign elem types", K(ret)); } else if (OB_FAIL(access_names_.assign(tmp.access_names_))) { - LOG_WARN("failed to assgin access names", K(ret)); + LOG_WARN("failed to assign access names", K(ret)); } } } @@ -5047,10 +5047,10 @@ int Bound::replace_expr(const common::ObIArray &other_exprs, int ret = OB_SUCCESS; if (OB_FAIL(ObTransformUtils::replace_expr( other_exprs, new_exprs, interval_expr_))) { - LOG_WARN("failed to repalce exprs", K(ret)); + LOG_WARN("failed to replace exprs", K(ret)); } else if (OB_FAIL(ObTransformUtils::replace_expr( other_exprs, new_exprs, date_unit_expr_))) { - LOG_WARN("failed to repalce exprs", K(ret)); + LOG_WARN("failed to replace exprs", K(ret)); } for (int i = 0; OB_SUCC(ret) && i < BOUND_EXPR_MAX; ++i) { if (OB_ISNULL(exprs_[i])) { @@ -5839,7 +5839,7 @@ bool ObExprParamCheckContext::compare_const(const ObConstRawExpr &left, const Ob const ObRawExpr *left_param = NULL; const ObRawExpr *right_param = NULL; if (OB_FAIL(get_calc_expr(left.get_value().get_unknown(), left_param))) { - LOG_WARN("faield to get calculable expr", K(ret)); + LOG_WARN("failed to get calculable expr", K(ret)); } else if (OB_FAIL(get_calc_expr(right.get_value().get_unknown(), right_param))) { LOG_WARN("failed to get calculable expr", K(ret)); } else if (OB_ISNULL(left_param) || OB_ISNULL(right_param)) { diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index b30dcd06dc..a5d10e5129 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -159,7 +159,7 @@ extern ObRawExpr *USELESS_POINTER; #define IS_SPATIAL_EXPR(op) \ ((op) >= T_FUN_SYS_ST_LONGITUDE && (op) <= T_FUN_SYS_ST_LATITUDE) -// ObSqlBitSet is a simple bitset, in order to avoid memory explosure +// ObSqlBitSet is a simple bitset, in order to avoid memory exposure // ObBitSet is too large just for a simple bitset const static int64_t DEFAULT_SQL_BITSET_SIZE = 32; templatecreate_raw_expr(expr_type, raw_expr))) { SQL_RESV_LOG(WARN, "failed to create raw expr by pl factory", K(ret)); } else { diff --git a/src/sql/resolver/expr/ob_raw_expr_canonicalizer_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_canonicalizer_impl.cpp index c0ed06b83a..3cc6339d43 100644 --- a/src/sql/resolver/expr/ob_raw_expr_canonicalizer_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_canonicalizer_impl.cpp @@ -453,7 +453,7 @@ int ObRawExprCanonicalizerImpl::push_not(ObRawExpr *&expr) } for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { if (OB_FAIL(SMART_CALL(push_not(expr->get_param_expr(i))))) { - LOG_WARN("fialed to push not expr", K(ret)); + LOG_WARN("failed to push not expr", K(ret)); } } } diff --git a/src/sql/resolver/expr/ob_raw_expr_copier.h b/src/sql/resolver/expr/ob_raw_expr_copier.h index 2ca648b6f3..fdc5c6674f 100644 --- a/src/sql/resolver/expr/ob_raw_expr_copier.h +++ b/src/sql/resolver/expr/ob_raw_expr_copier.h @@ -197,7 +197,7 @@ int ObRawExprCopier::copy_on_replace(const common::ObIArray &from_exprs, } } if (OB_SUCC(ret) && OB_FAIL(to_exprs.assign(tmp_arr))) { - SQL_RESV_LOG(WARN, "failed to assgin replaced results", K(ret)); + SQL_RESV_LOG(WARN, "failed to assign replaced results", K(ret)); } return ret; } diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp index d67fd2ba95..0745c39341 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp @@ -240,7 +240,7 @@ int ObRawExprDeduceType::calc_result_type_with_const_arg( /* Most expressions not accept lob type parameters. It reports an error in two situations before: * 1. report an error in calc_result_type function of expression; - * 2. cast lob to calc_type not suppoted/expected. + * 2. cast lob to calc_type not supported/expected. * Only a few expressions deal with lob type parameters in calc_result_type, so most errors caused by 2. * However, this makes some problems: * For example, cast lob to number is not supported before, and this results that nvl(lob, number) @@ -377,7 +377,7 @@ int ObRawExprDeduceType::calc_result_type(ObNonTerminalRawExpr &expr, // 以防calc_result_typeX没有对其进行设置 // 理想情况下，不应该要这个循环，所有calc_type的设置都在calc_result_typeX中完成 - // For avg(), internally it will call 'divison', which requires that both input are + // For avg(), internally it will call 'division', which requires that both input are // casted into number. However, this requirements are not remembered in the input_types // for the avg() expression but as the calc_type for the input expression itself. This // demands that we set the calculation type here. @@ -2361,7 +2361,11 @@ int ObRawExprDeduceType::visit(ObWinFunRawExpr &expr) // @TODO : nijia.nj, 细分各种window_funciton if (T_WIN_FUN_CUME_DIST == expr.get_func_type() || T_WIN_FUN_PERCENT_RANK == expr.get_func_type()) { - if (is_oracle_mode()) { + const uint64_t ob_version = GET_MIN_CLUSTER_VERSION(); + if (is_oracle_mode() || + !((ob_version >= CLUSTER_VERSION_2277 && ob_version < CLUSTER_VERSION_3000) + || (ob_version >= CLUSTER_VERSION_312 && ob_version < CLUSTER_VERSION_3200) + || ob_version >= CLUSTER_VERSION_3_2_3_0)) { result_type.set_accuracy(ObAccuracy::MAX_ACCURACY2[ORACLE_MODE][ObNumberType]); result_type.set_calc_accuracy(ObAccuracy::MAX_ACCURACY2[ORACLE_MODE][ObNumberType]); result_type.set_number(); @@ -2831,7 +2835,7 @@ int ObRawExprDeduceType::set_agg_udf_result_type(ObAggFunRawExpr &expr) int ret = OB_SUCCESS; ObIArray ¶m_exprs = expr.get_real_param_exprs_for_update(); common::ObSEArray udf_attributes; /* udf's input args' name */ - common::ObSEArray udf_attributes_types; /* udf's aatribute type */ + common::ObSEArray udf_attributes_types; /* udf's attribute type */ common::ObSEArray const_results; /* const input expr' result */ ObAggUdfFunction udf_func; const share::schema::ObUDFMeta &udf_meta = expr.get_udf_meta(); @@ -3295,7 +3299,7 @@ int ObRawExprDeduceType::add_implicit_cast(ObAggFunRawExpr &parent, ObRawExpr *&child_ptr = real_param_exprs.at(i); if (skip_cast_expr(parent, i)) { // do nothing - //兼容oralce行为,regr_sxx和regr_syy只需在计算的参数加cast,regr_sxy行为和regr_syy一致，比较诡异，暂时兼容 + //兼容oracle行为,regr_sxx和regr_syy只需在计算的参数加cast,regr_sxy行为和regr_syy一致，比较诡异，暂时兼容 } else if ((parent.get_expr_type() == T_FUN_REGR_SXX && i == 0) || (parent.get_expr_type() == T_FUN_REGR_SYY && i == 1) || (parent.get_expr_type() == T_FUN_REGR_SXY && i == 1) || diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.h b/src/sql/resolver/expr/ob_raw_expr_deduce_type.h index a390008344..b6fbb16e0c 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.h +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.h @@ -110,7 +110,7 @@ private: int add_implicit_cast_for_op_row(ObRawExpr *&child_ptr, const common::ObIArray &input_types, const ObCastMode &cast_mode); - // try add cast expr on subquery stmt's oubput && update column types. + // try add cast expr on subquery stmt's output && update column types. int add_implicit_cast_for_subquery(ObQueryRefRawExpr &expr, const common::ObIArray &input_types, const ObCastMode &cast_mode); diff --git a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp index 6ff94d4400..ec8e4948d2 100644 --- a/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_info_extractor.cpp @@ -515,7 +515,7 @@ int ObRawExprInfoExtractor::visit(ObSysFunRawExpr &expr) } else if (T_FUN_NORMAL_UDF == expr.get_expr_type() || T_FUN_AGG_UDF == expr.get_expr_type()) { /* - * it seems we have no chioce but to set the udf uncalculable. + * it seems we have no choice but to set the udf uncalculable. * we can not say a udf expr is const or not util we finish the xxx_init() function. * but we do the xxx_init() at the expr deduce type stage which was done after we * extractor info from expr. @@ -576,19 +576,19 @@ int ObRawExprInfoExtractor::visit(ObSysFunRawExpr &expr) || T_FUN_SYS_XML_ELEMENT == expr.get_expr_type() || T_FUN_SYS_XMLPARSE == expr.get_expr_type() || IS_LABEL_SE_POLICY_FUNC(expr.get_expr_type())) - && expr.has_flag(IS_CONST_EXPR)) { - expr.clear_flag(IS_CONST_EXPR); + && OB_FAIL(expr.clear_flag(IS_CONST_EXPR))) { + LOG_WARN("failed to clear flag", K(ret)); } if (OB_SUCC(ret) && T_FUN_SYS_JSON_VALUE == expr.get_expr_type()) { if (expr.get_param_count() >= 12) { ObRawExpr * sub_expr = expr.get_param_expr(7); - if (OB_NOT_NULL(sub_expr)) { - sub_expr->clear_flag(IS_CONST_EXPR); - } - sub_expr = expr.get_param_expr(4); - if (OB_NOT_NULL(sub_expr)) { - sub_expr->clear_flag(IS_CONST_EXPR); + if (OB_NOT_NULL(sub_expr) + && OB_FAIL(sub_expr->clear_flag(IS_CONST_EXPR))) { + LOG_WARN("failed to clear flag", K(ret)); + } else if (OB_NOT_NULL(sub_expr = expr.get_param_expr(4)) + && OB_FAIL(sub_expr->clear_flag(IS_CONST_EXPR))) { + LOG_WARN("failed to clear flag", K(ret)); } } } diff --git a/src/sql/resolver/expr/ob_raw_expr_modify_column_name.h b/src/sql/resolver/expr/ob_raw_expr_modify_column_name.h index d32c08d62c..f0e4fa83f9 100644 --- a/src/sql/resolver/expr/ob_raw_expr_modify_column_name.h +++ b/src/sql/resolver/expr/ob_raw_expr_modify_column_name.h @@ -37,7 +37,7 @@ public: int modifyColumnName(ObRawExpr &expr); - // interface of ObRawExprVistor + // interface of ObRawExprVisitor virtual int visit(ObConstRawExpr &expr); virtual int visit(ObExecParamRawExpr &expr); virtual int visit(ObVarRawExpr &expr); diff --git a/src/sql/resolver/expr/ob_raw_expr_printer.cpp b/src/sql/resolver/expr/ob_raw_expr_printer.cpp index 41a40a2fc5..124b6dca5f 100644 --- a/src/sql/resolver/expr/ob_raw_expr_printer.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_printer.cpp @@ -279,7 +279,7 @@ int ObRawExprPrinter::print(ObConstRawExpr *expr) } } else if (expr->get_expr_type() == T_DATE && OB_FAIL(databuff_printf(buf_, buf_len_, *pos_, "date "))) { - LOG_WARN("fail to print date strign", K(ret)); + LOG_WARN("fail to print date string", K(ret)); } else if (T_BOOL == expr->get_expr_type()) { /** * For SQL like "select * from T1 where C1 = 1 and C1 = 2", @@ -562,10 +562,10 @@ int ObRawExprPrinter::print(ObOpRawExpr *expr) case T_OP_NE: case T_OP_SQ_NE: SET_SYMBOL_IF_EMPTY("<>"); - case T_OP_IN: // in sub-query wille be rewrited as expr = ANY(sub-query) + case T_OP_IN: // in sub-query will be rewrited as expr = ANY(sub-query) SET_SYMBOL_IF_EMPTY("in"); case T_OP_NOT_IN: - SET_SYMBOL_IF_EMPTY("not in"); // not in sub-query wille be rewrited as expr != all(sub-query) + SET_SYMBOL_IF_EMPTY("not in"); // not in sub-query will be rewrited as expr != all(sub-query) case T_OP_BIT_OR: SET_SYMBOL_IF_EMPTY("|"); case T_OP_BIT_XOR: @@ -1080,6 +1080,10 @@ int ObRawExprPrinter::print(ObAggFunRawExpr *expr) SET_SYMBOL_IF_EMPTY("json_arrayagg"); case T_FUN_JSON_OBJECTAGG: SET_SYMBOL_IF_EMPTY("json_objectagg"); + case T_FUN_APPROX_COUNT_DISTINCT_SYNOPSIS: + SET_SYMBOL_IF_EMPTY("approx_count_distinct_synopsis"); + case T_FUN_APPROX_COUNT_DISTINCT_SYNOPSIS_MERGE: + SET_SYMBOL_IF_EMPTY("approx_count_distinct_synopsis_merge"); case T_FUN_PL_AGG_UDF:{ if (type == T_FUN_PL_AGG_UDF) { if (OB_ISNULL(expr->get_pl_agg_udf_expr()) || @@ -2853,7 +2857,7 @@ int ObRawExprPrinter::print(ObSysFunRawExpr *expr) PRINT_IDENT_WITH_QUOT(seq_expr->get_action()); } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("sequence should sepcify format as seqname.action", K(ret)); + LOG_WARN("sequence should specify format as seqname.action", K(ret)); } if (OB_SUCC(ret) && seq_expr->is_dblink_sys_func()) { DATA_PRINTF("@%.*s", LEN_AND_PTR(seq_expr->get_dblink_name())); @@ -3252,7 +3256,7 @@ int ObRawExprPrinter::print(ObUDFRawExpr *expr) if (params_type.at(i).is_null()) { // default parameter, do not print // do nothing ... } else if (0 == i && expr->get_is_udt_cons()) { - // do not print construnct null self argument + // do not print construct null self argument } else { if (!params_name.at(i).empty()) { PRINT_IDENT_WITH_QUOT(params_name.at(i)); @@ -3773,7 +3777,7 @@ int ObRawExprPrinter::print_partition_exprs(ObWinFunRawExpr *expr) for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { if (OB_ISNULL(expr->get_partition_exprs().at(i))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get partiton by exprs.", K(ret)); + LOG_WARN("failed to get partition by exprs.", K(ret)); } else { PRINT_EXPR(expr->get_partition_exprs().at(i)); if (i < N - 1) { diff --git a/src/sql/resolver/expr/ob_raw_expr_printer.h b/src/sql/resolver/expr/ob_raw_expr_printer.h index fee4e25d07..0be5fb2594 100644 --- a/src/sql/resolver/expr/ob_raw_expr_printer.h +++ b/src/sql/resolver/expr/ob_raw_expr_printer.h @@ -167,7 +167,7 @@ private: char *buf_; int64_t buf_len_; // avoid to update pos_ between different printers(mainly ObRawExprPrinter - // and ObSelectStmtPrinter), we definate pointer of pos_ rather than object + // and ObSelectStmtPrinter), we definite pointer of pos_ rather than object int64_t *pos_; ObStmtScope scope_; bool only_column_namespace_; diff --git a/src/sql/resolver/expr/ob_raw_expr_replacer.cpp b/src/sql/resolver/expr/ob_raw_expr_replacer.cpp index 74aad967f4..ad3de654e1 100644 --- a/src/sql/resolver/expr/ob_raw_expr_replacer.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_replacer.cpp @@ -51,7 +51,7 @@ int ObRawExprReplacer::replace(ObRawExpr *&expr) expr = new_expr; replace_happened_ = true; } else if (OB_FAIL(expr->preorder_accept(*this))) { - LOG_WARN("failed to preorder accecpt expr", K(ret)); + LOG_WARN("failed to preorder accept expr", K(ret)); } return ret; } diff --git a/src/sql/resolver/expr/ob_raw_expr_replacer.h b/src/sql/resolver/expr/ob_raw_expr_replacer.h index b3c8d6e9c9..1a1fd07e27 100644 --- a/src/sql/resolver/expr/ob_raw_expr_replacer.h +++ b/src/sql/resolver/expr/ob_raw_expr_replacer.h @@ -26,8 +26,8 @@ namespace sql * it is replaced when visit ObQueryRefRawExpr; * 3. the replaced expr and its children will be skipped when traverse the expr tree, for example * with replace rule [{c1 -> c1+c2+1}, {c2 -> c1+c2+2}] - * select c1, c2 from t1 will be transfromed into select c1+c2+1, c1+c2+2 from t1 - * In constrast, ObTransformUtils::replace_expr will return OB_SIZE_OVERFLOW unexpectedly; + * select c1, c2 from t1 will be transformed into select c1+c2+1, c1+c2+2 from t1 + * In contrast, ObTransformUtils::replace_expr will return OB_SIZE_OVERFLOW unexpectedly; */ class ObRawExprReplacer: public ObRawExprVisitor { @@ -87,7 +87,7 @@ private: hash::ObHashMap expr_replace_map_; bool replace_happened_; - //If true, skip param epxrs which are resolved as flags instead of exprs in mysql. + //If true, skip param exprs which are resolved as flags instead of exprs in mysql. //(e.g. the second param of IS expr) bool skip_bool_param_mysql_; }; diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index edc7ed8c71..6ba90a3b55 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -250,7 +250,7 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, ObRawExpr LOG_DEBUG("resolve item", "item_type", get_type_name(node->type_)); if (IS_DATATYPE_OR_QUESTIONMARK_OP(node->type_)) { if (OB_FAIL(process_datatype_or_questionmark(*node, expr))) { - LOG_WARN("fail to process datetype or questionmark", K(ret), K(node)); + LOG_WARN("fail to process datatype or questionmark", K(ret), K(node)); } } else { switch (node->type_) { @@ -599,7 +599,7 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, ObRawExpr case T_OP_REGEXP: case T_OP_NOT_REGEXP: { if (OB_FAIL(process_regexp_or_not_regexp_node(node, expr))) { - LOG_WARN("fail to process prgexp_or_not_pegexp node", K(ret), K(node)); + LOG_WARN("fail to process prgexp_or_not_regexp node", K(ret), K(node)); } break; } @@ -882,6 +882,16 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, ObRawExpr } break; } + case T_FUN_SYS_CURRENT_USER: { + ObSysFunRawExpr *f_expr = NULL; + if (OB_FAIL(ctx_.expr_factory_.create_raw_expr(T_FUN_SYS_CURRENT_USER, f_expr))) { + LOG_WARN("fail to create raw expr", K(ret)); + } else { + f_expr->set_func_name(ObString::make_string(N_CURRENT_USER)); + expr = f_expr; + } + break; + } case T_COLLATION: { // used in internal function `set_collation' to implement COLLATE clause if (OB_FAIL(process_collation_node(node, expr))) { @@ -1012,7 +1022,7 @@ int ObRawExprResolverImpl::do_recursive_resolve(const ParseNode *node, ObRawExpr } case T_FUN_UDF: { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("A BUG, Nerver Be Here!!!", K(ret)); + LOG_ERROR("A BUG, Never Be Here!!!", K(ret)); break; } case T_WINDOW_FUNCTION: { @@ -1546,7 +1556,7 @@ int ObRawExprResolverImpl::process_cursor_attr_node(const ParseNode &node, ObRaw } default: { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invlid node type", K(node.children_[0]->type_)); + LOG_WARN("invalid node type", K(node.children_[0]->type_)); } } } @@ -1560,7 +1570,7 @@ int ObRawExprResolverImpl::process_cursor_attr_node(const ParseNode &node, ObRaw } if (OB_SUCC(ret)) { if (T_FUN_PL_GET_CURSOR_ATTR == c_expr->get_expr_type()) { - // 注意: 替换为ColumnRef的目的是为了在DML语句中把CURSOR%ROWID替换为QuenstionMark + // 注意: 替换为ColumnRef的目的是为了在DML语句中把CURSOR%ROWID替换为QuestionMark // 参见: ObDMLResolver::resolve_qualified_identifier ObQualifiedName column_ref; ObObjAccessIdent access_ident; @@ -1611,7 +1621,7 @@ int ObRawExprResolverImpl::process_obj_access_node(const ParseNode &node, ObRawE ObQualifiedName column_ref; int64_t child_start = ctx_.columns_->count(); if (OB_FAIL(resolve_obj_access_idents(node, column_ref))) { - LOG_WARN("resolve obj acess idents failed", K(ret)); + LOG_WARN("resolve obj access idents failed", K(ret)); } else { column_ref.format_qualified_name(ctx_.case_mode_); column_ref.parents_expr_info_ = ctx_.parents_expr_info_; @@ -1733,7 +1743,7 @@ int ObRawExprResolverImpl::process_sys_connect_by_path_node(const ParseNode *nod if (OB_ISNULL(node)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpeced parameter", K(node), K(ret)); + LOG_WARN("unexpected parameter", K(node), K(ret)); } else if (OB_UNLIKELY(node->type_ != T_FUN_SYS_CONNECT_BY_PATH) || OB_UNLIKELY(node->num_child_ != 2) || OB_ISNULL(node->children_) @@ -2137,9 +2147,10 @@ int ObRawExprResolverImpl::resolve_func_node_of_obj_access_idents(const ParseNod std::pair param(index_expr, 0); if (OB_FAIL(access_ident.params_.push_back(param))) { LOG_WARN("push back error", K(ret)); + } else if (OB_FAIL(ctx_.parents_expr_info_.del_member(IS_PL_ACCESS_IDX))) { + LOG_WARN("failed to del member", K(ret)); } } - ctx_.parents_expr_info_.del_member(IS_PL_ACCESS_IDX); for (int64_t i = start_child; i < ctx_.columns_->count(); ++i) { ctx_.columns_->at(i).is_access_root_ = false; } @@ -2373,7 +2384,7 @@ int ObRawExprResolverImpl::process_connect_by_root_node(const ParseNode &node, O || OB_ISNULL(node.children_) || OB_UNLIKELY(node.type_ != T_OP_CONNECT_BY_ROOT)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid parametor", K(node.num_child_), K(node.type_), K(ret)); + LOG_WARN("invalid parameter", K(node.num_child_), K(node.type_), K(ret)); } else if (OB_ISNULL(ctx_.stmt_) || OB_UNLIKELY(false == ctx_.stmt_->is_select_stmt())) { ret = OB_ERR_CBY_CONNECT_BY_REQUIRED; @@ -2539,7 +2550,7 @@ int ObRawExprResolverImpl::process_datatype_or_questionmark(const ParseNode &nod } else if (ctx_.is_for_dbms_sql_ || (ctx_.is_for_dynamic_sql_ && OB_NOT_NULL(session_info->get_pl_context()))){ //NOTICE: only need to process PL dynamic sql and dbms sql - /*dynmaic and dbms sql already prepare question mark in parse stage.*/ + /*dynamic and dbms sql already prepare question mark in parse stage.*/ bool need_save = true; for (int64_t i = 0; OB_SUCC(ret) && i < ctx_.external_param_info_->count(); ++i) { CK (OB_NOT_NULL(ctx_.external_param_info_->at(i).first)); @@ -3014,7 +3025,7 @@ int ObRawExprResolverImpl::process_any_or_all_node(const ParseNode *node, ObRawE LOG_WARN("resolve sub-query failed", K(ret)); } else if (OB_ISNULL(sub_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("inavalid sub_expr", K(sub_expr)); + LOG_WARN("invalid sub_expr", K(sub_expr)); } else if (T_REF_QUERY == sub_expr->get_expr_type()) { ObQueryRefRawExpr *sub_ref = static_cast(sub_expr); sub_ref->set_is_set(true); @@ -3375,7 +3386,7 @@ int ObRawExprResolverImpl::process_regexp_or_not_regexp_node(const ParseNode *no ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(node)); } else if (OB_FAIL(process_node_with_children(node, num_child, t_expr))) { - LOG_WARN("fail to preocess node with children", K(ret)); + LOG_WARN("fail to process node with children", K(ret)); } else if (OB_ISNULL(t_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("process node fail with invalid expr", K(ret), K(t_expr)); @@ -3676,7 +3687,7 @@ int ObRawExprResolverImpl::process_like_node(const ParseNode *node, ObRawExpr *& IS_NO_BACKSLASH_ESCAPES(ctx_.session_info_->get_sql_mode(), no_escapes); } if (OB_FAIL(process_datatype_or_questionmark(escape_node, escape_expr))) { - LOG_WARN("fail to resolver defalut excape node", K(ret)); + LOG_WARN("fail to resolver default excape node", K(ret)); } else if (OB_FAIL(t_expr->add_param_expr(escape_expr))) { LOG_WARN("fail to set param expr"); } else if (lib::is_oracle_mode() || no_escapes) { @@ -3843,7 +3854,7 @@ int ObRawExprResolverImpl::process_case_node(const ParseNode *node, ObRawExpr *& if (OB_ISNULL(node) || OB_UNLIKELY(3 != node->num_child_) || OB_UNLIKELY(T_CASE != node->type_) || OB_ISNULL(ctx_.session_info_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid PraseNode or session_info_", K(node), KP(ctx_.session_info_)); + LOG_WARN("invalid ParseNode or session_info_", K(node), KP(ctx_.session_info_)); } else if (OB_FAIL(ctx_.expr_factory_.create_raw_expr(T_INVALID, case_expr))) { LOG_WARN("fail to create raw expr", K(ret)); } else if (OB_ISNULL(case_expr)) { @@ -4147,7 +4158,7 @@ int ObRawExprResolverImpl::process_agg_node(const ParseNode *node, ObRawExpr *&e order_item.order_type_ = NULLS_FIRST_ASC; if (OB_FAIL(agg_expr->add_order_item(order_item))) { LOG_WARN("fail to add median order item", K(ret)); - } else {/* do nothong */} + } else {/* do nothing */} } } } else if (T_FUN_GROUPING_ID == node->type_) { @@ -4334,7 +4345,7 @@ int ObRawExprResolverImpl::process_group_aggr_node(const ParseNode *node, ObRawE LOG_WARN("store aggr expr failed", K(ret)); } else if (OB_ISNULL(node->children_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("inalid group concat node", K(ret), K(node->children_)); + LOG_WARN("invalid group concat node", K(ret), K(node->children_)); } else { bool need_add_flag = !ctx_.parents_expr_info_.has_member(IS_AGG); ParseNode *expr_list_node = node->children_[1]; @@ -4975,7 +4986,7 @@ int ObRawExprResolverImpl::process_isnull_node(const ParseNode *node, ObRawExpr } else { ObRawExpr *obj_expr = NULL; if (OB_FAIL(SMART_CALL(recursive_resolve(node->children_[1]->children_[0], obj_expr)))) { - LOG_WARN("resolve child faield", K(ret)); + LOG_WARN("resolve child failed", K(ret)); } else if (OB_FAIL(ObRawExprUtils::build_is_not_null_expr(ctx_.expr_factory_, obj_expr, false/*is_not_null*/, @@ -5051,7 +5062,7 @@ int ObRawExprResolverImpl::process_sqlerrm_node(const ParseNode *node, ObRawExpr if (OB_ISNULL(node->children_[1]) || node->children_[1]->num_child_ != 1) { ret = OB_INVALID_ARGUMENT_NUM; - LOG_WARN("invlaid argument number for SQLERRM", K(node->children_[1]->num_child_)); + LOG_WARN("invalid argument number for SQLERRM", K(node->children_[1]->num_child_)); } if (OB_SUCC(ret)) { if (OB_ISNULL(node->children_[1]->children_[0])) { @@ -5933,7 +5944,7 @@ int ObRawExprResolverImpl::process_is_json_node(const ParseNode *node, ObRawExpr OZ(SMART_CALL(recursive_resolve(node->children_[i], para_expr))); CK(OB_NOT_NULL(para_expr)); if (OB_SUCC(ret)) { - para_expr->clear_flag(IS_CONST_EXPR); + OZ(para_expr->clear_flag(IS_CONST_EXPR)); OZ(func_expr->add_param_expr(para_expr)); } } //end for @@ -7441,7 +7452,7 @@ int ObRawExprResolverImpl::process_dll_udf_node(const ParseNode *node, ObRawExpr LOG_WARN("the udf info is null", K(ret)); } else if (!(udf_info->is_normal_udf() || udf_info->is_agg_udf())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("the udf schame is error", K(ret), K(udf_info->get_type())); + LOG_WARN("the udf schema is error", K(ret), K(udf_info->get_type())); } else if (udf_info->is_normal_udf()) { ObSysFunRawExpr *func_expr = NULL; ObCharset::casedn(CS_TYPE_UTF8MB4_GENERAL_CI, udf_name); @@ -7823,4 +7834,4 @@ int ObRawExprResolverImpl::process_odbc_time_literals(const ObItemType dst_time_ } //namespace sql -} //namespace oceabase +} //namespace oceanbase diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 5cce7dbf69..b92424802a 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -895,7 +895,7 @@ int ObRawExprUtils::resolve_udf_param_exprs(ObResolverParams ¶ms, // 通过名字指定参数统一记录在param_names_和param_exprs里面, 所以这里一定相等 if (udf_info.param_names_.count() != udf_info.param_exprs_.count()) { ret = OB_ERR_UNEXPECTED; - SQL_LOG(WARN, "names array not equle to exprs array count", + SQL_LOG(WARN, "names array not equal to exprs array count", K(ret), K(udf_info.param_names_.count()), K(udf_info.param_exprs_.count())); } else if ((udf_info.udf_param_num_ + udf_info.param_names_.count()) > func_info->get_param_count()) { ret = OB_ERR_SP_WRONG_ARG_NUM; @@ -1701,7 +1701,7 @@ int ObRawExprUtils::build_seq_nextval_expr(ObRawExpr *&expr, } else if (OB_FAIL(func_expr->add_flag(IS_SEQ_EXPR))) { LOG_WARN("failed to add flag", K(ret)); } else if (OB_FAIL(func_expr->add_param_expr(col_id_expr))) { - LOG_WARN("set funcation param expr failed", K(ret)); + LOG_WARN("set function param expr failed", K(ret)); } else if (OB_FAIL(func_expr->formalize(session_info))) { LOG_WARN("failed to extract info", K(ret)); } else if (NULL != stmt && OB_FAIL(stmt->get_pseudo_column_like_exprs().push_back(func_expr))) { @@ -7123,7 +7123,7 @@ int ObRawExprUtils::build_dup_data_expr(ObRawExprFactory &factory, if (OB_SUCC(ret)) { if (OB_ISNULL(name = (char *)factory.get_allocator().alloc(pos + 2))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("falied to allocate memory", K(ret)); + LOG_WARN("failed to allocate memory", K(ret)); } else { memcpy(name, name_buf, pos); name[pos] = ')'; diff --git a/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp b/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp index 650811d21e..2b91b10603 100644 --- a/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp @@ -32,7 +32,7 @@ int ObRawExprWrapEnumSet::wrap_enum_set(ObDMLStmt &stmt) int ret = OB_SUCCESS; cur_stmt_ = &stmt; if (stmt.is_select_stmt()) { - //handle the targest list of first level + //handle the target list of first level ObSelectStmt &select_stmt = static_cast(stmt); if (OB_FAIL(wrap_target_list(select_stmt))) { LOG_WARN("failed to wrap target list", K(ret)); diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 273a5a22e8..b4fe86dcaa 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -8101,7 +8101,12 @@ int ObResolverUtils::handle_varchar_charset(ObCharsetType charset_type, if ((T_HEX_STRING == node->type_ || T_VARCHAR == node->type_) && CHARSET_INVALID != charset_type) { ParseNode *charset_node = new_node(&allocator, T_CHARSET, 0); - ParseNode *varchar_node = new_non_terminal_node(&allocator, T_VARCHAR, 2, charset_node, node); + ParseNode *varchar_node = NULL; + if (T_HEX_STRING == node->type_) { + varchar_node = new_non_terminal_node(&allocator, T_VARCHAR, 1, charset_node); + } else if (T_VARCHAR == node->type_) { + varchar_node = new_non_terminal_node(&allocator, T_VARCHAR, 2, charset_node, node); + } if (OB_ISNULL(charset_node) || OB_ISNULL(varchar_node)) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/sql/resolver/ob_stmt.h b/src/sql/resolver/ob_stmt.h index f091be41a9..5aae913923 100644 --- a/src/sql/resolver/ob_stmt.h +++ b/src/sql/resolver/ob_stmt.h @@ -623,6 +623,8 @@ public: stmt = NULL; } else if (OB_FAIL(stmt->init_stmt(table_hash_allocator_, wrapper_allocator_))) { SQL_RESV_LOG(WARN, "failed to init tables hash", K(ret)); + stmt->~StmtType(); + stmt = NULL; } } return ret; diff --git a/src/sql/resolver/prepare/ob_deallocate_resolver.cpp b/src/sql/resolver/prepare/ob_deallocate_resolver.cpp index 2c1dd21026..d084a161a4 100644 --- a/src/sql/resolver/prepare/ob_deallocate_resolver.cpp +++ b/src/sql/resolver/prepare/ob_deallocate_resolver.cpp @@ -40,7 +40,7 @@ int ObDeallocateResolver::resolve(const ParseNode &parse_tree) ObString name(parse_tree.children_[0]->str_len_, parse_tree.children_[0]->str_value_); ObPsStmtId ps_id = OB_INVALID_ID; if (OB_FAIL(ob_simple_low_to_up(*params_.allocator_, name, stmt_name))) { - LOG_WARN("failed to write stirng", K(ret)); + LOG_WARN("failed to write string", K(ret)); } else if(OB_FAIL(session_info_->get_prepare_id(stmt_name, ps_id))) { LOG_WARN("failed to get prepare id", K(ret)); } else { diff --git a/src/sql/resolver/prepare/ob_execute_resolver.cpp b/src/sql/resolver/prepare/ob_execute_resolver.cpp index 1fe915e241..e796029c98 100644 --- a/src/sql/resolver/prepare/ob_execute_resolver.cpp +++ b/src/sql/resolver/prepare/ob_execute_resolver.cpp @@ -49,7 +49,7 @@ int ObExecuteResolver::resolve(const ParseNode &parse_tree) ObPsStmtId ps_id = OB_INVALID_ID; stmt::StmtType ps_type = stmt::T_NONE; if (OB_FAIL(ob_simple_low_to_up(*params_.allocator_, name, stmt_name))) { - LOG_WARN("failed to write stirng", K(ret)); + LOG_WARN("failed to write string", K(ret)); } else if(OB_FAIL(session_info_->get_prepare_id(stmt_name, ps_id))) { LOG_WARN("failed to get prepare id", K(ret)); } else if (OB_FAIL(session_info_->get_ps_session_info(ps_id, ps_session_info))) { @@ -76,7 +76,7 @@ int ObExecuteResolver::resolve(const ParseNode &parse_tree) for(int32_t i = 0; OB_SUCC(ret) && i < arguments->num_child_; ++i) { if (OB_ISNULL(arguments->children_[i])) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argumenet", K(arguments->children_[i]), K(ret)); + LOG_WARN("invalid argument", K(arguments->children_[i]), K(ret)); } else { ObRawExpr *param_expr = NULL; if (OB_FAIL(ObResolverUtils::resolve_const_expr(params_, *arguments->children_[i], param_expr, NULL))) { diff --git a/src/sql/rewrite/ob_predicate_deduce.h b/src/sql/rewrite/ob_predicate_deduce.h index 42e269fc29..5378ab37ba 100644 --- a/src/sql/rewrite/ob_predicate_deduce.h +++ b/src/sql/rewrite/ob_predicate_deduce.h @@ -28,7 +28,11 @@ class ObPredicateDeduce { }; public: - ObPredicateDeduce(ObDMLStmt &stmt) : stmt_(stmt) {} + ObPredicateDeduce(ObDMLStmt &stmt) : stmt_(stmt) { + graph_.set_attr(ObMemAttr(MTL_ID(), "Graph")); + type_safety_.set_attr(ObMemAttr(MTL_ID(), "TypeSafety")); + topo_order_.set_attr(ObMemAttr(MTL_ID(), "TopoOrder")); + } int add_predicate(ObRawExpr *pred, bool &is_added); diff --git a/src/sql/rewrite/ob_query_range.cpp b/src/sql/rewrite/ob_query_range.cpp index 4d05fad9ad..8527c9ec78 100644 --- a/src/sql/rewrite/ob_query_range.cpp +++ b/src/sql/rewrite/ob_query_range.cpp @@ -196,7 +196,7 @@ int ObQueryRange::init_query_range_ctx(ObIAllocator &allocator, LOG_WARN("expr result type is null", K(ret)); } else if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObKeyPartPos)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memeory for ObKeyPartPos", K(ret)); + LOG_WARN("failed to allocate memory for ObKeyPartPos", K(ret)); } else { ObExprResType tmp_expr_type = *expr_res_type; if (tmp_expr_type.is_lob_locator()) { @@ -972,7 +972,7 @@ int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part, bool use_in_opt } else { new_key->id_ = key_part->id_; key_part = new_key; - LOG_TRACE("refine lagre query range with full key", K(cur_range_size)); + LOG_TRACE("refine large query range with full key", K(cur_range_size)); } } else if (OB_ISNULL(first_keypart = pre_key_parts.at(0))) { ret = OB_ERR_UNEXPECTED; @@ -985,7 +985,7 @@ int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part, bool use_in_opt LOG_WARN("remove precise range expr failed", K(ret)); } else { // remove key part after pre key parts - LOG_TRACE("refine lagre query range remove some key parts", + LOG_TRACE("refine large query range remove some key parts", K(cur_range_size), K(redundant_offset)); ObKeyPart *cur = NULL;; ObKeyPart *and_next = NULL; @@ -1800,7 +1800,7 @@ int ObQueryRange::get_column_key_part(const ObRawExpr *l_expr, } else if(OB_FAIL(ObQueryRange::is_precise_like_range(val, escape_ch, query_range_ctx_->cur_expr_is_precise_))) { - LOG_WARN("failed to jugde whether is precise", K(ret)); + LOG_WARN("failed to judge whether is precise", K(ret)); } else if (OB_FAIL(add_precise_constraint(const_expr, query_range_ctx_->cur_expr_is_precise_))) { LOG_WARN("failed to add precise constraint", K(ret)); @@ -2261,7 +2261,7 @@ int ObQueryRange::get_like_const_range(const ObRawExpr *text_expr, // Under this kind of case, in fact we can ignore the second item and do next // 2) not real true, row(k1, k2, k2)>=(1, k1, 4), "k2>=k1" is run-time defined, // we can not know during parser, so true is returned. -// under this case, we can not ignore it. (k1=1 and k2=3) satisfied this condtion, +// under this case, we can not ignore it. (k1=1 and k2=3) satisfied this condition, // but not satisfied row(k1, k2)>=(1, 4) // we can not distinguish them, so do not do next. // 2. if item is always false, no need to do next @@ -2967,7 +2967,7 @@ int ObQueryRange::prepare_multi_in_info(const ObOpRawExpr *l_expr, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(l_expr), K(r_expr)); } else if (OB_UNLIKELY(l_expr->get_param_count() > MAX_EXTRACT_IN_COLUMN_NUMBER)) { - // do nothiing + // do nothing } else if (OB_FAIL(idx_pos_map.create(l_expr->get_param_count(), "IdxKeyMap", "IdxKeyMap"))) { LOG_WARN("fail to init hashmap", K(ret)); } else if (OB_FAIL(idx_param_map.create(l_expr->get_param_count(), "IdxParamMap", "IdxParamMap"))) { @@ -3553,7 +3553,7 @@ int ObQueryRange::set_geo_keypart_whole_range(ObKeyPart &out_key_part) ret = OB_ERR_UNEXPECTED; LOG_WARN("normal keypart is null", K(ret)); } else { - // set whole range when cosnt expr calculate failed in constant fold + // set whole range when const expr calculate failed in constant fold out_key_part.normal_keypart_->start_.set_min_value(); out_key_part.normal_keypart_->end_.set_max_value(); out_key_part.normal_keypart_->always_false_ = false; @@ -4248,7 +4248,7 @@ int ObQueryRange::intersect_border_from(const ObKeyPart *l_key_part, end_border_type = OB_FROM_RIGHT; } else if (cmp < 0) { end_border_type = OB_FROM_LEFT; - } else { // euqal + } else { // equal if (NULL == l_key_part->and_next_) { end_border_type = OB_FROM_LEFT; // lucky left } @@ -4414,7 +4414,7 @@ int ObQueryRange::set_partial_row_border( // // ROW(k1(s1, e1), k2(s2, e2)) and ROW(k1(s3, e4), k2(s3, e4)) // k1(s1, e1): means the first key part in row between s1 and e1. -// because the and operands is row, we can not AND each key part separtely. +// because the and operands is row, we can not AND each key part separately. // so the result is: // ROW(k1(ns1, ne1), k2(ns2, ne2)), if two row values has intersection. // if s1>s3, ns1 = s1 and ns2 = s2 else ns1 = s3 and ns2 = s4; @@ -4519,7 +4519,9 @@ int ObQueryRange::do_row_gt_and(ObKeyPart *l_gt, ObKeyPart *r_gt, ObKeyPart *&r } } - if (OB_SUCC(ret) && !is_reach_mem_limit_) { + if (OB_SUCC(ret) && !is_reach_mem_limit_ && + !result->is_always_false() && + !result->is_always_true()) { result->link_gt(rest); // link to the or_next_ list if (NULL != tail) { @@ -4545,7 +4547,7 @@ int ObQueryRange::do_row_gt_and(ObKeyPart *l_gt, ObKeyPart *r_gt, ObKeyPart *&r // AND single key part (itself) and items in item_next_ list -// Each item in item_next_ list must be unkown item untill physical plan open +// Each item in item_next_ list must be unkown item until physical plan open // // E.g. // (A1 and ?1 and ?2 and ... and ?m) and (A2 and ?I and ?II and ... and ?n) @@ -4897,7 +4899,7 @@ int ObQueryRange::and_single_gt_head_graphs( l_and_next = l_cur_gt; } else { // l_cur_gt and r_cur_gt belongs to same key if (OB_FAIL(do_gt_and(l_cur_gt, r_cur_gt, tmp_result))) { - LOG_WARN("Do AND of gerneral term failed", K(ret)); + LOG_WARN("Do AND of general term failed", K(ret)); } } if (OB_FAIL(ret)) { @@ -5358,7 +5360,8 @@ int ObQueryRange::or_single_head_graphs(ObKeyPartList &or_list, query_range_ctx_->cur_expr_is_precise_ = false; } } - if (OB_SUCC(ret) && !cur1->is_question_mark() && !cur2->is_question_mark()) { + if (OB_SUCC(ret) && !cur1->is_question_mark() && !cur2->is_question_mark() && + cur1->item_next_ == NULL && cur2->item_next_ == NULL) { if (OB_FAIL(union_in_with_in(or_list, cur1, cur2, exec_ctx, @@ -5381,7 +5384,8 @@ int ObQueryRange::or_single_head_graphs(ObKeyPartList &or_list, } } bool need_remove_normal = false; - if (OB_SUCC(ret) && !cur1->is_question_mark() && !cur2->is_question_mark()) { + if (OB_SUCC(ret) && !cur1->is_question_mark() && !cur2->is_question_mark() && + cur1->item_next_ == NULL && cur2->item_next_ == NULL) { if (OB_FAIL(union_in_with_normal(cur1, cur2, exec_ctx, dtc_params, @@ -5407,7 +5411,8 @@ int ObQueryRange::or_single_head_graphs(ObKeyPartList &or_list, query_range_ctx_->cur_expr_is_precise_ = false; } } - if (OB_SUCC(ret) && !cur1->is_question_mark() && !cur2->is_question_mark()) { + if (OB_SUCC(ret) && !cur1->is_question_mark() && !cur2->is_question_mark() && + cur1->item_next_ == NULL && cur2->item_next_ == NULL) { if (OB_FAIL(union_in_with_normal(cur2, cur1, exec_ctx, dtc_params, @@ -6188,7 +6193,7 @@ int ObQueryRange::generate_single_range(ObSearchState &search_state, LOG_ERROR("alloc memory for end_obj failed", K(ret)); } else { int64_t max_pred_index = search_state.max_exist_index_; - column_num = search_state.is_phy_rowid_range_ ? 1 : column_num;//physcial rowid range just use only one column + column_num = search_state.is_phy_rowid_range_ ? 1 : column_num;//physical rowid range just use only one column for (int i = 0; OB_SUCC(ret) && i < column_num; i++) { new(start + i) ObObj(); new(end + i) ObObj(); @@ -6372,7 +6377,7 @@ int ObQueryRange::and_first_search(ObSearchState &search_state, } // 3. to check if need to output - //copy_produec_range的作用是控制range能不能够输出，不是所有递归到最后都能输出 + //copy_produce_range的作用是控制range能不能够输出，不是所有递归到最后都能输出 //例如：a>1 and a<=2 and ((b>1 and b < 2) or (b > 4, and b < 5)) //这个例子不能抽成两段range，只能抽成一段range //因为如果抽成两段range->(1, max;2, 2) or (1, max;2, 5)这两段区间是有重叠的 @@ -7475,7 +7480,7 @@ int ObQueryRange::get_like_range(const ObObj &pattern, return ret; } -// Generaal term: if one or more same key parts with OR realtion have same and_next_, +// General term: if one or more same key parts with OR relation have same and_next_, // we call them general term(GT) // E.g. // A and B, A is GT @@ -8271,7 +8276,7 @@ int ObQueryRange::is_strict_equal_graph( } else { // check align if (-1 == max_pos) { max_pos = next_pos - 1; - } else if (cur_pos != max_pos) { + } else if (next_pos - 1 != max_pos) { is_strict_equal = false; } } @@ -8493,7 +8498,7 @@ bool ObQueryRange::check_like_range_precise(const ObString &pattern_str, if ((end_with_percent || match_without_wildcard) && (-1 == last_equal_idx || pattern_str_buf[last_equal_idx] != ' ')) { if (!is_oracle_mode() && match_without_wildcard) { - // in mysql, all operater will ignore trailing spaces except like. + // in mysql, all operator will ignore trailing spaces except like. // for example, 'abc ' = 'abc' is true, but 'abc ' like 'abc' is false } else { is_precise = true; diff --git a/src/sql/rewrite/ob_query_range.h b/src/sql/rewrite/ob_query_range.h index edc9e4a79f..f5799de10b 100644 --- a/src/sql/rewrite/ob_query_range.h +++ b/src/sql/rewrite/ob_query_range.h @@ -357,10 +357,10 @@ public: void reset(); - // preliminary_extract_query_range will prelininary extract query range + // preliminary_extract_query_range will preliminary extract query range // from query conditions, which is only occurred in generating the physical plan. // During this stage, some consts are not really known, for example, - // prepared params, session variables, global variables, now(), curret_timestamp(), + // prepared params, session variables, global variables, now(), current_timestamp(), // utc_timestamp, etc.. // final extraction may be need in physical plan open. @@ -399,14 +399,14 @@ public: const bool use_in_optimization = false); // final_extract_query_range extracts the final query range of its physical plan. - // It will get the real-time value of some const which are unknow during physical plan generating. + // It will get the real-time value of some const which are unknown during physical plan generating. // Query range can not be used until this function is called. int final_extract_query_range(ObExecContext &exec_ctx, const common::ObDataTypeCastParams &dtc_params); // get_tablet_ranges gets range of a index. - // This function can not be used untill physical plan is opened. + // This function can not be used until physical plan is opened. virtual int get_tablet_ranges(ObQueryRangeArray &ranges, bool &all_single_value_ranges, diff --git a/src/sql/rewrite/ob_transform_aggr_subquery.h b/src/sql/rewrite/ob_transform_aggr_subquery.h index cd239317b3..8fbf151547 100644 --- a/src/sql/rewrite/ob_transform_aggr_subquery.h +++ b/src/sql/rewrite/ob_transform_aggr_subquery.h @@ -75,7 +75,7 @@ private: ObSEArray is_null_prop_; // need case when for a expr ObArray not_null_const_; int64_t pullup_flag_; // the methods uesd when pulling up the ja query - ObRawExpr *not_null_expr_; // not null expr of subquery if corelated join happened + ObRawExpr *not_null_expr_; // not null expr of subquery if correlated join happened ObRawExpr *parent_expr_of_query_ref; // parent expr need to be modified for vector subquery comparison bool limit_for_exists_; int64_t limit_value_; diff --git a/src/sql/rewrite/ob_transform_count_to_exists.cpp b/src/sql/rewrite/ob_transform_count_to_exists.cpp index 58786aca7e..34447dd3b6 100644 --- a/src/sql/rewrite/ob_transform_count_to_exists.cpp +++ b/src/sql/rewrite/ob_transform_count_to_exists.cpp @@ -51,7 +51,7 @@ int ObTransformCountToExists::transform_one_stmt(common::ObIArrayget_condition_exprs()))) { - LOG_WARN("failed to assgin where condition exprs", K(ret)); + LOG_WARN("failed to assign where condition exprs", K(ret)); } else if (OB_FAIL(collect_trans_params(stmt, cond_exprs, trans_params))) { LOG_WARN("failed to transform count to exists", K(ret)); } else if (trans_params.empty()) { diff --git a/src/sql/rewrite/ob_transform_dblink.cpp b/src/sql/rewrite/ob_transform_dblink.cpp index 3bd1b31e40..298a3340b6 100644 --- a/src/sql/rewrite/ob_transform_dblink.cpp +++ b/src/sql/rewrite/ob_transform_dblink.cpp @@ -1614,6 +1614,9 @@ int ObTransformDBlink::formalize_bool_select_expr(ObDMLStmt *stmt) } else { case_when_expr->set_default_param_expr(null_expr); select_item.expr_ = case_when_expr; + if (OB_FAIL(case_when_expr->formalize(ctx_->session_info_))) { + LOG_WARN("failed to formalize expr", K(ret)); + } } } } diff --git a/src/sql/rewrite/ob_transform_left_join_to_anti.cpp b/src/sql/rewrite/ob_transform_left_join_to_anti.cpp index a0c6384252..fc86dd1055 100644 --- a/src/sql/rewrite/ob_transform_left_join_to_anti.cpp +++ b/src/sql/rewrite/ob_transform_left_join_to_anti.cpp @@ -315,7 +315,7 @@ int ObTransformLeftJoinToAnti::trans_stmt_to_anti(ObDMLStmt *stmt, JoinedTable * } } // do in-place modification - // a shared expr in semi_condition may be modified by the above replacemenet, + // a shared expr in semi_condition may be modified by the above replacement, // we revert the replacement in the following. if (OB_SUCC(ret)) { ObRawExprCopier copier(*ctx_->expr_factory_); @@ -337,7 +337,7 @@ int ObTransformLeftJoinToAnti::trans_stmt_to_anti(ObDMLStmt *stmt, JoinedTable * } else if (OB_FAIL(stmt->update_column_item_rel_id())) { LOG_WARN("failed to update column item rel id", K(ret)); } else if (OB_FAIL(stmt->formalize_stmt(ctx_->session_info_))) { - LOG_WARN("failed to formlize stmt", K(ret)); + LOG_WARN("failed to formalize stmt", K(ret)); } } return ret; @@ -353,9 +353,9 @@ int ObTransformLeftJoinToAnti::clear_for_update(TableItem *table) { table->for_update_ = false; } else if (table->is_joined_table()) { if (OB_FAIL(clear_for_update(static_cast(table)->left_table_))) { - LOG_WARN("fail to cleanr for update", K(ret)); + LOG_WARN("fail to clear for update", K(ret)); } else if (OB_FAIL(clear_for_update(static_cast(table)->right_table_))) { - LOG_WARN("fail to cleanr for update", K(ret)); + LOG_WARN("fail to clear for update", K(ret)); } } } @@ -523,7 +523,7 @@ int ObTransformLeftJoinToAnti::check_can_be_trans(ObDMLStmt *stmt, JoinedTable *right_joined_table = static_cast(right_table); if (is_contain(right_joined_table->single_table_ids_, table_info->table_id_)) { is_table_valid = false; - OPT_TRACE("right table is dml targe table"); + OPT_TRACE("right table is dml target table"); } } else if (table_info->table_id_ == right_table->table_id_) { is_table_valid = false; diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 6f68319d56..cb85e82cb6 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -9847,8 +9847,8 @@ int ObTransformPreProcess::check_is_correlated_cte(ObSelectStmt *stmt, ObIArray< if (OB_ISNULL(exec_param)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("exec param is null", K(ret)); - } else { - exec_param->clear_flag(BE_USED); + } else if (OB_FAIL(exec_param->clear_flag(BE_USED))) { + LOG_WARN("failed to clear flag", K(ret)); } } } diff --git a/src/sql/rewrite/ob_transform_simplify_expr.cpp b/src/sql/rewrite/ob_transform_simplify_expr.cpp index 8104bfc72c..fcabd4a809 100644 --- a/src/sql/rewrite/ob_transform_simplify_expr.cpp +++ b/src/sql/rewrite/ob_transform_simplify_expr.cpp @@ -774,14 +774,13 @@ int ObTransformSimplifyExpr::do_check_like_condition(ObRawExpr *&expr, } if (OB_SUCC(ret)) { if (text_expr->get_expr_type() == T_FUN_SYS_CAST && - text_expr->has_flag(IS_INNER_ADDED_EXPR)) {//avoid reconstuct sql is wrong. - text_expr->clear_flag(IS_INNER_ADDED_EXPR); - } - if (OB_FAIL(ObRawExprUtils::create_equal_expr(*ctx_->expr_factory_, - ctx_->session_info_, - pattern_expr, - text_expr, - eq_expr))) { + OB_FAIL(text_expr->clear_flag(IS_INNER_ADDED_EXPR))) { //avoid reconstuct sql is wrong. + LOG_WARN("failed to clear flag", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::create_equal_expr(*ctx_->expr_factory_, + ctx_->session_info_, + pattern_expr, + text_expr, + eq_expr))) { LOG_WARN("create equal expr failed", K(ret), K(pattern_expr)); } else if (OB_ISNULL(eq_expr)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/rewrite/ob_transform_subquery_coalesce.h b/src/sql/rewrite/ob_transform_subquery_coalesce.h index fab9573817..b8c2d21bc3 100644 --- a/src/sql/rewrite/ob_transform_subquery_coalesce.h +++ b/src/sql/rewrite/ob_transform_subquery_coalesce.h @@ -49,7 +49,7 @@ private: ObRawExpr *any_expr_; ObRawExpr *all_expr_; ObStmtMapInfo map_info_; - TransformFlag trans_flag_ = DEFAULT; //defalut value; + TransformFlag trans_flag_ = DEFAULT; //default value; TO_STRING_KV(K(exists_expr_), K(not_exists_expr_), diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 4f2a555ee3..130462960f 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -4325,12 +4325,8 @@ int ObTransformUtils::add_cast_for_replace(ObRawExprFactory &expr_factory, } else { ObSysFunRawExpr *cast_expr = NULL; ObCastMode cm; - if (OB_FAIL(ObSQLUtils::get_default_cast_mode(false,/* explicit_cast */ - 0, /* result_flag */ - session_info, cm))) { - LOG_WARN("failed to get default cast mode", K(ret)); - } else if (OB_FAIL(ObSQLUtils::set_cs_level_cast_mode(from_expr->get_collation_level(), cm))) { - LOG_WARN("failed to set cs level cast mode", K(ret)); + if (OB_FAIL(ObSQLUtils::get_cast_mode_for_replace(from_expr, session_info, cm))) { + LOG_WARN("failed to get cast mode for replace", K(ret)); } else if (OB_FAIL(ObRawExprUtils::create_cast_expr(expr_factory, to_expr, from_expr->get_result_type(), @@ -4359,7 +4355,8 @@ int ObTransformUtils::add_cast_for_replace_if_need(ObRawExprFactory &expr_factor bool need_cast = (src_type.get_type() != dst_type.get_type()) || (src_type.get_length() != dst_type.get_length()) || (src_type.get_precision() != dst_type.get_precision()) || - (src_type.get_scale() != dst_type.get_scale()); + (src_type.get_scale() != dst_type.get_scale()) || + from_expr->get_result_type().has_result_flag(ZEROFILL_FLAG); if (ob_is_string_or_lob_type(src_type.get_type())) { need_cast |= (src_type.get_collation_type() != dst_type.get_collation_type()) || (src_type.get_collation_level() != dst_type.get_collation_level()); diff --git a/src/sql/rewrite/ob_transform_win_magic.cpp b/src/sql/rewrite/ob_transform_win_magic.cpp index 649054aec2..a996d74af0 100644 --- a/src/sql/rewrite/ob_transform_win_magic.cpp +++ b/src/sql/rewrite/ob_transform_win_magic.cpp @@ -614,6 +614,9 @@ int ObTransformWinMagic::get_view_to_trans(ObDMLStmt *&stmt, LOG_WARN("check view table basic failed", K(ret)); } else if (!is_valid) { //do nothing + } else if (drill_down_view->get_from_item_size() != roll_up_view->get_from_item_size() + || drill_down_view->get_table_size() != roll_up_view->get_table_size()) { + is_valid = false; } else if (OB_FAIL(ObStmtComparer::compute_stmt_overlap(drill_down_view, roll_up_view, map_info))) { @@ -859,6 +862,15 @@ int ObTransformWinMagic::check_view_and_view(ObDMLStmt *main_stmt, LOG_WARN("check outer stmt condition failed", K(ret)); } + if (OB_SUCC(ret) && map_info.from_map_.count() != roll_up_view->get_from_item_size()) { + is_valid = false; + } + for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < map_info.from_map_.count(); i++) { + if (map_info.from_map_.at(i) == OB_INVALID_ID) { + is_valid = false; + } + } + //check if cnt agg for (int64_t i = 0; OB_SUCC(ret) && is_valid && i < roll_up_view->get_select_item_size(); i++) { ObRawExpr *expr = roll_up_view->get_select_item(i).expr_; diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index cbff711df2..55bdf69798 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -304,6 +304,11 @@ void ObBasicSessionInfo::clean_status() reset_session_changed_info(); } +void ObBasicSessionInfo::reset_user_var() +{ + user_var_val_map_.reuse(); +} + void ObBasicSessionInfo::reset(bool skip_sys_var) { set_valid(false); diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 64cedda847..ecffcabb68 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -396,6 +396,7 @@ public: virtual void destroy(); //called before put session to freelist: unlock/set invalid virtual void reset(bool skip_sys_var = false); + void reset_user_var(); void set_tenant_session_mgr(ObTenantSQLSessionMgr *tenant_session_mgr) { tenant_session_mgr_ = tenant_session_mgr; diff --git a/src/sql/session/ob_user_resource_mgr.cpp b/src/sql/session/ob_user_resource_mgr.cpp index f79e6c0a60..3498480e93 100644 --- a/src/sql/session/ob_user_resource_mgr.cpp +++ b/src/sql/session/ob_user_resource_mgr.cpp @@ -343,6 +343,14 @@ int ObConnectResourceMgr::on_user_connect( session.set_conn_res_user_id(user_id); } } + } else { + if (!session.has_got_tenant_conn_res()) { + if (OB_FAIL(apply_for_tenant_conn_resource(tenant_id, priv, UINT64_MAX))) { + LOG_WARN("reach teannt max connections", K(ret)); + } else { + session.set_got_tenant_conn_res(true); + } + } } return ret; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 118a7a9f7c..8e218d315b 100755 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -317,6 +317,7 @@ ob_set_subtarget(ob_storage tx tx/ob_dup_table_tablets.cpp tx/ob_dup_table_ts_sync.cpp tx/ob_dup_table_stat.cpp + tx/ob_dup_table_dump.cpp tx/ob_xa_ctx.cpp tx/ob_xa_ctx_mgr.cpp tx/ob_xa_dblink_service.cpp diff --git a/src/storage/access/ob_multiple_skip_scan_merge.cpp b/src/storage/access/ob_multiple_skip_scan_merge.cpp index 2eb242d930..094b3eb4b3 100644 --- a/src/storage/access/ob_multiple_skip_scan_merge.cpp +++ b/src/storage/access/ob_multiple_skip_scan_merge.cpp @@ -388,10 +388,8 @@ int ObMultipleSkipScanMerge::update_scan_rows_range(blocksstable::ObDatumRow &ro { int ret = OB_SUCCESS; range_allocator_.reuse(); - if (should_check_interrupt()) { - if (OB_FAIL(THIS_WORKER.check_status())) { - STORAGE_LOG(WARN, "query interrupt, ", K(ret)); - } + if (should_check_interrupt() && OB_FAIL(THIS_WORKER.check_status())) { + STORAGE_LOG(WARN, "query interrupt", K(ret)); } else if (should_retire_to_scan()) { // too many distinct prefix, retire to normal scan for (int64_t i = 0; OB_SUCC(ret) && i < ss_rowkey_prefix_cnt_; ++i) { @@ -515,4 +513,4 @@ int ObMultipleSkipScanMerge::check_range_exceeded(bool &exceeded) } } -} \ No newline at end of file +} diff --git a/src/storage/access/ob_table_scan_iterator.cpp b/src/storage/access/ob_table_scan_iterator.cpp index 2c9b16fbba..5fcec54d3b 100755 --- a/src/storage/access/ob_table_scan_iterator.cpp +++ b/src/storage/access/ob_table_scan_iterator.cpp @@ -542,7 +542,7 @@ int ObTableScanIterator::check_ls_offline_after_read() { int ret = OB_SUCCESS; - auto &acc_ctx = ctx_guard_.get_store_ctx().mvcc_acc_ctx_; + memtable::ObMvccAccessCtx &acc_ctx = ctx_guard_.get_store_ctx().mvcc_acc_ctx_; if (acc_ctx.tx_table_guards_.check_ls_offline()) { ret = OB_LS_OFFLINE; @@ -555,7 +555,7 @@ int ObTableScanIterator::check_ls_offline_after_read() int ObTableScanIterator::check_txn_status_if_read_uncommitted_() { int ret = OB_SUCCESS; - auto &acc_ctx = ctx_guard_.get_store_ctx().mvcc_acc_ctx_; + memtable::ObMvccAccessCtx &acc_ctx = ctx_guard_.get_store_ctx().mvcc_acc_ctx_; if (acc_ctx.snapshot_.tx_id_.is_valid() && acc_ctx.mem_ctx_) { if (acc_ctx.mem_ctx_->is_tx_rollbacked()) { if (acc_ctx.mem_ctx_->is_for_replay()) { diff --git a/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp b/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp index 17924816e5..e9f56e98ea 100644 --- a/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp @@ -73,11 +73,13 @@ public: ObDecoderCtxArray() : ctxs_() { ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtxArr"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; ctxs_.set_attr(attr); }; virtual ~ObDecoderCtxArray() { ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtx"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; FOREACH(it, ctxs_) { ObDecoderCtx *c = *it; OB_DELETE(ObDecoderCtx, attr, c); @@ -96,6 +98,7 @@ public: LOG_WARN("invalid argument", K(ret)); } else { ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtx"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; if (ctxs_.count() < size) { for (int64_t i = ctxs_.count(); OB_SUCC(ret) && i < size; ++i) { ObDecoderCtx *ctx = OB_NEW(ObDecoderCtx, attr); @@ -133,12 +136,14 @@ public: ObTLDecoderCtxArray() : ctxs_array_() { ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtxArr"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; ctxs_array_.set_attr(attr); } virtual ~ObTLDecoderCtxArray() { ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtx"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; FOREACH(it, ctxs_array_) { ObDecoderCtxArray *ctxs = *it; OB_DELETE(ObDecoderCtxArray, attr, ctxs); @@ -155,6 +160,7 @@ public: LOG_WARN("NULL instance", K(ret)); } else if (tl_array->ctxs_array_.empty()) { ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtx"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; ctxs = OB_NEW(ObDecoderCtxArray, attr); if (NULL == ctxs) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -171,6 +177,7 @@ public: { int ret = OB_SUCCESS; ObMemAttr attr(ob_thread_tenant_id(), "TLDecoderCtx"); + attr.sub_ctx_id_ = ObSubCtxIds::THREAD_LOCAL_DECODE_CTX_ID; ObTLDecoderCtxArray *tl_array = instance(); if (NULL == tl_array) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/storage/blockstore/ob_shared_block_reader_writer.cpp b/src/storage/blockstore/ob_shared_block_reader_writer.cpp index f9ea8a0c5a..f73261c781 100644 --- a/src/storage/blockstore/ob_shared_block_reader_writer.cpp +++ b/src/storage/blockstore/ob_shared_block_reader_writer.cpp @@ -17,6 +17,7 @@ namespace oceanbase { using namespace blocksstable; +using namespace common; namespace storage { diff --git a/src/storage/blockstore/ob_shared_block_reader_writer.h b/src/storage/blockstore/ob_shared_block_reader_writer.h index 124da687a3..57dcb40274 100644 --- a/src/storage/blockstore/ob_shared_block_reader_writer.h +++ b/src/storage/blockstore/ob_shared_block_reader_writer.h @@ -239,7 +239,7 @@ public: const ObSharedBlockWriteInfo &write_info, ObSharedBlockWriteHandle &block_handle); int async_batch_write( - const ObIArray &write_infos, + const common::ObIArray &write_infos, ObSharedBlockBatchHandle &block_handle); int async_link_write( const ObSharedBlockWriteInfo &write_infos, diff --git a/src/storage/compaction/ob_medium_compaction_mgr.cpp b/src/storage/compaction/ob_medium_compaction_mgr.cpp index a4e2fb1ad0..0968505eec 100644 --- a/src/storage/compaction/ob_medium_compaction_mgr.cpp +++ b/src/storage/compaction/ob_medium_compaction_mgr.cpp @@ -18,7 +18,8 @@ #include "logservice/ob_log_base_header.h" #include "storage/multi_data_source/mds_ctx.h" #include "storage/multi_data_source/mds_writer.h" -#include "src/storage/tx/ob_trans_define.h" +#include "storage/tx/ob_trans_define.h" +#include "storage/tablet/ob_tablet_obj_load_helper.h" #include "storage/tablet/ob_tablet_service_clog_replay_executor.h" namespace oceanbase @@ -448,7 +449,7 @@ int ObMediumCompactionInfoList::init( int ObMediumCompactionInfoList::init( common::ObIAllocator &allocator, const ObExtraMediumInfo &extra_medium_info, - const ObTabletDumpedMediumInfo &medium_info_list) + const ObTabletDumpedMediumInfo *medium_info_list) { int ret = OB_SUCCESS; @@ -457,32 +458,30 @@ int ObMediumCompactionInfoList::init( LOG_WARN("init twice", K(ret)); } else { allocator_ = &allocator; - const common::ObIArray &array = medium_info_list.medium_info_list_; - for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { - const ObMediumCompactionInfo *src_medium_info = array.at(i); - if (OB_ISNULL(src_medium_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(src_medium_info)); - } else { - ObMediumCompactionInfo *medium_info = nullptr; - void *buffer = allocator.alloc(sizeof(ObMediumCompactionInfo)); - if (OB_ISNULL(buffer)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", K(ret), "size", sizeof(ObMediumCompactionInfo)); - } else { - medium_info = new (buffer) ObMediumCompactionInfo(); - if (OB_FAIL(medium_info->init(allocator, *src_medium_info))) { - LOG_WARN("failed to copy medium info", K(ret), KPC(src_medium_info)); - } else if (OB_UNLIKELY(!medium_info_list_.add_last(medium_info))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to add last", K(ret), KPC(medium_info)); - } + if (nullptr == medium_info_list) { + // medium info is null, no need to copy + } else { + const common::ObIArray &array = medium_info_list->medium_info_list_; + ObMediumCompactionInfo *medium_info = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { + medium_info = nullptr; + const ObMediumCompactionInfo *src_medium_info = array.at(i); + if (OB_ISNULL(src_medium_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(src_medium_info)); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(medium_info->init(allocator, *src_medium_info))) { + LOG_WARN("failed to copy medium info", K(ret), KPC(src_medium_info)); + } else if (OB_UNLIKELY(!medium_info_list_.add_last(medium_info))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add last", K(ret), KPC(medium_info)); + } - if (OB_FAIL(ret)) { - if (OB_NOT_NULL(medium_info)) { - medium_info->~ObMediumCompactionInfo(); - allocator.free(medium_info); - } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(medium_info)) { + medium_info->~ObMediumCompactionInfo(); + allocator.free(medium_info); } } } diff --git a/src/storage/compaction/ob_medium_compaction_mgr.h b/src/storage/compaction/ob_medium_compaction_mgr.h index 6ed2d55495..c78b7d7a8a 100644 --- a/src/storage/compaction/ob_medium_compaction_mgr.h +++ b/src/storage/compaction/ob_medium_compaction_mgr.h @@ -101,7 +101,8 @@ public: int init( common::ObIAllocator &allocator, const ObExtraMediumInfo &extra_medium_info, - const ObTabletDumpedMediumInfo &medium_info_list); + const ObTabletDumpedMediumInfo *medium_info_list); + void reset(); OB_INLINE bool is_empty() const { return 0 == medium_info_list_.get_size(); } OB_INLINE int64_t size() const { return medium_info_list_.get_size(); } diff --git a/src/storage/compaction/ob_tablet_merge_ctx.cpp b/src/storage/compaction/ob_tablet_merge_ctx.cpp index 8d62b70a76..bca2c54a86 100755 --- a/src/storage/compaction/ob_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_tablet_merge_ctx.cpp @@ -831,16 +831,12 @@ int ObTabletMergeCtx::update_tablet_or_release_memtable(const ObGetMergeTablesRe } else if (OB_UNLIKELY(get_merge_table_result.scn_range_.end_scn_ > old_tablet->get_tablet_meta().clog_checkpoint_scn_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("can't have larger end_log_ts", K(ret), K(get_merge_table_result), K(old_tablet->get_tablet_meta())); - } else if (old_tablet->get_tablet_meta().snapshot_version_ >= get_merge_table_result.version_range_.snapshot_version_) { - // do nothing, no need to advance snapshot version on tablet meta. } else if (OB_FAIL(get_storage_schema_to_merge(get_merge_table_result.handle_))) { LOG_WARN("failed to get storage schema", K(ret), K(get_merge_table_result)); } else if (OB_ISNULL(schema_ctx_.storage_schema_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("storage schema is unexpected null", K(ret), KPC(this)); - } - - if (FAILEDx(update_tablet_directly(get_merge_table_result))) { + } else if (OB_FAIL(update_tablet_directly(get_merge_table_result))) { LOG_WARN("failed to update tablet directly", K(ret), K(get_merge_table_result)); } return ret; @@ -1052,6 +1048,8 @@ int ObTabletMergeCtx::get_storage_schema_to_merge(const ObTablesHandleArray &mer if (OB_FAIL(tablet_handle_.get_obj()->load_storage_schema(allocator_, schema_on_tablet))) { LOG_WARN("failed to load storage schema", K(ret), K_(tablet_handle)); + } else if (0 == merge_tables_handle.get_count()) { + // do nothing, only need to release memtable } else if (is_mini_merge(merge_type) && !param_.tablet_id_.is_ls_inner_tablet()) { if (OB_FAIL(schema_on_tablet->get_store_column_count(column_cnt_in_schema, true/*full_col*/))) { LOG_WARN("failed to get store column count", K(ret), K(column_cnt_in_schema)); diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index 04a2b430cb..a156467e24 100755 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -1277,11 +1277,8 @@ int ObTabletMergeFinishTask::add_sstable_for_merge(ObTabletMergeCtx &ctx) if (ctx.param_.tablet_id_.is_special_merge_tablet()) { param.multi_version_start_ = 1; } - // for mini merge, read all msd from frozen memtable - if (is_mini_merge(merge_type) && OB_FAIL(read_msd_from_memtable(ctx, param, allocator))) { - LOG_WARN("failed to read msd from memtable", K(ret), K(ctx)); - } else if (OB_FAIL(ctx.ls_handle_.get_ls()->update_tablet_table_store( - ctx.param_.tablet_id_, param, new_tablet_handle))) { + + if (OB_FAIL(ctx.ls_handle_.get_ls()->update_tablet_table_store(ctx.param_.tablet_id_, param, new_tablet_handle))) { LOG_WARN("failed to update tablet table store", K(ret), K(param)); } else if (FALSE_IT(ctx.time_guard_.click(ObCompactionTimeGuard::UPDATE_TABLET))) { } else if (is_mini_merge(merge_type)) { @@ -1344,73 +1341,6 @@ int ObTabletMergeFinishTask::try_report_tablet_stat_after_mini(ObTabletMergeCtx return ret; } -int ObTabletMergeFinishTask::read_msd_from_memtable( - ObTabletMergeCtx &ctx, - ObUpdateTableStoreParam ¶m, - ObArenaAllocator &allocator) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(traverse_all_memtables(ctx, ¶m.tx_data_, MultiSourceDataUnitType::TABLET_TX_DATA, allocator))) { - LOG_WARN("failed to read tx data from memtables", K(ret)); - } else if (OB_FAIL(traverse_all_memtables(ctx, ¶m.binding_info_, MultiSourceDataUnitType::TABLET_BINDING_INFO, allocator))) { - LOG_WARN("failed to read tx data from memtables", K(ret)); - } else if (OB_FAIL(traverse_all_memtables(ctx, ¶m.autoinc_seq_, MultiSourceDataUnitType::TABLET_SEQ, allocator))) { - LOG_WARN("failed to read tx data from memtables", K(ret)); - } else { - LOG_INFO("succeeded to read msd from memtable", K(ret), - "ls_id", ctx.param_.ls_id_, - "tablet_id", ctx.param_.tablet_id_, - "tx_data", param.tx_data_, - "binding_info", param.binding_info_, - "autoinc_seq", param.autoinc_seq_); - } - - return ret; -} - -int ObTabletMergeFinishTask::traverse_all_memtables( - ObTabletMergeCtx &ctx, - ObIMultiSourceDataUnit *msd, - const MultiSourceDataUnitType &type, - ObArenaAllocator &allocator) -{ - int ret = OB_SUCCESS; - ObITable *table = nullptr; - ObMemtable *memtable = nullptr; - - if (OB_ISNULL(msd)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret)); - } - - for (int64_t i = ctx.tables_handle_.get_count() - 1; OB_SUCC(ret) && i >= 0; --i) { - if (OB_ISNULL(table = ctx.tables_handle_.get_table(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is null", K(ret), K(ctx.tables_handle_), KP(table)); - } else if (OB_UNLIKELY(!table->is_memtable())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is not memtable", K(ret), K(ctx.tables_handle_), KPC(table)); - } else if (OB_UNLIKELY(!table->is_frozen_memtable())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table is not frozen memtable", K(ret), K(ctx.tables_handle_), KPC(table)); - } else if (table->is_data_memtable()) { - // TODO(lixia) delete this code - // memtable = static_cast(table); - // if (memtable->has_multi_source_data_unit(type)) { - // if (OB_FAIL(memtable->get_multi_source_data_unit(msd, &allocator))) { - // LOG_WARN("failed to get msd from memtable", K(ret), K(type)); - // } else { - // // succeeded to get msd, just break - // break; - // } - // } - } - } - - return ret; -} - int ObTabletMergeFinishTask::try_schedule_compaction_after_mini( ObTabletMergeCtx &ctx, ObTabletHandle &tablet_handle) diff --git a/src/storage/compaction/ob_tablet_merge_task.h b/src/storage/compaction/ob_tablet_merge_task.h index e65fea389e..5dfb4ff6c4 100644 --- a/src/storage/compaction/ob_tablet_merge_task.h +++ b/src/storage/compaction/ob_tablet_merge_task.h @@ -32,12 +32,6 @@ class ObTabletHandle; struct ObUpdateTableStoreParam; } -namespace memtable -{ -enum class MultiSourceDataUnitType; -class ObIMultiSourceDataUnit; -} - namespace blocksstable { class ObSSTable; @@ -164,15 +158,6 @@ private: int add_sstable_for_merge(ObTabletMergeCtx &ctx); int try_schedule_compaction_after_mini(ObTabletMergeCtx &ctx, storage::ObTabletHandle &tablet_handle); int try_report_tablet_stat_after_mini(ObTabletMergeCtx &ctx); - int read_msd_from_memtable( - ObTabletMergeCtx &ctx, - storage::ObUpdateTableStoreParam ¶m, - ObArenaAllocator &allocator); - int traverse_all_memtables( - ObTabletMergeCtx &ctx, - memtable::ObIMultiSourceDataUnit *msd, - const memtable::MultiSourceDataUnitType &type, - ObArenaAllocator &allocator); private: bool is_inited_; ObBasicTabletMergeDag *merge_dag_; diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index 70dea53a8a..acfe89256c 100755 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -2156,16 +2156,17 @@ int ObRemoteScan::prepare_iter(const ObSqlString &sql_string, common::ObCommonSq { int ret = OB_SUCCESS; ObSessionParam session_param; - ObSQLMode sql_mode = SMO_STRICT_ALL_TABLES | SMO_PAD_CHAR_TO_FULL_LENGTH; + ObSQLMode sql_mode = SMO_STRICT_ALL_TABLES; session_param.sql_mode_ = reinterpret_cast(&sql_mode); session_param.tz_info_wrap_ = nullptr; session_param.ddl_info_.set_is_ddl(true); session_param.ddl_info_.set_source_table_hidden(false); session_param.ddl_info_.set_dest_table_hidden(false); + const int64_t sql_total_timeout = max(7 * 24 * 60 * 60 * 1000 * 1000L, GCONF._ob_ddl_timeout); if (OB_ISNULL(sql_proxy)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(ret)); - } else if (OB_FAIL(sql_proxy->read(res_, tenant_id_, sql_string.ptr(), &session_param))) { + } else if (OB_FAIL(sql_proxy->read(res_, tenant_id_, sql_string.ptr(), &session_param, sql_total_timeout))) { LOG_WARN("fail to execute sql", K(ret), K_(tenant_id), K(sql_string)); } else if (OB_ISNULL(result_ = res_.get_result())) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/ddl/ob_ddl_clog.cpp b/src/storage/ddl/ob_ddl_clog.cpp index 51330764b2..3318de143c 100644 --- a/src/storage/ddl/ob_ddl_clog.cpp +++ b/src/storage/ddl/ob_ddl_clog.cpp @@ -143,6 +143,15 @@ ObDDLMacroBlockClogCb::ObDDLMacroBlockClogCb() } +ObDDLMacroBlockClogCb::~ObDDLMacroBlockClogCb() +{ + int ret = OB_SUCCESS; + if (macro_block_id_.is_valid() && OB_FAIL(OB_SERVER_BLOCK_MGR.dec_ref(macro_block_id_))) { + LOG_ERROR("dec ref failed", K(ret), K(macro_block_id_), K(common::lbt())); + } + macro_block_id_.reset(); +} + int ObDDLMacroBlockClogCb::init(const share::ObLSID &ls_id, const blocksstable::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, @@ -156,6 +165,8 @@ int ObDDLMacroBlockClogCb::init(const share::ObLSID &ls_id, } else if (OB_UNLIKELY(!ls_id.is_valid() || !redo_info.is_valid() || !macro_block_id.is_valid() || !tablet_handle.is_valid() || !ddl_kv_mgr_handle.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_id), K(redo_info), K(macro_block_id)); + } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.inc_ref(macro_block_id))) { + LOG_WARN("inc reference count failed", K(ret), K(macro_block_id)); } else { redo_info_.data_buffer_.assign(const_cast(redo_info.data_buffer_.ptr()), redo_info.data_buffer_.length()); redo_info_.block_type_ = redo_info.block_type_; diff --git a/src/storage/ddl/ob_ddl_clog.h b/src/storage/ddl/ob_ddl_clog.h index 7cb3d5e0bf..dfae841157 100644 --- a/src/storage/ddl/ob_ddl_clog.h +++ b/src/storage/ddl/ob_ddl_clog.h @@ -103,7 +103,7 @@ class ObDDLMacroBlockClogCb : public logservice::AppendCb { public: ObDDLMacroBlockClogCb(); - virtual ~ObDDLMacroBlockClogCb() = default; + virtual ~ObDDLMacroBlockClogCb(); int init(const share::ObLSID &ls_id, const blocksstable::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id, diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index e7cf32d14e..e2ec3c6bae 100755 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -1210,6 +1210,7 @@ int ObDDLSSTableRedoWriter::end_ddl_redo_and_create_ddl_sstable( SCN ddl_start_scn = get_start_scn(); SCN commit_scn = SCN::min_scn(); bool is_remote_write = false; + bool commit_by_this_execution = false; if (OB_UNLIKELY(!ls_id.is_valid() || !table_key.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid ls", K(ret), K(ls_id), K(table_key)); @@ -1225,6 +1226,8 @@ int ObDDLSSTableRedoWriter::end_ddl_redo_and_create_ddl_sstable( } else { LOG_WARN("fail write ddl commit log", K(ret), K(table_key)); } + } else { + commit_by_this_execution = true; } if (OB_TRANS_COMMITED == ret) { @@ -1280,7 +1283,7 @@ int ObDDLSSTableRedoWriter::end_ddl_redo_and_create_ddl_sstable( LOG_WARN("no major after wait merge success", K(ret), K(ls_id), K(tablet_id)); } else if (OB_FAIL(first_major_sstable->get_meta(sst_meta_hdl))) { LOG_WARN("fail to get sstable meta handle", K(ret)); - } else if (OB_UNLIKELY(first_major_sstable->get_key() != table_key)) { + } else if (commit_by_this_execution && OB_UNLIKELY(first_major_sstable->get_key() != table_key)) { ret = OB_SNAPSHOT_DISCARDED; LOG_WARN("ddl major sstable dropped, snapshot holding may have bug", K(ret), KPC(first_major_sstable), K(table_key), K(tablet_id), K(execution_id), K(ddl_task_id)); } else { diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp index 9d5e216270..216ed1c68a 100755 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp @@ -882,6 +882,8 @@ ObSSTableInsertTableContext::ObSSTableInsertTableContext() ObSSTableInsertTableContext::~ObSSTableInsertTableContext() { + remove_all_tablets_context(); // ignore error code. + tablet_ctx_map_.destroy(); } int ObSSTableInsertTableContext::init( @@ -1073,24 +1075,30 @@ int ObSSTableInsertTableContext::get_tablet_context( return ret; } -int ObSSTableInsertTableContext::remove_tablet_context( - const ObTabletID &tablet_id) +int ObSSTableInsertTableContext::remove_all_tablets_context() { int ret = OB_SUCCESS; - ObSSTableInsertTabletContext *context = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObTableInsertSSTableContext has not been inited", K(ret)); - } else if (OB_UNLIKELY(!tablet_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(tablet_id)); - } else if (OB_FAIL(tablet_ctx_map_.get_refactored(tablet_id, context))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_ENTRY_NOT_EXIST; - } } else { - context->~ObSSTableInsertTabletContext(); - allocator_.free(context); + GetManageTabletIDs get_tablet_ids_fn; + if (OB_FAIL(tablet_ctx_map_.foreach_refactored(get_tablet_ids_fn))) { + LOG_WARN("get tablet ids failed", K(ret)); + } else if (OB_FAIL(get_tablet_ids_fn.ret_code_)) { + LOG_WARN("get tablet ids failed", K(ret)); + } + for (int64_t i = 0; i < get_tablet_ids_fn.tablet_ids_.count(); ++i) { // ignore error code. + ObSSTableInsertTabletContext *tablet_context = nullptr; + const ObTabletID &tablet_id = get_tablet_ids_fn.tablet_ids_.at(i); + if (OB_FAIL(tablet_ctx_map_.erase_refactored(tablet_id, &tablet_context))) { + LOG_WARN("erase failed", K(ret), K(tablet_id)); + } else { + tablet_context->~ObSSTableInsertTabletContext(); + allocator_.free(tablet_context); + tablet_context = nullptr; + } + } } return ret; } @@ -1117,10 +1125,7 @@ int ObSSTableInsertTableContext::finish(const bool need_commit) LOG_WARN("create sstable failed", K(ret)); } } - for (int64_t i = 0; i < get_tablet_ids_fn.tablet_ids_.count(); ++i) { - const ObTabletID &tablet_id = get_tablet_ids_fn.tablet_ids_.at(i); - remove_tablet_context(tablet_id); // ignore ret - } + remove_all_tablets_context(); // ignore error code. } return ret; } @@ -1244,7 +1249,7 @@ ObSSTableInsertManager::ObSSTableInsertManager() ObSSTableInsertManager::~ObSSTableInsertManager() { - + destroy(); } ObSSTableInsertManager &ObSSTableInsertManager::get_instance() @@ -1377,17 +1382,17 @@ int ObSSTableInsertManager::finish_table_context(const int64_t context_id, const { int ret = OB_SUCCESS; ObSSTableInsertTableContext *table_context = nullptr; - if (OB_FAIL(get_context(context_id, table_context))) { + ObBucketHashWLockGuard guard(bucket_lock_, get_context_id_hash(context_id)); + if (OB_FAIL(get_context_no_lock(context_id, table_context))) { LOG_WARN("get context failed", K(ret)); } else if (OB_FAIL(table_context->finish(need_commit))) { LOG_WARN("finish table context failed", K(ret)); - } else if (OB_FAIL(table_ctx_map_.erase_refactored(context_id))) { - LOG_WARN("erase from map failed", K(ret), K(context_id)); } if (nullptr != table_context) { // ignore ret - table_context->~ObSSTableInsertTableContext(); - allocator_.free(table_context); - table_context = nullptr; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(remove_context_no_lock(context_id))) { + LOG_ERROR("erase factored failed", K(ret), K(tmp_ret), K(context_id)); + } } return ret; } @@ -1549,7 +1554,7 @@ int ObSSTableInsertManager::get_context_no_lock( return ret; } -int ObSSTableInsertManager::remove_context(const int64_t context_id) +int ObSSTableInsertManager::remove_context_no_lock(const int64_t context_id) { int ret = OB_SUCCESS; ObSSTableInsertTableContext *table_context = nullptr; @@ -1571,11 +1576,18 @@ int ObSSTableInsertManager::remove_context(const int64_t context_id) void ObSSTableInsertManager::destroy() { - ObSSTableInsertTableContext *context = nullptr; - for(TABLE_CTX_MAP::iterator iter = table_ctx_map_.begin(); iter != table_ctx_map_.end(); ++iter) { - if (OB_NOT_NULL(context = iter->second)) { - context->~ObSSTableInsertTableContext(); - allocator_.free(context); + int ret = OB_SUCCESS; + ObArray context_id_arr; + common::ObBucketWLockAllGuard lock_guard(bucket_lock_); + for(TABLE_CTX_MAP::iterator iter = table_ctx_map_.begin(); iter != table_ctx_map_.end(); ++iter) { // ignore error code. + if (OB_FAIL(context_id_arr.push_back(iter->first))) { + LOG_ERROR("push back failed", K(ret)); + } + } + for (int64_t i = 0; i < context_id_arr.count(); i++) { // ignore error code. + const int64_t context_id = context_id_arr.at(i); + if (OB_FAIL(remove_context_no_lock(context_id))) { + LOG_ERROR("remove context failed", K(ret), K(context_id)); } } table_ctx_map_.destroy(); diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx.h b/src/storage/ddl/ob_direct_insert_sstable_ctx.h index 1ebc07b319..6f7ed2d26a 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx.h @@ -160,9 +160,14 @@ private: const blocksstable::ObColDescIArray *col_descs_; int64_t snapshot_version_; ObDDLSSTableRedoWriter sstable_redo_writer_; - ObDDLRedoLogWriterCallback redo_log_writer_callback_; blocksstable::ObDataStoreDesc data_desc_; + /** + * ATTENTION! + * The deconstruction order of the `redo_log_writer_callback_` should be in front of the `macro_block_writer_` + * to ensure the safety-used of the ddl macro block. + */ blocksstable::ObMacroBlockWriter macro_block_writer_; + ObDDLRedoLogWriterCallback redo_log_writer_callback_; common::ObArenaAllocator allocator_; common::ObArenaAllocator lob_allocator_; int64_t lob_cnt_; @@ -265,7 +270,7 @@ private: void destroy(); int create_all_tablet_contexts(const common::ObIArray &ls_tablet_ids); int get_tablet_context(const common::ObTabletID &tablet_id, ObSSTableInsertTabletContext *&tablet_ctx); - int remove_tablet_context(const common::ObTabletID &tablet_id); + int remove_all_tablets_context(); private: typedef common::hash::ObHashMap< @@ -322,7 +327,7 @@ private: int get_context_no_lock( const int64_t context_id, ObSSTableInsertTableContext *&ctx); - int remove_context(const int64_t context_id); + int remove_context_no_lock(const int64_t context_id); int64_t alloc_context_id(); uint64_t get_context_id_hash(const int64_t context_id); private: diff --git a/src/storage/high_availability/ob_finish_transfer.cpp b/src/storage/high_availability/ob_finish_transfer.cpp index df2a115587..3081c9b5f9 100644 --- a/src/storage/high_availability/ob_finish_transfer.cpp +++ b/src/storage/high_availability/ob_finish_transfer.cpp @@ -373,7 +373,7 @@ int ObTxFinishTransfer::get_transfer_tablet_info_from_inner_table_( if (!task_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(task_id)); - } else if (OB_FAIL(ObTransferTaskOperator::get(*sql_proxy_, tenant_id, task_id, for_update, transfer_task))) { + } else if (OB_FAIL(ObTransferTaskOperator::get(*sql_proxy_, tenant_id, task_id, for_update, transfer_task, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to get transfer task", K(ret), K(tenant_id), K(task_id)); } else if (OB_FAIL(tablet_list.assign(transfer_task.get_tablet_list()))) { LOG_WARN("failed to assign tablet_list", KR(ret), K(transfer_task)); @@ -788,7 +788,7 @@ int ObTxFinishTransfer::unlock_ls_member_list_(const uint64_t tenant_id, const s { int ret = OB_SUCCESS; if (OB_FAIL(ObMemberListLockUtils::unlock_ls_member_list( - tenant_id, ls_id, task_id_.id(), member_list, status, *sql_proxy_))) { + tenant_id, ls_id, task_id_.id(), member_list, status, share::OBCG_STORAGE_HA_LEVEL2, *sql_proxy_))) { LOG_WARN("failed to unlock ls member list", K(ret), K(tenant_id), K(ls_id), K_(task_id), K(status)); } return ret; @@ -799,7 +799,7 @@ int ObTxFinishTransfer::lock_ls_member_list_(const uint64_t tenant_id, const sha { int ret = OB_SUCCESS; if (OB_FAIL(ObMemberListLockUtils::lock_ls_member_list( - tenant_id, ls_id, task_id_.id(), member_list, status, *sql_proxy_))) { + tenant_id, ls_id, task_id_.id(), member_list, status, share::OBCG_STORAGE_HA_LEVEL2, *sql_proxy_))) { LOG_WARN("failed to unlock ls member list", K(ret), K(ls_id), K(member_list)); } else { #ifdef ERRSIM @@ -888,16 +888,16 @@ int ObTxFinishTransfer::update_transfer_task_result_(const ObTransferTaskID &tas const bool for_update = true; ObTransferStatus next_status; next_status = OB_SUCCESS == result ? ObTransferStatus::COMPLETED : ObTransferStatus::DOING; - if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, transfer_task))) { + if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, transfer_task, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to get transfer task", K(ret), K(task_id), K(tenant_id)); } else if (transfer_task.get_start_scn() >= finish_scn) { ret = OB_ERR_UNEXPECTED; LOG_WARN("finish scn not expected", K(ret), K(transfer_task), K(finish_scn)); } else if (OB_FAIL(ObTransferTaskOperator::update_finish_scn( - trans, tenant_id, task_id, transfer_task.get_status(), finish_scn))) { + trans, tenant_id, task_id, transfer_task.get_status(), finish_scn, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to update finish scn", K(ret), K(tenant_id), K(task_id), K(finish_scn)); } else if (OB_FAIL(ObTransferTaskOperator::finish_task( - trans, tenant_id, task_id, transfer_task.get_status(), next_status, result, ObTransferTaskComment::EMPTY_COMMENT))) { + trans, tenant_id, task_id, transfer_task.get_status(), next_status, result, ObTransferTaskComment::EMPTY_COMMENT, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to finish task", K(ret), K(tenant_id), K(task_id)); } #ifdef ERRSIM @@ -1046,7 +1046,7 @@ int ObTxFinishTransfer::start_trans_( LOG_WARN("fail to set trx timeout", K(ret), K(stmt_timeout)); } else if (OB_FAIL(timeout_ctx.set_timeout(stmt_timeout))) { LOG_WARN("set timeout context failed", K(ret)); - } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id))) { + } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, false/*with_snapshot*/, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to start trans", K(ret), K(tenant_id)); } else { LOG_INFO("start trans", K(tenant_id)); @@ -1087,7 +1087,7 @@ int ObTxFinishTransfer::select_transfer_task_for_update_(const ObTransferTaskID if (!task_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid arg", K(ret), K(task_id)); - } else if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, task))) { + } else if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, task, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to get transfer task", K(ret), K(tenant_id), K(task_id)); } else if (!task.get_status().is_doing_status()) { ret = OB_STATE_NOT_MATCH; diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index d2f4f012b6..5d4409b2a5 100755 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -2406,12 +2406,12 @@ int ObStorageHATabletBuilderUtil::inner_update_tablet_table_store_with_major_( true/*need_check_sstable*/, true/*allow_duplicate_sstable*/, ObMergeType::MEDIUM_MERGE/*merge_type*/); - if (tablet_storage_schema->get_version() < storage_schema.get_version()) { + if (tablet_storage_schema->get_schema_version() < storage_schema.get_schema_version()) { SERVER_EVENT_ADD("storage_ha", "schema_change_need_merge_tablet_meta", "tenant_id", MTL_ID(), "tablet_id", tablet_id.id(), - "old_schema_version", tablet_storage_schema->get_version(), - "new_schema_version", storage_schema.get_version()); + "old_schema_version", tablet_storage_schema->get_schema_version(), + "new_schema_version", storage_schema.get_schema_version()); } #ifdef ERRSIM SERVER_EVENT_ADD("storage_ha", "update_major_tablet_table_store", diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index 47cea8ac0a..c2a86964d2 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -2124,7 +2124,7 @@ int ObTabletRestoreTask::process() LOG_WARN("failed to record server event", K(tmp_ret)); } #ifdef ERRSIM - if (OB_SUCC(ret) && !tablet_restore_ctx_->is_leader_ + if (OB_SUCC(ret) && tablet_restore_ctx_->is_leader_ && ObTabletRestoreAction::is_restore_minor(tablet_restore_ctx_->action_) && tablet_restore_ctx_->ls_id_.is_user_ls()) { SERVER_EVENT_SYNC_ADD("storage_ha", "follower_restore_major_errsim", "tablet_id", tablet_restore_ctx_->tablet_id_.id()); diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index 3ef69cb0a9..ad5fc4e22e 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -170,7 +170,7 @@ int ObTransferHandler::get_transfer_task_from_inner_table_( if (! task_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid arg", K(ret), K(task_id)); - } else if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, task))) { + } else if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, task, share::OBCG_STORAGE_HA_LEVEL1))) { LOG_WARN("failed to get transfer task", K(ret), K(tenant_id), K(task_id)); } else if (OB_FAIL(task_info.convert_from(tenant_id, task))) { LOG_WARN("failed to convert from transfer task", K(ret), K(task)); @@ -218,7 +218,7 @@ int ObTransferHandler::fetch_transfer_task_from_inner_table_by_src_ls_( const ObLSID &src_ls_id = ls_->get_ls_id(); ObTransferTask task; if (OB_FAIL(ObTransferTaskOperator::get_by_src_ls( - *sql_proxy_, tenant_id, src_ls_id, task))) { + *sql_proxy_, tenant_id, src_ls_id, task, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to get transfer task", K(ret), K(tenant_id), K(src_ls_id)); } else if (OB_FAIL(task_info.convert_from(tenant_id, task))) { LOG_WARN("failed to convert from transfer task", K(ret), K(task)); @@ -246,7 +246,7 @@ int ObTransferHandler::fetch_transfer_task_from_inner_table_by_dest_ls_( const ObLSID &dest_ls_id = ls_->get_ls_id(); ObTransferTask task; if (OB_FAIL(ObTransferTaskOperator::get_by_dest_ls( - *sql_proxy_, tenant_id, dest_ls_id, task))) { + *sql_proxy_, tenant_id, dest_ls_id, task, share::OBCG_STORAGE_HA_LEVEL2))) { LOG_WARN("failed to get transfer task by dest ls", K(ret), K(tenant_id), K(dest_ls_id)); } else if (OB_FAIL(task_info.convert_from(tenant_id, task))) { LOG_WARN("failed to convert from transfer task", K(ret), K(task)); @@ -565,7 +565,7 @@ int ObTransferHandler::lock_src_and_dest_ls_member_list_( } else if (OB_FAIL(lock_ls_list.push_back(dest_ls_id))) { LOG_WARN("failed to push back", K(ret), K(dest_ls_id)); } else if (OB_FAIL(ObMemberListLockUtils::batch_lock_ls_member_list(tenant_id, task_id, - lock_ls_list, member_list, status, *sql_proxy_))) { + lock_ls_list, member_list, status, share::OBCG_STORAGE_HA_LEVEL2, *sql_proxy_))) { LOG_WARN("failed to batch lock ls member list", K(ret)); } else if (OB_FAIL(check_ls_member_list_same_(src_ls_id, dest_ls_id, member_list, is_same))) { LOG_WARN("failed to check ls member listsame", K(ret), K(src_ls_id), K(dest_ls_id)); @@ -647,7 +647,7 @@ int ObTransferHandler::inner_unlock_ls_member_list_( const int64_t task_id = task_info.task_id_.id(); const ObTransferLockStatus status(ObTransferLockStatus::START); if (OB_FAIL(ObMemberListLockUtils::unlock_ls_member_list( - tenant_id, ls_id, task_id, member_list, status, *sql_proxy_))) { + tenant_id, ls_id, task_id, member_list, status, share::OBCG_STORAGE_HA_LEVEL2, *sql_proxy_))) { LOG_WARN("failed to lock ls member list", K(ret), K(task_info), K(ls_id), K(member_list)); } return ret; @@ -926,6 +926,8 @@ int ObTransferHandler::start_trans_( omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); int64_t stmt_timeout = 10_s; const int64_t LOCK_MEMBER_LIST_TIMEOUT = 10_s; + const bool with_snapshot = false; + const int32_t group_id = share::OBCG_STORAGE_HA_LEVEL1; if (tenant_config.is_valid()) { stmt_timeout = tenant_config->_transfer_start_trans_timeout + LOCK_MEMBER_LIST_TIMEOUT; if (tenant_config->_enable_balance_kill_transaction) { @@ -941,7 +943,7 @@ int ObTransferHandler::start_trans_( LOG_WARN("fail to set trx timeout", K(ret), K(stmt_timeout)); } else if (OB_FAIL(timeout_ctx.set_timeout(stmt_timeout))) { LOG_WARN("set timeout context failed", K(ret)); - } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id))) { + } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, with_snapshot, group_id))) { LOG_WARN("failed to start trans", K(ret)); } return ret; @@ -1508,7 +1510,7 @@ int ObTransferHandler::update_all_tablet_to_ls_( const ObTransferTabletInfo &tablet_info = task_info.tablet_list_.at(i); if (OB_FAIL(ObTabletToLSTableOperator::update_ls_id_and_transfer_seq(trans, task_info.tenant_id_, tablet_info.tablet_id_, tablet_info.transfer_seq_, task_info.src_ls_id_, - tablet_info.transfer_seq_ + 1, task_info.dest_ls_id_))) { + tablet_info.transfer_seq_ + 1, task_info.dest_ls_id_, share::OBCG_STORAGE_HA_LEVEL1))) { LOG_WARN("failed to update ls id and transfer seq", K(ret), K(tablet_info), K(task_info)); } } @@ -1577,7 +1579,7 @@ int ObTransferHandler::update_transfer_status_( } else if (!task_info.is_valid() || !next_status.is_valid()) { LOG_WARN("update transfer status get invalid argument", K(ret), K(task_info), K(next_status)); } else { - if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, transfer_task))) { + if (OB_FAIL(ObTransferTaskOperator::get(trans, tenant_id, task_id, for_update, transfer_task, share::OBCG_STORAGE_HA_LEVEL1))) { LOG_WARN("failed to get transfer task", K(ret), K(task_id), K(tenant_id)); } else if (task_info.status_ != transfer_task.get_status() || task_info.src_ls_id_ != transfer_task.get_src_ls() @@ -1586,10 +1588,10 @@ int ObTransferHandler::update_transfer_status_( LOG_WARN("task info in not equal to inner table transfer task, unexpected", K(ret), K(task_info), K(transfer_task)); } else if (start_scn.is_valid() && OB_FAIL(ObTransferTaskOperator::update_start_scn( - trans, tenant_id, task_id, transfer_task.get_status(), start_scn))) { + trans, tenant_id, task_id, transfer_task.get_status(), start_scn, share::OBCG_STORAGE_HA_LEVEL1))) { LOG_WARN("failed to update finish scn", K(ret), K(tenant_id), K(task_id), K(start_scn)); } else if (OB_FAIL(ObTransferTaskOperator::update_status_and_result( - trans, tenant_id, task_id, transfer_task.get_status(), next_status, result))) { + trans, tenant_id, task_id, transfer_task.get_status(), next_status, result, share::OBCG_STORAGE_HA_LEVEL1))) { LOG_WARN("failed to finish task", K(ret), K(tenant_id), K(task_id)); } else { #ifdef ERRSIM diff --git a/src/storage/high_availability/ob_transfer_lock_info_operator.cpp b/src/storage/high_availability/ob_transfer_lock_info_operator.cpp index f15fa34a1e..fdfae61e1f 100644 --- a/src/storage/high_availability/ob_transfer_lock_info_operator.cpp +++ b/src/storage/high_availability/ob_transfer_lock_info_operator.cpp @@ -27,13 +27,14 @@ namespace storage { /* ObTransferLockInfoOperator */ -int ObTransferLockInfoOperator::insert(const ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy) +int ObTransferLockInfoOperator::insert(const ObTransferTaskLockInfo &lock_info, + const int32_t group_id, common::ObISQLClient &sql_proxy) { int ret = OB_SUCCESS; const uint64_t tenant_id = lock_info.tenant_id_; - if (!lock_info.is_valid()) { + if (!lock_info.is_valid() || group_id < 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(lock_info)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(lock_info), K(group_id)); } else { ObSqlString sql; ObDMLSqlSplicer dml_splicer; @@ -44,7 +45,7 @@ int ObTransferLockInfoOperator::insert(const ObTransferTaskLockInfo &lock_info, LOG_WARN("failed to finish row", K(ret), K(tenant_id), K(lock_info)); } else if (OB_FAIL(dml_splicer.splice_insert_sql(OB_ALL_LS_TRANSFER_MEMBER_LIST_LOCK_INFO_TNAME, sql))) { LOG_WARN("failed to splice insert sql", K(ret), K(tenant_id), K(sql)); - } else if (OB_FAIL(sql_proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), group_id, affected_rows))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { ret = OB_ENTRY_EXIST; } else { @@ -67,14 +68,14 @@ int ObTransferLockInfoOperator::insert(const ObTransferTaskLockInfo &lock_info, } int ObTransferLockInfoOperator::remove(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const ObTransferLockStatus &status, common::ObISQLClient &sql_proxy) + const ObTransferLockStatus &status, const int32_t group_id, common::ObISQLClient &sql_proxy) { int ret = OB_SUCCESS; ObSqlString sql; int64_t affected_rows = 0; - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid())) { + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || !ls_id.is_valid() || group_id < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id), K(group_id)); } else if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE tenant_id = %lu and ls_id = %ld" " and task_id = %ld and status = '%s'", OB_ALL_LS_TRANSFER_MEMBER_LIST_LOCK_INFO_TNAME, @@ -83,7 +84,7 @@ int ObTransferLockInfoOperator::remove(const uint64_t tenant_id, const share::Ob task_id, status.str()))) { LOG_WARN("failed to assign sql", K(ret), K(tenant_id), K(ls_id), K(task_id), K(status)); - } else if (OB_FAIL(sql_proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affected_rows))) { + } else if (OB_FAIL(sql_proxy.write(gen_meta_tenant_id(tenant_id), sql.ptr(), group_id, affected_rows))) { LOG_WARN("failed to write sql", K(ret), K(tenant_id), K(sql), K(affected_rows)); } else if (OB_UNLIKELY(0 == affected_rows)) { ret = OB_ENTRY_NOT_EXIST; @@ -106,14 +107,14 @@ int ObTransferLockInfoOperator::remove(const uint64_t tenant_id, const share::Ob } int ObTransferLockInfoOperator::get(const ObTransferLockInfoRowKey &row_key, const int64_t task_id, - const ObTransferLockStatus &status, const bool for_update, ObTransferTaskLockInfo &lock_info, + const ObTransferLockStatus &status, const bool for_update, const int32_t group_id, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy) { int ret = OB_SUCCESS; const uint64_t tenant_id = row_key.tenant_id_; - if (OB_UNLIKELY(!row_key.is_valid())) { + if (OB_UNLIKELY(!row_key.is_valid() || group_id < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id), K(for_update)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id), K(for_update), K(group_id)); } else { ObSqlString sql; SMART_VAR(ObISQLClient::ReadResult, result) @@ -127,7 +128,7 @@ int ObTransferLockInfoOperator::get(const ObTransferLockInfoRowKey &row_key, con status.str(), for_update ? " FOR UPDATE" : ""))) { LOG_WARN("fail to assign sql", K(ret), K(tenant_id), K(row_key), K(task_id), K(status), K(for_update)); - } else if (OB_FAIL(sql_proxy.read(result, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + } else if (OB_FAIL(sql_proxy.read(result, gen_meta_tenant_id(tenant_id), sql.ptr(), group_id))) { LOG_WARN("execute sql failed", K(ret), K(tenant_id), K(sql)); } else if (OB_ISNULL(result.get_result())) { ret = OB_ERR_UNEXPECTED; @@ -143,12 +144,12 @@ int ObTransferLockInfoOperator::get(const ObTransferLockInfoRowKey &row_key, con } int ObTransferLockInfoOperator::fetch_all(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, - common::ObArray &lock_infos) + const int32_t group_id, common::ObArray &lock_infos) { int ret = OB_SUCCESS; - if (OB_INVALID_ID == tenant_id) { + if (OB_INVALID_ID == tenant_id || group_id < 0) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id)); + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(group_id)); } else { ObSqlString sql; SMART_VAR(ObISQLClient::ReadResult, result) @@ -157,7 +158,7 @@ int ObTransferLockInfoOperator::fetch_all(common::ObISQLClient &sql_proxy, const OB_ALL_LS_TRANSFER_MEMBER_LIST_LOCK_INFO_TNAME, tenant_id))) { LOG_WARN("fail to assign sql", K(ret), K(tenant_id)); - } else if (OB_FAIL(sql_proxy.read(result, gen_meta_tenant_id(tenant_id), sql.ptr()))) { + } else if (OB_FAIL(sql_proxy.read(result, gen_meta_tenant_id(tenant_id), sql.ptr(), group_id))) { LOG_WARN("execute sql failed", K(ret), K(tenant_id), K(sql)); } else if (OB_ISNULL(result.get_result())) { ret = OB_ERR_UNEXPECTED; @@ -275,4 +276,4 @@ int ObTransferLockInfoOperator::parse_sql_results_( } } // namespace storage -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/storage/high_availability/ob_transfer_lock_info_operator.h b/src/storage/high_availability/ob_transfer_lock_info_operator.h index 61bda8ebe2..170d5e556b 100644 --- a/src/storage/high_availability/ob_transfer_lock_info_operator.h +++ b/src/storage/high_availability/ob_transfer_lock_info_operator.h @@ -28,12 +28,12 @@ public: ObTransferLockInfoOperator() = default; ~ObTransferLockInfoOperator() = default; - static int insert(const ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy); + static int insert(const ObTransferTaskLockInfo &lock_info, const int32_t group_id, common::ObISQLClient &sql_proxy); static int remove(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const ObTransferLockStatus &status, common::ObISQLClient &sql_proxy); + const ObTransferLockStatus &status, const int32_t group_id, common::ObISQLClient &sql_proxy); static int get(const ObTransferLockInfoRowKey &row_key, const int64_t task_id, const ObTransferLockStatus &status, - const bool for_update, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy); - static int fetch_all(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, + const bool for_update, const int32_t group_id, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy); + static int fetch_all(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, const int32_t group_id, common::ObArray &lock_infos); private: @@ -46,4 +46,4 @@ private: } // namespace storage } // namespace oceanbase -#endif \ No newline at end of file +#endif diff --git a/src/storage/high_availability/ob_transfer_lock_utils.cpp b/src/storage/high_availability/ob_transfer_lock_utils.cpp index ac94dd6f3f..dca1369324 100644 --- a/src/storage/high_availability/ob_transfer_lock_utils.cpp +++ b/src/storage/high_availability/ob_transfer_lock_utils.cpp @@ -53,7 +53,7 @@ static int get_ls_handle(const uint64_t tenant_id, const share::ObLSID &ls_id, s int ObMemberListLockUtils::batch_lock_ls_member_list(const uint64_t tenant_id, const int64_t task_id, const common::ObArray &lock_ls_list, const common::ObMemberList &member_list, - const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy) + const ObTransferLockStatus &status, const int32_t group_id, common::ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; ObArray sorted_ls_list; @@ -63,7 +63,7 @@ int ObMemberListLockUtils::batch_lock_ls_member_list(const uint64_t tenant_id, c std::sort(sorted_ls_list.begin(), sorted_ls_list.end()); for (int64_t i = 0; OB_SUCC(ret) && i < sorted_ls_list.count(); ++i) { const share::ObLSID &ls_id = sorted_ls_list.at(i); - if (OB_FAIL(lock_ls_member_list(tenant_id, ls_id, task_id, member_list, status, sql_proxy))) { + if (OB_FAIL(lock_ls_member_list(tenant_id, ls_id, task_id, member_list, status, group_id, sql_proxy))) { LOG_WARN("failed to lock ls member list", K(ret), K(ls_id), K(member_list)); } } @@ -73,7 +73,7 @@ int ObMemberListLockUtils::batch_lock_ls_member_list(const uint64_t tenant_id, c int ObMemberListLockUtils::lock_ls_member_list(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, const common::ObMemberList &member_list, const ObTransferLockStatus &status, - common::ObMySQLProxy &sql_proxy) + const int32_t group_id, common::ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; int64_t lock_owner = -1; @@ -83,14 +83,14 @@ int ObMemberListLockUtils::lock_ls_member_list(const uint64_t tenant_id, const s if (!ls_id.is_valid() || !member_list.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(ls_id)); - } else if (OB_FAIL(unlock_ls_member_list(tenant_id, ls_id, task_id, member_list, status, sql_proxy))) { + } else if (OB_FAIL(unlock_ls_member_list(tenant_id, ls_id, task_id, member_list, status, group_id, sql_proxy))) { LOG_WARN("failed to unlock ls member list", K(ret)); - } else if (OB_FAIL(get_lock_owner(tenant_id, ls_id, task_id, status, sql_proxy, lock_owner))) { + } else if (OB_FAIL(get_lock_owner(tenant_id, ls_id, task_id, status, group_id, sql_proxy, lock_owner))) { LOG_WARN("failed to get lock owner", K(ret), K(tenant_id), K(ls_id), K(task_id), K(status)); } else if (OB_FAIL(get_member_list_str(member_list, member_list_str))) { LOG_WARN("failed to get member list str", K(ret), K(member_list)); } else if (OB_FAIL( - insert_lock_info(tenant_id, ls_id, task_id, status, lock_owner, member_list_str.ptr(), real_lock_owner, sql_proxy))) { + insert_lock_info(tenant_id, ls_id, task_id, status, lock_owner, member_list_str.ptr(), group_id, real_lock_owner, sql_proxy))) { LOG_WARN("failed to insert lock info", K(ret), K(ls_id), K(task_id)); } else { #ifdef ERRSIM @@ -105,11 +105,11 @@ int ObMemberListLockUtils::lock_ls_member_list(const uint64_t tenant_id, const s bool need_lock_palf = false; if (OB_FAIL(lock_info.set(tenant_id, ls_id, task_id, status, real_lock_owner, member_list_str.ptr()))) { LOG_WARN("failed to set lock info", K(ret), K(tenant_id), K(ls_id), K(task_id), K(status), K(real_lock_owner)); - } else if (OB_FAIL(get_config_change_lock_stat_(lock_info, palf_lock_owner, palf_is_locked))) { + } else if (OB_FAIL(get_config_change_lock_stat_(lock_info, group_id, palf_lock_owner, palf_is_locked))) { LOG_WARN("failed to get config change lock stat", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(check_lock_status_(palf_is_locked, palf_lock_owner, real_lock_owner, need_lock_palf))) { LOG_WARN("failed to check lock status", K(ret), K(palf_is_locked), K(palf_lock_owner), K(real_lock_owner)); - } else if (need_lock_palf && OB_FAIL(try_lock_config_change_(lock_info, lock_timeout))) { + } else if (need_lock_palf && OB_FAIL(try_lock_config_change_(lock_info, lock_timeout, group_id))) { LOG_WARN("failed to try lock config config", K(ret), K(tenant_id), @@ -128,7 +128,7 @@ int ObMemberListLockUtils::lock_ls_member_list(const uint64_t tenant_id, const s int ObMemberListLockUtils::unlock_ls_member_list(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, const common::ObMemberList &member_list, const ObTransferLockStatus &status, - common::ObMySQLProxy &sql_proxy) + const int32_t group_id, common::ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -142,7 +142,7 @@ int ObMemberListLockUtils::unlock_ls_member_list(const uint64_t tenant_id, const const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); row_key.tenant_id_ = tenant_id; row_key.ls_id_ = ls_id; - if (OB_FAIL(trans.start(&sql_proxy, meta_tenant_id))) { + if (OB_FAIL(trans.start(&sql_proxy, meta_tenant_id, false/*with_snapshot*/, group_id))) { LOG_WARN("failed to start trans", K(ret), K(meta_tenant_id)); } else { int64_t lock_owner = -1; @@ -153,7 +153,7 @@ int ObMemberListLockUtils::unlock_ls_member_list(const uint64_t tenant_id, const bool need_unlock = true; bool need_relock_before_unlock = false; - if (OB_FAIL(ObTransferLockInfoOperator::get(row_key, task_id, status, for_update, lock_info, trans))) { + if (OB_FAIL(ObTransferLockInfoOperator::get(row_key, task_id, status, for_update, group_id, lock_info, trans))) { if (OB_ENTRY_NOT_EXIST == ret) { // palf need to be unlocked ret = OB_SUCCESS; LOG_INFO("member list already unlocked", @@ -166,18 +166,18 @@ int ObMemberListLockUtils::unlock_ls_member_list(const uint64_t tenant_id, const } else { LOG_WARN("failed to get lock info", K(ret), K(tenant_id), K(row_key)); } - } else if (OB_FAIL(get_config_change_lock_stat_(lock_info, palf_lock_owner, palf_is_locked))) { + } else if (OB_FAIL(get_config_change_lock_stat_(lock_info, group_id, palf_lock_owner, palf_is_locked))) { LOG_WARN("failed to get config change lock stat"); } else if (OB_FAIL(check_unlock_status_(palf_is_locked, palf_lock_owner, lock_info.lock_owner_, need_unlock, need_relock_before_unlock))) { LOG_WARN("failed to check unlock status", K(ret), K(palf_is_locked), K(palf_lock_owner), K(lock_info)); } else if (FALSE_IT(lock_owner = lock_info.lock_owner_)) { // assign lock owner - } else if (need_relock_before_unlock && OB_FAIL(relock_before_unlock_(lock_info, palf_lock_owner, lock_timeout))) { + } else if (need_relock_before_unlock && OB_FAIL(relock_before_unlock_(lock_info, palf_lock_owner, lock_timeout, group_id))) { LOG_WARN("failed to relock config change", K(ret), K(lock_info), K(palf_lock_owner)); - } else if (need_unlock && OB_FAIL(unlock_config_change_(lock_info, lock_timeout))) { + } else if (need_unlock && OB_FAIL(unlock_config_change_(lock_info, lock_timeout, group_id))) { LOG_WARN("failed to get paxos member list", K(ret), K(lock_info)); - } else if (OB_FAIL(ObTransferLockInfoOperator::remove(tenant_id, ls_id, task_id, status, trans))) { + } else if (OB_FAIL(ObTransferLockInfoOperator::remove(tenant_id, ls_id, task_id, status, group_id, trans))) { LOG_WARN("failed to update lock info", K(ret), K(row_key), K(lock_info), K(palf_lock_owner), K(palf_is_locked)); } else { #ifdef ERRSIM @@ -228,11 +228,12 @@ int ObMemberListLockUtils::unlock_ls_member_list(const uint64_t tenant_id, const return ret; } -int ObMemberListLockUtils::unlock_member_list_when_switch_to_standby(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy) +int ObMemberListLockUtils::unlock_member_list_when_switch_to_standby( + const uint64_t tenant_id, const int32_t group_id, common::ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; ObArray lock_infos; - if (OB_FAIL(ObTransferLockInfoOperator::fetch_all(sql_proxy, tenant_id, lock_infos))) { + if (OB_FAIL(ObTransferLockInfoOperator::fetch_all(sql_proxy, tenant_id, group_id, lock_infos))) { LOG_WARN("failed to fetch all lock info", K(ret), K(tenant_id)); } else if (lock_infos.empty()) { LOG_INFO("no need unlock member list when switch to standby", K(tenant_id)); @@ -249,6 +250,7 @@ int ObMemberListLockUtils::unlock_member_list_when_switch_to_standby(const uint6 task_id, fake_member_list, status, + group_id, sql_proxy))) { LOG_WARN("failed to unlock ls member list", K(ret), K(lock_info)); } @@ -258,7 +260,7 @@ int ObMemberListLockUtils::unlock_member_list_when_switch_to_standby(const uint6 } int ObMemberListLockUtils::try_lock_config_change_( - const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout) + const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id) { int ret = OB_SUCCESS; bool ls_exist = false; @@ -270,7 +272,7 @@ int ObMemberListLockUtils::try_lock_config_change_( storage::ObStorageRpc storage_rpc; if (OB_FAIL(init_storage_rpc_(storage_svr_rpc_proxy, storage_rpc))) { LOG_WARN("failed to init storage rpc", K(ret)); - } else if (OB_FAIL(inner_try_lock_config_change_(lock_info, lock_timeout, storage_rpc))) { + } else if (OB_FAIL(inner_try_lock_config_change_(lock_info, lock_timeout, group_id, storage_rpc))) { LOG_WARN("failed to try lock config change fallback", K(ret), K(lock_info)); } else { LOG_INFO("try lock config change fallback", K(lock_info), K(lock_timeout)); @@ -289,7 +291,7 @@ int ObMemberListLockUtils::try_lock_config_change_( } int ObMemberListLockUtils::inner_try_lock_config_change_( - const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, + const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id, storage::ObStorageRpc &storage_rpc) { int ret = OB_SUCCESS; @@ -301,14 +303,14 @@ int ObMemberListLockUtils::inner_try_lock_config_change_( } else if (OB_FAIL(ObStorageHAUtils::get_ls_leader(lock_info.tenant_id_, lock_info.ls_id_, src_info.src_addr_))) { LOG_WARN("failed to get ls leader", K(ret), K(lock_info)); } else if (OB_FAIL(storage_rpc.lock_config_change(lock_info.tenant_id_, src_info, lock_info.ls_id_, - lock_info.lock_owner_, lock_timeout))) { + lock_info.lock_owner_, lock_timeout, group_id))) { LOG_WARN("failed to try lock config config", K(ret), K(lock_info)); } return ret; } int ObMemberListLockUtils::get_config_change_lock_stat_( - const ObTransferTaskLockInfo &lock_info, int64_t &palf_lock_owner, bool &is_locked) + const ObTransferTaskLockInfo &lock_info, const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked) { int ret = OB_SUCCESS; bool ls_exist = false; @@ -319,7 +321,7 @@ int ObMemberListLockUtils::get_config_change_lock_stat_( storage::ObStorageRpc storage_rpc; if (OB_FAIL(init_storage_rpc_(storage_svr_rpc_proxy, storage_rpc))) { LOG_WARN("failed to init storage rpc", K(ret)); - } else if (OB_FAIL(get_config_change_lock_stat_fallback_(lock_info, palf_lock_owner, is_locked, storage_rpc))) { + } else if (OB_FAIL(get_config_change_lock_stat_fallback_(lock_info, group_id, palf_lock_owner, is_locked, storage_rpc))) { LOG_WARN("failed to get lock config change fallback", K(ret), K(lock_info)); } else { LOG_INFO("get lock config change stat fallback", K(lock_info), K(palf_lock_owner), K(is_locked)); @@ -338,7 +340,7 @@ int ObMemberListLockUtils::get_config_change_lock_stat_( } int ObMemberListLockUtils::get_config_change_lock_stat_fallback_( - const ObTransferTaskLockInfo &lock_info, int64_t &palf_lock_owner, + const ObTransferTaskLockInfo &lock_info, const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked, storage::ObStorageRpc &storage_rpc) { int ret = OB_SUCCESS; @@ -352,14 +354,14 @@ int ObMemberListLockUtils::get_config_change_lock_stat_fallback_( } else if (OB_FAIL(ObStorageHAUtils::get_ls_leader(lock_info.tenant_id_, lock_info.ls_id_, src_info.src_addr_))) { LOG_WARN("failed to get ls leader", K(ret), K(lock_info)); } else if (OB_FAIL(storage_rpc.get_config_change_lock_stat(lock_info.tenant_id_, src_info, - lock_info.ls_id_, palf_lock_owner, is_locked))) { + lock_info.ls_id_, group_id, palf_lock_owner, is_locked))) { LOG_WARN("failed to get config change lock stat", K(ret), K(lock_info)); } return ret; } int ObMemberListLockUtils::unlock_config_change_( - const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout) + const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id) { int ret = OB_SUCCESS; bool ls_exist = false; @@ -371,7 +373,7 @@ int ObMemberListLockUtils::unlock_config_change_( storage::ObStorageRpc storage_rpc; if (OB_FAIL(init_storage_rpc_(storage_svr_rpc_proxy, storage_rpc))) { LOG_WARN("failed to init storage rpc", K(ret)); - } else if (OB_FAIL(unlock_config_change_fallback_(lock_info, lock_timeout, storage_rpc))) { + } else if (OB_FAIL(unlock_config_change_fallback_(lock_info, lock_timeout, group_id, storage_rpc))) { LOG_WARN("failed to try lock config change fallback", K(ret), K(lock_info)); } else { LOG_INFO("unlock lock config change fallback", K(lock_info), K(lock_timeout)); @@ -390,7 +392,8 @@ int ObMemberListLockUtils::unlock_config_change_( } int ObMemberListLockUtils::unlock_config_change_fallback_( - const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, storage::ObStorageRpc &storage_rpc) + const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id, + storage::ObStorageRpc &storage_rpc) { int ret = OB_SUCCESS; ObStorageHASrcInfo src_info; @@ -401,7 +404,7 @@ int ObMemberListLockUtils::unlock_config_change_fallback_( } else if (OB_FAIL(ObStorageHAUtils::get_ls_leader(lock_info.tenant_id_, lock_info.ls_id_, src_info.src_addr_))) { LOG_WARN("failed to get ls leader", K(ret), K(lock_info)); } else if (OB_FAIL(storage_rpc.unlock_config_change(lock_info.tenant_id_, src_info, lock_info.ls_id_, - lock_info.lock_owner_, lock_timeout))) { + lock_info.lock_owner_, lock_timeout, group_id))) { LOG_WARN("failed to try lock config config", K(ret), K(lock_info)); } return ret; @@ -410,7 +413,7 @@ int ObMemberListLockUtils::unlock_config_change_fallback_( /* ObMemberListLockUtils */ int ObMemberListLockUtils::get_lock_owner(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const ObTransferLockStatus &status, common::ObMySQLProxy &proxy, int64_t &lock_owner) + const ObTransferLockStatus &status, const int32_t group_id, common::ObMySQLProxy &proxy, int64_t &lock_owner) { int ret = OB_SUCCESS; share::ObCommonID common_id; @@ -419,7 +422,7 @@ int ObMemberListLockUtils::get_lock_owner(const uint64_t tenant_id, const share: if (!is_valid_tenant_id(tenant_id) || !ls_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant id not valid", K(ret), K(tenant_id)); - } else if (OB_FAIL(ObCommonIDUtils::gen_monotonic_id(meta_tenant_id, id_type, proxy, common_id))) { + } else if (OB_FAIL(ObCommonIDUtils::gen_monotonic_id(meta_tenant_id, id_type, group_id, proxy, common_id))) { LOG_WARN("failed to gen monotonic id", K(ret), K(meta_tenant_id)); } else { lock_owner = common_id.id(); @@ -454,7 +457,7 @@ int ObMemberListLockUtils::get_member_list_str( int ObMemberListLockUtils::insert_lock_info(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, const ObTransferLockStatus &status, const int64_t lock_owner, const common::ObString &comment, - int64_t &real_lock_owner, common::ObISQLClient &sql_proxy) + const int32_t group_id, int64_t &real_lock_owner, common::ObISQLClient &sql_proxy) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -465,7 +468,7 @@ int ObMemberListLockUtils::insert_lock_info(const uint64_t tenant_id, const shar const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); if (OB_FAIL(lock_info.set(tenant_id, ls_id, task_id, status, lock_owner, comment))) { LOG_WARN("failed to set lock info", K(ret), K(tenant_id), K(ls_id), K(status), K(real_lock_owner)); - } else if (OB_FAIL(trans.start(&sql_proxy, meta_tenant_id))) { + } else if (OB_FAIL(trans.start(&sql_proxy, meta_tenant_id, false/*with_snapshot*/, group_id))) { LOG_WARN("failed to start trans", K(ret), K(meta_tenant_id)); } else { if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans, true/*for_update*/, tenant_info))) { @@ -473,11 +476,11 @@ int ObMemberListLockUtils::insert_lock_info(const uint64_t tenant_id, const shar } else if (!tenant_info.is_primary()) { ret = OB_OP_NOT_ALLOW; LOG_WARN("tenant is not primary, do not allow insert lock info", K(tenant_id), K(tenant_info)); - } else if (OB_FAIL(ObTransferLockInfoOperator::insert(lock_info, trans))) { + } else if (OB_FAIL(ObTransferLockInfoOperator::insert(lock_info, group_id, trans))) { if (OB_ENTRY_EXIST == ret) { ret = OB_SUCCESS; ObTransferTaskLockInfo tmp_lock_info; - if (OB_FAIL(get_lock_info(tenant_id, ls_id, task_id, status, tmp_lock_info, trans))) { + if (OB_FAIL(get_lock_info(tenant_id, ls_id, task_id, status, group_id, tmp_lock_info, trans))) { LOG_WARN("failed to get lock info", K(ret), K(ls_id)); } else { real_lock_owner = tmp_lock_info.lock_owner_; @@ -497,7 +500,7 @@ int ObMemberListLockUtils::insert_lock_info(const uint64_t tenant_id, const shar } int ObMemberListLockUtils::get_lock_info(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const ObTransferLockStatus &status, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy) + const ObTransferLockStatus &status, const int32_t group_id, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy) { int ret = OB_SUCCESS; lock_info.reset(); @@ -508,7 +511,7 @@ int ObMemberListLockUtils::get_lock_info(const uint64_t tenant_id, const share:: if (!row_key.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(row_key)); - } else if (OB_FAIL(ObTransferLockInfoOperator::get(row_key, task_id, status, for_update, lock_info, sql_proxy))) { + } else if (OB_FAIL(ObTransferLockInfoOperator::get(row_key, task_id, status, for_update, group_id, lock_info, sql_proxy))) { LOG_WARN("failed to get lock info", K(ret), K(row_key), K(task_id), K(status)); } return ret; @@ -588,10 +591,10 @@ int ObMemberListLockUtils::check_unlock_status_( } int ObMemberListLockUtils::relock_before_unlock_(const ObTransferTaskLockInfo &lock_info, - const int64_t palf_lock_owner, const int64_t lock_timeout) + const int64_t palf_lock_owner, const int64_t lock_timeout, const int32_t group_id) { int ret = OB_SUCCESS; - if (OB_FAIL(try_lock_config_change_(lock_info, lock_timeout))) { + if (OB_FAIL(try_lock_config_change_(lock_info, lock_timeout, group_id))) { LOG_WARN("failed to try lock config change", K(ret), K(lock_info)); } else { LOG_WARN("relock before unlock", K(ret), K(lock_info)); diff --git a/src/storage/high_availability/ob_transfer_lock_utils.h b/src/storage/high_availability/ob_transfer_lock_utils.h index 8fdda729f4..9c422c7ddd 100644 --- a/src/storage/high_availability/ob_transfer_lock_utils.h +++ b/src/storage/high_availability/ob_transfer_lock_utils.h @@ -27,42 +27,42 @@ public: /* member list*/ static int batch_lock_ls_member_list(const uint64_t tenant_id, const int64_t task_id, const common::ObArray &lock_ls_list, const common::ObMemberList &member_list, - const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy); + const ObTransferLockStatus &status, const int32_t group_id, common::ObMySQLProxy &sql_proxy); static int lock_ls_member_list(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const common::ObMemberList &member_list, const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy); + const common::ObMemberList &member_list, const ObTransferLockStatus &status, const int32_t group_id, common::ObMySQLProxy &sql_proxy); static int unlock_ls_member_list(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const common::ObMemberList &member_list, const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy); + const common::ObMemberList &member_list, const ObTransferLockStatus &status, const int32_t group_id, common::ObMySQLProxy &sql_proxy); static int unlock_for_ob_admin(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t lock_id); public: /* interface used for primary switch over to standby */ - static int unlock_member_list_when_switch_to_standby(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy); + static int unlock_member_list_when_switch_to_standby(const uint64_t tenant_id, const int32_t group_id, common::ObMySQLProxy &sql_proxy); private: /* sql operator */ static int insert_lock_info(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, const ObTransferLockStatus &status, const int64_t lock_owner, const common::ObString &comment, - int64_t &real_lock_owner, common::ObISQLClient &sql_proxy); + const int32_t group_id, int64_t &real_lock_owner, common::ObISQLClient &sql_proxy); static int get_lock_info(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const ObTransferLockStatus &status, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy); + const ObTransferLockStatus &status, const int32_t group_id, ObTransferTaskLockInfo &lock_info, common::ObISQLClient &sql_proxy); private: /* monotonic id*/ static int get_lock_owner(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t task_id, - const ObTransferLockStatus &status, common::ObMySQLProxy &sql_proxy, int64_t &lock_owner); + const ObTransferLockStatus &status, const int32_t group_id, common::ObMySQLProxy &sql_proxy, int64_t &lock_owner); static int get_member_list_str(const common::ObMemberList &member_list, ObSqlString &member_list_str); private: /* palf lock config*/ - static int try_lock_config_change_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout); + static int try_lock_config_change_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id); static int inner_try_lock_config_change_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, - storage::ObStorageRpc &storage_rpc); - static int get_config_change_lock_stat_(const ObTransferTaskLockInfo &lock_info, + const int32_t group_id, storage::ObStorageRpc &storage_rpc); + static int get_config_change_lock_stat_(const ObTransferTaskLockInfo &lock_info, const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked); - static int get_config_change_lock_stat_fallback_(const ObTransferTaskLockInfo &lock_info, + static int get_config_change_lock_stat_fallback_(const ObTransferTaskLockInfo &lock_info, const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked, storage::ObStorageRpc &storage_rpc); - static int unlock_config_change_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout); - static int unlock_config_change_fallback_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, + static int unlock_config_change_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id); + static int unlock_config_change_fallback_(const ObTransferTaskLockInfo &lock_info, const int64_t lock_timeout, const int32_t group_id, storage::ObStorageRpc &storage_rpc); private: @@ -71,7 +71,7 @@ private: static int check_unlock_status_(const bool palf_is_locked, const int64_t palf_lock_owner, const int64_t inner_table_lock_owner, bool &need_unlock, bool &need_relock_before_unlock); static int relock_before_unlock_(const ObTransferTaskLockInfo &lock_info, const int64_t palf_lock_owner, - const int64_t lock_timeout); + const int64_t lock_timeout, const int32_t group_id); static int init_storage_rpc_(obrpc::ObStorageRpcProxy &storage_svr_rpc_proxy, storage::ObStorageRpc &storage_rpc); static void destory_storage_rpc_(obrpc::ObStorageRpcProxy &storage_svr_rpc_proxy, storage::ObStorageRpc &storage_rpc); diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index a6196dfc06..f261a85cc0 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -430,12 +430,65 @@ int ObLobManager::is_remote(ObLobAccessParam& param, bool& is_remote, common::Ob return ret; } +bool ObLobManager::is_remote_ret_can_retry(int ret) +{ + return (ret == OB_NOT_MASTER); +} + +int ObLobManager::lob_remote_query_with_retry( + ObLobAccessParam ¶m, + common::ObAddr& dst_addr, + ObLobQueryArg& arg, + int64_t timeout, + common::ObDataBuffer& rpc_buffer, + obrpc::ObStorageRpcProxy::SSHandle& handle) +{ + int ret = OB_SUCCESS; + ObLSService *ls_service = (MTL(ObLSService *)); + obrpc::ObStorageRpcProxy *svr_rpc_proxy = ls_service->get_storage_rpc_proxy(); + int64_t retry_max = REMOTE_LOB_QUERY_RETRY_MAX; + int64_t retry_cnt = 0; + bool is_continue = true; + do { + ret = svr_rpc_proxy->to(dst_addr).by(arg.tenant_id_) + .dst_cluster_id(GCONF.cluster_id) + .ratelimit(true).bg_flow(obrpc::ObRpcProxy::BACKGROUND_FLOW) + .timeout(timeout) + .lob_query(arg, rpc_buffer, handle); + if (OB_FAIL(ret)) { + LOG_WARN("failed to do remote query", K(ret), K(arg), K(dst_addr), K(timeout)); + if (is_remote_ret_can_retry(ret)) { + retry_cnt++; + switch (ret) { + case OB_NOT_MASTER: { + bool remote_bret = false; + // refresh leader + if (OB_FAIL(is_remote(param, remote_bret, dst_addr))) { + LOG_WARN("fail to refresh leader addr", K(ret), K(param)); + is_continue = false; + } else { + LOG_INFO("refresh leader location", K(retry_cnt), K(retry_max), K(remote_bret), K(dst_addr), K(param)); + } + break; + } + default: { + LOG_INFO("do nothing, just retry", K(ret), K(retry_cnt), K(retry_max)); + } + } + } else { + is_continue = false; + } + } else { + is_continue = false; + } + } while (is_continue && retry_cnt <= retry_max); + return ret; +} + int ObLobManager::query_remote(ObLobAccessParam& param, common::ObAddr& dst_addr, ObString& data) { int ret = OB_SUCCESS; ObLobLocatorV2 *lob_locator = param.lob_locator_; - ObLSService *ls_service = (MTL(ObLSService *)); - obrpc::ObStorageRpcProxy *svr_rpc_proxy = ls_service->get_storage_rpc_proxy(); obrpc::ObStorageRpcProxy::SSHandle handle; common::ObDataBuffer rpc_buffer; ObLobQueryRemoteReader reader; @@ -460,12 +513,7 @@ int ObLobManager::query_remote(ObLobAccessParam& param, common::ObAddr& dst_addr if (timeout < ObStorageRpcProxy::STREAM_RPC_TIMEOUT) { timeout = ObStorageRpcProxy::STREAM_RPC_TIMEOUT; } - ret = svr_rpc_proxy->to(dst_addr).by(arg.tenant_id_) - .dst_cluster_id(GCONF.cluster_id) - .ratelimit(true).bg_flow(obrpc::ObRpcProxy::BACKGROUND_FLOW) - .timeout(timeout) - .lob_query(arg, rpc_buffer, handle); - if (OB_FAIL(ret)) { + if (OB_FAIL(lob_remote_query_with_retry(param, dst_addr, arg, timeout, rpc_buffer, handle))) { LOG_WARN("failed to do remote query", K(ret), K(arg)); } else { ObLobQueryBlock block; @@ -1815,8 +1863,6 @@ int ObLobManager::getlength_remote(ObLobAccessParam& param, common::ObAddr& dst_ { int ret = OB_SUCCESS; ObLobLocatorV2 *lob_locator = param.lob_locator_; - ObLSService *ls_service = (MTL(ObLSService *)); - obrpc::ObStorageRpcProxy *svr_rpc_proxy = ls_service->get_storage_rpc_proxy(); obrpc::ObStorageRpcProxy::SSHandle handle; common::ObDataBuffer rpc_buffer; ObLobQueryBlock header; @@ -1847,13 +1893,8 @@ int ObLobManager::getlength_remote(ObLobAccessParam& param, common::ObAddr& dst_ if (timeout < ObStorageRpcProxy::STREAM_RPC_TIMEOUT) { timeout = ObStorageRpcProxy::STREAM_RPC_TIMEOUT; } - ret = svr_rpc_proxy->to(dst_addr).by(arg.tenant_id_) - .dst_cluster_id(GCONF.cluster_id) - .ratelimit(true).bg_flow(obrpc::ObRpcProxy::BACKGROUND_FLOW) - .timeout(timeout) - .lob_query(arg, rpc_buffer, handle); - if (OB_FAIL(ret)) { - LOG_WARN("failed to do remote query", K(ret)); + if (OB_FAIL(lob_remote_query_with_retry(param, dst_addr, arg, timeout, rpc_buffer, handle))) { + LOG_WARN("failed to do remote query", K(ret), K(arg)); } else { int64_t cur_position = rpc_buffer.get_position(); while (OB_SUCC(ret) && handle.has_more()) { @@ -3348,9 +3389,7 @@ int ObLobQueryIter::open(ObLobAccessParam ¶m, common::ObAddr dst_addr) } else if (OB_FAIL(remote_reader_.open(param, rpc_buffer_))) { LOG_WARN("failed to open remote reader", K(ret)); } else { - ObLSService *ls_service = (MTL(ObLSService *)); - obrpc::ObStorageRpcProxy *svr_rpc_proxy = ls_service->get_storage_rpc_proxy(); - const int64_t cluster_id = GCONF.cluster_id; + ObLobManager *lob_manager = MTL(ObLobManager*); // build arg query_arg_.tenant_id_ = param.tenant_id_; query_arg_.offset_ = param.offset_; @@ -3365,12 +3404,7 @@ int ObLobQueryIter::open(ObLobAccessParam ¶m, common::ObAddr dst_addr) if (timeout < ObStorageRpcProxy::STREAM_RPC_TIMEOUT) { timeout = ObStorageRpcProxy::STREAM_RPC_TIMEOUT; } - ret = svr_rpc_proxy->to(dst_addr).by(query_arg_.tenant_id_) - .dst_cluster_id(cluster_id) - .ratelimit(true).bg_flow(obrpc::ObRpcProxy::BACKGROUND_FLOW) - .timeout(timeout) - .lob_query(query_arg_, rpc_buffer_, handle_); - if (OB_FAIL(ret)) { + if (OB_FAIL(lob_manager->lob_remote_query_with_retry(param, dst_addr, query_arg_, timeout, rpc_buffer_, handle_))) { LOG_WARN("failed to do remote query", K(ret), K(query_arg_)); } else { param_ = param; diff --git a/src/storage/lob/ob_lob_manager.h b/src/storage/lob/ob_lob_manager.h index 02949f3b5a..a40ba5575f 100644 --- a/src/storage/lob/ob_lob_manager.h +++ b/src/storage/lob/ob_lob_manager.h @@ -139,6 +139,7 @@ public: static const int64_t LOB_WITH_OUTROW_CTX_SIZE = sizeof(ObLobCommon) + sizeof(ObLobData) + sizeof(ObLobDataOutRowCtx); static const int64_t LOB_OUTROW_FULL_SIZE = sizeof(ObLobCommon) + sizeof(ObLobData) + sizeof(ObLobDataOutRowCtx) + sizeof(uint64_t); static const uint64_t LOB_READ_BUFFER_LEN = 1024L*1024L; // 1M + static const uint64_t REMOTE_LOB_QUERY_RETRY_MAX = 10L; // 1M private: explicit ObLobManager(const uint64_t tenant_id) : tenant_id_(tenant_id), @@ -181,6 +182,14 @@ public: const ObString &data, common::ObCollationType coll_type, ObLobLocatorV2 &out); + int lob_remote_query_with_retry( + ObLobAccessParam ¶m, + common::ObAddr& dst_addr, + ObLobQueryArg& arg, + int64_t timeout, + common::ObDataBuffer& rpc_buffer, + obrpc::ObStorageRpcProxy::SSHandle& handle); + bool is_remote_ret_can_retry(int ret); // Tmp Delta Lob locator interface int process_delta(ObLobAccessParam& param, ObLobLocatorV2& lob_locator); diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index a69a967acc..78e703d45f 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -1394,6 +1394,25 @@ int ObLS::block_tx_start() return ret; } +int ObLS::block_all() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls is not inited", K(ret)); + } else { + int64_t read_lock = 0; + int64_t write_lock = LSLOCKSTORAGE | LSLOCKTX; + ObLSLockGuard lock_myself(this, lock_, read_lock, write_lock); + // protect with lock_ to make sure there is no tablet transfer in process doing. + // transfer in must use this lock too. + if (OB_FAIL(ls_tx_svr_.block_all())) { + LOG_WARN("block_all failed", K(get_ls_id())); + } + } + return ret; +} + int ObLS::tablet_transfer_in(const ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -2301,6 +2320,24 @@ int ObLS::set_restore_status( return ret; } +int ObLS::set_gc_state(const logservice::LSGCState &gc_state) +{ + SCN invalid_scn; + return set_gc_state(gc_state, invalid_scn); +} + +int ObLS::set_gc_state(const logservice::LSGCState &gc_state, const share::SCN &offline_scn) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ls is not inited", K(ret), K(ls_meta_)); + } else { + ret = ls_meta_.set_gc_state(gc_state, offline_scn); + } + return ret; +} + int ObLS::set_ls_rebuild() { int ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 3dfd872e05..e527dd1a2b 100755 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -322,6 +322,7 @@ public: // for ls gc int block_tablet_transfer_in(); int block_tx_start(); + int block_all(); // for tablet transfer // this function is used for tablet transfer in // it will check if it is allowed to transfer in and then @@ -380,7 +381,8 @@ public: int set_ls_rebuild(); // protect in ls lock // int set_gc_state(const logservice::LSGCState &gc_state); - DELEGATE_WITH_RET(ls_meta_, set_gc_state, int); + int set_gc_state(const logservice::LSGCState &gc_state); + int set_gc_state(const logservice::LSGCState &gc_state, const share::SCN &offline_scn); // int set_clog_checkpoint(const palf::LSN &clog_checkpoint_lsn, // const share::SCN &clog_checkpoint_scn, // const bool write_slog = true); @@ -414,10 +416,6 @@ public: // @param [in] gc state. // int get_gc_state(LSGCState &status); DELEGATE_WITH_RET(ls_meta_, get_gc_state, int); - // set offline ts - // @param [in] offline ts. - // int set_offline_scn(const int64_t offline_scn); - DELEGATE_WITH_RET(ls_meta_, set_offline_scn, int); // get offline ts // @param [in] offline ts. // int get_offline_scn(const share::SCN &offline_scn); diff --git a/src/storage/ls/ob_ls_meta.cpp b/src/storage/ls/ob_ls_meta.cpp index f9a5f218cc..b813948668 100644 --- a/src/storage/ls/ob_ls_meta.cpp +++ b/src/storage/ls/ob_ls_meta.cpp @@ -304,22 +304,26 @@ int ObLSMeta::get_migration_status(ObMigrationStatus &migration_status) const return ret; } -int ObLSMeta::set_gc_state(const logservice::LSGCState &gc_state) +int ObLSMeta::set_gc_state(const logservice::LSGCState &gc_state, const SCN &scn) { int ret = OB_SUCCESS; ObSpinLockTimeGuard guard(lock_); if (OB_FAIL(check_can_update_())) { LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else if (!ObGCHandler::is_valid_ls_gc_state(gc_state)) { + } else if (!ObGCHandler::is_valid_ls_gc_state(gc_state) + || (ObGCHandler::is_ls_offline_gc_state(gc_state) && !scn.is_valid()) + || (!ObGCHandler::is_ls_offline_gc_state(gc_state) && scn.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("gc_state invalid", K(ret), K(gc_state)); } else { ObLSMeta tmp(*this); tmp.gc_state_ = gc_state; + tmp.offline_scn_ = scn; if (OB_FAIL(write_slog_(tmp))) { LOG_WARN("gc_state write slog failed", K(ret)); } else { gc_state_ = gc_state; + offline_scn_ = scn; } } return ret; @@ -338,19 +342,6 @@ int ObLSMeta::get_gc_state(logservice::LSGCState &gc_state) return ret; } -int ObLSMeta::set_offline_scn(const SCN &offline_scn) -{ - // 不主动写slog - int ret = OB_SUCCESS; - ObSpinLockTimeGuard guard(lock_); - if (OB_FAIL(check_can_update_())) { - LOG_WARN("ls meta cannot update", K(ret), K(*this)); - } else { - offline_scn_ = offline_scn; - } - return ret; -} - int ObLSMeta::get_offline_scn(SCN &offline_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls_meta.h b/src/storage/ls/ob_ls_meta.h index 7a50508f7a..e3e1d4a5fe 100644 --- a/src/storage/ls/ob_ls_meta.h +++ b/src/storage/ls/ob_ls_meta.h @@ -60,9 +60,8 @@ public: int set_migration_status(const ObMigrationStatus &migration_status, const bool write_slog = true); int get_migration_status (ObMigrationStatus &migration_status) const; - int set_gc_state(const logservice::LSGCState &gc_state); + int set_gc_state(const logservice::LSGCState &gc_state, const share::SCN &offline_scn); int get_gc_state(logservice::LSGCState &gc_state); - int set_offline_scn(const share::SCN &offline_scn); int get_offline_scn(share::SCN &offline_scn); int set_restore_status(const share::ObLSRestoreStatus &restore_status); diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index c875d8d369..2d6283f1fe 100755 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -6180,7 +6180,7 @@ int ObLSTabletService::check_need_rollback_in_transfer_for_4377_(const transacti // read from origin tablet, if there's inconsistent, it should throw 4377 error // CASE 2.2: transfer_scn_ is valid, the transfer_scn_ is backfilled, it means that // tablet has started to transfer out. If it's the first transfer, we should - // throw 4377 erorr, otherwise we should compare the transfer_scn_ with the tx_scn, + // throw 4377 error, otherwise we should compare the transfer_scn_ with the tx_scn, // and decide to rollback the transaction or throw 4377 error. // However, we can not be sure about whether this transfer out is the first transfer or not. // So we consider all the transfer out status as not the first transfer here, to avoid diff --git a/src/storage/memtable/ob_lock_wait_mgr.cpp b/src/storage/memtable/ob_lock_wait_mgr.cpp index bd57418a13..f141bca4a7 100644 --- a/src/storage/memtable/ob_lock_wait_mgr.cpp +++ b/src/storage/memtable/ob_lock_wait_mgr.cpp @@ -36,7 +36,7 @@ namespace obmysql void request_finish_callback() { bool unused = false; - auto lock_wait_mgr = MTL(memtable::ObLockWaitMgr*); + memtable::ObLockWaitMgr *lock_wait_mgr = MTL(memtable::ObLockWaitMgr*); if (OB_ISNULL(lock_wait_mgr)) { TRANS_LOG(TRACE, "MTL(lock wait mgr) is null", K(MTL_ID())); } else { @@ -513,18 +513,18 @@ ObLink* ObLockWaitMgr::check_timeout() iter->on_retry_lock(hash); TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "LOCK_MGR: req wait lock cost too much time", K(curr_lock_seq), K(last_lock_seq), K(*iter)); } else { - auto tx_desc = session_info->get_tx_desc(); + transaction::ObTxDesc *&tx_desc = session_info->get_tx_desc(); bool ac = false, has_explicit_start_tx = session_info->has_explicit_start_trans(); session_info->get_autocommit(ac); if (OB_ISNULL(tx_desc) && (!ac || has_explicit_start_tx)) { - auto session_id = session_info->get_sessid(); - auto &trace_id = session_info->get_current_trace_id(); + uint32_t session_id = session_info->get_sessid(); + const common::ObCurTraceId::TraceId &trace_id = session_info->get_current_trace_id(); TRANS_LOG(WARN, "LOG_MGR: found session ac = 0 or has_explicit_start_trans but txDesc was released!", K(session_id), K(trace_id), K(ac), K(has_explicit_start_tx)); } } if (OB_NOT_NULL(session_info)) { - auto tx_desc = session_info->get_tx_desc(); + transaction::ObTxDesc *&tx_desc = session_info->get_tx_desc(); TRANS_LOG(INFO, "check transaction state", KP(tx_desc)); } } @@ -583,15 +583,15 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, Key key(&row_key); uint64_t &hold_key = get_thread_hold_key(); if (OB_TRY_LOCK_ROW_CONFLICT == tmp_ret) { - auto row_hash = hash_rowkey(tablet_id, key); - auto tx_hash = hash_trans(holder_tx_id); - auto row_lock_seq = get_seq(row_hash); - auto tx_lock_seq = get_seq(tx_hash); + uint64_t row_hash = hash_rowkey(tablet_id, key); + uint64_t tx_hash = hash_trans(holder_tx_id); + int64_t row_lock_seq = get_seq(row_hash); + int64_t tx_lock_seq = get_seq(tx_hash); bool locked = false, wait_on_row = true; if (OB_FAIL(rechecker(locked, wait_on_row))) { TRANS_LOG(WARN, "recheck lock fail", K(key), K(holder_tx_id)); } else if (locked) { - auto hash = wait_on_row ? row_hash : tx_hash; + uint64_t hash = wait_on_row ? row_hash : tx_hash; if (hold_key == hash) { hold_key = 0; } @@ -636,7 +636,7 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, TRANS_LOG(WARN, "lock wait mgr not inited", K(ret)); } else if (NULL == (node = get_thread_node())) { } else if (OB_TRY_LOCK_ROW_CONFLICT == tmp_ret) { - auto hash = LockHashHelper::hash_lock_id(lock_id); + uint64_t hash = LockHashHelper::hash_lock_id(lock_id); const bool need_delay = is_remote_sql; char lock_id_buf[common::MAX_LOCK_ID_BUF_LENGTH]; lock_id.to_string(lock_id_buf, sizeof(lock_id_buf)); @@ -648,7 +648,7 @@ int ObLockWaitMgr::post_lock(const int tmp_ret, if (need_delay) { delay_header_node_run_ts(hash); } - auto lock_seq = get_seq(hash); + int64_t lock_seq = get_seq(hash); bool need_wait = false; if (OB_FAIL(check_need_wait(need_wait))) { TRANS_LOG(WARN, "check need wait failed", K(ret)); diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 817a7a23e0..1ac856c798 100755 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -756,14 +756,14 @@ int ObMemtable::get( char *trans_info_ptr = nullptr; if (param.need_trans_info()) { int64_t length = concurrency_control::ObTransStatRow::MAX_TRANS_STRING_SIZE; - if (OB_ISNULL(trans_info_ptr = static_cast(context.allocator_->alloc(length)))) { + if (OB_ISNULL(trans_info_ptr = static_cast(context.stmt_allocator_->alloc(length)))) { ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to alloc memory", K(ret)); } } if (OB_FAIL(ret)) { // do nothing - } else if (OB_FAIL(row.init(*context.allocator_, request_cnt, trans_info_ptr))) { + } else if (OB_FAIL(row.init(*context.stmt_allocator_, request_cnt, trans_info_ptr))) { STORAGE_LOG(WARN, "Failed to init datum row", K(ret), K(param.need_trans_info())); } } @@ -1121,7 +1121,7 @@ int ObMemtable::lock_row_on_frozen_stores_( int ret = OB_SUCCESS; ObStoreRowLockState &lock_state = res.lock_state_; ObStoreCtx &ctx = *(context.store_ctx_); - const auto reader_seq_no = ctx.mvcc_acc_ctx_.snapshot_.scn_; + const ObTxSEQ reader_seq_no = ctx.mvcc_acc_ctx_.snapshot_.scn_; if (OB_ISNULL(value) || !ctx.mvcc_acc_ctx_.is_write() || NULL == key) { TRANS_LOG(WARN, "invalid param", KP(value), K(ctx), KP(key)); ret = OB_INVALID_ARGUMENT; @@ -2636,7 +2636,7 @@ int ObMemtable::set_( ObStoreRowkey tmp_key; ObMemtableKey mtk; ObStoreCtx &ctx = *(context.store_ctx_); - auto *mem_ctx = ctx.mvcc_acc_ctx_.get_mem_ctx(); + ObMemtableCtx *mem_ctx = ctx.mvcc_acc_ctx_.get_mem_ctx(); //set_begin(ctx.mvcc_acc_ctx_); @@ -2938,7 +2938,7 @@ int ObMemtable::mvcc_write_( // cannot be serializable when transaction set violation if (OB_TRANSACTION_SET_VIOLATION == ret) { - auto iso = ctx.mvcc_acc_ctx_.tx_desc_->get_isolation_level(); + ObTxIsolationLevel iso = ctx.mvcc_acc_ctx_.tx_desc_->get_isolation_level(); if (ObTxIsolationLevel::SERIAL == iso || ObTxIsolationLevel::RR == iso) { ret = OB_TRANS_CANNOT_SERIALIZE; } @@ -2957,8 +2957,8 @@ int ObMemtable::post_row_write_conflict_(ObMvccAccessCtx &acc_ctx, { int ret = OB_TRY_LOCK_ROW_CONFLICT; ObLockWaitMgr *lock_wait_mgr = NULL; - auto conflict_tx_id = lock_state.lock_trans_id_; - auto mem_ctx = acc_ctx.get_mem_ctx(); + ObTransID conflict_tx_id = lock_state.lock_trans_id_; + ObMemtableCtx *mem_ctx = acc_ctx.get_mem_ctx(); int64_t current_ts = common::ObClockGenerator::getClock(); int64_t lock_wait_start_ts = mem_ctx->get_lock_wait_start_ts() > 0 ? mem_ctx->get_lock_wait_start_ts() @@ -2975,15 +2975,15 @@ int ObMemtable::post_row_write_conflict_(ObMvccAccessCtx &acc_ctx, mem_ctx->add_conflict_trans_id(conflict_tx_id); mem_ctx->on_wlock_retry(row_key, conflict_tx_id); int tmp_ret = OB_SUCCESS; - auto tx_ctx = acc_ctx.tx_ctx_; - auto tx_id = acc_ctx.get_tx_id(); + transaction::ObPartTransCtx *tx_ctx = acc_ctx.tx_ctx_; + transaction::ObTransID tx_id = acc_ctx.get_tx_id(); bool remote_tx = tx_ctx->get_scheduler() != tx_ctx->get_addr(); ObFunction recheck_func([&](bool &locked, bool &wait_on_row) -> int { int ret = OB_SUCCESS; lock_state.is_locked_ = false; if (lock_state.is_delayed_cleanout_) { - auto lock_data_sequence = lock_state.lock_data_sequence_; - auto &tx_table_guards = acc_ctx.get_tx_table_guards(); + transaction::ObTxSEQ lock_data_sequence = lock_state.lock_data_sequence_; + storage::ObTxTableGuards &tx_table_guards = acc_ctx.get_tx_table_guards(); if (OB_FAIL(tx_table_guards.check_row_locked( tx_id, conflict_tx_id, lock_data_sequence, lock_state.trans_scn_, lock_state))) { TRANS_LOG(WARN, "re-check row locked via tx_table fail", K(ret), K(tx_id), K(lock_state)); diff --git a/src/storage/memtable/ob_memtable_iterator.cpp b/src/storage/memtable/ob_memtable_iterator.cpp index 33b013e8fc..66606e49e0 100755 --- a/src/storage/memtable/ob_memtable_iterator.cpp +++ b/src/storage/memtable/ob_memtable_iterator.cpp @@ -80,7 +80,7 @@ int ObMemtableGetIterator::init( const ObITableReadInfo *read_info = param.get_read_info(context.use_fuse_row_cache_); if (param.need_trans_info()) { int64_t length = concurrency_control::ObTransStatRow::MAX_TRANS_STRING_SIZE; - if (OB_ISNULL(trans_info_ptr = static_cast(context.allocator_->alloc(length)))) { + if (OB_ISNULL(trans_info_ptr = static_cast(context.stmt_allocator_->alloc(length)))) { ret = OB_ALLOCATE_MEMORY_FAILED; TRANS_LOG(WARN, "fail to alloc memory", K(ret)); } @@ -90,7 +90,7 @@ int ObMemtableGetIterator::init( } else if (OB_UNLIKELY(nullptr == read_info || !read_info->is_valid())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "Unexpected read info", K(ret), KPC(read_info)); - } else if (OB_FAIL(cur_row_.init(*context.allocator_, read_info->get_request_count(), trans_info_ptr))) { + } else if (OB_FAIL(cur_row_.init(*context.stmt_allocator_, read_info->get_request_count(), trans_info_ptr))) { STORAGE_LOG(WARN, "Failed to init datum row", K(ret)); } else { param_ = ¶m; @@ -206,7 +206,7 @@ int ObMemtableScanIterator::init( if (param.need_trans_info()) { int64_t length = concurrency_control::ObTransStatRow::MAX_TRANS_STRING_SIZE; - if (OB_ISNULL(trans_info_ptr = static_cast(context.allocator_->alloc(length)))) { + if (OB_ISNULL(trans_info_ptr = static_cast(context.stmt_allocator_->alloc(length)))) { ret = OB_ALLOCATE_MEMORY_FAILED; TRANS_LOG(WARN, "fail to alloc memory", K(ret)); } @@ -216,7 +216,7 @@ int ObMemtableScanIterator::init( } else if (OB_ISNULL(read_info_ = param.get_read_info(false))) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "Unexpected null read info", K(ret), K(param)); - } else if (OB_FAIL(row_.init(*context.allocator_, + } else if (OB_FAIL(row_.init(*context.stmt_allocator_, read_info_->get_request_count(), trans_info_ptr))) { TRANS_LOG(WARN, "Failed to init datum row", K(ret), K(param.need_trans_info())); @@ -476,7 +476,7 @@ int ObMemtableMGetIterator::init( const ObITableReadInfo *read_info = param.get_read_info(); if (param.need_trans_info()) { int64_t length = concurrency_control::ObTransStatRow::MAX_TRANS_STRING_SIZE; - if (OB_ISNULL(trans_info_ptr = static_cast(context.allocator_->alloc(length)))) { + if (OB_ISNULL(trans_info_ptr = static_cast(context.stmt_allocator_->alloc(length)))) { ret = OB_ALLOCATE_MEMORY_FAILED; TRANS_LOG(WARN, "fail to alloc memory", K(ret)); } @@ -489,7 +489,7 @@ int ObMemtableMGetIterator::init( } else if (OB_UNLIKELY(nullptr == read_info || !read_info->is_valid())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "Unexpected read info", K(ret), KPC(read_info)); - } else if (OB_FAIL(cur_row_.init(*context.allocator_, read_info->get_request_count(), trans_info_ptr))) { + } else if (OB_FAIL(cur_row_.init(*context.stmt_allocator_, read_info->get_request_count(), trans_info_ptr))) { TRANS_LOG(WARN, "Failed to init datum row", K(ret)); } else { const ObColDescIArray &out_cols = read_info->get_columns_desc(); @@ -812,7 +812,7 @@ int ObMemtableMultiVersionScanIterator::init( TRANS_LOG(WARN, "mvcc engine scan fail", K(ret), K(mvcc_scan_range)); } else if (OB_FAIL(bitmap_.init(read_info_->get_request_count(), read_info_->get_rowkey_count()))) { TRANS_LOG(WARN, "init nop bitmap fail, ", K(ret)); - } else if (OB_FAIL(row_.init(*context.allocator_, read_info_->get_request_count()))) { + } else if (OB_FAIL(row_.init(*context.stmt_allocator_, read_info_->get_request_count()))) { TRANS_LOG(WARN, "Failed to init datum row", K(ret)); } else { TRANS_LOG(INFO, "multi version scan iterator init succ", K(param.table_id_), K(range), KPC(read_info_), K(row_)); diff --git a/src/storage/memtable/ob_row_conflict_handler.cpp b/src/storage/memtable/ob_row_conflict_handler.cpp index 488ffc4fee..08f08ba3d7 100644 --- a/src/storage/memtable/ob_row_conflict_handler.cpp +++ b/src/storage/memtable/ob_row_conflict_handler.cpp @@ -100,7 +100,6 @@ int ObRowConflictHandler::post_row_read_conflict(ObMvccAccessCtx &acc_ctx, int ret = OB_TRY_LOCK_ROW_CONFLICT; ObLockWaitMgr *lock_wait_mgr = NULL; ObTransID conflict_tx_id = lock_state.lock_trans_id_; - // auto mem_ctx = acc_ctx.get_mem_ctx(); ObTxDesc *tx_desc = acc_ctx.get_tx_desc(); int64_t current_ts = common::ObClockGenerator::getClock(); int64_t lock_wait_start_ts = acc_ctx.get_lock_wait_start_ts() > 0 @@ -134,8 +133,8 @@ int ObRowConflictHandler::post_row_read_conflict(ObMvccAccessCtx &acc_ctx, int ret = OB_SUCCESS; lock_state.is_locked_ = false; if (lock_state.is_delayed_cleanout_) { - auto lock_data_sequence = lock_state.lock_data_sequence_; - auto &tx_table_guards = acc_ctx.get_tx_table_guards(); + ObTxSEQ lock_data_sequence = lock_state.lock_data_sequence_; + ObTxTableGuards &tx_table_guards = acc_ctx.get_tx_table_guards(); if (OB_FAIL(tx_table_guards.check_row_locked( tx_id, conflict_tx_id, lock_data_sequence, trans_scn, lock_state))) { TRANS_LOG(WARN, "re-check row locked via tx_table fail", K(ret), K(tx_id), K(lock_state)); diff --git a/src/storage/multi_data_source/mds_table_handle.h b/src/storage/multi_data_source/mds_table_handle.h index 83f2c0fb12..e03e40fa04 100644 --- a/src/storage/multi_data_source/mds_table_handle.h +++ b/src/storage/multi_data_source/mds_table_handle.h @@ -45,7 +45,6 @@ public: int fill_virtual_info(ObIArray &mds_node_info_array) const; int mark_removed_from_t3m(ObTabletPointer *pointer) const; int mark_switched_to_empty_shell() const; - int is_switched_to_empty_shell(bool &is_switched_to_empty_shell) const; template int forcely_reset_mds_table(const char (&reason)[N]); /******************************Single Key Unit Access Interface**********************************/ diff --git a/src/storage/multi_data_source/mds_table_handle.ipp b/src/storage/multi_data_source/mds_table_handle.ipp index 1c7a1f8287..924e761c04 100644 --- a/src/storage/multi_data_source/mds_table_handle.ipp +++ b/src/storage/multi_data_source/mds_table_handle.ipp @@ -711,19 +711,6 @@ inline int MdsTableHandle::mark_switched_to_empty_shell() const return ret; } -inline int MdsTableHandle::is_switched_to_empty_shell(bool &is_switched_to_empty_shell) const -{ - bool ret = OB_SUCCESS; - CHECK_MDS_TABLE_INIT(); - if (!p_mds_table_base_.is_valid()) { - ret = OB_BAD_NULL_ERROR; - MDS_LOG(WARN, "p_mds_table_base_ is invalid", K(*this)); - } else { - is_switched_to_empty_shell = p_mds_table_base_->is_switched_to_empty_shell(); - } - return ret; -} - template inline int MdsTableHandle::forcely_reset_mds_table(const char (&reason)[N]) { diff --git a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp index 386dc6ad51..79433ddb7d 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp +++ b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.cpp @@ -89,6 +89,7 @@ int ObTenantMdsAllocator::init() { int ret = OB_SUCCESS; ObMemAttr mem_attr; + // TODO : @gengli new ctx id? mem_attr.tenant_id_ = MTL_ID(); mem_attr.ctx_id_ = ObCtxIds::MDS_DATA_ID; mem_attr.label_ = "MdsTable"; @@ -102,34 +103,17 @@ int ObTenantMdsAllocator::init() return ret; } -constexpr int64_t HEADER_SIZE = sizeof(const char *) + sizeof(uint16_t); - void *ObTenantMdsAllocator::alloc(const int64_t size) { - const char **related_mds_type = nullptr; - uint16_t *alloc_size = 0; - char *obj = nullptr;// C++ standard: sizeof(char) == 1 - if (size >= UINT16_MAX) { - MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, - "too big mds data size, if you make sure you need so big data, do not use mds allocator", K(size)); - } else { - obj = (char *)allocator_.alloc(HEADER_SIZE + size); - related_mds_type = (const char **)obj; - obj += sizeof(const char *); - alloc_size = (uint16_t *)obj; - obj += sizeof(uint16_t); - *related_mds_type = __thread_mds_alloc_type__; - *alloc_size = size; - alloc_info_map_.record_alloc_info(*related_mds_type, *alloc_size); - MDS_LOG(DEBUG, "mds alloc ", K(size), KP(obj)); - if (OB_NOT_NULL(obj)) { - MTL(ObTenantMdsService*)->record_alloc_backtrace(obj, - __thread_mds_tag__, - __thread_mds_alloc_type__, - __thread_mds_alloc_file__, - __thread_mds_alloc_func__, - __thread_mds_alloc_line__);// for debug mem leak - } + void *obj = allocator_.alloc(size); + MDS_LOG(DEBUG, "mds alloc ", K(size), KP(obj)); + if (OB_NOT_NULL(obj)) { + MTL(ObTenantMdsService*)->record_alloc_backtrace(obj, + __thread_mds_tag__, + __thread_mds_alloc_type__, + __thread_mds_alloc_file__, + __thread_mds_alloc_func__, + __thread_mds_alloc_line__);// for debug mem leak } return obj; } @@ -144,14 +128,8 @@ void *ObTenantMdsAllocator::alloc(const int64_t size, const ObMemAttr &attr) void ObTenantMdsAllocator::free(void *ptr) { - char *free_ptr = (char *)ptr; - if (OB_NOT_NULL(free_ptr)) { - alloc_info_map_.record_free_info(*(const char **)(free_ptr - sizeof(uint16_t) - sizeof(const char *)), - *(uint16_t*)(free_ptr - sizeof(uint16_t))); - free_ptr -= HEADER_SIZE; - allocator_.free(free_ptr); - MTL(ObTenantMdsService*)->erase_alloc_backtrace(ptr); - } + allocator_.free(ptr); + MTL(ObTenantMdsService*)->erase_alloc_backtrace(ptr); } void ObTenantMdsAllocator::set_attr(const ObMemAttr &attr) @@ -223,6 +201,10 @@ int ObTenantMdsService::mtl_start(ObTenantMdsService *&mds_service) 3_s, [mds_service]() -> bool { ObCurTraceId::init(GCONF.self_addr_); + if (REACH_TIME_INTERVAL(30_s)) { + observer::ObMdsEventBuffer::dump_statistics(); + mds_service->dump_map_holding_item(5_min); + } mds_service->mds_timer_.try_recycle_mds_table_task(); return false;// won't stop until tenant exit } @@ -238,19 +220,6 @@ int ObTenantMdsService::mtl_start(ObTenantMdsService *&mds_service) } ))) { MDS_LOG(ERROR, "fail to register dump mds table status task to timer", KR(ret), KPC(mds_service)); - } else if (MDS_FAIL(mds_service->mds_timer_.timer_.schedule_task_repeat( - mds_service->mds_timer_.dump_memory_statistics_task_handle_, - 30_s, - [mds_service]() -> bool { - mds_service->mds_allocator_.dump(); - mds_service->dump_map_holding_item(5_min); - if (REACH_TIME_INTERVAL(30_s)) { - observer::ObMdsEventBuffer::dump_statistics(); - } - return false;// won't stop until tenant exit - } - ))) { - MDS_LOG(ERROR, "fail to register memory dump timer task", KR(ret), KPC(mds_service)); } return ret; } @@ -260,7 +229,6 @@ void ObTenantMdsService::mtl_stop(ObTenantMdsService *&mds_service) if (nullptr != mds_service) { mds_service->mds_timer_.recycle_task_handle_.stop(); mds_service->mds_timer_.dump_special_mds_table_status_task_handle_.stop(); - mds_service->mds_timer_.dump_memory_statistics_task_handle_.stop(); } } @@ -269,7 +237,6 @@ void ObTenantMdsService::mtl_wait(ObTenantMdsService *&mds_service) if (nullptr != mds_service) { mds_service->mds_timer_.recycle_task_handle_.wait(); mds_service->mds_timer_.dump_special_mds_table_status_task_handle_.wait(); - mds_service->mds_timer_.dump_memory_statistics_task_handle_.wait(); } } diff --git a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h index 4403e5f5bf..82701799ad 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h +++ b/src/storage/multi_data_source/runtime_utility/mds_tenant_service.h @@ -98,83 +98,6 @@ struct ObMdsMemoryLeakDebugInfo class ObTenantMdsAllocator : public ObIAllocator { friend class ObTenantMdsService; - static constexpr int64_t ALLOC_INFO_BUCKET_SIZE = (1ULL << 7); - struct AllocInfoBucket { - AllocInfoBucket() - : alloc_type_(nullptr), - total_alloc_size_(0), - total_free_size_(0), - total_alloc_times_(0), - total_free_times_(0) {} - void record_alloc_info(int64_t alloc_size) { - ATOMIC_AAF(&total_alloc_times_, 1); - ATOMIC_AAF(&total_alloc_size_, alloc_size); - } - void record_free_info(int64_t free_size) { - ATOMIC_AAF(&total_free_times_, 1); - ATOMIC_AAF(&total_free_size_, free_size); - } - const char *alloc_type_; - int64_t total_alloc_size_; - int64_t total_free_size_; - int64_t total_alloc_times_; - int64_t total_free_times_; - }; - struct AllockInfoMap { - AllockInfoMap() = default; - void record_alloc_info(const char *type, int64_t alloc_size) { - int64_t idx = find_idx_(type); - if (idx != -1) { - bucket[idx].record_alloc_info(alloc_size); - } - } - void record_free_info(const char *type, int64_t free_size) { - int64_t idx = find_idx_(type); - if (idx != -1) { - bucket[idx].record_free_info(free_size); - } - } - int64_t find_idx_(const char *type) { - int64_t ret_idx = -1; - if (OB_NOT_NULL(type)) { - constexpr int64_t MASK = ALLOC_INFO_BUCKET_SIZE - 1; - int64_t hash_idx = ((int64_t)type); - for (int64_t idx = hash_idx; idx < hash_idx + ALLOC_INFO_BUCKET_SIZE && ret_idx == -1; ++idx) { - int64_t redirected_idx = (idx & MASK); - const char *alloc_type = ATOMIC_LOAD(&bucket[redirected_idx].alloc_type_); - if (OB_LIKELY(alloc_type == type)) {// find existing bucket - ret_idx = redirected_idx; - } else if (OB_LIKELY(alloc_type == nullptr)) {// take a new place - if (nullptr == ATOMIC_CAS(&bucket[redirected_idx].alloc_type_, nullptr, type)) { - ret_idx = redirected_idx; - } - } - } - } - return ret_idx; - } - void dump() const { - for (int64_t idx = 0; idx < ALLOC_INFO_BUCKET_SIZE; ++idx) { - const char *alloc_type = ATOMIC_LOAD(&bucket[idx].alloc_type_); - int64_t total_alloc_size = ATOMIC_LOAD(&bucket[idx].total_alloc_size_); - int64_t total_free_size = ATOMIC_LOAD(&bucket[idx].total_free_size_); - int64_t total_alloc_times = ATOMIC_LOAD(&bucket[idx].total_alloc_times_); - int64_t total_free_times = ATOMIC_LOAD(&bucket[idx].total_free_times_); - int64_t active_size = total_alloc_size - total_free_size; - int64_t active_alloc_times = total_alloc_times - total_free_times; - if (OB_NOT_NULL(alloc_type)) { - MDS_LOG_RET(INFO, OB_SUCCESS, "[MDS_MEMORY]dump memory useage", K(alloc_type), - "active_alloc_size", to_cstring(ObSizeLiteralPrettyPrinter(active_size)), - "active_alloc_times", active_alloc_times, - "average_active_size", active_alloc_times == 0 ? "INVALID" : to_cstring(ObSizeLiteralPrettyPrinter(active_size / active_alloc_times)), - "history_alloc_size", to_cstring(ObSizeLiteralPrettyPrinter(total_alloc_size)), - "history_alloc_times", total_alloc_times, - "average_history_size", total_alloc_times == 0 ? "INVALID" : to_cstring(ObSizeLiteralPrettyPrinter(total_alloc_size/total_alloc_times))); - } - } - } - AllocInfoBucket bucket[ALLOC_INFO_BUCKET_SIZE]; - }; private: static const int64_t MDS_ALLOC_CONCURRENCY = 32; public: @@ -186,12 +109,10 @@ public: virtual void free(void *ptr) override; virtual void set_attr(const ObMemAttr &attr) override; int64_t hold() { return allocator_.hold(); } - void dump() { alloc_info_map_.dump(); } TO_STRING_KV(KP(this)); private: common::ObBlockAllocMgr block_alloc_; common::ObVSliceAlloc allocator_; - AllockInfoMap alloc_info_map_; }; struct ObTenantBufferCtxAllocator : public ObIAllocator// for now, it is just a wrapper of mtl_malloc @@ -213,7 +134,6 @@ struct ObTenantMdsTimer TO_STRING_KV(KP(this), K_(recycle_task_handle)) common::ObOccamTimerTaskRAIIHandle recycle_task_handle_; common::ObOccamTimerTaskRAIIHandle dump_special_mds_table_status_task_handle_; - common::ObOccamTimerTaskRAIIHandle dump_memory_statistics_task_handle_; common::ObOccamTimer timer_; private: int process_with_tablet_(ObTablet &tablet); diff --git a/src/storage/ob_common_id_utils.cpp b/src/storage/ob_common_id_utils.cpp index 2b8379dfe7..6aa847e254 100644 --- a/src/storage/ob_common_id_utils.cpp +++ b/src/storage/ob_common_id_utils.cpp @@ -70,12 +70,13 @@ int ObCommonIDUtils::gen_unique_id_by_rpc(const uint64_t tenant_id, ObCommonID & int ObCommonIDUtils::gen_monotonic_id(const uint64_t tenant_id, const ObMaxIdType id_type, + const int32_t group_id, common::ObMySQLProxy &proxy, share::ObCommonID &id) { int ret = OB_SUCCESS; uint64_t ret_id = OB_INVALID_ID; - ObMaxIdFetcher id_fetcher(proxy); + ObMaxIdFetcher id_fetcher(proxy, group_id); id.reset(); diff --git a/src/storage/ob_common_id_utils.h b/src/storage/ob_common_id_utils.h index 47dff0b281..1aae4bde9e 100644 --- a/src/storage/ob_common_id_utils.h +++ b/src/storage/ob_common_id_utils.h @@ -60,6 +60,7 @@ public: // @param [out] id generated monotonically increasing ID static int gen_monotonic_id(const uint64_t tenant_id, const share::ObMaxIdType id_type, + const int32_t group_id, common::ObMySQLProxy &proxy, share::ObCommonID &id); }; diff --git a/src/storage/ob_partition_range_spliter.cpp b/src/storage/ob_partition_range_spliter.cpp index e38cdb7e75..24f56dad79 100644 --- a/src/storage/ob_partition_range_spliter.cpp +++ b/src/storage/ob_partition_range_spliter.cpp @@ -391,7 +391,13 @@ int ObPartitionParallelRanger::init_macro_iters(ObRangeSplitInfo &range_info) int64_t endkey_cnt = 0; void *buf = nullptr; ObITable *table = nullptr; - if (is_micro_level_) { + if (OB_ISNULL(table = range_info.tables_->at(i))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected null pointer to table", K(ret), KP(table)); + } else if (OB_UNLIKELY(!table->is_sstable())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected table type", K(ret), KPC(table)); + } else if (is_micro_level_ && !table->is_ddl_mem_sstable()) { // ddl kv not support endkey iterator of micro block if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObMicroEndkeyIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "Failed to alloc memory for endkey iter", K(ret)); @@ -408,12 +414,6 @@ int ObPartitionParallelRanger::init_macro_iters(ObRangeSplitInfo &range_info) } if (OB_FAIL(ret)) { - } else if (OB_ISNULL(table = range_info.tables_->at(i))) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected null pointer to table", K(ret), KP(table)); - } else if (OB_UNLIKELY(!table->is_sstable())) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected table type", K(ret), KPC(table)); } else if (OB_FAIL(endkey_iter->open( sample_cnt_, iter_idx, *(static_cast(table)), range_info))) { STORAGE_LOG(WARN, "Failed to open endkey iter", K(ret), KP(table), K(sample_cnt_), K(iter_idx)); diff --git a/src/storage/ob_storage_rpc.cpp b/src/storage/ob_storage_rpc.cpp index 82a049de3b..a7cb3b6db0 100644 --- a/src/storage/ob_storage_rpc.cpp +++ b/src/storage/ob_storage_rpc.cpp @@ -3288,6 +3288,7 @@ int ObStorageRpc::check_start_transfer_tablets( } else if (OB_FAIL(rpc_proxy_->to(src_info.src_addr_) .by(tenant_id) .dst_cluster_id(src_info.cluster_id_) + .group_id(share::OBCG_STORAGE_HA_LEVEL2) .check_start_transfer_tablets(arg))) { LOG_WARN("failed to check src transfer tablets", K(ret), K(src_info), K(arg)); } @@ -3317,6 +3318,7 @@ int ObStorageRpc::get_ls_active_trans_count( if (OB_FAIL(rpc_proxy_->to(src_info.src_addr_) .by(tenant_id) .dst_cluster_id(src_info.cluster_id_) + .group_id(share::OBCG_STORAGE_HA_LEVEL2) .get_ls_active_trans_count(arg, res))) { LOG_WARN("failed to get ls active trans count", K(ret), K(src_info), K(arg)); } else { @@ -3358,6 +3360,7 @@ int ObStorageRpc::get_transfer_start_scn( .by(tenant_id) .dst_cluster_id(src_info.cluster_id_) .timeout(get_transfer_start_scn_timeout) + .group_id(share::OBCG_STORAGE_HA_LEVEL1) .get_transfer_start_scn(arg, res))) { LOG_WARN("failed to get transfer start scn", K(ret), K(src_info), K(arg)); } else { @@ -3389,6 +3392,7 @@ int ObStorageRpc::fetch_ls_replay_scn( if (OB_FAIL(rpc_proxy_->to(src_info.src_addr_) .by(tenant_id) .dst_cluster_id(src_info.cluster_id_) + .group_id(share::OBCG_STORAGE_HA_LEVEL2) .fetch_ls_replay_scn(arg, res))) { LOG_WARN("failed to fetch ls replay scn", K(ret), K(src_info), K(arg)); } else { @@ -3422,6 +3426,7 @@ int ObStorageRpc::check_tablets_logical_table_replaced( } else if (OB_FAIL(rpc_proxy_->to(src_info.src_addr_) .by(tenant_id) .dst_cluster_id(src_info.cluster_id_) + .group_id(share::OBCG_STORAGE_HA_LEVEL2) .check_transfer_tablet_backfill_completed(arg, res))) { LOG_WARN("failed to check tablets backfill completed", K(ret), K(src_info), K(arg)); } else { @@ -3475,15 +3480,16 @@ int ObStorageRpc::lock_config_change( const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, const int64_t lock_owner, - const int64_t lock_timeout) + const int64_t lock_timeout, + const int32_t group_id) { int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "storage rpc is not inited", K(ret)); - } else if (tenant_id == OB_INVALID_ID || !src_info.is_valid() || !ls_id.is_valid()) { + } else if (tenant_id == OB_INVALID_ID || !src_info.is_valid() || !ls_id.is_valid() || group_id < 0) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(src_info), K(ls_id)); + STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(src_info), K(ls_id), K(group_id)); } else { ObStorageConfigChangeOpArg arg; ObStorageConfigChangeOpRes res; @@ -3497,6 +3503,7 @@ int ObStorageRpc::lock_config_change( .by(tenant_id) .timeout(timeout) .dst_cluster_id(src_info.cluster_id_) + .group_id(group_id) .lock_config_change(arg, res))) { LOG_WARN("failed to replace member", K(ret), K(src_info), K(arg)); } @@ -3509,15 +3516,16 @@ int ObStorageRpc::unlock_config_change( const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, const int64_t lock_owner, - const int64_t lock_timeout) + const int64_t lock_timeout, + const int32_t group_id) { int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "storage rpc is not inited", K(ret)); - } else if (tenant_id == OB_INVALID_ID || !src_info.is_valid() || !ls_id.is_valid()) { + } else if (tenant_id == OB_INVALID_ID || !src_info.is_valid() || !ls_id.is_valid() || group_id < 0) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(src_info), K(ls_id)); + STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(src_info), K(ls_id), K(group_id)); } else { ObStorageConfigChangeOpArg arg; ObStorageConfigChangeOpRes res; @@ -3531,6 +3539,7 @@ int ObStorageRpc::unlock_config_change( .by(tenant_id) .timeout(timeout) .dst_cluster_id(src_info.cluster_id_) + .group_id(group_id) .unlock_config_change(arg, res))) { LOG_WARN("failed to replace member", K(ret), K(src_info), K(arg)); } @@ -3542,6 +3551,7 @@ int ObStorageRpc::get_config_change_lock_stat( const uint64_t tenant_id, const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, + const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked) { @@ -3549,9 +3559,9 @@ int ObStorageRpc::get_config_change_lock_stat( if (!is_inited_) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "storage rpc is not inited", K(ret)); - } else if (tenant_id == OB_INVALID_ID || !src_info.is_valid() || !ls_id.is_valid()) { + } else if (tenant_id == OB_INVALID_ID || !src_info.is_valid() || !ls_id.is_valid() || group_id < 0) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(src_info), K(ls_id)); + STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(src_info), K(ls_id), K(group_id)); } else { ObStorageConfigChangeOpArg arg; ObStorageConfigChangeOpRes res; @@ -3563,6 +3573,7 @@ int ObStorageRpc::get_config_change_lock_stat( .by(tenant_id) .timeout(timeout) .dst_cluster_id(src_info.cluster_id_) + .group_id(group_id) .get_config_change_lock_stat(arg, res))) { LOG_WARN("failed to replace member", K(ret), K(src_info), K(arg)); } else { @@ -3591,6 +3602,7 @@ int ObStorageRpc::wakeup_transfer_service( if (OB_FAIL(rpc_proxy_->to(src_info.src_addr_) .by(tenant_id) .dst_cluster_id(src_info.cluster_id_) + .group_id(share::OBCG_STORAGE_HA_LEVEL2) .wakeup_transfer_service(arg))) { LOG_WARN("failed to wakeup transfer service", K(ret), K(src_info), K(arg)); } diff --git a/src/storage/ob_storage_rpc.h b/src/storage/ob_storage_rpc.h index 76a883098a..12a2c22fde 100755 --- a/src/storage/ob_storage_rpc.h +++ b/src/storage/ob_storage_rpc.h @@ -1138,17 +1138,20 @@ public: const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, const int64_t lock_owner, - const int64_t lock_timeout) = 0; + const int64_t lock_timeout, + const int32_t group_id) = 0; virtual int unlock_config_change( const uint64_t tenant_id, const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, const int64_t lock_owner, - const int64_t lock_timeout) = 0; + const int64_t lock_timeout, + const int32_t group_id) = 0; virtual int get_config_change_lock_stat( const uint64_t tenant_id, const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, + const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked) = 0; virtual int wakeup_transfer_service( @@ -1249,17 +1252,20 @@ public: const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, const int64_t lock_owner, - const int64_t lock_timeout); + const int64_t lock_timeout, + const int32_t group_id); virtual int unlock_config_change( const uint64_t tenant_id, const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, const int64_t lock_owner, - const int64_t lock_timeout); + const int64_t lock_timeout, + const int32_t group_id); virtual int get_config_change_lock_stat( const uint64_t tenant_id, const ObStorageHASrcInfo &src_info, const share::ObLSID &ls_id, + const int32_t group_id, int64_t &palf_lock_owner, bool &is_locked); virtual int wakeup_transfer_service( diff --git a/src/storage/ob_storage_schema.cpp b/src/storage/ob_storage_schema.cpp index 882bc2a5df..caaf1a7b9c 100644 --- a/src/storage/ob_storage_schema.cpp +++ b/src/storage/ob_storage_schema.cpp @@ -290,21 +290,6 @@ int ObStorageSchema::deep_copy_column_array( return ret; } -int ObStorageSchema::deep_copy(const memtable::ObIMultiSourceDataUnit *src, ObIAllocator *allocator) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(nullptr == src || nullptr == allocator)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "invalid src info or allocator", K(ret), K(src), K(allocator)); - } else if (OB_UNLIKELY(src->type() != type())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid type", K(ret), K(type()), KPC(src)); - } else { - ret = init(*allocator, *static_cast(src)); - } - return ret; -} - void ObStorageSchema::reset() { storage_schema_version_ = 0; diff --git a/src/storage/ob_storage_schema.h b/src/storage/ob_storage_schema.h index e8da9fe0c7..c494e5082d 100644 --- a/src/storage/ob_storage_schema.h +++ b/src/storage/ob_storage_schema.h @@ -15,7 +15,6 @@ #include "lib/container/ob_fixed_array.h" #include "share/schema/ob_table_schema.h" -#include "storage/memtable/ob_multi_source_data.h" namespace oceanbase { @@ -93,7 +92,7 @@ public: ObObj orig_default_value_; }; -class ObStorageSchema : public share::schema::ObMergeSchema, public memtable::ObIMultiSourceDataUnit +class ObStorageSchema : public share::schema::ObMergeSchema { public: ObStorageSchema(); @@ -113,17 +112,9 @@ public: common::ObIAllocator &allocator, const ObStorageSchema &src_schema, const int64_t copy_array_cnt); + void reset(); + bool is_valid() const; - // ObIMultiSourceDataUnit section - virtual int deep_copy(const ObIMultiSourceDataUnit *src, ObIAllocator *allocator) override; - virtual void reset() override; - virtual bool is_valid() const override; - virtual inline int64_t get_data_size() const override { return sizeof(ObStorageSchema); } - virtual inline memtable::MultiSourceDataUnitType type() const override - { - return memtable::MultiSourceDataUnitType::STORAGE_SCHEMA; - } - virtual int64_t get_version() const override { return get_schema_version(); } // serialize & deserialize int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; int deserialize( @@ -195,7 +186,7 @@ public: return store_column_cnt_ < input_schema.store_column_cnt_; } - INHERIT_TO_STRING_KV("ObIMultiSourceDataUnit", ObIMultiSourceDataUnit, KP(this), K_(storage_schema_version), K_(version), + VIRTUAL_TO_STRING_KV(KP(this), K_(storage_schema_version), K_(version), K_(is_use_bloomfilter), K_(column_info_simplified), K_(compat_mode), K_(table_type), K_(index_type), K_(index_status), K_(row_store_type), K_(schema_version), K_(column_cnt), K_(store_column_cnt), K_(tablet_size), K_(pctfree), K_(block_size), K_(progressive_merge_round), diff --git a/src/storage/ob_storage_schema_recorder.cpp b/src/storage/ob_storage_schema_recorder.cpp index 9a436119b1..243cf68727 100644 --- a/src/storage/ob_storage_schema_recorder.cpp +++ b/src/storage/ob_storage_schema_recorder.cpp @@ -204,7 +204,6 @@ int ObStorageSchemaRecorder::try_update_storage_schema( void ObStorageSchemaRecorder::sync_clog_failed_for_leader() { - dec_ref_on_memtable(false/*sync_finish*/); } int ObStorageSchemaRecorder::sync_clog_succ_for_leader(const int64_t update_version) @@ -216,8 +215,6 @@ int ObStorageSchemaRecorder::sync_clog_succ_for_leader(const int64_t update_vers } else if (OB_UNLIKELY(storage_schema_->get_schema_version() != update_version)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("schema version not match", K(storage_schema_), K(update_version)); - } else if (OB_FAIL(dec_ref_on_memtable(true/*sync_finish*/))) { - LOG_WARN("failed to save storage schema", K_(tablet_id), K(storage_schema_)); } else { LOG_INFO("success to update storage schema", K(ret), K_(ls_id), K_(tablet_id), K(storage_schema_), K(update_version), K_(clog_scn)); @@ -225,25 +222,6 @@ int ObStorageSchemaRecorder::sync_clog_succ_for_leader(const int64_t update_vers return ret; } -int ObStorageSchemaRecorder::dec_ref_on_memtable(const bool sync_finish) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(nullptr == storage_schema_ - || nullptr == tablet_handle_ptr_ - || !tablet_handle_ptr_->is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("storage schema or tablet handle is unexpected null", K(ret), K_(ls_id), K_(tablet_id), - KP_(storage_schema), K_(tablet_handle_ptr)); - } else { - storage_schema_->set_sync_finish(sync_finish); - // if (OB_FAIL(tablet_handle_ptr_->get_obj()->save_multi_source_data_unit(storage_schema_, clog_scn_, - // false/*for_replay*/, memtable::MemtableRefOp::DEC_REF, true/*is_callback*/))) { - // LOG_WARN("failed to save storage schema", K(ret), K_(tablet_id), K(storage_schema_)); - // } - } - return ret; -} - int ObStorageSchemaRecorder::prepare_struct_in_lock( int64_t &update_version, ObIAllocator *allocator, @@ -366,18 +344,8 @@ int ObStorageSchemaRecorder::submit_log( ret = OB_ERR_UNEXPECTED; LOG_WARN("log handler or storage_schema is null", K(ret), KP(storage_schema_), KP(clog_buf), K(clog_len), K(tablet_handle_ptr_)); - } else if (FALSE_IT(storage_schema_->set_sync_finish(false))) { - //} else if (OB_FAIL(tablet_handle_ptr_->get_obj()->save_multi_source_data_unit(storage_schema_, - // SCN::max_scn(), false/*for_replay*/, memtable::MemtableRefOp::INC_REF))) { - // if (OB_BLOCK_FROZEN != ret) { - // LOG_WARN("failed to inc ref for storage schema", K(ret), K_(tablet_id), K(storage_schema_)); - // } } else if (OB_FAIL(write_clog(clog_buf, clog_len))) { - LOG_WARN("fail to submit log", K(ret), K_(tablet_id)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(dec_ref_on_memtable(false))) { - LOG_ERROR("failed to dec ref on memtable", K(tmp_ret), K_(ls_id), K_(tablet_id)); - } + LOG_WARN("fail to submit log", K(ret), K_(ls_id), K_(tablet_id)); } else { LOG_INFO("submit schema log succeed", K(ret), K_(ls_id), K_(tablet_id), K(clog_scn_), "schema_version", storage_schema_->get_schema_version()); diff --git a/src/storage/ob_storage_schema_recorder.h b/src/storage/ob_storage_schema_recorder.h index fec7a16491..b49edfe1a4 100644 --- a/src/storage/ob_storage_schema_recorder.h +++ b/src/storage/ob_storage_schema_recorder.h @@ -99,7 +99,6 @@ private: int64_t calc_schema_log_size() const; void free_allocated_info(); int try_update_with_lock(const int64_t table_id, const int64_t table_version, const int64_t expire_ts); - OB_INLINE int dec_ref_on_memtable(const bool sync_finish); bool is_inited_; bool ignore_storage_schema_; diff --git a/src/storage/ob_value_row_iterator.cpp b/src/storage/ob_value_row_iterator.cpp index 11cbfd225f..17778457a5 100644 --- a/src/storage/ob_value_row_iterator.cpp +++ b/src/storage/ob_value_row_iterator.cpp @@ -292,7 +292,7 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row) } if (OB_SUCC(ret) || OB_ITER_END == ret) { // check txn status not aborted, which cause readout incorrect result - auto &acc_ctx = store_ctx_->mvcc_acc_ctx_; + memtable::ObMvccAccessCtx &acc_ctx = store_ctx_->mvcc_acc_ctx_; if (acc_ctx.snapshot_.tx_id_.is_valid() && acc_ctx.mem_ctx_ && acc_ctx.mem_ctx_->is_tx_rollbacked()) { diff --git a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp index 7c30652616..95b46ee238 100755 --- a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp +++ b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp @@ -821,16 +821,28 @@ int ObTenantCheckpointSlogHandler::replay_create_tablets_per_task( tablet_transfer_info.reset(); if (OB_FAIL(ATOMIC_LOAD(&replay_create_tablet_errcode_))) { LOG_WARN("replay create has already failed", K(ret)); - } else if (OB_FAIL(read_from_disk(addr, allocator, buf, buf_len))) { - LOG_WARN("fail to read from disk", K(ret), K(addr), KP(buf), K(buf_len)); - } else if (OB_FAIL(get_tablet_svr(key.ls_id_, ls_tablet_svr, ls_handle))) { - LOG_WARN("fail to get ls tablet service", K(ret)); - } else if (OB_FAIL(ls_tablet_svr->replay_create_tablet(addr, buf, buf_len, key.tablet_id_, tablet_transfer_info))) { - LOG_WARN("fail to create tablet for replay", K(ret), K(key), K(addr)); - } else if (tablet_transfer_info.has_transfer_table() && OB_FAIL(record_ls_transfer_info(ls_handle, key.tablet_id_, tablet_transfer_info))) { - LOG_WARN("fail to record_ls_transfer_info", K(ret), K(key), K(tablet_transfer_info)); + } else { + // io maybe timeout, so need retry + int64_t max_retry_time = 5; + do { + if (OB_FAIL(read_from_disk(addr, allocator, buf, buf_len))) { + LOG_WARN("fail to read from disk", K(ret), K(addr), KP(buf), K(buf_len)); + } else if (OB_FAIL(get_tablet_svr(key.ls_id_, ls_tablet_svr, ls_handle))) { + LOG_WARN("fail to get ls tablet service", K(ret)); + } else if (OB_FAIL(ls_tablet_svr->replay_create_tablet(addr, buf, buf_len, key.tablet_id_, tablet_transfer_info))) { + LOG_WARN("fail to create tablet for replay", K(ret), K(key), K(addr)); + } + } while (OB_FAIL(ret) && OB_TIMEOUT == ret && max_retry_time-- > 0); + + if (OB_SUCC(ret)) { + if (tablet_transfer_info.has_transfer_table() && + OB_FAIL(record_ls_transfer_info(ls_handle, key.tablet_id_, tablet_transfer_info))) { + LOG_WARN("fail to record_ls_transfer_info", K(ret), K(key), K(tablet_transfer_info)); + } + } } } + if (OB_SUCC(ret)) { inc_finished_replay_tablet_cnt(tablet_addr_arr.count()); } diff --git a/src/storage/tablelock/ob_lock_memtable.cpp b/src/storage/tablelock/ob_lock_memtable.cpp index 889c61f5f7..af919f758a 100644 --- a/src/storage/tablelock/ob_lock_memtable.cpp +++ b/src/storage/tablelock/ob_lock_memtable.cpp @@ -170,7 +170,7 @@ int ObLockMemtable::lock_( if (lock_op.lock_id_.obj_type_ == ObLockOBJType::OBJ_TYPE_DBMS_LOCK) { ret = OB_OBJ_LOCK_EXIST; } - LOG_INFO("lock is exist", K(ret), K(lock_op)); + LOG_DEBUG("lock is exist", K(ret), K(lock_op)); } else if (FALSE_IT(lock_upgrade(lock_mode_in_same_trans, lock_op))) { } else if (OB_FAIL(obj_lock_map_.lock(param, ctx, lock_op, lock_mode_in_same_trans, conflict_tx_set))) { if (ret != OB_TRY_LOCK_ROW_CONFLICT && @@ -473,7 +473,7 @@ int ObLockMemtable::check_lock_conflict( if (lock_op.lock_id_.obj_type_ == ObLockOBJType::OBJ_TYPE_DBMS_LOCK) { ret = OB_OBJ_LOCK_EXIST; } - LOG_INFO("lock is exist", K(ret), K(lock_op)); + LOG_DEBUG("lock is exist", K(ret), K(lock_op)); } else if (OB_FAIL(obj_lock_map_.check_allow_lock(lock_op, lock_mode_in_same_trans, conflict_tx_set, diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.h b/src/storage/tablet/ob_i_tablet_mds_interface.h index b121b68088..93d22a4057 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.h +++ b/src/storage/tablet/ob_i_tablet_mds_interface.h @@ -71,7 +71,6 @@ protected:// implemented by ObTablet virtual int get_mds_table_handle_(mds::MdsTableHandle &handle, const bool create_if_not_exist) const = 0; virtual ObTabletPointer *get_tablet_ponter_() const = 0; - virtual int fetch_autoinc_seq(ObTabletMemberWrapper &wrapper) const = 0; template int get_mds_data_from_tablet(const common::ObFunction &read_op) const; diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.ipp b/src/storage/tablet/ob_i_tablet_mds_interface.ipp index b833a3f041..045d0836af 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.ipp +++ b/src/storage/tablet/ob_i_tablet_mds_interface.ipp @@ -75,13 +75,13 @@ inline int ObITabletMdsInterface::get_ddl_data(const share::SCN &snapshot, ret = OB_NOT_SUPPORTED; MDS_LOG_GET(WARN, "only support read latest data currently"); } else if (CLICK_FAIL(get_snapshot( - [&data](const ObTabletBindingMdsUserData &user_data) -> int { return data.assign(user_data); }, - snapshot, 0, timeout))) { + [&data](const ObTabletBindingMdsUserData &user_data) -> int { return data.assign(user_data); }, + snapshot, 0, timeout))) { if (OB_EMPTY_RESULT == ret) { - data.reset(); // use default value + data.set_default_value(); // use default value ret = OB_SUCCESS; } else { - MDS_LOG_GET(WARN, "fail to get_ddl_data"); + MDS_LOG_GET(WARN, "fail to get ddl data"); } } return ret; @@ -183,9 +183,8 @@ inline int ObITabletMdsInterface::get_mds_data_from_tabletv_.user_data_; int64_t pos = 0; @@ -220,34 +219,44 @@ inline int ObITabletMdsInterface::get_mds_data_from_tablet &aux_tablet_info_addr = get_mds_data_().aux_tablet_info_.committed_kv_; const ObTabletBindingMdsUserData& aux_tablet_info_cache = get_mds_data_().aux_tablet_info_cache_; - // TODO(@chenqingxiang.cqx): remove read from IO after cache ready - if (aux_tablet_info_addr.is_disk_object() && !aux_tablet_info_cache.is_valid()) { - if (CLICK_FAIL(ObTabletMdsData::load_mds_dump_kv(allocator, aux_tablet_info_addr, kv))) { - MDS_LOG_GET(WARN, "failed to load mds dump kv"); - } else if (OB_ISNULL(kv)) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "unexpected error, mds dump kv is null"); - } else { - const common::ObString &user_data = kv->v_.user_data_; - int64_t pos = 0; - if (user_data.empty()) { - ret = OB_EMPTY_RESULT; - } else if (CLICK_FAIL(data.deserialize(user_data.ptr(), user_data.length(), pos))) { - MDS_LOG_GET(WARN, "failed to deserialize", K(user_data), - "user_data_length", user_data.length(), - "user_hash:%x", user_data.hash(), - "crc_check_number", kv->v_.crc_check_number_); - } else if (CLICK_FAIL(read_op(data))) { + if (aux_tablet_info_addr.is_memory_object()) { + if (CLICK_FAIL(read_op(aux_tablet_info_cache))) { + MDS_LOG_GET(WARN, "failed to read_op"); + } + } else if (aux_tablet_info_addr.is_none_object()) { + ret = OB_EMPTY_RESULT; + } else if (aux_tablet_info_addr.is_disk_object()) { + if (aux_tablet_info_cache.is_valid()) { + if (CLICK_FAIL(read_op(aux_tablet_info_cache))) { MDS_LOG_GET(WARN, "failed to read_op"); } - } + } else { + if (CLICK_FAIL(ObTabletMdsData::load_mds_dump_kv(allocator, aux_tablet_info_addr, kv))) { + MDS_LOG_GET(WARN, "failed to load mds dump kv"); + } else if (nullptr == kv) { + ret = OB_EMPTY_RESULT; + } else { + const common::ObString &user_data = kv->v_.user_data_; + int64_t pos = 0; + if (user_data.empty()) { + ret = OB_EMPTY_RESULT; + } else if (CLICK_FAIL(data.deserialize(user_data.ptr(), user_data.length(), pos))) { + MDS_LOG_GET(WARN, "failed to deserialize", K(user_data), + "user_data_length", user_data.length(), + "user_hash:%x", user_data.hash(), + "crc_check_number", kv->v_.crc_check_number_); + } else if (CLICK_FAIL(read_op(data))) { + MDS_LOG_GET(WARN, "failed to read_op"); + } + } - ObTabletMdsData::free_mds_dump_kv(allocator, kv); - } else { - if (CLICK_FAIL(read_op(aux_tablet_info_cache))) { - MDS_LOG_GET(WARN, "failed to read_op aux_tablet_info_cache", K(aux_tablet_info_cache)); + ObTabletMdsData::free_mds_dump_kv(allocator, kv); } + } else { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "unexpected addr", K(aux_tablet_info_addr)); } + return ret; #undef PRINT_WRAPPER } @@ -261,16 +270,18 @@ inline int ObITabletMdsInterface::get_mds_data_from_tablet &auto_inc_seq_addr = get_mds_data_().auto_inc_seq_; const share::ObTabletAutoincSeq *auto_inc_seq = nullptr; - ObTabletMemberWrapper wrapper; + ObArenaAllocator allocator("mds_reader"); - if (CLICK_FAIL(ObTabletMdsData::fetch_auto_inc_seq(auto_inc_seq_addr, wrapper))) { - MDS_LOG_GET(WARN, "failed to fetch auto inc seq"); - } else if (CLICK_FAIL(wrapper.get_member(auto_inc_seq))) { - MDS_LOG_GET(WARN, "failed to get member"); + if (CLICK_FAIL(ObTabletMdsData::load_auto_inc_seq(allocator, auto_inc_seq_addr, auto_inc_seq))) { + MDS_LOG_GET(WARN, "failed to load auto inc seq"); + } else if (nullptr == auto_inc_seq) { + ret = OB_EMPTY_RESULT; } else if (CLICK_FAIL(read_op(*auto_inc_seq))) { MDS_LOG_GET(WARN, "failed to read_op"); } + ObTabletMdsData::free_auto_inc_seq(allocator, auto_inc_seq); + return ret; #undef PRINT_WRAPPER } @@ -680,17 +691,14 @@ int ObITabletMdsInterface::fill_virtual_info_by_complex_addr_(const ObTabletComp const mds::MdsDumpKV *dump_kv = nullptr; if (CLICK_FAIL(mds_node_info_array.push_back(mds::MdsNodeInfoForVirtualTable()))) { - MDS_LOG_GET(WARN, "fatil to push_back"); + MDS_LOG_GET(WARN, "fail to push_back"); } else if (FALSE_IT(cur_virtual_info = &mds_node_info_array.at(mds_node_info_array.count() - 1))) { } else { if (CLICK_FAIL(ObTabletMdsData::load_mds_dump_kv(allocator, addr, dump_kv))) { MDS_LOG_GET(WARN, "fatil to read tablet_status_addr"); - } else if (!dump_kv->is_valid()) { + } else if (nullptr == dump_kv || (nullptr != dump_kv && !dump_kv->is_valid())) { ret = OB_ENTRY_NOT_EXIST; MDS_LOG_GET(INFO, "dump kv not exist"); - if (OB_NOT_NULL(dump_kv)) { - MDS_LOG_GET(INFO, "xuwang.txw debug", K(dump_kv->k_), K(dump_kv->v_)); - } } else if (CLICK_FAIL(obj_to_string_holder_(dump_kv->k_, cur_virtual_info->user_key_))) { MDS_LOG_GET(WARN, "fatil to fill string holder"); } else if (CLICK_FAIL(dump_kv->v_.convert_to_user_mds_node(user_mds_node, get_tablet_meta_().ls_id_, get_tablet_meta_().tablet_id_))) { diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 089636e530..7636407be9 100755 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -239,24 +239,24 @@ int ObTablet::init_for_first_time_creation( ALLOC_AND_INIT(allocator, storage_schema_addr_, table_schema, compat_mode, true/*skip_column_info*/, ObStorageSchema::STORAGE_SCHEMA_VERSION_V2); } - if (OB_SUCC(ret)) { - if (OB_FAIL(check_sstable_column_checksum())) { - LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); - } else if (OB_FAIL(build_read_info(allocator))) { - LOG_WARN("failed to build read info", K(ret)); - } else if (OB_FAIL(mds_data_.init(allocator))) { - LOG_WARN("failed to init mds data", K(ret)); - } else if (is_ls_inner_tablet() && - OB_FAIL(mds_data_.set_tablet_status(allocator_, ObTabletStatus::Status::NORMAL, ObTabletMdsUserDataType::CREATE_TABLET))) { - LOG_WARN("failed to set normal status for ls inner tablet", K(ret)); - } else if (FALSE_IT(set_mem_addr())) { - } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { - LOG_WARN("failed to increase macro ref cnt", K(ret)); - } else { - is_inited_ = true; - LOG_INFO("succeeded to init tablet for first time creation", K(ret), KPC(sstable), K(*this)); - } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(check_sstable_column_checksum())) { + LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); + } else if (OB_FAIL(build_read_info(allocator))) { + LOG_WARN("failed to build read info", K(ret)); + } else if (!is_ls_inner_tablet() && OB_FAIL(mds_data_.init_for_first_creation(allocator))) { + LOG_WARN("failed to init mds data", K(ret)); + } else if (is_ls_inner_tablet() && OB_FAIL(mds_data_.init_with_tablet_status(allocator, ObTabletStatus::NORMAL, ObTabletMdsUserDataType::CREATE_TABLET))) { + LOG_WARN("failed to init mds data for ls inner tablet", K(ret)); + } else if (FALSE_IT(set_mem_addr())) { + } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { + LOG_WARN("failed to increase macro ref cnt", K(ret)); + } else { + is_inited_ = true; + LOG_INFO("succeeded to init tablet for first time creation", K(ret), KPC(sstable), K(*this)); } + if (OB_UNLIKELY(!is_inited_)) { reset(); } @@ -331,7 +331,7 @@ int ObTablet::init_for_merge( LOG_WARN("failed to update table store flag", K(ret), K(param), K(table_store_addr_)); } else if (OB_FAIL(get_finish_medium_scn(finish_medium_scn))) { LOG_WARN("failed to get finish medium scn", K(ret)); - } else if (OB_FAIL(mds_data_.init(allocator, old_mds_data, finish_medium_scn, param.merge_type_))) { + } else if (OB_FAIL(mds_data_.init_for_evict_medium_info(allocator, old_mds_data, finish_medium_scn, param.merge_type_))) { LOG_WARN("failed to init mds data", K(ret), K(old_mds_data), K(finish_medium_scn), "merge_type", param.merge_type_); } else if (OB_FAIL(build_read_info(allocator))) { LOG_WARN("failed to build read info", K(ret)); @@ -412,7 +412,7 @@ int ObTablet::init_for_mds_table_dump( LOG_WARN("fail to init table store", K(ret), KPC(old_table_store)); } else if (CLICK_FAIL(get_finish_medium_scn(finish_medium_scn))) { LOG_WARN("failed to get finish medium scn", K(ret)); - } else if (CLICK_FAIL(mds_data_.init(allocator, mds_table_data, base_data, finish_medium_scn))) { + } else if (CLICK_FAIL(mds_data_.init_for_mds_table_dump(allocator, mds_table_data, base_data, finish_medium_scn))) { LOG_WARN("failed to init mds data", K(ret), K(finish_medium_scn)); } else { ddl_kvs_ = ddl_kvs_addr; @@ -496,7 +496,7 @@ int ObTablet::init_with_migrate_param( LOG_INFO("succeeded to init empty shell tablet", K(ret), K(param), KPC(this)); } } else { - if (OB_FAIL(mds_data_.init(*allocator_, param.mds_data_))) { + if (OB_FAIL(mds_data_.init_by_full_memory_mds_data(*allocator_, param.mds_data_))) { LOG_WARN("failed to assign mds data", K(ret), K(param)); } else if (OB_FAIL(pull_memtables(allocator, ddl_kvs_addr, ddl_kv_count))) { LOG_WARN("fail to pull memtable", K(ret)); @@ -580,7 +580,7 @@ int ObTablet::init_for_defragment( } if (OB_FAIL(ret)) { - } else if (OB_FAIL(mds_data_.init(allocator, old_mds_data, 0/*finish_medium_scn*/))) { + } else if (OB_FAIL(mds_data_.init_for_evict_medium_info(allocator, old_mds_data, 0/*finish_medium_scn*/))) { LOG_WARN("failed to init mds data", K(ret), K(old_mds_data)); } else if (OB_FAIL(build_read_info(*allocator_))) { LOG_WARN("fail to build read info", K(ret)); @@ -663,10 +663,10 @@ int ObTablet::init_for_sstable_replace( } else if (OB_FAIL(get_finish_medium_scn(finish_medium_scn))) { LOG_WARN("failed to get finish medium scn", K(ret)); } else if (nullptr != param.tablet_meta_ - && OB_FAIL(mds_data_.init(allocator, old_tablet.mds_data_, param.tablet_meta_->mds_data_.medium_info_list_, finish_medium_scn))) { + && OB_FAIL(mds_data_.init_for_merge_with_full_mds_data(allocator, old_tablet.mds_data_, param.tablet_meta_->mds_data_.medium_info_list_, finish_medium_scn))) { LOG_WARN("failed to init mds data", K(ret), "mds_data", param.tablet_meta_->mds_data_, K(finish_medium_scn)); } else if (nullptr == param.tablet_meta_ - && OB_FAIL(mds_data_.init(allocator, old_tablet.mds_data_, finish_medium_scn))) { + && OB_FAIL(mds_data_.init_for_evict_medium_info(allocator, old_tablet.mds_data_, finish_medium_scn))) { LOG_WARN("failed to init mds data", K(ret), "mds_data", old_tablet.mds_data_, K(finish_medium_scn)); } else if (OB_FAIL(build_read_info(*allocator_))) { LOG_WARN("failed to build read info", K(ret)); @@ -835,7 +835,7 @@ int ObTablet::read_medium_info_list( if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(fused_medium_info_list.init(allocator))) { + } else if (OB_FAIL(fused_medium_info_list.init_for_first_creation(allocator))) { LOG_WARN("failed to init medium info list", K(ret)); } else if (OB_FAIL(get_finish_medium_scn(finish_medium_scn))) { LOG_WARN("failed to get finish medium scn", K(ret)); @@ -850,7 +850,8 @@ int ObTablet::read_medium_info_list( if (OB_FAIL(ret)) { } else if (OB_FAIL(ObTabletMdsData::load_medium_info_list(allocator, complex_addr, base_medium_info_list))) { LOG_WARN("failed to load medium info list", K(ret)); - } else if (OB_FAIL(ObTabletMdsData::copy_medium_info_list(finish_medium_scn, *base_medium_info_list, fused_medium_info_list))) { + } else if (nullptr != base_medium_info_list + && OB_FAIL(ObTabletMdsData::copy_medium_info_list(finish_medium_scn, *base_medium_info_list, fused_medium_info_list))) { LOG_WARN("failed to copy base medium info list", K(ret)); } else if (OB_FAIL(ObTabletMdsData::copy_medium_info_list(finish_medium_scn, mds_table_medium_info_list, fused_medium_info_list))) { LOG_WARN("failed to copy mds table medium info list", K(ret)); @@ -858,7 +859,7 @@ int ObTablet::read_medium_info_list( compaction::ObMediumCompactionInfoList *tmp_list = nullptr; if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, tmp_list))) { LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(tmp_list->init(allocator, mds_data_.extra_medium_info_, fused_medium_info_list))) { + } else if (OB_FAIL(tmp_list->init(allocator, mds_data_.extra_medium_info_, &fused_medium_info_list))) { LOG_WARN("failed to init", K(ret)); } else { medium_info_list = tmp_list; @@ -1748,8 +1749,8 @@ int ObTablet::inner_inc_macro_ref_cnt() } else { hold_ref_cnt_ = true; } - LOG_DEBUG("the tablet that inner increases ref cnt is", - K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_.is_valid()), + FLOG_INFO("the tablet that inner increases ref cnt is", + K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_), @@ -1800,12 +1801,6 @@ void ObTablet::dec_macro_ref_cnt() const ObTabletComplexAddr &medium_info_list_addr = mds_data_.medium_info_list_; const ObTabletComplexAddr &auto_inc_seq_addr = mds_data_.auto_inc_seq_; // We don't need to recursively decrease macro ref cnt, since we will push both them to gc queue - LOG_DEBUG("the tablet that decreases ref cnt is", - K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_.is_valid()), - K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), - K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), - K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_), - K(tablet_addr_), KP(this), K(lbt())); if (OB_UNLIKELY(!hold_ref_cnt_)) { FLOG_INFO("tablet doesn't hold ref cnt, no need to dec ref cnt", K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_.is_valid()), @@ -1816,6 +1811,12 @@ void ObTablet::dec_macro_ref_cnt() } else if (OB_FAIL(check_meta_addr())) { LOG_WARN("fail to check meta addrs", K(ret)); } else { + FLOG_INFO("the tablet that decreases ref cnt is", + K(is_inited_), K(tablet_meta_.ls_id_), K(tablet_meta_.tablet_id_), K(table_store_addr_.addr_), + K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), + K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), + K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_), + K(tablet_addr_), KP(this), K(lbt())); // the order can't be changed, must be sstable blocks' ref cnt -> tablet meta blocks' ref cnt dec_linked_block_ref_cnt(medium_info_list_addr.addr_); dec_table_store_ref_cnt(); @@ -1994,6 +1995,9 @@ void ObTablet::set_mem_addr() int ObTablet::check_meta_addr() const { int ret = OB_SUCCESS; + const share::ObLSID &ls_id = tablet_meta_.ls_id_; + const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; + const ObTabletComplexAddr &tablet_status_uncommitted_kv_addr = mds_data_.tablet_status_.uncommitted_kv_; const ObTabletComplexAddr &tablet_status_committed_kv_addr = mds_data_.tablet_status_.committed_kv_; const ObTabletComplexAddr &aux_tablet_info_uncommitted_kv_addr = mds_data_.aux_tablet_info_.uncommitted_kv_; @@ -2008,7 +2012,7 @@ int ObTablet::check_meta_addr() const || !aux_tablet_info_committed_kv_addr.is_valid())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("meta addrs are invalid", K(ret), K(tablet_addr_), K(table_store_addr_.addr_), + LOG_WARN("meta addrs are invalid", K(ret), K(ls_id), K(tablet_id), K(tablet_addr_), K(table_store_addr_.addr_), K(auto_inc_seq_addr.addr_), K(storage_schema_addr_.addr_), K(medium_info_list_addr.addr_), K(tablet_status_uncommitted_kv_addr.addr_), K(tablet_status_committed_kv_addr.addr_), K(aux_tablet_info_uncommitted_kv_addr.addr_), K(aux_tablet_info_committed_kv_addr.addr_)); @@ -2017,8 +2021,8 @@ int ObTablet::check_meta_addr() const && ((tablet_addr_.is_block() ^ table_store_addr_.addr_.is_none()) || (tablet_addr_.is_block() ^ storage_schema_addr_.addr_.is_none()))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("meta addrs are inconsistent", - K(ret), K(tablet_addr_), K(table_store_addr_.addr_), K(storage_schema_addr_.addr_)); + LOG_WARN("meta addrs are inconsistent", K(ret), K(ls_id), K(tablet_id), + K(tablet_addr_), K(table_store_addr_.addr_), K(storage_schema_addr_.addr_)); } return ret; @@ -4622,7 +4626,6 @@ int ObTablet::get_mds_table_rec_log_scn(SCN &rec_scn) int ret = OB_SUCCESS; const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; - bool is_switched_to_empty_shell = false; mds::MdsTableHandle mds_table; rec_scn = SCN::max_scn(); if (IS_NOT_INIT) { @@ -4643,11 +4646,6 @@ int ObTablet::get_mds_table_rec_log_scn(SCN &rec_scn) } else if (!rec_scn.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get invalid scn from mds table", K(ret)); - } else if (OB_FAIL(mds_table.is_switched_to_empty_shell(is_switched_to_empty_shell))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get mds table empty shell status", K(ret)); - } else if (is_switched_to_empty_shell) { - rec_scn = share::SCN::max_scn(); } return ret; } @@ -4713,8 +4711,8 @@ int ObTablet::get_storage_schema_for_transfer_in( } if (OB_FAIL(ret)) { - } else if (OB_FAIL(storage_schema.deep_copy(tablet_storage_schema, &allocator))) { - LOG_WARN("failed to get tx data from tablet", K(ret), K(ls_id), K(tablet_id), KPC(tablet_storage_schema)); + } else if (OB_FAIL(storage_schema.init(allocator, *tablet_storage_schema))) { + LOG_WARN("failed to init storage schema", K(ret), K(ls_id), K(tablet_id), KPC(tablet_storage_schema)); } else { int64_t old_column_cnt = storage_schema.get_column_count(); int64_t old_schema_version = storage_schema.get_schema_version(); @@ -4762,6 +4760,8 @@ int ObTablet::check_and_set_initial_state() int ObTablet::check_medium_list() const { int ret = OB_SUCCESS; + const share::ObLSID &ls_id = tablet_meta_.ls_id_; + const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; if (tablet_meta_.ha_status_.is_none()) { ObTabletMemberWrapper table_store_wrapper; ObITable *last_major = nullptr; @@ -4771,21 +4771,20 @@ int ObTablet::check_medium_list() const ObArenaAllocator arena_allocator("check_medium", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); const ObTabletDumpedMediumInfo *dumped_list = nullptr; if (OB_FAIL(ObTabletMdsData::load_medium_info_list(arena_allocator, mds_data_.medium_info_list_, dumped_list))) { - LOG_WARN("failed to load medium info list", K(ret), K(mds_data_)); - } else if (OB_ISNULL(dumped_list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, list is null", K(ret), KP(dumped_list)); + LOG_WARN("failed to load medium info list", K(ret), K(ls_id), K(tablet_id), K(mds_data_)); + } else if (nullptr == dumped_list) { + // do nothing + LOG_INFO("skip check medium list for empty dumped medium info list", KR(ret), K(ls_id), K(tablet_id)); } else if (OB_FAIL(ObMediumListChecker::validate_medium_info_list( mds_data_.extra_medium_info_, dumped_list->medium_info_list_, last_major->get_snapshot_version()))) { - LOG_WARN("fail to validate medium info list", K(ret), K(mds_data_), KPC(dumped_list), KPC(last_major)); + LOG_WARN("fail to validate medium info list", K(ret), K(ls_id), K(tablet_id), K(mds_data_), KPC(dumped_list), KPC(last_major)); } ObTabletMdsData::free_medium_info_list(arena_allocator, dumped_list); } } else { - LOG_INFO("skip check medium list for non empty ha_status", KR(ret), - "tablet_id", tablet_meta_.tablet_id_, K(tablet_meta_.ha_status_)); + LOG_INFO("skip check medium list for non empty ha_status", KR(ret), K(ls_id), K(tablet_id), "ha_status", tablet_meta_.ha_status_); } return ret; } @@ -4979,7 +4978,7 @@ int ObTablet::build_transfer_tablet_param( if (OB_FAIL(ret)) { } else if (OB_FAIL(get_finish_medium_scn(finish_medium_scn))) { LOG_WARN("failed to get finish medium scn", K(ret)); - } else if (OB_FAIL(new_mds_data.init(mig_tablet_param.allocator_, mds_table_data, mds_data_, finish_medium_scn))) { + } else if (OB_FAIL(new_mds_data.init_for_mds_table_dump(mig_tablet_param.allocator_, mds_table_data, mds_data_, finish_medium_scn))) { LOG_WARN("failed to init new mds data", K(ret), K(finish_medium_scn)); } else if (OB_FAIL(build_transfer_in_tablet_status_(user_data, new_mds_data, mig_tablet_param.allocator_))) { LOG_WARN("failed to build transfer in tablet status", K(new_mds_data), K(mds_data_), KPC(this)); @@ -5149,11 +5148,17 @@ int ObTablet::build_transfer_in_tablet_status_( } else if (OB_FAIL(new_user_data.serialize(buffer, length, pos))) { LOG_WARN("failed to serialize user data", K(ret)); } else { - mds::MdsDumpKV *uncommitted_kv = mds_data.tablet_status_.uncommitted_kv_.ptr_; + mds::MdsDumpKV *&uncommitted_kv = mds_data.tablet_status_.uncommitted_kv_.ptr_; const mds::MdsDumpKV *committed_kv = mds_data.tablet_status_.committed_kv_.ptr_; - if (OB_ISNULL(uncommitted_kv) || OB_ISNULL(committed_kv)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("dump kv is null", K(ret), KP(uncommitted_kv), KP(committed_kv)); + if (nullptr == uncommitted_kv) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, uncommitted_kv))) { + LOG_WARN("failed to alloc and new", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (nullptr == committed_kv) { + // do nothing, no need to copy } else if (OB_FAIL(uncommitted_kv->assign(*committed_kv, allocator))) { LOG_WARN("failed to copy committed kv to uncommitted kv", K(ret)); } else { @@ -5255,7 +5260,7 @@ int ObTablet::read_mds_table(common::ObIAllocator &allocator, if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (CLICK_FAIL(mds_data.init(allocator))) { + } else if (CLICK_FAIL(mds_data.init_for_first_creation(allocator))) { LOG_WARN("failed to init mds data", K(ret)); } else if (CLICK_FAIL(inner_get_mds_table(mds_table_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -5288,7 +5293,7 @@ int ObTablet::read_mds_table_medium_info_list( medium_info_list.reset(); mds::MdsTableHandle mds_table_handle; - if (CLICK_FAIL(medium_info_list.init(allocator))) { + if (CLICK_FAIL(medium_info_list.init_for_first_creation(allocator))) { LOG_WARN("failed to init medium info list", K(ret)); } else if (CLICK_FAIL(inner_get_mds_table(mds_table_handle))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -5668,10 +5673,7 @@ int ObTablet::load_medium_info_list( if (OB_FAIL(ObTabletMdsData::load_medium_info_list(allocator, complex_addr, list))) { LOG_WARN("failed to load medium info list", K(ret), K(complex_addr)); - } else if (OB_ISNULL(list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, list is null", K(ret), KP(list)); - } else if (OB_FAIL(medium_info_list.init(allocator, extra_info, *list))) { + } else if (OB_FAIL(medium_info_list.init(allocator, extra_info, list))) { LOG_WARN("failed to init", K(ret)); } @@ -5704,7 +5706,7 @@ int ObTablet::get_fused_medium_info_list( if (OB_FAIL(ret)) { } else if (OB_FAIL(get_finish_medium_scn(finish_medium_scn))) { LOG_WARN("failed to get finish medium scn", K(ret)); - } else if (OB_FAIL(new_mds_data.init(allocator, mds_table_data, mds_data_, finish_medium_scn))) { + } else if (OB_FAIL(new_mds_data.init_for_mds_table_dump(allocator, mds_table_data, mds_data_, finish_medium_scn))) { LOG_WARN("failed to init new mds data", K(ret), K(finish_medium_scn)); } else if (OB_FAIL(validate_medium_info_list(finish_medium_scn, new_mds_data))) { LOG_WARN("failed to validate medium info list", K(ret), K(finish_medium_scn)); @@ -5724,9 +5726,11 @@ int ObTablet::validate_medium_info_list( const ObTabletDumpedMediumInfo *medium_info_list = mds_data.medium_info_list_.ptr_; const ObExtraMediumInfo &extra_info = mds_data.extra_medium_info_; - if (OB_ISNULL(medium_info_list)) { + if (mds_data.medium_info_list_.is_none_object()) { + LOG_INFO("medium info list addr is none, no need to validate", K(ret), K(ls_id), K(tablet_id), K(finish_medium_scn), "medium_info_list_complex_addr", mds_data.medium_info_list_); + } else if (OB_ISNULL(medium_info_list)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("medium info list is null", K(ret), K(mds_data), KP(medium_info_list)); + LOG_WARN("medium info list is null", K(ret), K(ls_id), K(tablet_id), K(finish_medium_scn), KP(medium_info_list)); } else if (OB_FAIL(ObMediumListChecker::validate_medium_info_list(extra_info, medium_info_list->medium_info_list_, finish_medium_scn))) { LOG_WARN("failed to validate medium info list", KR(ret), K(ls_id), K(tablet_id), K(mds_data), K(finish_medium_scn)); } @@ -5866,20 +5870,25 @@ int ObTablet::get_auto_inc_seq_mds_dump_kv( mds::MdsDumpKV *&kv) { int ret = OB_SUCCESS; + kv = nullptr; const ObTabletComplexAddr &complex_addr = mds_data_.auto_inc_seq_; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); + } else if (complex_addr.is_none_object()) { + // do nothing } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, kv))) { LOG_WARN("failed to alloc and new", K(ret)); } else { - ObTabletMemberWrapper wrapper; const share::ObTabletAutoincSeq *auto_inc_seq = nullptr; - if (OB_FAIL(ObTabletMdsData::fetch_auto_inc_seq(complex_addr, wrapper))) { - LOG_WARN("failed to fetch auto inc seq", K(ret)); - } else if (OB_FAIL(wrapper.get_member(auto_inc_seq))) { - LOG_WARN("failed to get member", K(ret)); + if (OB_FAIL(ObTabletMdsData::load_auto_inc_seq(allocator, complex_addr, auto_inc_seq))) { + LOG_WARN("failed to load auto inc seq", K(ret), K(complex_addr)); + } else if (nullptr == auto_inc_seq) { + // do nothing + if (nullptr != kv) { + allocator.free(kv); + } } else if (OB_FAIL(convert_to_mds_dump_kv(allocator, *auto_inc_seq, *kv))) { LOG_WARN("failed to convert to mds dump kv", K(ret)); } diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index c01a38d15d..037d90cfba 100755 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -102,7 +102,6 @@ class ObDDLKVHandle; class ObStorageSchema; class ObTabletTableIterator; class ObMetaDiskAddr; -class ObTabletMdsDumpStruct; class ObTabletCreateDeleteMdsUserData; class ObTabletBindingMdsUserData; class ObMemtableArray; diff --git a/src/storage/tablet/ob_tablet_binding_mds_user_data.cpp b/src/storage/tablet/ob_tablet_binding_mds_user_data.cpp index cbed04c895..c220de8eeb 100644 --- a/src/storage/tablet/ob_tablet_binding_mds_user_data.cpp +++ b/src/storage/tablet/ob_tablet_binding_mds_user_data.cpp @@ -20,8 +20,8 @@ namespace oceanbase namespace storage { ObTabletBindingMdsUserData::ObTabletBindingMdsUserData() - : snapshot_version_(0), - schema_version_(0), + : snapshot_version_(INT64_MAX), + schema_version_(INT64_MAX), data_tablet_id_(), hidden_tablet_id_(), lob_meta_tablet_id_(), @@ -31,6 +31,17 @@ ObTabletBindingMdsUserData::ObTabletBindingMdsUserData() } void ObTabletBindingMdsUserData::reset() +{ + redefined_ = false; + snapshot_version_ = INT64_MAX; + schema_version_ = INT64_MAX; + data_tablet_id_.reset(); + hidden_tablet_id_.reset(); + lob_meta_tablet_id_.reset(); + lob_piece_tablet_id_.reset(); +} + +void ObTabletBindingMdsUserData::set_default_value() { redefined_ = false; snapshot_version_ = 0; @@ -43,15 +54,7 @@ void ObTabletBindingMdsUserData::reset() bool ObTabletBindingMdsUserData::is_valid() const { - bool valid = true; - - if (0 == snapshot_version_) { - valid = false; - } else if (0 == schema_version_) { - valid = false; - } - - return valid; + return snapshot_version_ != INT64_MAX && schema_version_ != INT64_MAX; } int ObTabletBindingMdsUserData::assign(const ObTabletBindingMdsUserData &other) diff --git a/src/storage/tablet/ob_tablet_binding_mds_user_data.h b/src/storage/tablet/ob_tablet_binding_mds_user_data.h index 56abe32184..26a37345e5 100644 --- a/src/storage/tablet/ob_tablet_binding_mds_user_data.h +++ b/src/storage/tablet/ob_tablet_binding_mds_user_data.h @@ -50,6 +50,7 @@ public: void on_commit(const share::SCN &commit_version, const share::SCN &commit_scn); void reset(); + void set_default_value(); TO_STRING_KV(K_(redefined), K_(snapshot_version), diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.cpp b/src/storage/tablet/ob_tablet_create_mds_helper.cpp index edb91c7520..7d6e5c9fd5 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_mds_helper.cpp @@ -207,7 +207,7 @@ int ObTabletCreateMdsHelper::on_replay( return ret; } -int ObTabletCreateMdsHelper::check_create_new_tablets(const int64_t inc_tablet_cnt) +int ObTabletCreateMdsHelper::check_create_new_tablets(const int64_t inc_tablet_cnt, const bool is_soft_limit) { int ret = OB_SUCCESS; const uint64_t tenant_id = MTL_ID(); @@ -222,6 +222,7 @@ int ObTabletCreateMdsHelper::check_create_new_tablets(const int64_t inc_tablet_c LOG_ERROR("get invalid tenant config", K(ret)); } else { tablet_cnt_per_gb = tenant_config->_max_tablet_cnt_per_gb; + tablet_cnt_per_gb = !is_soft_limit ? tablet_cnt_per_gb : MAX(tablet_cnt_per_gb, 30000); } } @@ -237,13 +238,12 @@ int ObTabletCreateMdsHelper::check_create_new_tablets(const int64_t inc_tablet_c const int64_t max_tablet_cnt = memory_limit / (1 << 30) * tablet_cnt_per_gb; const int64_t cur_tablet_cnt = t3m->get_total_tablet_cnt(); - if (OB_UNLIKELY(cur_tablet_cnt + inc_tablet_cnt >= max_tablet_cnt)) { + if (OB_UNLIKELY(cur_tablet_cnt + inc_tablet_cnt > max_tablet_cnt)) { ret = OB_TOO_MANY_PARTITIONS_ERROR; - LOG_WARN("too many partitions of tenant", K(ret), K(tenant_id), K(memory_limit), K(tablet_cnt_per_gb), + LOG_WARN("too many partitions of tenant", K(ret), K(tenant_id), K(is_soft_limit), K(memory_limit), K(tablet_cnt_per_gb), K(max_tablet_cnt), K(cur_tablet_cnt), K(inc_tablet_cnt)); } } - return ret; } @@ -251,21 +251,43 @@ int ObTabletCreateMdsHelper::check_create_new_tablets(const obrpc::ObBatchCreate { int ret = OB_SUCCESS; bool skip_check = !arg.need_check_tablet_cnt_; + bool is_truncate = false; - // skip hidden tablet creation or truncate tablet creation + // skip hidden tablet creation for (int64_t i = 0; OB_SUCC(ret) && !skip_check && i < arg.table_schemas_.count(); ++i) { - if (arg.table_schemas_[i].is_user_hidden_table() - || OB_INVALID_VERSION != arg.table_schemas_[i].get_truncate_version()) { + if (arg.table_schemas_[i].is_user_hidden_table()) { skip_check = true; + } else if (OB_INVALID_VERSION != arg.table_schemas_[i].get_truncate_version()) { + is_truncate = true; } } if (OB_FAIL(ret)) { } else if (skip_check) { + } else if (is_truncate) { /* retry for truncate */ + bool need_wait = false; + const int64_t timeout = THIS_WORKER.get_timeout_remain(); + const int64_t start_time = ObTimeUtility::fast_current_time(); + do { + if (need_wait) { + ob_usleep(1000 * 1000L); // sleep 1s + } + need_wait = false; + if (ObTimeUtility::fast_current_time() - start_time >= timeout) { + ret = OB_TIMEOUT; + LOG_WARN("too many partitions, retry timeout", K(ret)); + break; + } else if (OB_FAIL(check_create_new_tablets(arg.get_tablet_count(), true/*is_soft_limit*/))) { + if (OB_TOO_MANY_PARTITIONS_ERROR != ret) { + LOG_WARN("fail to check create new tablets", K(ret)); + } else { + need_wait = true; + } + } + } while (need_wait); } else if (OB_FAIL(check_create_new_tablets(arg.get_tablet_count()))) { - LOG_WARN("check create new tablet fail", K(ret)); + LOG_WARN("fail to create new tablets", K(ret)); } - return ret; } diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.h b/src/storage/tablet/ob_tablet_create_mds_helper.h index 2dc1b0fc46..583fa0c32f 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.h +++ b/src/storage/tablet/ob_tablet_create_mds_helper.h @@ -76,7 +76,7 @@ public: const obrpc::ObBatchCreateTabletArg &arg, const share::SCN &scn, mds::BufferCtx &ctx); - static int check_create_new_tablets(const int64_t inc_tablet_cnt); + static int check_create_new_tablets(const int64_t inc_tablet_cnt, const bool is_soft_limit = false); private: static int check_create_new_tablets(const obrpc::ObBatchCreateTabletArg &arg); static int check_create_arg( diff --git a/src/storage/tablet/ob_tablet_dumped_medium_info.cpp b/src/storage/tablet/ob_tablet_dumped_medium_info.cpp index aa48ba7f32..6dc64980bc 100644 --- a/src/storage/tablet/ob_tablet_dumped_medium_info.cpp +++ b/src/storage/tablet/ob_tablet_dumped_medium_info.cpp @@ -53,7 +53,7 @@ void ObTabletDumpedMediumInfo::reset() is_inited_ = false; } -int ObTabletDumpedMediumInfo::init(common::ObIAllocator &allocator) +int ObTabletDumpedMediumInfo::init_for_first_creation(common::ObIAllocator &allocator) { int ret = OB_SUCCESS; @@ -69,6 +69,139 @@ int ObTabletDumpedMediumInfo::init(common::ObIAllocator &allocator) return ret; } +int ObTabletDumpedMediumInfo::init_for_evict_medium_info( + common::ObIAllocator &allocator, + const int64_t finish_medium_scn, + const ObTabletDumpedMediumInfo &other) +{ + int ret = OB_SUCCESS; + const common::ObIArray &array = other.medium_info_list_; + + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K_(is_inited)); + } else { + allocator_ = &allocator; + for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { + const compaction::ObMediumCompactionInfo *src_info = array.at(i); + if (OB_ISNULL(src_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(src_info)); + } else if (src_info->medium_snapshot_ <= finish_medium_scn) { + // medium snapshot no bigger than finish medium scn(which is from last major sstable), + // no need to copy it + } else if (src_info->medium_snapshot_ <= get_max_medium_snapshot()) { + // medium info no bigger than current max medium snapshot, + // no need to copy it + } else if (OB_FAIL(do_append(*src_info))) { + LOG_WARN("failed to append medium info", K(ret), K(i), KPC(src_info)); + } + } + + if (OB_FAIL(ret)) { + reset(); + } else { + is_inited_ = true; + } + } + + return ret; +} + +int ObTabletDumpedMediumInfo::init_for_mds_table_dump( + common::ObIAllocator &allocator, + const int64_t finish_medium_scn, + const ObTabletDumpedMediumInfo &other1, + const ObTabletDumpedMediumInfo &other2) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K_(is_inited)); + } else { + allocator_ = &allocator; + common::ObSEArray array1; + common::ObSEArray array2; + + if (OB_FAIL(array1.assign(other1.medium_info_list_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(array2.assign(other2.medium_info_list_))) { + LOG_WARN("failed to assign", K(ret)); + } else { + // sort first + std::sort(array1.begin(), array1.end(), ObTabletDumpedMediumInfo::compare); + std::sort(array2.begin(), array2.end(), ObTabletDumpedMediumInfo::compare); + + // merge + bool contain = false; + int64_t i = 0; + int64_t j = 0; + while (OB_SUCC(ret) && i < array1.count() && j < array2.count()) { + const compaction::ObMediumCompactionInfo *info1 = array1.at(i); + const compaction::ObMediumCompactionInfo *info2 = array2.at(j); + const compaction::ObMediumCompactionInfo *chosen_info = nullptr; + + if (OB_ISNULL(info1) || OB_ISNULL(info2)) { + LOG_WARN("medium info is null", K(ret), K(i), K(j), KP(info1), KP(info2)); + } else if (info1->medium_snapshot_ < info2->medium_snapshot_) { + chosen_info = info1; + ++i; + } else if (info1->medium_snapshot_ > info2->medium_snapshot_) { + chosen_info = info2; + ++j; + } else { + chosen_info = info2; + ++i; + ++j; + } + + if (OB_FAIL(ret)) { + } else if (chosen_info->medium_snapshot_ <= finish_medium_scn) { + // medium snapshot no bigger than finish medium scn(which is from last major sstable), + // no need to copy it + } else if (chosen_info->medium_snapshot_ <= get_max_medium_snapshot()) { + // do nothing + } else if (OB_FAIL(do_append(*chosen_info))) { + LOG_WARN("failed to append medium info", K(ret), K(i), K(j), KPC(chosen_info)); + } + } + + for (; OB_SUCC(ret) && i < array1.count(); ++i) { + const compaction::ObMediumCompactionInfo *info = array1.at(i); + if (info->medium_snapshot_ <= finish_medium_scn) { + // medium snapshot no bigger than finish medium scn(which is from last major sstable), + // no need to copy it + } else if (info->medium_snapshot_ <= get_max_medium_snapshot()) { + // do nothing + } else if (OB_FAIL(do_append(*info))) { + LOG_WARN("failed to append medium info", K(ret), K(i), KPC(info)); + } + } + + for (; OB_SUCC(ret) && j < array2.count(); ++j) { + const compaction::ObMediumCompactionInfo *info = array2.at(j); + if (info->medium_snapshot_ <= finish_medium_scn) { + // medium snapshot no bigger than finish medium scn(which is from last major sstable), + // no need to copy it + } else if (info->medium_snapshot_ <= get_max_medium_snapshot()) { + // do nothing + } else if (OB_FAIL(do_append(*info))) { + LOG_WARN("failed to append medium info", K(ret), K(j), KPC(info)); + } + } + } + + if (OB_FAIL(ret)) { + reset(); + } else { + is_inited_ = true; + } + } + + return ret; +} + int ObTabletDumpedMediumInfo::assign(const ObTabletDumpedMediumInfo &other, common::ObIAllocator &allocator) { int ret = OB_SUCCESS; @@ -152,7 +285,19 @@ int ObTabletDumpedMediumInfo::append(const compaction::ObMediumCompactionInfo &m if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(*allocator_, info))) { + } else if (OB_FAIL(do_append(medium_info))) { + LOG_WARN("failed to do append", K(ret)); + } + + return ret; +} + +int ObTabletDumpedMediumInfo::do_append(const compaction::ObMediumCompactionInfo &medium_info) +{ + int ret = OB_SUCCESS; + compaction::ObMediumCompactionInfo *info = nullptr; + + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(*allocator_, info))) { LOG_WARN("failed to alloc and new", K(ret)); } else if (OB_FAIL(info->assign(*allocator_, medium_info))) { LOG_WARN("failed to copy medium info", K(ret), K(medium_info)); @@ -202,7 +347,7 @@ int ObTabletDumpedMediumInfo::get_min_medium_info_key(compaction::ObMediumCompac } else { ObTabletDumpedMediumInfoIterator iter; ObArenaAllocator arena_allocator("iter"); - if (OB_FAIL(iter.init(arena_allocator, *this))) { + if (OB_FAIL(iter.init(arena_allocator, this))) { LOG_WARN("failed to init", K(ret)); } else { while (OB_SUCC(ret)) { @@ -234,7 +379,7 @@ int ObTabletDumpedMediumInfo::get_max_medium_info_key(compaction::ObMediumCompac } else { ObTabletDumpedMediumInfoIterator iter; ObArenaAllocator arena_allocator("iter"); - if (OB_FAIL(iter.init(arena_allocator, *this))) { + if (OB_FAIL(iter.init(arena_allocator, this))) { LOG_WARN("failed to init", K(ret)); } else if (OB_FAIL(iter.get_next_key(key))) { LOG_WARN("failed to get next medium info", K(ret)); @@ -479,7 +624,7 @@ ObTabletDumpedMediumInfoIterator::~ObTabletDumpedMediumInfoIterator() int ObTabletDumpedMediumInfoIterator::init( common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo &dumped_medium_info) + const ObTabletDumpedMediumInfo *dumped_medium_info) { int ret = OB_SUCCESS; @@ -487,25 +632,30 @@ int ObTabletDumpedMediumInfoIterator::init( ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); } else { - const common::ObSEArray &array = dumped_medium_info.medium_info_list_; - compaction::ObMediumCompactionInfo* info = nullptr; - for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { - compaction::ObMediumCompactionInfo* src_medium_info = array.at(i); - info = nullptr; - if (OB_ISNULL(src_medium_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, src medium info is null", K(ret), K(i), KP(src_medium_info)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, info))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(info->assign(allocator, *src_medium_info))) { - LOG_WARN("failed to copy medium info", K(ret), KPC(src_medium_info)); - } else if (OB_FAIL(medium_info_list_.push_back(info))) { - LOG_WARN("failed to push back to array", K(ret)); - } + if (nullptr == dumped_medium_info) { + // no need to copy medium info + } else { + const common::ObSEArray &array = dumped_medium_info->medium_info_list_; + compaction::ObMediumCompactionInfo *info = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { + info = nullptr; + compaction::ObMediumCompactionInfo *src_medium_info = array.at(i); + if (OB_ISNULL(src_medium_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, src medium info is null", K(ret), K(i), KP(src_medium_info)); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, info))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(info->assign(allocator, *src_medium_info))) { + LOG_WARN("failed to copy medium info", K(ret), KPC(src_medium_info)); + } else if (OB_FAIL(medium_info_list_.push_back(info))) { + LOG_WARN("failed to push back to array", K(ret)); + } - if (OB_FAIL(ret)) { - if (nullptr != info) { - allocator.free(info); + if (OB_FAIL(ret)) { + if (nullptr != info) { + info->compaction::ObMediumCompactionInfo::~ObMediumCompactionInfo(); + allocator.free(info); + } } } } diff --git a/src/storage/tablet/ob_tablet_dumped_medium_info.h b/src/storage/tablet/ob_tablet_dumped_medium_info.h index c6d7da1af1..40102abd9f 100644 --- a/src/storage/tablet/ob_tablet_dumped_medium_info.h +++ b/src/storage/tablet/ob_tablet_dumped_medium_info.h @@ -43,7 +43,16 @@ public: ObTabletDumpedMediumInfo(const ObTabletDumpedMediumInfo &) = delete; ObTabletDumpedMediumInfo &operator=(const ObTabletDumpedMediumInfo &) = delete; public: - int init(common::ObIAllocator &allocator); + int init_for_first_creation(common::ObIAllocator &allocator); + int init_for_evict_medium_info( + common::ObIAllocator &allocator, + const int64_t finish_medium_scn, + const ObTabletDumpedMediumInfo &other); + int init_for_mds_table_dump( + common::ObIAllocator &allocator, + const int64_t finish_medium_scn, + const ObTabletDumpedMediumInfo &other1, + const ObTabletDumpedMediumInfo &other2); void reset(); // key order in array: big -> small @@ -70,6 +79,8 @@ public: int64_t simple_to_string(char* buf, const int64_t buf_len, int64_t &pos) const; public: static bool compare(const compaction::ObMediumCompactionInfo *lhs, const compaction::ObMediumCompactionInfo *rhs); +private: + int do_append(const compaction::ObMediumCompactionInfo &medium_info); public: bool is_inited_; common::ObIAllocator *allocator_; @@ -86,7 +97,7 @@ public: public: int init( common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo &dumped_medium_info); + const ObTabletDumpedMediumInfo *dumped_medium_info); void reset(); int get_next_key(compaction::ObMediumCompactionInfoKey &key); int get_next_medium_info( diff --git a/src/storage/tablet/ob_tablet_full_memory_mds_data.cpp b/src/storage/tablet/ob_tablet_full_memory_mds_data.cpp index 2d1a4126c4..7b6b42730f 100644 --- a/src/storage/tablet/ob_tablet_full_memory_mds_data.cpp +++ b/src/storage/tablet/ob_tablet_full_memory_mds_data.cpp @@ -47,7 +47,7 @@ int ObTabletFullMemoryMdsData::init(common::ObArenaAllocator &allocator) if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); - } else if (OB_FAIL(medium_info_list_.medium_info_list_.init(allocator))) { + } else if (OB_FAIL(medium_info_list_.medium_info_list_.init_for_first_creation(allocator))) { LOG_WARN("failed to init medium info list", K(ret)); } else { is_inited_ = true; @@ -136,9 +136,9 @@ int ObTabletFullMemoryMdsData::read_mds_dump_kv( LOG_WARN("invalid args", K(ret), K(mds_dump_kv_addr)); } else if (OB_FAIL(ObTabletMdsData::load_mds_dump_kv(allocator, mds_dump_kv_addr, ptr))) { LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_ISNULL(ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, ptr is null", K(ret), KP(ptr)); + } else if (nullptr == ptr) { + // do nothing + dump_kv.reset(); } else if (OB_FAIL(dump_kv.assign(*ptr, allocator))) { LOG_WARN("failed to copy mds dump kv", K(ret)); } @@ -161,11 +161,10 @@ int ObTabletFullMemoryMdsData::read_medium_info_list( LOG_WARN("invalid args", K(ret), K(medium_info_list_addr)); } else if (OB_FAIL(ObTabletMdsData::load_medium_info_list(allocator, medium_info_list_addr, ptr))) { LOG_WARN("failed to load medium info list", K(ret)); - } else if (OB_ISNULL(ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, ptr is null", K(ret), KP(ptr)); - } else if (OB_FAIL(medium_info_list.assign(*ptr, allocator))) { - LOG_WARN("failed to medium info list", K(ret)); + } else if (nullptr == ptr && OB_FAIL(medium_info_list.init_for_first_creation(allocator))) { + LOG_WARN("failed to init medium info list", K(ret)); + } else if (nullptr != ptr && OB_FAIL(medium_info_list.assign(*ptr, allocator))) { + LOG_WARN("failed to copy medium info list", K(ret)); } ObTabletMdsData::free_medium_info_list(allocator, ptr); @@ -180,15 +179,11 @@ int ObTabletFullMemoryMdsData::read_auto_inc_seq( { int ret = OB_SUCCESS; const share::ObTabletAutoincSeq *ptr = nullptr; - ObTabletMemberWrapper auto_inc_seq_wrapper; - if (OB_FAIL(ObTabletMdsData::fetch_auto_inc_seq(auto_inc_seq_addr, auto_inc_seq_wrapper))) { - LOG_WARN("failed to fetch auto inc seq", K(ret)); - } else if (OB_FAIL(auto_inc_seq_wrapper.get_member(ptr))) { - LOG_WARN("ObTabletMemberWrapper get member failed", K(ret)); - } else if (OB_ISNULL(ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, ptr is null", K(ret), KP(ptr)); + if (OB_FAIL(ObTabletMdsData::load_auto_inc_seq(allocator, auto_inc_seq_addr, ptr))) { + LOG_WARN("failed to load auto inc seq", K(ret), K(auto_inc_seq_addr)); + } else if (nullptr == ptr) { + // do nothing } else if (OB_FAIL(auto_inc_seq.assign(allocator, *ptr))) { LOG_WARN("failed to copy auto inc seq", K(ret)); } @@ -231,7 +226,6 @@ int ObTabletFullMemoryMdsData::deserialize(common::ObIAllocator &allocator, cons LOG_WARN("failed to deserialize", K(ret)); } else { is_inited_ = true; - LOG_INFO("succeeded to deserialize full memory mds data", K(ret), KPC(this)); } return ret; diff --git a/src/storage/tablet/ob_tablet_mds_data.cpp b/src/storage/tablet/ob_tablet_mds_data.cpp index a00f6d2809..6d4470bfc1 100644 --- a/src/storage/tablet/ob_tablet_mds_data.cpp +++ b/src/storage/tablet/ob_tablet_mds_data.cpp @@ -17,11 +17,11 @@ #include "lib/utility/ob_macro_utils.h" #include "storage/blocksstable/ob_storage_cache_suite.h" #include "storage/compaction/ob_medium_compaction_mgr.h" +#include "storage/compaction/ob_medium_list_checker.h" #include "storage/compaction/ob_extra_medium_info.h" #include "storage/tablet/ob_tablet_full_memory_mds_data.h" #include "storage/tablet/ob_tablet_obj_load_helper.h" -#include "ob_i_tablet_mds_interface.h" -#include "storage/compaction/ob_medium_list_checker.h" +#include "storage/tablet/ob_i_tablet_mds_interface.h" #define USING_LOG_PREFIX MDS @@ -116,13 +116,62 @@ void ObTabletMdsData::reset() is_inited_ = false; } +void ObTabletMdsData::reset(common::ObIAllocator &allocator) +{ + free_auto_inc_seq(allocator, auto_inc_seq_.ptr_); + free_medium_info_list(allocator, medium_info_list_.ptr_); + free_mds_dump_kv(allocator, aux_tablet_info_.uncommitted_kv_.ptr_); + free_mds_dump_kv(allocator, aux_tablet_info_.committed_kv_.ptr_); + free_mds_dump_kv(allocator, tablet_status_.uncommitted_kv_.ptr_); + free_mds_dump_kv(allocator, tablet_status_.committed_kv_.ptr_); + + auto_inc_seq_.ptr_ = nullptr; + auto_inc_seq_.addr_.reset(); + medium_info_list_.ptr_ = nullptr; + medium_info_list_.addr_.reset(); + aux_tablet_info_.uncommitted_kv_.ptr_ = nullptr; + aux_tablet_info_.uncommitted_kv_.addr_.reset(); + aux_tablet_info_.committed_kv_.ptr_ = nullptr; + aux_tablet_info_.committed_kv_.addr_.reset(); + tablet_status_.uncommitted_kv_.ptr_ = nullptr; + tablet_status_.uncommitted_kv_.addr_.reset(); + tablet_status_.committed_kv_.ptr_ = nullptr; + tablet_status_.committed_kv_.addr_.reset(); + + is_inited_ = false; +} + bool ObTabletMdsData::is_valid() const { // TODO(@bowen.gbw): add more check rules return is_inited_; } -int ObTabletMdsData::init(common::ObIAllocator &allocator) +int ObTabletMdsData::init_for_first_creation(common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K_(is_inited)); + } else { + auto_inc_seq_.addr_.set_none_addr(); + medium_info_list_.addr_.set_none_addr(); + aux_tablet_info_.uncommitted_kv_.addr_.set_none_addr(); + aux_tablet_info_.committed_kv_.addr_.set_none_addr(); + tablet_status_.uncommitted_kv_.addr_.set_none_addr(); + tablet_status_.committed_kv_.addr_.set_none_addr(); + + is_inited_ = true; + } + + return ret; +} + +int ObTabletMdsData::init_with_tablet_status( + common::ObIAllocator &allocator, + const ObTabletStatus::Status &tablet_status, + const ObTabletMdsUserDataType &data_type) { int ret = OB_SUCCESS; @@ -131,23 +180,23 @@ int ObTabletMdsData::init(common::ObIAllocator &allocator) LOG_WARN("init twice", K(ret), K_(is_inited)); } else if (OB_FAIL(tablet_status_.init(allocator))) { LOG_WARN("failed to init tablet status", K(ret)); - } else if (OB_FAIL(aux_tablet_info_.init(allocator))) { - LOG_WARN("failed to init aux tablet info", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_.ptr_))) { - LOG_WARN("failed to alloc and new medium info list", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, auto_inc_seq_.ptr_))) { - LOG_WARN("failed to alloc and new auto inc seq", K(ret)); - } else if (OB_FAIL(medium_info_list_.ptr_->init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); + } else if (OB_FAIL(set_tablet_status(allocator, tablet_status, data_type))) { + LOG_WARN("failed to set tablet status", K(ret)); } else { - set_mem_addr(); + auto_inc_seq_.addr_.set_none_addr(); + medium_info_list_.addr_.set_none_addr(); + aux_tablet_info_.uncommitted_kv_.addr_.set_none_addr(); + aux_tablet_info_.committed_kv_.addr_.set_none_addr(); + tablet_status_.uncommitted_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + tablet_status_.committed_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + is_inited_ = true; } return ret; } -int ObTabletMdsData::init( +int ObTabletMdsData::init_for_mds_table_dump( common::ObIAllocator &allocator, const ObTabletMdsData &mds_table_data, const ObTabletMdsData &base_data, @@ -158,208 +207,558 @@ int ObTabletMdsData::init( if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); - } else if (OB_FAIL(tablet_status_.init(allocator))) { - LOG_WARN("failed to init tablet status", K(ret)); - } else if (OB_FAIL(aux_tablet_info_.init(allocator))) { - LOG_WARN("failed to init aux tablet info", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_.ptr_))) { - LOG_WARN("failed to alloc and new medium info list", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, auto_inc_seq_.ptr_))) { - LOG_WARN("failed to alloc and new auto inc seq", K(ret)); - } else if (OB_FAIL(medium_info_list_.ptr_->init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(fuse_mds_dump_node(allocator, mds_table_data.tablet_status_, base_data.tablet_status_, tablet_status_))) { - LOG_WARN("failed to fuse", K(ret)); - } else if (OB_FAIL(fuse_mds_dump_node(allocator, mds_table_data.aux_tablet_info_, base_data.aux_tablet_info_, aux_tablet_info_))) { - LOG_WARN("failed to fuse", K(ret)); - } else if (OB_FAIL(fuse_mds_dump_node(allocator, finish_medium_scn, mds_table_data.medium_info_list_, base_data.medium_info_list_, medium_info_list_))) { - LOG_WARN("failed to fuse", K(ret)); - } else if (OB_FAIL(fuse_mds_dump_node(allocator, mds_table_data.auto_inc_seq_, base_data.auto_inc_seq_, auto_inc_seq_))) { - LOG_WARN("failed to fuse", K(ret)); } else { - // always use base data to set extra medium info - extra_medium_info_.last_compaction_type_ = base_data.extra_medium_info_.last_compaction_type_; - extra_medium_info_.last_medium_scn_ = base_data.extra_medium_info_.last_medium_scn_; - extra_medium_info_.wait_check_flag_ = base_data.extra_medium_info_.wait_check_flag_; + if (OB_FAIL(init_single_complex_addr(allocator, mds_table_data.tablet_status_.uncommitted_kv_, base_data.tablet_status_.uncommitted_kv_, tablet_status_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, mds_table_data.tablet_status_.committed_kv_, base_data.tablet_status_.committed_kv_, tablet_status_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, mds_table_data.aux_tablet_info_.uncommitted_kv_, base_data.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, mds_table_data.aux_tablet_info_.committed_kv_, base_data.aux_tablet_info_.committed_kv_, aux_tablet_info_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, mds_table_data.auto_inc_seq_, base_data.auto_inc_seq_, auto_inc_seq_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, mds_table_data.medium_info_list_, base_data.medium_info_list_, finish_medium_scn, medium_info_list_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", tablet_status_.committed_kv_); + } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + } else { + // always use base data to set extra medium info + extra_medium_info_ = base_data.extra_medium_info_; + is_inited_ = true; + LOG_INFO("succeeded to init mds data", K(ret), + "mds_table_medium_info", mds_table_data.medium_info_list_, + "base_medium_info", base_data.medium_info_list_, + K(finish_medium_scn), K_(medium_info_list), K_(extra_medium_info)); + } - LOG_INFO("succeeded to init medium info list", K(ret), - "mds_table_medium_info", mds_table_data.medium_info_list_, - "base_medium_info", base_data.medium_info_list_, - K(finish_medium_scn), K_(medium_info_list), K_(extra_medium_info)); - } - - if (OB_FAIL(ret)) { - } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { - LOG_WARN("failed to update user data cache", K(ret), "complex_addr", tablet_status_.committed_kv_); - } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { - LOG_WARN("failed to update user data cache", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); - } - - if (OB_SUCC(ret)) { - set_mem_addr(); - - is_inited_ = true; + if (OB_FAIL(ret)) { + reset(allocator); + } } return ret; } -int ObTabletMdsData::init( +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &mds_table_data, + const ObTabletComplexAddr &base_data, + ObTabletComplexAddr &fused_data) +{ + int ret = OB_SUCCESS; + fused_data.reset(); + mds::MdsDumpKV *&fused_data_ptr = fused_data.ptr_; + const mds::MdsDumpKV *mds_table_ptr = nullptr; + const mds::MdsDumpKV *base_ptr = nullptr; + + if (OB_FAIL(load_mds_dump_kv(allocator, mds_table_data, mds_table_ptr))) { + LOG_WARN("failed to load mds dump kv", K(ret)); + } else if (OB_FAIL(load_mds_dump_kv(allocator, base_data, base_ptr))) { + LOG_WARN("failed to load mds dump kv", K(ret)); + } else if (nullptr == mds_table_ptr && nullptr == base_ptr) { + // data in mds table and base is both empty, so set addr to NONE + fused_data.addr_.set_none_addr(); + } else { + if (nullptr == mds_table_ptr && nullptr != base_ptr) { + fused_data_ptr = const_cast(base_ptr); + } else if (nullptr != mds_table_ptr) { + // data in mds table is not empty, ignore data in base, use data in mds table as fused data + fused_data_ptr = const_cast(mds_table_ptr); + free_mds_dump_kv(allocator, base_ptr); + } + + if (OB_FAIL(ret)) { + fused_data_ptr = nullptr; + free_mds_dump_kv(allocator, mds_table_ptr); + free_mds_dump_kv(allocator, base_ptr); + } else { + fused_data.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + } + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &mds_table_data, + const ObTabletComplexAddr &base_data, + ObTabletComplexAddr &fused_data) +{ + int ret = OB_SUCCESS; + fused_data.reset(); + share::ObTabletAutoincSeq *&fused_data_ptr = fused_data.ptr_; + const share::ObTabletAutoincSeq *mds_table_ptr = nullptr; + const share::ObTabletAutoincSeq *base_ptr = nullptr; + + if (OB_FAIL(load_auto_inc_seq(allocator, mds_table_data, mds_table_ptr))) { + LOG_WARN("failed to load auto inc seq", K(ret), K(mds_table_data)); + } else if (OB_FAIL(load_auto_inc_seq(allocator, base_data, base_ptr))) { + LOG_WARN("failed to load auto inc seq", K(ret), K(base_data)); + } else if (nullptr == mds_table_ptr && nullptr == base_ptr) { + // both data in mds table and base is invalid, no need to fuse, set fused data to NONE + fused_data.addr_.set_none_addr(); + } else { + if (nullptr == mds_table_ptr && nullptr != base_ptr) { + fused_data_ptr = const_cast(base_ptr); + } else if (nullptr != mds_table_ptr) { + // data in mds table is not empty, ignore data in base, use data in mds table as fused data + fused_data_ptr = const_cast(mds_table_ptr); + free_auto_inc_seq(allocator, base_ptr); + } + + if (OB_FAIL(ret)) { + fused_data_ptr = nullptr; + free_auto_inc_seq(allocator, mds_table_ptr); + free_auto_inc_seq(allocator, base_ptr); + } else { + fused_data.addr_.set_mem_addr(0, sizeof(share::ObTabletAutoincSeq)); + } + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &mds_table_data, + const ObTabletComplexAddr &base_data, + const int64_t finish_medium_scn, + ObTabletComplexAddr &fused_data) +{ + int ret = OB_SUCCESS; + fused_data.reset(); + ObTabletDumpedMediumInfo *&fused_data_ptr = fused_data.ptr_; + const ObTabletDumpedMediumInfo *mds_table_ptr = nullptr; + const ObTabletDumpedMediumInfo *base_ptr = nullptr; + + if (OB_FAIL(load_medium_info_list(allocator, mds_table_data, mds_table_ptr))) { + LOG_WARN("failed to load medium info list", K(ret), K(mds_table_data)); + } else if (OB_FAIL(load_medium_info_list(allocator, base_data, base_ptr))) { + LOG_WARN("failed to load medium info list", K(ret), K(base_data)); + } else if (nullptr == mds_table_ptr && nullptr == base_ptr) { + // data in mds table and base is both empty, set fused data to NONE + fused_data.addr_.set_none_addr(); + } else { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, fused_data_ptr))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (nullptr == mds_table_ptr && nullptr != base_ptr) { + if (OB_FAIL(fused_data_ptr->init_for_evict_medium_info(allocator, finish_medium_scn, *base_ptr))) { + LOG_WARN("failed to init", K(ret)); + } + } else if (nullptr != mds_table_ptr && nullptr == base_ptr) { + if (OB_FAIL(fused_data_ptr->init_for_evict_medium_info(allocator, finish_medium_scn, *mds_table_ptr))) { + LOG_WARN("failed to init", K(ret)); + } + } else if (nullptr != mds_table_ptr && nullptr != base_ptr) { + if (OB_FAIL(fused_data_ptr->init_for_mds_table_dump(allocator, finish_medium_scn, *mds_table_ptr, *base_ptr))) { + LOG_WARN("failed to init", K(ret)); + } + } + + if (OB_FAIL(ret)) { + free_medium_info_list(allocator, fused_data_ptr); + fused_data_ptr = nullptr; + } else { + fused_data.addr_.set_mem_addr(0, sizeof(ObTabletDumpedMediumInfo)); + } + + // always free medium info list after usage + free_medium_info_list(allocator, mds_table_ptr); + free_medium_info_list(allocator, base_ptr); + } + + return ret; +} + +int ObTabletMdsData::init_for_evict_medium_info( common::ObIAllocator &allocator, const ObTabletMdsData &other, const int64_t finish_medium_scn, const ObMergeType merge_type) { int ret = OB_SUCCESS; - ObArenaAllocator arena_allocator("mds_data"); - const mds::MdsDumpKV *tablet_status_uncommitted_kv = nullptr; - const mds::MdsDumpKV *tablet_status_committed_kv = nullptr; - const mds::MdsDumpKV *aux_tablet_info_uncommitted_kv = nullptr; - const mds::MdsDumpKV *aux_tablet_info_committed_kv = nullptr; - const ObTabletDumpedMediumInfo *medium_info_list = nullptr; - const share::ObTabletAutoincSeq *auto_inc_seq = nullptr; - ObTabletMemberWrapper auto_inc_seq_wrapper; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); - } - - // load or fetch - if (OB_FAIL(ret)) { - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.tablet_status_.uncommitted_kv_, tablet_status_uncommitted_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.tablet_status_.committed_kv_, tablet_status_committed_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_uncommitted_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.aux_tablet_info_.committed_kv_, aux_tablet_info_committed_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_medium_info_list(arena_allocator, other.medium_info_list_, medium_info_list))) { - LOG_WARN("failed to load medium info list", K(ret)); - } else if (OB_FAIL(fetch_auto_inc_seq(other.auto_inc_seq_, auto_inc_seq_wrapper))) { - LOG_WARN("failed to fetch auto inc seq", K(ret)); - } else if (OB_FAIL(auto_inc_seq_wrapper.get_member(auto_inc_seq))) { - LOG_WARN("ObTabletMemberWrapper get member failed", K(ret)); - } - - // do initialization - if (OB_FAIL(ret)) { - } else if (OB_FAIL(do_init(allocator, - tablet_status_uncommitted_kv, tablet_status_committed_kv, - aux_tablet_info_uncommitted_kv, aux_tablet_info_committed_kv, - auto_inc_seq))) { - LOG_WARN("failed to do init", K(ret)); - } else if (OB_FAIL(init_medium_info_list(allocator, medium_info_list, other.extra_medium_info_, - finish_medium_scn, merge_type))) { - LOG_WARN("failed to init medium info list", K(ret)); } else { - set_mem_addr(); - is_inited_ = true; - } + if (OB_FAIL(init_single_complex_addr(allocator, other.tablet_status_.uncommitted_kv_, tablet_status_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.tablet_status_.committed_kv_, tablet_status_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.aux_tablet_info_.committed_kv_, aux_tablet_info_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.auto_inc_seq_, auto_inc_seq_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.medium_info_list_, finish_medium_scn, medium_info_list_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { + LOG_WARN("failed to update user data", K(ret), "complex_addr", tablet_status_.committed_kv_); + } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { + LOG_WARN("failed to update user data", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + } else if (is_major_merge_type(merge_type)) { + extra_medium_info_.last_compaction_type_ = is_major_merge(merge_type) ? compaction::ObMediumCompactionInfo::MAJOR_COMPACTION : compaction::ObMediumCompactionInfo::MEDIUM_COMPACTION; + extra_medium_info_.last_medium_scn_ = finish_medium_scn; + extra_medium_info_.wait_check_flag_ = true; + } else { + extra_medium_info_ = other.extra_medium_info_; + } - ObTabletMdsData::free_mds_dump_kv(arena_allocator, tablet_status_uncommitted_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, tablet_status_committed_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, aux_tablet_info_uncommitted_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, aux_tablet_info_committed_kv); - ObTabletMdsData::free_medium_info_list(arena_allocator, medium_info_list); + if (OB_FAIL(ret)) { + reset(allocator); + } else { + is_inited_ = true; + LOG_INFO("succeeded to init mds data", K(ret), K(finish_medium_scn), + "src_medium_info_list", other.medium_info_list_, + "src_extra_medium_info", other.extra_medium_info_, + K_(medium_info_list), K_(extra_medium_info)); + } + } return ret; } -int ObTabletMdsData::init( +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &src_addr, + ObTabletComplexAddr &dst_addr) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + const mds::MdsDumpKV *ptr = nullptr; + + if (OB_FAIL(load_mds_dump_kv(allocator, src_addr, ptr))) { + LOG_WARN("failed to load mds dump kv", K(ret), K(src_addr)); + } else if (nullptr == ptr) { + dst_addr.addr_.set_none_addr(); + } else { + dst_addr.ptr_ = const_cast(ptr); + dst_addr.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &src_addr, + ObTabletComplexAddr &dst_addr) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + const share::ObTabletAutoincSeq *ptr = nullptr; + share::ObTabletAutoincSeq *&dst_data = dst_addr.ptr_; + + if (OB_FAIL(load_auto_inc_seq(allocator, src_addr, ptr))) { + LOG_WARN("failed to load auto inc seq", K(ret), K(src_addr)); + } else if (nullptr == ptr) { + // ptr is emtpy, no need to copy it, set dst addr to NONE + dst_addr.addr_.set_none_addr(); + } else { + dst_addr.ptr_ = const_cast(ptr); + dst_addr.addr_.set_mem_addr(0, sizeof(share::ObTabletAutoincSeq)); + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &src_addr, + const int64_t finish_medium_scn, + ObTabletComplexAddr &dst_addr) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + const ObTabletDumpedMediumInfo *ptr = nullptr; + ObTabletDumpedMediumInfo *&dst_data = dst_addr.ptr_; + + if (OB_FAIL(load_medium_info_list(allocator, src_addr, ptr))) { + LOG_WARN("failed to load medium info list", K(ret), K(src_addr)); + } else if (nullptr == ptr) { + dst_addr.addr_.set_none_addr(); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, dst_data))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(dst_data->init_for_evict_medium_info(allocator, finish_medium_scn, *ptr))) { + LOG_WARN("failed to init", K(ret)); + } else if (dst_data->medium_info_list_.empty()) { + // dst data is empty, so no need to keep it + free_medium_info_list(allocator, dst_data); + dst_data = nullptr; + dst_addr.addr_.set_none_addr(); + } else { + dst_addr.addr_.set_mem_addr(0, sizeof(ObTabletDumpedMediumInfo)); + } + + if (OB_FAIL(ret)) { + free_medium_info_list(allocator, dst_data); + dst_data = nullptr; + } + + // always free ptr after usage + free_medium_info_list(allocator, ptr); + + return ret; +} + +int ObTabletMdsData::init_by_full_memory_mds_data( common::ObIAllocator &allocator, const ObTabletFullMemoryMdsData &full_memory_mds_data) { int ret = OB_SUCCESS; - if (OB_FAIL(alloc_and_new(allocator))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(tablet_status_.uncommitted_kv_.ptr_->assign(full_memory_mds_data.tablet_status_uncommitted_kv_, allocator))) { - LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(tablet_status_.committed_kv_.ptr_->assign(full_memory_mds_data.tablet_status_committed_kv_, allocator))) { - LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(aux_tablet_info_.uncommitted_kv_.ptr_->assign(full_memory_mds_data.aux_tablet_info_uncommitted_kv_, allocator))) { - LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(aux_tablet_info_.committed_kv_.ptr_->assign(full_memory_mds_data.aux_tablet_info_committed_kv_, allocator))) { - LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(auto_inc_seq_.ptr_->assign(allocator, full_memory_mds_data.auto_inc_seq_))) { - LOG_WARN("failed to assign auto inc seq", K(ret), "auto_inc_seq", full_memory_mds_data.auto_inc_seq_); - } else if (OB_FAIL(init_medium_info_list(allocator, &full_memory_mds_data.medium_info_list_.medium_info_list_, full_memory_mds_data.medium_info_list_.extra_medium_info_))) { - LOG_WARN("failed to init medium info list", K(ret)); - } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { - LOG_WARN("failed to update user data cache", K(ret), "complex_addr", tablet_status_.committed_kv_); - } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { - LOG_WARN("failed to update user data cache", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K_(is_inited)); } else { - set_mem_addr(); - is_inited_ = true; + if (OB_FAIL(init_single_complex_addr(allocator, full_memory_mds_data.tablet_status_uncommitted_kv_, tablet_status_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, full_memory_mds_data.tablet_status_committed_kv_, tablet_status_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, full_memory_mds_data.aux_tablet_info_uncommitted_kv_, aux_tablet_info_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, full_memory_mds_data.aux_tablet_info_committed_kv_, aux_tablet_info_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, full_memory_mds_data.auto_inc_seq_, auto_inc_seq_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, full_memory_mds_data.medium_info_list_.medium_info_list_, medium_info_list_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", tablet_status_.committed_kv_); + } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + } else { + extra_medium_info_ = full_memory_mds_data.medium_info_list_.extra_medium_info_; + } + + if (OB_FAIL(ret)) { + reset(allocator); + } else { + is_inited_ = true; + LOG_INFO("succeeded to init mds data", K(ret), + "src_medium_info_list", full_memory_mds_data.medium_info_list_.medium_info_list_, + "src_extra_medium_info", full_memory_mds_data.medium_info_list_.extra_medium_info_, + K_(medium_info_list), K_(extra_medium_info)); + } } return ret; } -int ObTabletMdsData::init( +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const mds::MdsDumpKV &src_data, + ObTabletComplexAddr &dst_addr) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + mds::MdsDumpKV *&dst_data = dst_addr.ptr_; + const mds::MdsDumpNode &dump_node = src_data.v_; + + if (dump_node.user_data_.empty()) { + // src data is empty actually, dst should be set to NONE + dst_addr.addr_.set_none_addr(); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, dst_data))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(dst_data->assign(src_data, allocator))) { + LOG_WARN("failed to copy", K(ret), K(src_data)); + } else { + dst_addr.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + } + + if (OB_FAIL(ret)) { + free_mds_dump_kv(allocator, dst_data); + dst_data = nullptr; + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const share::ObTabletAutoincSeq &src_data, + ObTabletComplexAddr &dst_addr) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + share::ObTabletAutoincSeq *&dst_data = dst_addr.ptr_; + + if (!src_data.is_valid()) { + // src data is invalid, dst should be set to NONE + dst_addr.addr_.set_none_addr(); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, dst_data))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(dst_data->assign(allocator, src_data))) { + LOG_WARN("failed to copy", K(ret), K(src_data)); + } else { + dst_addr.addr_.set_mem_addr(0, sizeof(share::ObTabletAutoincSeq)); + } + + if (OB_FAIL(ret)) { + free_auto_inc_seq(allocator, dst_data); + dst_data = nullptr; + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletDumpedMediumInfo &src_data, + ObTabletComplexAddr &dst_addr) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + ObTabletDumpedMediumInfo *&dst_data = dst_addr.ptr_; + + if (!src_data.is_valid() || src_data.medium_info_list_.empty()) { + // src data is invalid or medium info list is empty, dst should be set to NONE + dst_addr.addr_.set_none_addr(); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, dst_data))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(dst_data->assign(src_data, allocator))) { + LOG_WARN("failed to copy", K(ret)); + } else { + dst_addr.addr_.set_mem_addr(0, sizeof(ObTabletDumpedMediumInfo)); + } + + if (OB_FAIL(ret)) { + free_medium_info_list(allocator, dst_data); + dst_data = nullptr; + } + + return ret; +} + +int ObTabletMdsData::init_single_complex_addr_and_extra_info( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &src_addr, + const compaction::ObExtraMediumInfo &src_addr_extra_info, + const ObTabletDumpedMediumInfo &src_data, + const compaction::ObExtraMediumInfo &src_data_extra_info, + const int64_t finish_medium_scn, + ObTabletComplexAddr &dst_addr, + compaction::ObExtraMediumInfo &dst_extra_info) +{ + int ret = OB_SUCCESS; + dst_addr.reset(); + const ObTabletDumpedMediumInfo *ptr = nullptr; + ObTabletDumpedMediumInfo *&dst_data = dst_addr.ptr_; + bool empty_data_from_src_data = (!src_data.is_valid() || src_data.medium_info_list_.empty()); + + if (OB_FAIL(load_medium_info_list(allocator, src_addr, ptr))) { + LOG_WARN("failed to load medium info list", K(ret), K(src_addr)); + } else if ((nullptr == ptr) && empty_data_from_src_data) { + dst_addr.addr_.set_none_addr(); + } else { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, dst_data))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (nullptr == ptr && !empty_data_from_src_data) { + if (OB_FAIL(dst_data->init_for_evict_medium_info(allocator, finish_medium_scn, src_data))) { + LOG_WARN("failed to copy medium info", K(ret), K(finish_medium_scn)); + } + } else if ((nullptr != ptr) && empty_data_from_src_data) { + if (OB_FAIL(dst_data->init_for_evict_medium_info(allocator, finish_medium_scn, *ptr))) { + LOG_WARN("failed to copy medium info", K(ret), K(finish_medium_scn)); + } + } else if ((nullptr != ptr) && !empty_data_from_src_data) { + if (OB_FAIL(dst_data->init_for_mds_table_dump(allocator, finish_medium_scn, *ptr, src_data))) { + LOG_WARN("failed to copy medium info", K(ret), K(finish_medium_scn)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected branch", K(ret), K(src_addr), KPC(ptr), K(src_data)); + } + + if (OB_FAIL(ret)) { + free_medium_info_list(allocator, dst_data); + dst_data = nullptr; + } else if (dst_data->medium_info_list_.empty()) { + free_medium_info_list(allocator, dst_data); + dst_data = nullptr; + dst_addr.addr_.set_none_addr(); + } else { + dst_addr.addr_.set_mem_addr(0, sizeof(ObTabletDumpedMediumInfo)); + } + } + + if (OB_FAIL(ret)) { + } else if (nullptr == ptr + || finish_medium_scn < src_addr_extra_info.last_medium_scn_ + || src_addr_extra_info.last_medium_scn_ < src_data_extra_info.last_medium_scn_) { + dst_extra_info.last_compaction_type_ = src_data_extra_info.last_compaction_type_; + dst_extra_info.last_medium_scn_ = src_data_extra_info.last_medium_scn_; + if (0 == src_data_extra_info.last_medium_scn_) { + dst_extra_info.wait_check_flag_ = false; + } else { + dst_extra_info.wait_check_flag_ = true; + } + } else { + dst_extra_info = src_addr_extra_info; + } + + // always free ptr after usage + free_medium_info_list(allocator, ptr); + + return ret; +} + +int ObTabletMdsData::init_for_merge_with_full_mds_data( common::ObIAllocator &allocator, const ObTabletMdsData &other, const ObTabletFullMediumInfo &full_memory_medium_info_list, const int64_t finish_medium_scn) { int ret = OB_SUCCESS; - ObArenaAllocator arena_allocator("mds_data"); - const mds::MdsDumpKV *tablet_status_uncommitted_kv = nullptr; - const mds::MdsDumpKV *tablet_status_committed_kv = nullptr; - const mds::MdsDumpKV *aux_tablet_info_uncommitted_kv = nullptr; - const mds::MdsDumpKV *aux_tablet_info_committed_kv = nullptr; - const ObTabletDumpedMediumInfo *medium_info_list = nullptr; - const share::ObTabletAutoincSeq *auto_inc_seq = nullptr; - ObTabletMemberWrapper auto_inc_seq_wrapper; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); - } - - // load or fetch - if (OB_FAIL(ret)) { - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.tablet_status_.uncommitted_kv_, tablet_status_uncommitted_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.tablet_status_.committed_kv_, tablet_status_committed_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_uncommitted_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.aux_tablet_info_.committed_kv_, aux_tablet_info_committed_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_medium_info_list(arena_allocator, other.medium_info_list_, medium_info_list))) { - LOG_WARN("failed to load medium info list", K(ret)); - } else if (OB_FAIL(fetch_auto_inc_seq(other.auto_inc_seq_, auto_inc_seq_wrapper))) { - LOG_WARN("failed to fetch auto inc seq", K(ret)); - } else if (OB_FAIL(auto_inc_seq_wrapper.get_member(auto_inc_seq))) { - LOG_WARN("ObTabletMemberWrapper get member failed", K(ret)); - } - - // do initialization - if (OB_FAIL(ret)) { - } else if (OB_FAIL(do_init(allocator, - tablet_status_uncommitted_kv, tablet_status_committed_kv, - aux_tablet_info_uncommitted_kv, aux_tablet_info_committed_kv, - auto_inc_seq))) { - LOG_WARN("failed to do init", K(ret)); - } else if (OB_FAIL(init_medium_info_list(allocator, medium_info_list, full_memory_medium_info_list, other.extra_medium_info_, finish_medium_scn))) { - LOG_WARN("failed to init medium info list", K(ret)); } else { - set_mem_addr(); - is_inited_ = true; - } + if (OB_FAIL(init_single_complex_addr(allocator, other.tablet_status_.uncommitted_kv_, tablet_status_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.tablet_status_.committed_kv_, tablet_status_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.aux_tablet_info_.committed_kv_, aux_tablet_info_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.auto_inc_seq_, auto_inc_seq_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", tablet_status_.committed_kv_); + } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + } else if (OB_FAIL(init_single_complex_addr_and_extra_info( + allocator, + other.medium_info_list_, + other.extra_medium_info_, + full_memory_medium_info_list.medium_info_list_, + full_memory_medium_info_list.extra_medium_info_, + finish_medium_scn, + medium_info_list_, + extra_medium_info_))) { + LOG_WARN("failed to init single complex addr", K(ret), K(finish_medium_scn)); + } - ObTabletMdsData::free_mds_dump_kv(arena_allocator, tablet_status_uncommitted_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, tablet_status_committed_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, aux_tablet_info_uncommitted_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, aux_tablet_info_committed_kv); - ObTabletMdsData::free_medium_info_list(arena_allocator, medium_info_list); + if (OB_FAIL(ret)) { + reset(allocator); + } else { + is_inited_ = true; + LOG_INFO("succeeded to init mds data", K(ret), K(finish_medium_scn), + "other_memory_medium_info_list", other.medium_info_list_, + "other_memory_extra_medium_info", other.extra_medium_info_, + "full_memory_medium_info_list", full_memory_medium_info_list.medium_info_list_, + "full_memory_extra_medium_info", full_memory_medium_info_list.extra_medium_info_, + K_(medium_info_list), K_(extra_medium_info)); + } + } return ret; } @@ -369,62 +768,48 @@ int ObTabletMdsData::init_with_update_medium_info( const ObTabletMdsData &other) { int ret = OB_SUCCESS; - ObArenaAllocator arena_allocator("mds_data"); - const mds::MdsDumpKV *tablet_status_uncommitted_kv = nullptr; - const mds::MdsDumpKV *tablet_status_committed_kv = nullptr; - const mds::MdsDumpKV *aux_tablet_info_uncommitted_kv = nullptr; - const mds::MdsDumpKV *aux_tablet_info_committed_kv = nullptr; - const ObTabletDumpedMediumInfo *medium_info_list = nullptr; - const share::ObTabletAutoincSeq *auto_inc_seq = nullptr; - ObTabletMemberWrapper auto_inc_seq_wrapper; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K_(is_inited)); - } - - // load or fetch - if (OB_FAIL(ret)) { - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.tablet_status_.uncommitted_kv_, tablet_status_uncommitted_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.tablet_status_.committed_kv_, tablet_status_committed_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_uncommitted_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_mds_dump_kv(arena_allocator, other.aux_tablet_info_.committed_kv_, aux_tablet_info_committed_kv))) { - LOG_WARN("failed to load mds dump kv", K(ret)); - } else if (OB_FAIL(load_medium_info_list(arena_allocator, other.medium_info_list_, medium_info_list))) { - LOG_WARN("failed to load medium info list", K(ret)); - } else if (OB_FAIL(fetch_auto_inc_seq(other.auto_inc_seq_, auto_inc_seq_wrapper))) { - LOG_WARN("failed to fetch auto inc seq", K(ret)); - } else if (OB_FAIL(auto_inc_seq_wrapper.get_member(auto_inc_seq))) { - LOG_WARN("ObTabletMemberWrapper get member failed", K(ret)); - } - - // do initialization - if (OB_FAIL(ret)) { - } else if (OB_FAIL(do_init(allocator, - tablet_status_uncommitted_kv, tablet_status_committed_kv, - aux_tablet_info_uncommitted_kv, aux_tablet_info_committed_kv, - auto_inc_seq))) { - LOG_WARN("failed to do init", K(ret)); - } else if (OB_FAIL(init_with_update_medium_info(allocator, medium_info_list, other.extra_medium_info_))) { - LOG_WARN("failed to init medium info list", K(ret)); } else { - set_mem_addr(); - is_inited_ = true; - } + if (OB_FAIL(init_single_complex_addr(allocator, other.tablet_status_.uncommitted_kv_, tablet_status_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.tablet_status_.committed_kv_, tablet_status_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.aux_tablet_info_.uncommitted_kv_, aux_tablet_info_.uncommitted_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.aux_tablet_info_.committed_kv_, aux_tablet_info_.committed_kv_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(init_single_complex_addr(allocator, other.auto_inc_seq_, auto_inc_seq_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", tablet_status_.committed_kv_); + } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { + LOG_WARN("failed to update user data cache", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + } else { + const int64_t finish_medium_scn = other.extra_medium_info_.last_medium_scn_; + if (OB_FAIL(init_single_complex_addr(allocator, other.medium_info_list_, finish_medium_scn, medium_info_list_))) { + LOG_WARN("failed to init single complex addr", K(ret)); + } else { + extra_medium_info_.last_compaction_type_ = other.extra_medium_info_.last_compaction_type_; + extra_medium_info_.last_medium_scn_ = other.extra_medium_info_.last_medium_scn_; + extra_medium_info_.wait_check_flag_ = false; + } + } - ObTabletMdsData::free_mds_dump_kv(arena_allocator, tablet_status_uncommitted_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, tablet_status_committed_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, aux_tablet_info_uncommitted_kv); - ObTabletMdsData::free_mds_dump_kv(arena_allocator, aux_tablet_info_committed_kv); - ObTabletMdsData::free_medium_info_list(arena_allocator, medium_info_list); + if (OB_FAIL(ret)) { + reset(allocator); + } else { + is_inited_ = true; + } + } return ret; } -int ObTabletMdsData::init_empty_shell(const ObTabletCreateDeleteMdsUserData &tablet_status) +int ObTabletMdsData::init_empty_shell( + const ObTabletCreateDeleteMdsUserData &tablet_status) { int ret = OB_SUCCESS; @@ -440,7 +825,6 @@ int ObTabletMdsData::init_empty_shell(const ObTabletCreateDeleteMdsUserData &tab aux_tablet_info_.committed_kv_.addr_.set_none_addr(); extra_medium_info_.reset(); medium_info_list_.addr_.set_none_addr(); - auto_inc_seq_.ptr_ = nullptr; auto_inc_seq_.addr_.set_none_addr(); is_inited_ = true; @@ -449,205 +833,41 @@ int ObTabletMdsData::init_empty_shell(const ObTabletCreateDeleteMdsUserData &tab return ret; } -int ObTabletMdsData::alloc_and_new(common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, tablet_status_.uncommitted_kv_.ptr_))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, tablet_status_.committed_kv_.ptr_))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, aux_tablet_info_.uncommitted_kv_.ptr_))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, aux_tablet_info_.committed_kv_.ptr_))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_.ptr_))) { - LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, auto_inc_seq_.ptr_))) { - LOG_WARN("failed to alloc and new", K(ret)); - } - - return ret; -} - void ObTabletMdsData::set_mem_addr() { auto_inc_seq_.addr_.set_mem_addr(0, sizeof(share::ObTabletAutoincSeq)); medium_info_list_.addr_.set_mem_addr(0, sizeof(ObTabletDumpedMediumInfo)); - aux_tablet_info_.committed_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); aux_tablet_info_.uncommitted_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); - tablet_status_.committed_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + aux_tablet_info_.committed_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); tablet_status_.uncommitted_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); + tablet_status_.committed_kv_.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); } -int ObTabletMdsData::do_init( +int ObTabletMdsData::init_single_mds_dump_kv( common::ObIAllocator &allocator, - const mds::MdsDumpKV *tablet_status_uncommitted_kv, - const mds::MdsDumpKV *tablet_status_committed_kv, - const mds::MdsDumpKV *aux_tablet_info_uncommitted_kv, - const mds::MdsDumpKV *aux_tablet_info_committed_kv, - const share::ObTabletAutoincSeq *auto_inc_seq) + const mds::MdsDumpKV *input_kv, + ObTabletComplexAddr &kv) { int ret = OB_SUCCESS; - if (OB_ISNULL(tablet_status_uncommitted_kv) - || OB_ISNULL(tablet_status_committed_kv) - || OB_ISNULL(aux_tablet_info_uncommitted_kv) - || OB_ISNULL(aux_tablet_info_committed_kv) - || OB_ISNULL(auto_inc_seq)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), - KP(tablet_status_uncommitted_kv), KP(tablet_status_committed_kv), - KP(aux_tablet_info_uncommitted_kv), KP(aux_tablet_info_committed_kv), - KP(auto_inc_seq)); - } else if (OB_FAIL(alloc_and_new(allocator))) { + if (nullptr == input_kv) { + kv.ptr_ = nullptr; + kv.addr_.set_none_addr(); + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, kv.ptr_))) { LOG_WARN("failed to alloc and new", K(ret)); - } else if (OB_FAIL(tablet_status_.uncommitted_kv_.ptr_->assign(*tablet_status_uncommitted_kv, allocator))) { - LOG_WARN("failed to assign tablet status uncommitted kv", K(ret)); - } else if (OB_FAIL(tablet_status_.committed_kv_.ptr_->assign(*tablet_status_committed_kv, allocator))) { - LOG_WARN("failed to assign tablet status committed kv", K(ret)); - } else if (OB_FAIL(aux_tablet_info_.uncommitted_kv_.ptr_->assign(*aux_tablet_info_uncommitted_kv, allocator))) { - LOG_WARN("failed to assign aux tablet info uncommitted kv", K(ret)); - } else if (OB_FAIL(aux_tablet_info_.committed_kv_.ptr_->assign(*aux_tablet_info_committed_kv, allocator))) { - LOG_WARN("failed to assign aux tablet info committed kv", K(ret)); - } else if (OB_FAIL(auto_inc_seq_.ptr_->assign(allocator, *auto_inc_seq))) { - LOG_WARN("failed to assign auto inc seq kv", K(ret)); - } else if (OB_FAIL(update_user_data_from_complex_addr(tablet_status_.committed_kv_, tablet_status_cache_))) { - LOG_WARN("failed to update user data", K(ret), "complex_addr", tablet_status_.committed_kv_); - } else if (OB_FAIL(update_user_data_from_complex_addr(aux_tablet_info_.committed_kv_, aux_tablet_info_cache_))) { - LOG_WARN("failed to update user data", K(ret), "complex_addr", aux_tablet_info_.committed_kv_); + } else if (OB_FAIL(kv.ptr_->assign(*input_kv, allocator))) { + LOG_WARN("failed to copy mds dump kv", K(ret)); + } else { + kv.addr_.set_mem_addr(0, sizeof(mds::MdsDumpKV)); } if (OB_FAIL(ret)) { - reset(); - } - - return ret; -} - -int ObTabletMdsData::init_medium_info_list( - common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo *old_medium_info_list, - const compaction::ObExtraMediumInfo &old_extra_medium_info, - const int64_t finish_medium_scn, - const ObMergeType merge_type) -{ - int ret = OB_SUCCESS; - ObTabletDumpedMediumInfo *cur_medium_info_list = medium_info_list_.ptr_; - - if (OB_ISNULL(cur_medium_info_list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("medium info list is null", K(ret), KP(cur_medium_info_list)); - } else if (OB_FAIL(cur_medium_info_list->init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); - } else if (nullptr == old_medium_info_list) { - // no need to copy, do nothing - extra_medium_info_.reset(); - } else if (OB_FAIL(copy_medium_info_list(finish_medium_scn, *old_medium_info_list, *cur_medium_info_list))) { - LOG_WARN("failed to copy medium info list", K(ret), K(finish_medium_scn), KPC(old_medium_info_list)); - } else if (is_major_merge_type(merge_type)) { - extra_medium_info_.last_compaction_type_ = is_major_merge(merge_type) ? compaction::ObMediumCompactionInfo::MAJOR_COMPACTION : compaction::ObMediumCompactionInfo::MEDIUM_COMPACTION; - extra_medium_info_.last_medium_scn_ = finish_medium_scn; - extra_medium_info_.wait_check_flag_ = true; - } else { - extra_medium_info_.last_compaction_type_ = old_extra_medium_info.last_compaction_type_; - extra_medium_info_.last_medium_scn_ = old_extra_medium_info.last_medium_scn_; - extra_medium_info_.wait_check_flag_ = old_extra_medium_info.wait_check_flag_; - } - - if (OB_SUCC(ret)) { - LOG_INFO("succeeded to init medium info list", K(ret), - KPC(old_medium_info_list), K(old_extra_medium_info), - K(finish_medium_scn), K(merge_type), - KPC(cur_medium_info_list), K_(extra_medium_info)); - } - - return ret; -} - -int ObTabletMdsData::init_medium_info_list( - common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo *old_medium_info_list, - const ObTabletFullMediumInfo &full_memory_medium_info_list, - const compaction::ObExtraMediumInfo &old_extra_medium_info, - const int64_t finish_medium_scn) -{ - int ret = OB_SUCCESS; - ObTabletDumpedMediumInfo *cur_medium_info_list = medium_info_list_.ptr_; - - if (OB_ISNULL(cur_medium_info_list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("medium info list is null", K(ret), KP(cur_medium_info_list)); - } else if (OB_FAIL(cur_medium_info_list->init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); - } else if (nullptr != old_medium_info_list - && OB_FAIL(copy_medium_info_list(finish_medium_scn, *old_medium_info_list, full_memory_medium_info_list.medium_info_list_, *cur_medium_info_list))) { - LOG_WARN("failed to copy medium info", K(ret)); - } else if (nullptr == old_medium_info_list - && OB_FAIL(copy_medium_info_list(finish_medium_scn, full_memory_medium_info_list.medium_info_list_, *cur_medium_info_list))) { - LOG_WARN("failed to copy medium info", K(ret)); - } else if (OB_FAIL(compaction::ObMediumListChecker::check_continue(cur_medium_info_list->medium_info_list_))) { - LOG_WARN("failed to check medium info continuity", K(ret), K(finish_medium_scn), KPC(cur_medium_info_list)); - } else { - /* - * finish_medium_scn = last_major->get_snapshot_version() - * if finish_medium_scn < old_extra_medium_info.last_medium_scn_, means local extra_medium_info is invalid, - * use input medium list to replace - */ - if (nullptr == old_medium_info_list - || finish_medium_scn < old_extra_medium_info.last_medium_scn_ - || old_extra_medium_info.last_medium_scn_ < full_memory_medium_info_list.extra_medium_info_.last_medium_scn_) { - extra_medium_info_.last_compaction_type_ = full_memory_medium_info_list.extra_medium_info_.last_compaction_type_; - extra_medium_info_.last_medium_scn_ = full_memory_medium_info_list.extra_medium_info_.last_medium_scn_; - if (0 == full_memory_medium_info_list.extra_medium_info_.last_medium_scn_) { - extra_medium_info_.wait_check_flag_ = false; - } else { - extra_medium_info_.wait_check_flag_ = true; - } - } else { - extra_medium_info_.last_compaction_type_ = old_extra_medium_info.last_compaction_type_; - extra_medium_info_.last_medium_scn_ = old_extra_medium_info.last_medium_scn_; - extra_medium_info_.wait_check_flag_ = old_extra_medium_info.wait_check_flag_; + if (nullptr != kv.ptr_) { + allocator.free(kv.ptr_); + kv.ptr_ = nullptr; } } - if (OB_SUCC(ret)) { - LOG_INFO("succeeded to init medium info list", K(ret), - KPC(old_medium_info_list), K(old_extra_medium_info), - K(full_memory_medium_info_list), K(finish_medium_scn), - KPC(cur_medium_info_list), K_(extra_medium_info)); - } - - return ret; -} - -int ObTabletMdsData::init_with_update_medium_info( - common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo *old_medium_info_list, - const compaction::ObExtraMediumInfo &old_extra_medium_info) -{ - int ret = OB_SUCCESS; - const int64_t finish_medium_scn = old_extra_medium_info.last_medium_scn_; - ObTabletDumpedMediumInfo *cur_medium_info_list = medium_info_list_.ptr_; - - if (OB_ISNULL(cur_medium_info_list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("medium info list is null", K(ret), KP(cur_medium_info_list)); - } else if (OB_FAIL(cur_medium_info_list->init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); - } else if (OB_FAIL(copy_medium_info_list(finish_medium_scn, *old_medium_info_list, *cur_medium_info_list))) { - LOG_WARN("failed to copy medium info", K(ret)); - } else { - extra_medium_info_.last_compaction_type_ = old_extra_medium_info.last_compaction_type_; - extra_medium_info_.last_medium_scn_ = old_extra_medium_info.last_medium_scn_; - extra_medium_info_.wait_check_flag_ = false; - } - - if (OB_SUCC(ret)) { - LOG_INFO("succeeded to init medium info list", K(ret), KPC(old_medium_info_list), K(old_extra_medium_info), - KPC(cur_medium_info_list), K_(extra_medium_info)); - } - return ret; } @@ -765,170 +985,29 @@ int ObTabletMdsData::copy_medium_info_list( return ret; } -int ObTabletMdsData::fuse_mds_dump_node( - common::ObIAllocator &allocator, - const ObTabletComplexAddr &mds_table_data, - const ObTabletComplexAddr &base_data, - ObTabletComplexAddr &fused_data) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!fused_data.is_memory_object())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("fused data must be memory object", K(ret), K(fused_data)); - } else if (OB_UNLIKELY(!mds_table_data.is_memory_object())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("mds table data is not in memory", K(ret), K(mds_table_data)); - } else if (OB_UNLIKELY(!base_data.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("base data is invalid", K(ret), K(base_data)); - } else { - const mds::MdsDumpKV *mds_dump_kv = mds_table_data.ptr_; - const common::ObString &mds_user_data = mds_dump_kv->v_.user_data_; - - if (mds_user_data.empty()) { - // mds data in mds table is empty, use that in base data - ObArenaAllocator arena_allocator("mds_reader"); - char *buf = nullptr; - int64_t len = 0; - int64_t pos = 0; - if (base_data.is_memory_object()) { - if (OB_FAIL(fused_data.ptr_->assign(*base_data.ptr_, allocator))) { - LOG_WARN("failed to copy", K(ret), K(base_data)); - } - } else if (OB_FAIL(ObTabletObjLoadHelper::read_from_addr(arena_allocator, base_data.addr_, buf, len))) { - LOG_WARN("failed to read mds data from block addr", K(ret)); - } else if (OB_FAIL(fused_data.ptr_->deserialize(allocator, buf, len, pos))) { - LOG_WARN("failed to deserialize", K(ret)); - } - } else { - // mds data in mds table is valid, just copy it - if (OB_FAIL(fused_data.ptr_->assign(*mds_dump_kv, allocator))) { - LOG_WARN("failed to copy", K(ret), KPC(mds_dump_kv)); - } - } - } - - return ret; -} - -int ObTabletMdsData::fuse_mds_dump_node( - common::ObIAllocator &allocator, - const ObTabletMdsDumpStruct &mds_table_data, - const ObTabletMdsDumpStruct &base_data, - ObTabletMdsDumpStruct &fused_data) -{ - int ret = OB_SUCCESS; - const ObTabletComplexAddr &mds_uncommitted_kv = mds_table_data.uncommitted_kv_; - const ObTabletComplexAddr &mds_committed_kv = mds_table_data.committed_kv_; - const ObTabletComplexAddr &base_uncommitted_kv = base_data.uncommitted_kv_; - const ObTabletComplexAddr &base_committed_kv = base_data.committed_kv_; - - if (OB_FAIL(fuse_mds_dump_node(allocator, mds_uncommitted_kv, base_uncommitted_kv, fused_data.uncommitted_kv_))) { - LOG_WARN("failed to fuse complex addr", K(ret)); - } else if (OB_FAIL(fuse_mds_dump_node(allocator, mds_committed_kv, base_committed_kv, fused_data.committed_kv_))) { - LOG_WARN("failed to fuse complex addr", K(ret)); - } - - return ret; -} - -int ObTabletMdsData::fuse_mds_dump_node( - common::ObIAllocator &allocator, - const ObTabletComplexAddr &mds_table_data, - const ObTabletComplexAddr &base_data, - ObTabletComplexAddr &fused_data) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!fused_data.is_memory_object())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("fused data must be memory object", K(ret), K(fused_data)); - } else if (OB_UNLIKELY(!mds_table_data.is_memory_object())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("mds table data is not in memory", K(ret), K(mds_table_data)); - } else if (OB_UNLIKELY(!base_data.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("base data is invalid", K(ret), K(base_data)); - } else { - const share::ObTabletAutoincSeq *mds_table_auto_inc_seq = mds_table_data.ptr_; - - if (mds_table_auto_inc_seq->is_valid()) { - if (OB_FAIL(fused_data.ptr_->assign(allocator, *mds_table_auto_inc_seq))) { - LOG_WARN("failed to assign", K(ret), KPC(mds_table_auto_inc_seq)); - } - } else { - // auto inc seq in mds table is not valid, use that in base data - ObTabletMemberWrapper auto_inc_seq_wrapper; - const share::ObTabletAutoincSeq *auto_inc_seq = nullptr; - if (OB_FAIL(fetch_auto_inc_seq(base_data, auto_inc_seq_wrapper))) { - LOG_WARN("failed to fetch auto inc seq", K(ret), K(base_data)); - } else if (OB_FAIL(auto_inc_seq_wrapper.get_member(auto_inc_seq))) { - LOG_WARN("failed to get member", K(ret)); - } else if (OB_FAIL(fused_data.ptr_->assign(allocator, *auto_inc_seq))) { - LOG_WARN("failed to assign", K(ret), KPC(auto_inc_seq)); - } - } - } - - return ret; -} - -int ObTabletMdsData::fuse_mds_dump_node( - common::ObIAllocator &allocator, - const int64_t finish_medium_scn, - const ObTabletComplexAddr &mds_table_data, - const ObTabletComplexAddr &base_data, - ObTabletComplexAddr &fused_data) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!fused_data.is_memory_object())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("fused data must be memory object", K(ret), K(fused_data)); - } else if (OB_UNLIKELY(!mds_table_data.is_memory_object())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("mds table data is not in memory", K(ret), K(mds_table_data)); - } else if (OB_UNLIKELY(!base_data.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("base data is invalid", K(ret), K(base_data)); - } else { - const ObTabletDumpedMediumInfo *mds_table_medium_info_list = mds_table_data.ptr_; - const ObTabletDumpedMediumInfo *base_medium_info_list = base_data.ptr_; - ObTabletDumpedMediumInfo *fused_medium_info_list = fused_data.ptr_; - - if (OB_FAIL(load_medium_info_list(allocator, base_data, base_medium_info_list))) { - LOG_WARN("failed to laod medium info list", K(ret), K(base_data)); - } else if (OB_ISNULL(base_medium_info_list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, base medium info list is null", K(ret), KP(base_medium_info_list)); - } else if (OB_FAIL(copy_medium_info_list(finish_medium_scn, *base_medium_info_list, *fused_medium_info_list))) { - LOG_WARN("failed to copy base medium info list", K(ret)); - } else if (OB_FAIL(copy_medium_info_list(finish_medium_scn, *mds_table_medium_info_list, *fused_medium_info_list))) { - LOG_WARN("failed to copy mds table medium info list", K(ret)); - } - - ObTabletMdsData::free_medium_info_list(allocator, base_medium_info_list); - } - - return ret; -} - int ObTabletMdsData::load_mds_dump_kv( common::ObIAllocator &allocator, const ObTabletComplexAddr &complex_addr, const mds::MdsDumpKV *&kv) { int ret = OB_SUCCESS; + kv = nullptr; mds::MdsDumpKV *ptr = nullptr; - if (OB_UNLIKELY(!complex_addr.is_valid() || complex_addr.is_none_object())) { + if (OB_UNLIKELY(!complex_addr.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid addr", K(ret), K(complex_addr)); + } else if (complex_addr.is_none_object()) { + // do nothing } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, ptr))) { LOG_WARN("failed to alloc and new", K(ret)); } else if (complex_addr.is_memory_object()) { - if (OB_FAIL(ptr->assign(*complex_addr.ptr_, allocator))) { + const mds::MdsDumpKV *src_kv = complex_addr.ptr_; + if (src_kv->v_.user_data_.empty()) { + free_mds_dump_kv(allocator, ptr); + ptr = nullptr; + LOG_INFO("read empty user data", K(ret), K(complex_addr)); + } else if (OB_FAIL(ptr->assign(*src_kv, allocator))) { LOG_WARN("failed to copy mds dump kv", K(ret)); } } else if (complex_addr.is_disk_object()) { @@ -941,6 +1020,16 @@ int ObTabletMdsData::load_mds_dump_kv( LOG_WARN("failed to read from addr", K(ret), K(addr)); } else if (OB_FAIL(ptr->deserialize(allocator, buf, len, pos))) { LOG_WARN("failed to deserialize", K(ret)); + } else { + // for compat logic + // if user data is empty, we consider that nothing is read from disk, + // and ptr should be set to null + const mds::MdsDumpNode &dump_node = ptr->v_; + if (dump_node.user_data_.empty()) { + free_mds_dump_kv(allocator, ptr); + ptr = nullptr; + LOG_INFO("read empty user data", K(ret), K(complex_addr)); + } } } else { ret = OB_ERR_UNEXPECTED; @@ -964,24 +1053,32 @@ int ObTabletMdsData::load_medium_info_list( const ObTabletDumpedMediumInfo *&medium_info_list) { int ret = OB_SUCCESS; + medium_info_list = nullptr; ObTabletDumpedMediumInfo *ptr = nullptr; if (OB_UNLIKELY(!complex_addr.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid addr", K(ret), K(complex_addr)); + } else if (complex_addr.is_none_object()) { + // do nothing } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, ptr))) { LOG_WARN("failed to alloc and new", K(ret)); - } else if (complex_addr.is_none_object()) { - if (OB_FAIL(ptr->init(allocator))) { - LOG_WARN("failed to init medium info list", K(ret)); - } } else if (complex_addr.is_memory_object()) { - if (OB_FAIL(ptr->assign(*complex_addr.ptr_, allocator))) { + const ObTabletDumpedMediumInfo *src_medium_info = complex_addr.ptr_; + if (src_medium_info->medium_info_list_.empty()) { + free_medium_info_list(allocator, ptr); + ptr = nullptr; + LOG_INFO("read empty medium info", K(ret), K(complex_addr)); + } else if (OB_FAIL(ptr->assign(*src_medium_info, allocator))) { LOG_INFO("failed to copy medium info list", K(ret)); } } else if (complex_addr.is_disk_object()) { if (OB_FAIL(read_medium_info(allocator, complex_addr.addr_, ptr->medium_info_list_))) { LOG_WARN("failed to read medium info", K(ret), "addr", complex_addr.addr_); + } else if (ptr->medium_info_list_.empty()) { + free_medium_info_list(allocator, ptr); + ptr = nullptr; + LOG_INFO("read empty medium info", K(ret), K(complex_addr)); } else { std::sort(ptr->medium_info_list_.begin(), ptr->medium_info_list_.end(), ObTabletDumpedMediumInfo::compare); @@ -1004,6 +1101,69 @@ int ObTabletMdsData::load_medium_info_list( return ret; } +int ObTabletMdsData::load_auto_inc_seq( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &complex_addr, + const share::ObTabletAutoincSeq *&auto_inc_seq) +{ + int ret = OB_SUCCESS; + auto_inc_seq = nullptr; + share::ObTabletAutoincSeq *ptr = nullptr; + + if (OB_UNLIKELY(!complex_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid addr", K(ret), K(complex_addr)); + } else if (complex_addr.is_none_object()) { + // do nothing + } else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, ptr))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (complex_addr.is_memory_object()) { + const share::ObTabletAutoincSeq *src_auto_inc_seq = complex_addr.ptr_; + if (!src_auto_inc_seq->is_valid()) { + free_auto_inc_seq(allocator, ptr); + ptr = nullptr; + LOG_INFO("read empty auto inc seq", K(ret), K(complex_addr)); + } else if (OB_FAIL(ptr->assign(allocator, *src_auto_inc_seq))) { + LOG_WARN("failed to copy auto inc seq", K(ret)); + } + } else if (complex_addr.is_disk_object()) { + ObTabletMemberWrapper wrapper; + ObStorageMetaHandle handle; + ObStorageMetaKey meta_key(MTL_ID(), complex_addr.addr_); + const ObTablet *tablet = nullptr; // no use here + const share::ObTabletAutoincSeq *auto_inc_seq_cache = nullptr; + if (OB_FAIL(OB_STORE_CACHE.get_storage_meta_cache().get_meta(ObStorageMetaValue::MetaType::AUTO_INC_SEQ, + meta_key, handle, tablet))) { + LOG_WARN("get meta failed", K(ret), K(meta_key)); + } else if (OB_FAIL(wrapper.set_cache_handle(handle))) { + LOG_WARN("wrapper set cache handle failed", K(ret), K(meta_key), K(complex_addr)); + } else if (OB_ISNULL(auto_inc_seq_cache = wrapper.get_member())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null member", K(ret)); + } else if (!auto_inc_seq_cache->is_valid()) { + // no need to copy + free_auto_inc_seq(allocator, ptr); + ptr = nullptr; + LOG_INFO("empty auto inc seq", K(ret)); + } else if (OB_FAIL(ptr->assign(allocator, *auto_inc_seq_cache))) { + LOG_WARN("failed to copy", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected complex addr type", K(ret), K(complex_addr)); + } + + if (OB_FAIL(ret)) { + if (nullptr != ptr) { + allocator.free(ptr); + } + } else { + auto_inc_seq = ptr; + } + + return ret; +} + void ObTabletMdsData::free_mds_dump_kv( common::ObIAllocator &allocator, const mds::MdsDumpKV *kv) @@ -1014,6 +1174,16 @@ void ObTabletMdsData::free_mds_dump_kv( } } +void ObTabletMdsData::free_auto_inc_seq( + common::ObIAllocator &allocator, + const share::ObTabletAutoincSeq *auto_inc_seq) +{ + if (nullptr != auto_inc_seq) { + auto_inc_seq->share::ObTabletAutoincSeq::~ObTabletAutoincSeq(); + allocator.free(const_cast(auto_inc_seq)); + } +} + void ObTabletMdsData::free_medium_info_list( common::ObIAllocator &allocator, const ObTabletDumpedMediumInfo *medium_info_list) @@ -1076,32 +1246,6 @@ int ObTabletMdsData::read_medium_info( return ret; } -int ObTabletMdsData::fetch_auto_inc_seq( - const ObTabletComplexAddr &auto_inc_seq_addr, - ObTabletMemberWrapper &wrapper) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!auto_inc_seq_addr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid auto inc seq addr", K(ret)); - } else if (auto_inc_seq_addr.is_memory_object()) { - wrapper.set_member(auto_inc_seq_addr.get_ptr()); - } else { - ObStorageMetaHandle handle; - ObStorageMetaKey meta_key(MTL_ID(), auto_inc_seq_addr.addr_); - const ObTablet *tablet = nullptr; // no use here - if (OB_FAIL(OB_STORE_CACHE.get_storage_meta_cache().get_meta(ObStorageMetaValue::MetaType::AUTO_INC_SEQ, - meta_key, handle, tablet))) { - LOG_WARN("get meta failed", K(ret), K(meta_key)); - } else if (OB_FAIL(wrapper.set_cache_handle(handle))) { - LOG_WARN("wrapper set cache handle failed", K(ret), K(meta_key), K(auto_inc_seq_addr)); - } - } - - return ret; -} - int ObTabletMdsData::build_mds_data( common::ObArenaAllocator &allocator, const share::ObTabletAutoincSeq &auto_inc_seq, @@ -1125,7 +1269,7 @@ int ObTabletMdsData::build_mds_data( const compaction::ObMediumCompactionInfoList::MediumInfoList &medium_info_list = info_list.get_list(); if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, mds_data.medium_info_list_.ptr_))) { LOG_WARN("failed to alloc and new mda data medium info list", K(ret)); - } else if (OB_FAIL(mds_data.medium_info_list_.ptr_->init(allocator))) { + } else if (OB_FAIL(mds_data.medium_info_list_.ptr_->init_for_first_creation(allocator))) { LOG_WARN("failed to init mda data medium info list", K(ret)); } @@ -1375,7 +1519,7 @@ int ObTabletMdsData::build_auto_inc_seq( } int ObTabletMdsData::set_tablet_status( - ObArenaAllocator *allocator, + common::ObIAllocator &allocator, const ObTabletStatus::Status &tablet_status, const ObTabletMdsUserDataType &data_type) { @@ -1385,7 +1529,7 @@ int ObTabletMdsData::set_tablet_status( user_data.data_type_ = data_type; const int64_t length = user_data.get_serialize_size(); - char *buffer = static_cast(allocator->alloc(length)); + char *buffer = static_cast(allocator.alloc(length)); int64_t pos = 0; if (OB_ISNULL(buffer)) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1396,13 +1540,13 @@ int ObTabletMdsData::set_tablet_status( LOG_WARN("failed to assign tablet status cache", K(ret)); } else { mds::MdsDumpNode &node = tablet_status_.committed_kv_.get_ptr()->v_; - node.allocator_ = allocator; + node.allocator_ = &allocator; node.user_data_.assign(buffer, length); } if (OB_FAIL(ret)) { if (nullptr != buffer) { - allocator->free(buffer); + allocator.free(buffer); } } return ret; diff --git a/src/storage/tablet/ob_tablet_mds_data.h b/src/storage/tablet/ob_tablet_mds_data.h index d3689d3482..271a9a3dc4 100644 --- a/src/storage/tablet/ob_tablet_mds_data.h +++ b/src/storage/tablet/ob_tablet_mds_data.h @@ -18,6 +18,7 @@ #include "lib/string/ob_string.h" #include "share/ob_tablet_autoincrement_param.h" #include "storage/compaction/ob_compaction_util.h" +#include "storage/compaction/ob_extra_medium_info.h" #include "storage/multi_data_source/adapter_define/mds_dump_node.h" #include "storage/tablet/ob_tablet_complex_addr.h" #include "storage/tablet/ob_tablet_member_wrapper.h" @@ -35,7 +36,6 @@ namespace compaction class ObMediumCompactionInfoKey; class ObMediumCompactionInfo; class ObMediumCompactionInfoList; -class ObExtraMediumInfo; } namespace storage @@ -71,21 +71,25 @@ public: ObTabletMdsData &operator=(const ObTabletMdsData&) = delete; public: void reset(); - int init(common::ObIAllocator &allocator); - int init( + int init_for_first_creation(common::ObIAllocator &allocator); + int init_with_tablet_status( + common::ObIAllocator &allocator, + const ObTabletStatus::Status &tablet_status, + const ObTabletMdsUserDataType &data_type); + int init_by_full_memory_mds_data( common::ObIAllocator &allocator, const ObTabletFullMemoryMdsData &full_memory_mds_data); - int init( + int init_for_mds_table_dump( common::ObIAllocator &allocator, const ObTabletMdsData &mds_table_data, const ObTabletMdsData &base_data, const int64_t finish_medium_scn); - int init( + int init_for_evict_medium_info( common::ObIAllocator &allocator, const ObTabletMdsData &other, const int64_t finish_medium_scn, const ObMergeType merge_type = ObMergeType::MERGE_TYPE_MAX); - int init( + int init_for_merge_with_full_mds_data( common::ObIAllocator &allocator, const ObTabletMdsData &other, const ObTabletFullMediumInfo &full_memory_medium_info_list, @@ -93,11 +97,8 @@ public: int init_with_update_medium_info( common::ObIAllocator &allocator, const ObTabletMdsData &other); - int init_empty_shell(const ObTabletCreateDeleteMdsUserData &tablet_status); - int set_tablet_status( - ObArenaAllocator *allocator, - const ObTabletStatus::Status &tablet_status, - const ObTabletMdsUserDataType &data_type); + int init_empty_shell( + const ObTabletCreateDeleteMdsUserData &tablet_status); bool is_valid() const; void set_mem_addr(); public: @@ -122,15 +123,19 @@ public: common::ObIAllocator &allocator, const ObTabletComplexAddr &complex_addr, const ObTabletDumpedMediumInfo *&medium_info_list); + static int load_auto_inc_seq( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &complex_addr, + const share::ObTabletAutoincSeq *&auto_inc_seq); static void free_mds_dump_kv( common::ObIAllocator &allocator, const mds::MdsDumpKV *kv); + static void free_auto_inc_seq( + common::ObIAllocator &allocator, + const share::ObTabletAutoincSeq *auto_inc_seq); static void free_medium_info_list( common::ObIAllocator &allocator, const ObTabletDumpedMediumInfo *medium_info_list); - static int fetch_auto_inc_seq( - const ObTabletComplexAddr &auto_inc_seq_addr, - ObTabletMemberWrapper &wrapper); static int build_tablet_status( common::ObArenaAllocator &allocator, const ObTabletTxMultiSourceDataUnit &tx_data, @@ -156,50 +161,60 @@ public: const compaction::ObMediumCompactionInfoList &info_list, ObTabletMdsData &mds_data); private: - int alloc_and_new(common::ObIAllocator &allocator); - int do_init( + void reset(common::ObIAllocator &allocator); + int set_tablet_status( common::ObIAllocator &allocator, - const mds::MdsDumpKV *tablet_status_uncommitted_kv, - const mds::MdsDumpKV *tablet_status_committed_kv, - const mds::MdsDumpKV *aux_tablet_info_uncommitted_kv, - const mds::MdsDumpKV *aux_tablet_info_committed_kv, - const share::ObTabletAutoincSeq *auto_inc_seq); - int init_medium_info_list( + const ObTabletStatus::Status &tablet_status, + const ObTabletMdsUserDataType &data_type); + static int init_single_complex_addr( common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo *old_medium_info_list, - const compaction::ObExtraMediumInfo &old_extra_medium_info, - const int64_t finish_medium_scn = 0, - const ObMergeType merge_type = ObMergeType::MERGE_TYPE_MAX); - int init_medium_info_list( + const ObTabletComplexAddr &src_addr, + ObTabletComplexAddr &dst_addr); + static int init_single_complex_addr( common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo *old_medium_info_list, - const ObTabletFullMediumInfo &full_memory_medium_info_list, - const compaction::ObExtraMediumInfo &old_extra_medium_info, - const int64_t finish_medium_scn); - int init_with_update_medium_info( + const ObTabletComplexAddr &src_addr, + ObTabletComplexAddr &dst_addr); + static int init_single_complex_addr( common::ObIAllocator &allocator, - const ObTabletDumpedMediumInfo *old_medium_info_list, - const compaction::ObExtraMediumInfo &old_extra_medium_info); - static int fuse_mds_dump_node( + const ObTabletComplexAddr &src_addr, + const int64_t finish_medium_scn, + ObTabletComplexAddr &dst_addr); + static int init_single_complex_addr( + common::ObIAllocator &allocator, + const mds::MdsDumpKV &src_data, + ObTabletComplexAddr &dst_addr); + static int init_single_complex_addr( + common::ObIAllocator &allocator, + const share::ObTabletAutoincSeq &src_data, + ObTabletComplexAddr &dst_addr); + static int init_single_complex_addr( + common::ObIAllocator &allocator, + const ObTabletDumpedMediumInfo &src_data, + ObTabletComplexAddr &dst_addr); + static int init_single_complex_addr_and_extra_info( + common::ObIAllocator &allocator, + const ObTabletComplexAddr &src_addr, + const compaction::ObExtraMediumInfo &src_addr_extra_info, + const ObTabletDumpedMediumInfo &src_data, + const compaction::ObExtraMediumInfo &src_data_extra_info, + const int64_t finish_medium_scn, + ObTabletComplexAddr &dst_addr, + compaction::ObExtraMediumInfo &dst_extra_info); + static int init_single_complex_addr( common::ObIAllocator &allocator, const ObTabletComplexAddr &mds_table_data, const ObTabletComplexAddr &base_data, ObTabletComplexAddr &fused_data); - static int fuse_mds_dump_node( - common::ObIAllocator &allocator, - const ObTabletMdsDumpStruct &mds_table_data, - const ObTabletMdsDumpStruct &base_data, - ObTabletMdsDumpStruct &fused_data); - static int fuse_mds_dump_node( + static int init_single_complex_addr( common::ObIAllocator &allocator, const ObTabletComplexAddr &mds_table_data, const ObTabletComplexAddr &base_data, ObTabletComplexAddr &fused_data); - static int fuse_mds_dump_node( + static int init_single_complex_addr( common::ObIAllocator &allocator, - const int64_t finish_medium_scn, const ObTabletComplexAddr &mds_table_data, const ObTabletComplexAddr &base_data, + const int64_t finish_medium_scn, ObTabletComplexAddr &fused_data); static int read_medium_info( common::ObIAllocator &allocator, @@ -214,6 +229,10 @@ private: const ObTabletDumpedMediumInfo &input_medium_info_list1, const ObTabletDumpedMediumInfo &input_medium_info_list2, ObTabletDumpedMediumInfo &medium_info_list); + static int init_single_mds_dump_kv( + common::ObIAllocator &allocator, + const mds::MdsDumpKV *input_kv, + ObTabletComplexAddr &kv); template static int update_user_data_from_complex_addr( const ObTabletComplexAddr &complex_addr, @@ -238,9 +257,11 @@ int ObTabletMdsData::update_user_data_from_complex_addr( { int ret = common::OB_SUCCESS; - if (OB_UNLIKELY(!complex_addr.is_memory_object())) { + if (OB_UNLIKELY(!complex_addr.is_memory_object() && !complex_addr.is_none_object())) { ret = common::OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "complex addr is not memory type", K(ret), K(complex_addr)); + STORAGE_LOG(WARN, "complex addr is not memory or none type", K(ret), K(complex_addr)); + } else if (complex_addr.is_none_object()) { + // do nothing } else { const mds::MdsDumpKV *kv = complex_addr.ptr_; const common::ObString &str = kv->v_.user_data_; diff --git a/src/storage/tablet/ob_tablet_mds_node_dump_operator.cpp b/src/storage/tablet/ob_tablet_mds_node_dump_operator.cpp index 33cedaae47..91e1a9caa2 100644 --- a/src/storage/tablet/ob_tablet_mds_node_dump_operator.cpp +++ b/src/storage/tablet/ob_tablet_mds_node_dump_operator.cpp @@ -18,6 +18,7 @@ #include "share/ob_tablet_autoincrement_param.h" #include "storage/tablet/ob_tablet_mds_data.h" #include "storage/tablet/ob_tablet_create_delete_mds_user_data.h" +#include "storage/tablet/ob_tablet_obj_load_helper.h" #include "storage/multi_data_source/adapter_define/mds_dump_node.h" #include "storage/multi_data_source/compile_utility/mds_dummy_key.h" #include "storage/multi_data_source/mds_table_handle.h" @@ -43,6 +44,8 @@ int ObTabletDumpMdsNodeOperator::dump>::value; const mds::MdsDumpKey &key = kv.k_; const mds::MdsDumpNode &node = kv.v_; + ObTabletComplexAddr &uncommitted_tablet_status = mds_data_.tablet_status_.uncommitted_kv_; + ObTabletComplexAddr &committed_tablet_status = mds_data_.tablet_status_.committed_kv_; if (table_id == key.mds_table_id_ && unit_id == key.mds_unit_id_) { const mds::TwoPhaseCommitState &state = node.status_.get_state(); @@ -51,13 +54,29 @@ int ObTabletDumpMdsNodeOperator::dumpassign(kv, allocator_))) { - LOG_WARN("failed to copy mds dump kv", K(ret)); + { + if (!uncommitted_tablet_status.is_memory_object()) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator_, uncommitted_tablet_status.ptr_))) { + LOG_WARN("failed to alloc and new", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(uncommitted_tablet_status.ptr_->assign(kv, allocator_))) { + LOG_WARN("failed to copy mds dump kv", K(ret)); + } } break; case mds::TwoPhaseCommitState::ON_COMMIT: - if (OB_FAIL(mds_data_.tablet_status_.committed_kv_.ptr_->assign(kv, allocator_))) { - LOG_WARN("failed to copy mds dump kv", K(ret)); + { + if (!committed_tablet_status.is_memory_object()) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator_, committed_tablet_status.ptr_))) { + LOG_WARN("failed to alloc and new", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(committed_tablet_status.ptr_->assign(kv, allocator_))) { + LOG_WARN("failed to copy mds dump kv", K(ret)); + } } break; case mds::TwoPhaseCommitState::STATE_END: @@ -82,6 +101,8 @@ int ObTabletDumpMdsNodeOperator::dump constexpr uint8_t unit_id = mds::TupleTypeIdx>::value; const mds::MdsDumpKey &key = kv.k_; const mds::MdsDumpNode &node = kv.v_; + ObTabletComplexAddr &uncommitted_aux_tablet_info = mds_data_.aux_tablet_info_.uncommitted_kv_; + ObTabletComplexAddr &committed_aux_tablet_info = mds_data_.aux_tablet_info_.committed_kv_; if (table_id == key.mds_table_id_ && unit_id == key.mds_unit_id_) { const mds::TwoPhaseCommitState &state = node.status_.get_state(); @@ -90,13 +111,29 @@ int ObTabletDumpMdsNodeOperator::dump case mds::TwoPhaseCommitState::BEFORE_PREPARE: case mds::TwoPhaseCommitState::ON_ABORT: case mds::TwoPhaseCommitState::ON_PREPARE: - if (OB_FAIL(mds_data_.aux_tablet_info_.uncommitted_kv_.ptr_->assign(kv, allocator_))) { - LOG_WARN("failed to copy mds dump kv", K(ret)); + { + if (!uncommitted_aux_tablet_info.is_memory_object()) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator_, uncommitted_aux_tablet_info.ptr_))) { + LOG_WARN("failed to alloc and new", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(uncommitted_aux_tablet_info.ptr_->assign(kv, allocator_))) { + LOG_WARN("failed to copy mds dump kv", K(ret)); + } } break; case mds::TwoPhaseCommitState::ON_COMMIT: - if (OB_FAIL(mds_data_.aux_tablet_info_.committed_kv_.ptr_->assign(kv, allocator_))) { - LOG_WARN("failed to copy mds dump kv", K(ret)); + { + if (!committed_aux_tablet_info.is_memory_object()) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator_, committed_aux_tablet_info.ptr_))) { + LOG_WARN("failed to alloc and new", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(committed_aux_tablet_info.ptr_->assign(kv, allocator_))) { + LOG_WARN("failed to copy mds dump kv", K(ret)); + } } break; case mds::TwoPhaseCommitState::STATE_END: @@ -121,16 +158,23 @@ int ObTabletDumpMdsNodeOperator::dump( constexpr uint8_t unit_id = mds::TupleTypeIdx>::value; const mds::MdsDumpKey &key = kv.k_; const mds::MdsDumpNode &node = kv.v_; + ObTabletComplexAddr &auto_inc_seq = mds_data_.auto_inc_seq_; if (table_id == key.mds_table_id_ && unit_id == key.mds_unit_id_) { const mds::TwoPhaseCommitState &state = node.status_.get_state(); const common::ObString &user_data = kv.v_.user_data_; int64_t pos = 0; - share::ObTabletAutoincSeq *auto_inc_seq = mds_data_.auto_inc_seq_.ptr_; if (OB_UNLIKELY(state != mds::TwoPhaseCommitState::ON_COMMIT)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid state", K(ret), K(state)); - } else if (OB_FAIL(auto_inc_seq->deserialize(allocator_, user_data.ptr(), user_data.length(), pos))) { + } else if (!auto_inc_seq.is_memory_object()) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator_, auto_inc_seq.ptr_))) { + LOG_WARN("failed to alloc and new", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(auto_inc_seq.ptr_->deserialize(allocator_, user_data.ptr(), user_data.length(), pos))) { LOG_WARN("failed to deserialize", K(ret)); } else { dumped = true; @@ -148,14 +192,23 @@ int ObTabletDumpMdsNodeOperator::dump>::value; const mds::MdsDumpKey &key = kv.k_; const mds::MdsDumpNode &node = kv.v_; + ObTabletComplexAddr &medium_info_list = mds_data_.medium_info_list_; if (table_id == key.mds_table_id_ && unit_id == key.mds_unit_id_) { const mds::TwoPhaseCommitState &state = node.status_.get_state(); - ObTabletDumpedMediumInfo *medium_info_list = mds_data_.medium_info_list_.ptr_; if (OB_UNLIKELY(state != mds::TwoPhaseCommitState::ON_COMMIT)) { ret = OB_SUCCESS; LOG_WARN("invalid state", K(ret), K(state)); - } else if (OB_FAIL(medium_info_list->append(key, node))) { + } else if (!medium_info_list.is_memory_object()) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator_, medium_info_list.ptr_))) { + LOG_WARN("failed to alloc and new", K(ret)); + } else if (OB_FAIL(medium_info_list.ptr_->init_for_first_creation(allocator_))) { + LOG_WARN("failed to init medium info list", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(medium_info_list.ptr_->append(key, node))) { LOG_WARN("failed to copy mds dump node", K(ret)); } else { dumped = true; diff --git a/src/storage/tablet/ob_tablet_medium_info_reader.cpp b/src/storage/tablet/ob_tablet_medium_info_reader.cpp index 676195739e..cae85c5df9 100644 --- a/src/storage/tablet/ob_tablet_medium_info_reader.cpp +++ b/src/storage/tablet/ob_tablet_medium_info_reader.cpp @@ -46,35 +46,32 @@ ObTabletMediumInfoReader::~ObTabletMediumInfoReader() int ObTabletMediumInfoReader::init(common::ObArenaAllocator &allocator) { int ret = OB_SUCCESS; + const share::ObLSID &ls_id = tablet_.get_tablet_meta().ls_id_; + const common::ObTabletID &tablet_id = tablet_.get_tablet_meta().tablet_id_; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret), K_(is_inited)); + LOG_WARN("init twice", K(ret), K(ls_id), K(tablet_id), K_(is_inited)); } else { mds::MdsTableHandle mds_table; const ObTabletDumpedMediumInfo *dumped_medium_info = nullptr; if (OB_FAIL(tablet_.inner_get_mds_table(mds_table, false/*not_exist_create*/))) { if (OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("failed to get mds table", K(ret)); + LOG_WARN("failed to get mds table", K(ret), K(ls_id), K(tablet_id)); } else { mds_end_ = true; // no mds table, directly iter end ret = OB_SUCCESS; - LOG_DEBUG("no mds table", K(ret), K_(mds_end)); + LOG_DEBUG("no mds table", K(ret), K(ls_id), K(tablet_id), K_(mds_end)); } } else if (OB_FAIL(mds_iter_.init(mds_table))) { - LOG_WARN("failed to init mds iter", K(ret)); + LOG_WARN("failed to init mds iter", K(ret), K(ls_id), K(tablet_id)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(ObTabletMdsData::load_medium_info_list(allocator, tablet_.mds_data_.medium_info_list_, dumped_medium_info))) { - LOG_WARN("failed to load medium info list", K(ret), - "ls_id", tablet_.get_tablet_meta().ls_id_, - "tablet_id", tablet_.get_tablet_meta().tablet_id_); - } else if (OB_ISNULL(dumped_medium_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, medium info is null", K(ret), KP(dumped_medium_info)); - } else if (OB_FAIL(dump_iter_.init(allocator, *dumped_medium_info))) { - LOG_WARN("failed to init dumped iter", K(ret)); + LOG_WARN("failed to load medium info list", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(dump_iter_.init(allocator, dumped_medium_info))) { + LOG_WARN("failed to init dumped iter", K(ret), K(ls_id), K(tablet_id)); } else { allocator_ = &allocator; } diff --git a/src/storage/tablet/ob_tablet_persister.cpp b/src/storage/tablet/ob_tablet_persister.cpp index dd7af3bc46..98335352ae 100644 --- a/src/storage/tablet/ob_tablet_persister.cpp +++ b/src/storage/tablet/ob_tablet_persister.cpp @@ -69,6 +69,12 @@ void ObTabletTransformArg::reset() auto_inc_seq_addr_.reset(); tablet_status_cache_.reset(); aux_tablet_info_cache_.reset(); + ddl_kvs_ = nullptr; + ddl_kv_count_ = 0; + for (int64_t i = 0; i < MAX_MEMSTORE_CNT; ++i) { + memtables_[i] = nullptr; + } + memtable_count_ = 0; } bool ObTabletTransformArg::is_valid() const @@ -92,7 +98,7 @@ int ObTabletPersister::persist_and_transform_tablet( { TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; - common::ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "PesistTranf")); + common::ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "PATF")); common::ObSEArray tablet_meta_write_ctxs; common::ObSEArray sstable_meta_write_ctxs; @@ -176,13 +182,13 @@ int ObTabletPersister::convert_tablet_to_mem_arg( } return ret; } + int ObTabletPersister::convert_tablet_to_disk_arg( common::ObArenaAllocator &allocator, const ObTablet &tablet, common::ObIArray &tablet_meta_write_ctxs, common::ObIArray &sstable_meta_write_ctxs, ObTabletPoolType &type, - ObTabletMemberWrapper &auto_inc_seq, ObTabletTransformArg &arg) { TIMEGUARD_INIT(STORAGE, 10_ms); @@ -191,18 +197,7 @@ int ObTabletPersister::convert_tablet_to_disk_arg( common::ObSEArray write_infos; // fetch member wrapper - ObTabletMemberWrapper table_store; - // fetch member function - FetchTableStore fetch_table_store = - std::bind(&ObTablet::fetch_table_store, &tablet, _1); - FetchAutoincSeq fetch_auto_inc_seq = - std::bind(&ObTabletMdsData::fetch_auto_inc_seq, std::cref(tablet.mds_data_.auto_inc_seq_), _1); - - // load member - const ObStorageSchema *storage_schema = nullptr; - // load member function - LoadStorageSchema load_storage_schema = - std::bind(&ObTablet::load_storage_schema, &tablet, _1, _2); + ObTabletMemberWrapper table_store_wrapper; // load new mds data const ObTabletComplexAddr &uncommitted_tablet_status_addr = tablet.mds_data_.tablet_status_.uncommitted_kv_; @@ -219,40 +214,36 @@ int ObTabletPersister::convert_tablet_to_disk_arg( LOG_WARN("fail to assign tablet meta", K(ret), K(tablet)); } else if (FALSE_IT(arg.rowkey_read_info_ptr_ = tablet.rowkey_read_info_)) { } else if (FALSE_IT(arg.extra_medium_info_ = tablet.mds_data_.extra_medium_info_)) { - } else if (CLICK_FAIL(fetch_wrapper_and_write_info(allocator, fetch_table_store, table_store, write_infos, sstable_meta_write_ctxs))) { - LOG_WARN("fail to fetch table store wrapper and write info", K(ret)); - } else if (CLICK_FAIL(fetch_wrapper_and_write_info( - allocator, fetch_auto_inc_seq, auto_inc_seq, write_infos, tablet_meta_write_ctxs))) { - LOG_WARN("fail to fetch auto inc seq wrapper and write info", K(ret)); - } else if (FALSE_IT(arg.auto_inc_seq_ptr_ = auto_inc_seq.get_member())) { + } else if (CLICK_FAIL(fetch_table_store_and_write_info(tablet, allocator, table_store_wrapper, write_infos, sstable_meta_write_ctxs))) { + LOG_WARN("fail to fetch table store and write info", K(ret)); + } else if (CLICK_FAIL(load_auto_inc_seq_and_write_info(allocator, tablet.mds_data_.auto_inc_seq_, arg.auto_inc_seq_ptr_, write_infos, arg.auto_inc_seq_addr_))) { + LOG_WARN("fail to load auto inc seq and write info", K(ret)); } else if (FALSE_IT(arg.ddl_kvs_ = tablet.ddl_kvs_)) { } else if (FALSE_IT(arg.ddl_kv_count_ = tablet.ddl_kv_count_)) { } else if (FALSE_IT(arg.memtable_count_ = tablet.memtable_count_)) { } else if (OB_ISNULL(MEMCPY(arg.memtables_, tablet.memtables_, sizeof(arg.memtables_)))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to memcpy memtables", K(ret), KP(arg.memtables_), KP(tablet.memtables_)); - } else if (CLICK_FAIL(load_member_and_write_info( - allocator, load_storage_schema, storage_schema, write_infos))) { - LOG_WARN("fail to load storage schema and write info", K(ret)); - } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, uncommitted_tablet_status_addr, write_infos))) { + } else if (CLICK_FAIL(load_storage_schema_and_fill_write_info(tablet, allocator, write_infos))) { + LOG_WARN("fail to load storage schema and fill write info", K(ret)); + } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, uncommitted_tablet_status_addr, write_infos, arg.tablet_status_uncommitted_kv_addr_))) { LOG_WARN("fail to load tablet status uncommitted kv", K(ret), K(uncommitted_tablet_status_addr)); - } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, committed_tablet_status_addr, write_infos))) { + } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, committed_tablet_status_addr, write_infos, arg.tablet_status_committed_kv_addr_))) { LOG_WARN("fail to load tablet status committed kv", K(ret), K(committed_tablet_status_addr)); - } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, uncommitted_aux_tablet_info_addr, write_infos))) { + } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, uncommitted_aux_tablet_info_addr, write_infos, arg.aux_tablet_info_uncommitted_kv_addr_))) { LOG_WARN("fail to load aux tablet info uncommitted kv", K(ret), K(uncommitted_aux_tablet_info_addr)); - } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, committed_aux_tablet_info_addr, write_infos))) { + } else if (CLICK_FAIL(load_dump_kv_and_fill_write_info(allocator, committed_aux_tablet_info_addr, write_infos, arg.aux_tablet_info_committed_kv_addr_))) { LOG_WARN("fail to load aux tablet info committed kv", K(ret), K(committed_aux_tablet_info_addr)); } else if (CLICK_FAIL(write_and_fill_args(write_infos, arg, tablet_meta_write_ctxs))) { - LOG_WARN("fail to write and fill address", K(ret), K(write_infos), K(auto_inc_seq)); - } else if (CLICK_FAIL(load_medium_info_list_and_write(allocator, medium_info_list_addr, arg.medium_info_list_addr_, tablet_meta_write_ctxs))) { + LOG_WARN("fail to write and fill address", K(ret)); + } else if (CLICK_FAIL(load_medium_info_list_and_write(allocator, medium_info_list_addr, tablet_meta_write_ctxs, arg.medium_info_list_addr_))) { LOG_WARN("fail to load medium info list and write", K(ret), K(medium_info_list_addr)); } else { - const int64_t try_cache_size = tablet.get_try_cache_size() + table_store.get_member()->get_deep_copy_size(); + const int64_t try_cache_size = tablet.get_try_cache_size() + table_store_wrapper.get_member()->get_deep_copy_size(); if (try_cache_size > ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE) { type = ObTabletPoolType::TP_LARGE; } } - ObTablet::free_storage_schema(allocator, storage_schema); return ret; } @@ -266,7 +257,6 @@ int ObTabletPersister::persist_and_fill_tablet( { TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; - common::ObSEArray write_infos; ObTabletTransformArg arg; const ObTabletMeta &tablet_meta = old_tablet.get_tablet_meta(); @@ -280,13 +270,14 @@ int ObTabletPersister::persist_and_fill_tablet( LOG_WARN("fail to conver tablet to mem arg", K(ret), K(old_tablet)); } } else if (CLICK_FAIL(convert_tablet_to_disk_arg( - allocator, old_tablet, tablet_meta_write_ctxs, sstable_meta_write_ctxs, type, auto_inc_seq, arg))) { + allocator, old_tablet, tablet_meta_write_ctxs, sstable_meta_write_ctxs, type, arg))) { LOG_WARN("fail to conver tablet to disk arg", K(ret), K(old_tablet)); } else if (old_tablet.get_try_cache_size() > ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE) { try_smaller_pool = false; } - if (CLICK() && FAILEDx(acquire_tablet(type, key, try_smaller_pool, new_handle))) { + if (OB_FAIL(ret)) { + } else if (CLICK_FAIL(acquire_tablet(type, key, try_smaller_pool, new_handle))) { LOG_WARN("fail to acquire tablet", K(ret), K(key), K(type)); } else if (CLICK_FAIL(transform(arg, new_handle.get_buf(), new_handle.get_buf_len()))) { LOG_WARN("fail to transform old tablet", K(ret), K(arg), K(new_handle), K(type)); @@ -366,17 +357,14 @@ int ObTabletPersister::persist_4k_tablet(common::ObArenaAllocator &allocator, Ob TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; ObTablet *new_tablet = new_handle.get_obj(); - ObTenantCheckpointSlogHandler *ckpt_slog_hanlder = MTL(ObTenantCheckpointSlogHandler*); + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); common::ObSEArray write_infos; ObSharedBlockWriteHandle handle; ObSharedBlocksWriteCtx write_ctx; if (CLICK_FAIL(fill_write_info(allocator, new_tablet, write_infos))) { LOG_WARN("fail to fill write info", K(ret), KPC(new_tablet)); - } else if (OB_ISNULL(ckpt_slog_hanlder)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, ckpt slog handler is nullptr", K(ret), KP(ckpt_slog_hanlder)); - } else if (CLICK_FAIL(ckpt_slog_hanlder->get_shared_block_reader_writer().async_write(write_infos.at(0), handle))) { - LOG_WARN("fail to async write", K(ret), K(write_infos)); + } else if (CLICK_FAIL(ckpt_slog_handler->get_shared_block_reader_writer().async_write(write_infos.at(0), handle))) { + LOG_WARN("fail to async write", K(ret), "write_info", write_infos.at(0)); } else if (CLICK_FAIL(handle.get_write_ctx(write_ctx))) { LOG_WARN("fail to batch get address", K(ret), K(handle)); } else if (FALSE_IT(new_tablet->set_tablet_addr(write_ctx.addr_))) { @@ -410,8 +398,7 @@ int ObTabletPersister::convert_arg_to_tablet( tablet.mds_data_.tablet_status_.committed_kv_.addr_ = arg.tablet_status_committed_kv_addr_; tablet.mds_data_.aux_tablet_info_.uncommitted_kv_.addr_ = arg.aux_tablet_info_uncommitted_kv_addr_; tablet.mds_data_.aux_tablet_info_.committed_kv_.addr_ = arg.aux_tablet_info_committed_kv_addr_; - tablet.mds_data_.extra_medium_info_.info_ = arg.extra_medium_info_.info_; - tablet.mds_data_.extra_medium_info_.last_medium_scn_ = arg.extra_medium_info_.last_medium_scn_; + tablet.mds_data_.extra_medium_info_ = arg.extra_medium_info_; tablet.mds_data_.medium_info_list_.addr_ = arg.medium_info_list_addr_; tablet.mds_data_.auto_inc_seq_.addr_ = arg.auto_inc_seq_addr_; tablet.mds_data_.is_inited_ = true; @@ -526,6 +513,7 @@ int ObTabletPersister::transform( if (OB_SUCC(ret)) { if (OB_ISNULL(arg.auto_inc_seq_ptr_)) { tiny_tablet->mds_data_.auto_inc_seq_.ptr_ = nullptr; + tiny_tablet->mds_data_.auto_inc_seq_.addr_.set_none_addr(); } else { LOG_DEBUG("TINY TABLET: tablet + rowkey_read_info + tablet store", KP(buf), K(start_pos), K(remain)); ObIStorageMetaObj *auto_inc_obj = nullptr; @@ -590,11 +578,11 @@ int ObTabletPersister::fetch_and_persist_sstable( ret = OB_SUCCESS; } if (OB_SUCC(ret) && write_infos.count() > 0) { - ObTenantCheckpointSlogHandler *ckpt_slog_hanlder = MTL(ObTenantCheckpointSlogHandler*); - if (OB_ISNULL(ckpt_slog_hanlder)) { + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + if (OB_ISNULL(ckpt_slog_handler)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, ckpt slog handler is nullptr", K(ret), KP(ckpt_slog_hanlder)); - } else if (OB_FAIL(ckpt_slog_hanlder->get_shared_block_reader_writer().async_batch_write(write_infos, handle))) { + LOG_WARN("unexpected error, ckpt slog handler is nullptr", K(ret), KP(ckpt_slog_handler)); + } else if (OB_FAIL(ckpt_slog_handler->get_shared_block_reader_writer().async_batch_write(write_infos, handle))) { LOG_WARN("fail to batch async write", K(ret), K(write_infos)); } else if (OB_FAIL(handle.batch_get_write_ctx(write_ctxs))) { LOG_WARN("fail to batch get addr", K(ret), K(handle)); @@ -624,8 +612,8 @@ int ObTabletPersister::write_and_fill_args( common::ObIArray &meta_write_ctxs) { int ret = OB_SUCCESS; - ObTenantCheckpointSlogHandler *ckpt_slog_hanlder = MTL(ObTenantCheckpointSlogHandler*); - ObSharedBlockReaderWriter &reader_writer = ckpt_slog_hanlder->get_shared_block_reader_writer(); + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + ObSharedBlockReaderWriter &reader_writer = ckpt_slog_handler->get_shared_block_reader_writer(); ObSharedBlockBatchHandle handle; ObMetaDiskAddr* addr[] = { // NOTE: The order must be the same as the batch async write. &arg.table_store_addr_, @@ -636,29 +624,43 @@ int ObTabletPersister::write_and_fill_args( &arg.aux_tablet_info_uncommitted_kv_addr_, &arg.aux_tablet_info_committed_kv_addr_, }; - common::ObSEArray write_ctxs; - if (OB_UNLIKELY(sizeof(addr)/sizeof(addr[0]) != write_infos.count())) { + constexpr int64_t total_addr_cnt = sizeof(addr) / sizeof(addr[0]); + int64_t none_addr_cnt = 0; + for (int64_t i = 0; i < total_addr_cnt; ++i) { + if (addr[i]->is_none()) { + ++none_addr_cnt; + } + } + + common::ObSEArray write_ctxs; + if (OB_UNLIKELY(total_addr_cnt != write_infos.count() + none_addr_cnt)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), "write info count", write_infos.count(), K(write_infos)); - } else if (OB_ISNULL(ckpt_slog_hanlder)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, ckpt slog handler is nullptr", K(ret), KP(ckpt_slog_hanlder)); - } else if (OB_FAIL(ckpt_slog_hanlder->get_shared_block_reader_writer().async_batch_write(write_infos, handle))) { - LOG_WARN("fail to batch async write", K(ret), K(write_infos)); + LOG_WARN("invalid arguments", K(ret), K(total_addr_cnt), "write_info_count", write_infos.count(), K(none_addr_cnt)); + } else if (OB_FAIL(reader_writer.async_batch_write(write_infos, handle))) { + LOG_WARN("fail to batch async write", K(ret)); } else if (OB_FAIL(handle.batch_get_write_ctx(write_ctxs))) { LOG_WARN("fail to batch get addr", K(ret), K(handle)); - } else if (OB_UNLIKELY(sizeof(addr)/sizeof(addr[0]) != write_ctxs.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), "write ctx count", write_ctxs.count(), K(write_ctxs), K(handle)); + } else if (OB_UNLIKELY(write_infos.count() != write_ctxs.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write ctx count does not equal to write info count", K(ret), + "write_info_count", write_infos.count(), + "write_ctx_count", write_ctxs.count(), + K(write_ctxs), K(handle)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < write_ctxs.count(); ++i) { - if (OB_UNLIKELY(!write_ctxs.at(i).is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected invalid addr", K(ret), K(i), K(write_ctxs.at(i)), K(handle)); - } else if (OB_FAIL(meta_write_ctxs.push_back(write_ctxs.at(i)))) { - LOG_WARN("fail to push write ctx to array", K(ret), K(i), K(write_ctxs.at(i))); + int64_t pos = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < total_addr_cnt; ++i) { + if (addr[i]->is_none()) { + // skip none addr } else { - *addr[i] = write_ctxs.at(i).addr_; + const ObSharedBlocksWriteCtx &write_ctx = write_ctxs.at(pos++); + if (OB_UNLIKELY(!write_ctx.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected write ctx", K(ret), K(i), K(write_ctx), K(handle)); + } else if (OB_FAIL(meta_write_ctxs.push_back(write_ctx))) { + LOG_WARN("fail to push write ctx to array", K(ret), K(i), K(write_ctx)); + } else { + *addr[i] = write_ctx.addr_; + } } } } @@ -669,20 +671,21 @@ int ObTabletPersister::write_and_fill_args( int ObTabletPersister::load_dump_kv_and_fill_write_info( common::ObArenaAllocator &allocator, const ObTabletComplexAddr &complex_addr, - common::ObIArray &write_infos) + common::ObIArray &write_infos, + ObMetaDiskAddr &addr) { int ret = OB_SUCCESS; const mds::MdsDumpKV *kv = nullptr; - if (complex_addr.addr_.is_none()) { - // do nothing - } else if (OB_FAIL(ObTabletMdsData::load_mds_dump_kv(allocator, complex_addr, kv))) { + if (OB_FAIL(ObTabletMdsData::load_mds_dump_kv(allocator, complex_addr, kv))) { LOG_WARN("fail to load mds dump kv", K(ret), K(complex_addr)); - } else if (OB_ISNULL(kv)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, kv is null", K(ret), KP(kv)); - } else if (OB_FAIL(fill_write_info(allocator, kv, write_infos))) { - LOG_WARN("fail to fill write info", K(ret), KPC(kv)); + } else if (nullptr == kv) { + // read nothing from complex addr, so disk addr is set to NONE + addr.set_none_addr(); + } else { + if (OB_FAIL(fill_write_info(allocator, kv, write_infos))) { + LOG_WARN("fail to fill write info", K(ret), KPC(kv)); + } } ObTabletMdsData::free_mds_dump_kv(allocator, kv); @@ -693,19 +696,20 @@ int ObTabletPersister::load_dump_kv_and_fill_write_info( int ObTabletPersister::load_medium_info_list_and_write( common::ObArenaAllocator &allocator, const ObTabletComplexAddr &complex_addr, - ObMetaDiskAddr &addr, - common::ObIArray &meta_write_ctxs) + common::ObIArray &meta_write_ctxs, + ObMetaDiskAddr &addr) { int ret = OB_SUCCESS; const ObTabletDumpedMediumInfo *medium_info_list = nullptr; if (OB_FAIL(ObTabletMdsData::load_medium_info_list(allocator, complex_addr, medium_info_list))) { LOG_WARN("fail to load medium info list", K(ret), K(complex_addr)); - } else if (OB_ISNULL(medium_info_list)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, medium info list is null", K(ret), KP(medium_info_list)); - } else if (OB_FAIL(link_write_medium_info_list(*medium_info_list, addr, meta_write_ctxs))) { - LOG_WARN("failed to link write medium info list", K(ret)); + } else if (nullptr == medium_info_list) { + addr.set_none_addr(); + } else { + if (OB_FAIL(link_write_medium_info_list(medium_info_list, meta_write_ctxs, addr))) { + LOG_WARN("failed to link write medium info list", K(ret)); + } } ObTabletMdsData::free_medium_info_list(allocator, medium_info_list); @@ -714,68 +718,76 @@ int ObTabletPersister::load_medium_info_list_and_write( } int ObTabletPersister::link_write_medium_info_list( - const ObTabletDumpedMediumInfo &medium_info_list, - ObMetaDiskAddr &addr, - common::ObIArray &meta_write_ctxs) + const ObTabletDumpedMediumInfo *medium_info_list, + common::ObIArray &meta_write_ctxs, + ObMetaDiskAddr &addr) { int ret = OB_SUCCESS; - ObTenantCheckpointSlogHandler *ckpt_slog_hanlder = MTL(ObTenantCheckpointSlogHandler*); - ObSharedBlockReaderWriter &reader_writer = ckpt_slog_hanlder->get_shared_block_reader_writer(); + ObTenantCheckpointSlogHandler *ckpt_slog_handler = MTL(ObTenantCheckpointSlogHandler*); + ObSharedBlockReaderWriter &reader_writer = ckpt_slog_handler->get_shared_block_reader_writer(); common::ObArenaAllocator arena_allocator(common::ObMemAttr(MTL_ID(), "serializer")); ObSharedBlockWriteInfo write_info; ObSharedBlockLinkHandle write_handle; - const common::ObIArray &array = medium_info_list.medium_info_list_; - for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { - const compaction::ObMediumCompactionInfo *medium_info = array.at(i); - if (OB_ISNULL(medium_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(medium_info)); - } else { - const int64_t size = medium_info->get_serialize_size(); - - if (0 == size) { - LOG_INFO("medium info serialize size is 0, just skip", K(ret)); + if (nullptr == medium_info_list) { + // no need to do link write, just return NONE addr + addr.set_none_addr(); + } else { + const common::ObIArray &array = medium_info_list->medium_info_list_; + for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) { + const compaction::ObMediumCompactionInfo *medium_info = array.at(i); + if (OB_ISNULL(medium_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(medium_info)); } else { - int64_t pos = 0; - char *buffer = static_cast(arena_allocator.alloc(size)); - if (OB_ISNULL(buffer)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret), K(size)); - } else if (OB_FAIL(medium_info->serialize(buffer, size, pos))) { - LOG_WARN("failed to serialize medium info", K(ret)); - } else { - write_info.reset(); - write_info.buffer_ = buffer; - write_info.offset_ = 0; - write_info.size_ = size; - write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); - if (OB_FAIL(reader_writer.async_link_write(write_info, write_handle))) { - LOG_WARN("failed to do async link write", K(ret), K(write_info)); - } else if (OB_UNLIKELY(!write_handle.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, write handle is invalid", K(ret), K(write_handle)); - } - } + const int64_t size = medium_info->get_serialize_size(); - if (nullptr != buffer) { - arena_allocator.free(buffer); + if (0 == size) { + LOG_INFO("medium info serialize size is 0, just skip", K(ret)); + } else { + int64_t pos = 0; + char *buffer = static_cast(arena_allocator.alloc(size)); + if (OB_ISNULL(buffer)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(size)); + } else if (OB_FAIL(medium_info->serialize(buffer, size, pos))) { + LOG_WARN("failed to serialize medium info", K(ret)); + } else { + write_info.reset(); + write_info.buffer_ = buffer; + write_info.offset_ = 0; + write_info.size_ = size; + write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE); + if (OB_FAIL(reader_writer.async_link_write(write_info, write_handle))) { + LOG_WARN("failed to do async link write", K(ret), K(write_info)); + } else if (OB_UNLIKELY(!write_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, write handle is invalid", K(ret), K(write_handle)); + } + } + + if (nullptr != buffer) { + arena_allocator.free(buffer); + } } } } - } - if (OB_FAIL(ret)) { - } else if (array.empty()) { - addr.set_none_addr(); - } else { - ObSharedBlocksWriteCtx write_ctx; - if (OB_FAIL(write_handle.get_write_ctx(write_ctx))) { - LOG_WARN("failed to get write ctx", K(ret), K(write_handle)); - } else if (OB_FAIL(meta_write_ctxs.push_back(write_ctx))) { - LOG_WARN("failed to push back write ctx", K(ret), K(write_ctx)); + if (OB_FAIL(ret)) { + } else if (array.empty()) { + addr.set_none_addr(); } else { - addr = write_ctx.addr_; + ObSharedBlocksWriteCtx write_ctx; + if (OB_FAIL(write_handle.get_write_ctx(write_ctx))) { + LOG_WARN("failed to get write ctx", K(ret), K(write_handle)); + } else if (OB_UNLIKELY(!write_ctx.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("write ctx is invalid", K(ret), K(write_ctx)); + } else if (OB_FAIL(meta_write_ctxs.push_back(write_ctx))) { + LOG_WARN("failed to push back write ctx", K(ret), K(write_ctx)); + } else { + addr = write_ctx.addr_; + } } } @@ -847,5 +859,80 @@ int ObTabletPersister::transform_tablet_memory_footprint( return ret; } +int ObTabletPersister::fetch_table_store_and_write_info( + const ObTablet &tablet, + common::ObArenaAllocator &allocator, + ObTabletMemberWrapper &wrapper, + common::ObIArray &write_infos, + common::ObIArray &meta_write_ctxs) +{ + int ret = OB_SUCCESS; + ObTabletTableStore new_table_store; + const ObTabletTableStore *table_store = nullptr; + ObTableStoreIterator table_iter; + if (OB_FAIL(tablet.fetch_table_store(wrapper))) { + LOG_WARN("fail to fetch table store", K(ret)); + } else if (OB_FAIL(wrapper.get_member(table_store))) { + LOG_WARN("fail to get table store from wrapper", K(ret), K(wrapper)); + } else if (OB_FAIL(table_store->get_all_sstable(table_iter))) { + LOG_WARN("fail to get all sstable iterator", K(ret), KPC(table_store)); + } else if (OB_FAIL(fetch_and_persist_sstable(allocator, table_iter, new_table_store, meta_write_ctxs))) { + LOG_WARN("fail to fetch and persist sstable", K(ret), K(table_iter)); + } else if (OB_FAIL(fill_write_info(allocator, &new_table_store, write_infos))) { + LOG_WARN("fail to fill table store write info", K(ret), K(new_table_store)); + } + return ret; +} + +int ObTabletPersister::load_auto_inc_seq_and_write_info( + common::ObArenaAllocator &allocator, + const ObTabletComplexAddr &complex_addr, + const share::ObTabletAutoincSeq *&auto_inc_seq, + common::ObIArray &write_infos, + ObMetaDiskAddr &addr) +{ + int ret = OB_SUCCESS; + auto_inc_seq = nullptr; + const share::ObTabletAutoincSeq *ptr = nullptr; + + if (OB_UNLIKELY(!complex_addr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), K(complex_addr)); + } else if (OB_FAIL(ObTabletMdsData::load_auto_inc_seq(allocator, complex_addr, ptr))) { + LOG_WARN("failed to load auto inc seq", K(ret), K(complex_addr)); + } else if (nullptr == ptr) { + addr.set_none_addr(); + } else if (OB_FAIL(fill_write_info(allocator, ptr, write_infos))) { + LOG_WARN("fail to fill empty write info", K(ret)); + } else { + auto_inc_seq = ptr; + } + + if (OB_FAIL(ret)) { + ObTabletMdsData::free_auto_inc_seq(allocator, ptr); + } + + return ret; +} + +int ObTabletPersister::load_storage_schema_and_fill_write_info( + const ObTablet &tablet, + common::ObArenaAllocator &allocator, + common::ObIArray &write_infos) +{ + int ret = OB_SUCCESS; + const ObStorageSchema *storage_schema = nullptr; + if (OB_FAIL(tablet.load_storage_schema(allocator, storage_schema))) { + LOG_WARN("fail to load storage schema", K(ret)); + } else if (OB_ISNULL(storage_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("storage schema is null", K(ret), KP(storage_schema)); + } else if (OB_FAIL(fill_write_info(allocator, storage_schema, write_infos))) { + LOG_WARN("fail to fill write info", K(ret), KP(storage_schema)); + } + ObTablet::free_storage_schema(allocator, storage_schema); + return ret; +} + } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_persister.h b/src/storage/tablet/ob_tablet_persister.h index 64d22fa9a6..db5e8ebe87 100644 --- a/src/storage/tablet/ob_tablet_persister.h +++ b/src/storage/tablet/ob_tablet_persister.h @@ -90,12 +90,6 @@ public: const ObTablet &old_tablet, char *buf, const int64_t len); -private: - using FetchTableStore = std::function &)>; - using FetchAutoincSeq = std::function &)>; - using LoadStorageSchema = std::function; - using LoadMediumInfoList = std::function; - using LoadMdsDumpKV = std::function; private: static int check_tablet_meta_ids( const common::ObIArray &tablet_meta_write_ctxs, @@ -115,7 +109,6 @@ private: common::ObIArray &tablet_meta_write_ctxs, common::ObIArray &sstable_meta_write_ctxs, ObTabletPoolType &type, - ObTabletMemberWrapper &auto_inc_seq, ObTabletTransformArg &arg); static int convert_arg_to_tablet( const ObTabletTransformArg &arg, @@ -142,32 +135,36 @@ private: ObTableStoreIterator &table_iter, ObTabletTableStore &new_table_store, common::ObIArray &meta_write_ctxs); - template - static int fetch_wrapper_and_write_info( + static int load_auto_inc_seq_and_write_info( common::ObArenaAllocator &allocator, - Fetch &fetch, - ObTabletMemberWrapper &wrapper, + const ObTabletComplexAddr &complex_addr, + const share::ObTabletAutoincSeq *&auto_inc_seq, + common::ObIArray &write_infos, + ObMetaDiskAddr &addr); + static int fetch_table_store_and_write_info( + const ObTablet &tablet, + common::ObArenaAllocator &allocator, + ObTabletMemberWrapper &wrapper, common::ObIArray &write_infos, common::ObIArray &meta_write_ctxs); - template - static int load_member_and_write_info( + static int load_storage_schema_and_fill_write_info( + const ObTablet &tablet, common::ObArenaAllocator &allocator, - Load &load, - T *&t, common::ObIArray &write_infos); static int load_dump_kv_and_fill_write_info( common::ObArenaAllocator &allocator, const ObTabletComplexAddr &complex_addr, - common::ObIArray &write_infos); + common::ObIArray &write_infos, + ObMetaDiskAddr &addr); static int load_medium_info_list_and_write( common::ObArenaAllocator &allocator, const ObTabletComplexAddr &complex_addr, - ObMetaDiskAddr &addr, - common::ObIArray &meta_write_ctxs); + common::ObIArray &meta_write_ctxs, + ObMetaDiskAddr &addr); static int link_write_medium_info_list( - const ObTabletDumpedMediumInfo &medium_info_list, - ObMetaDiskAddr &addr, - common::ObIArray &meta_write_ctxs); + const ObTabletDumpedMediumInfo *medium_info_list, + common::ObIArray &meta_write_ctxs, + ObMetaDiskAddr &addr); template static int fill_write_info( common::ObArenaAllocator &allocator, @@ -191,78 +188,6 @@ private: ObTabletTableStore *&table_store); }; -template -int ObTabletPersister::fetch_wrapper_and_write_info( - common::ObArenaAllocator &allocator, - Fetch &fetch, - ObTabletMemberWrapper &wrapper, - common::ObIArray &write_infos, - common::ObIArray &meta_write_ctxs) -{ - UNUSED(meta_write_ctxs); - int ret = common::OB_SUCCESS; - const T *member = nullptr; - if (OB_FAIL(fetch(wrapper))) { - STORAGE_LOG(WARN, "fail to fetch tablet wrapper", K(ret)); - } else if (OB_FAIL(wrapper.get_member(member))) { - STORAGE_LOG(WARN, "fail to get tablet member", K(ret), K(wrapper)); - } else if (OB_ISNULL(member)) { - ret = common::OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, member is nullptr", K(ret), KP(member)); - } else if (OB_FAIL(fill_write_info(allocator, member, write_infos))) { - STORAGE_LOG(WARN, "fail to fill write info", K(ret), KP(member)); - } - return ret; -}; - -template <> -inline int ObTabletPersister::fetch_wrapper_and_write_info( - common::ObArenaAllocator &allocator, - FetchTableStore &fetch, - ObTabletMemberWrapper &wrapper, - common::ObIArray &write_infos, - common::ObIArray &meta_write_ctxs) -{ - int ret = common::OB_SUCCESS; - ObTabletTableStore new_table_store; - const ObTabletTableStore *table_store = nullptr; - ObTableStoreIterator table_iter; - if (OB_FAIL(fetch(wrapper))) { - STORAGE_LOG(WARN, "fail to fetch table store", K(ret)); - } else if (OB_FAIL(wrapper.get_member(table_store))) { - STORAGE_LOG(WARN, "fail to get table store from wrapper", K(ret), K(wrapper)); - } else if (OB_ISNULL(table_store)) { - ret = common::OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, table store is nullptr", K(ret), KP(table_store)); - } else if (OB_FAIL(table_store->get_all_sstable(table_iter))) { - STORAGE_LOG(WARN, "fail to get all sstable iterator", K(ret), KPC(table_store)); - } else if (OB_FAIL(fetch_and_persist_sstable(allocator, table_iter, new_table_store, meta_write_ctxs))) { - STORAGE_LOG(WARN, "fail to fetch and persist sstable", K(ret), K(table_iter)); - } else if (OB_FAIL(fill_write_info(allocator, &new_table_store, write_infos))) { - STORAGE_LOG(WARN, "fail to fill table store write info", K(ret), K(new_table_store)); - } - return ret; -} - -template -int ObTabletPersister::load_member_and_write_info( - common::ObArenaAllocator &allocator, - Load &load, - T *&t, - common::ObIArray &write_infos) -{ - int ret = common::OB_SUCCESS; - if (OB_FAIL(load(allocator, t))) { - STORAGE_LOG(WARN, "fail to load tablet member", K(ret)); - } else if (OB_ISNULL(t)) { - ret = common::OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, t is nullptr", K(ret), KP(t)); - } else if (OB_FAIL(fill_write_info(allocator, t, write_infos))) { - STORAGE_LOG(WARN, "fail to fill write info", K(ret), KP(t)); - } - return ret; -} - template int ObTabletPersister::fill_write_info( common::ObArenaAllocator &allocator, @@ -270,9 +195,10 @@ int ObTabletPersister::fill_write_info( common::ObIArray &write_infos) { int ret = common::OB_SUCCESS; + if (OB_ISNULL(t)) { - ret = common::OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected error, tablet member is nullptr", K(ret), KP(t)); + ret = common::OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid args", K(ret), KP(t)); } else { const int64_t size = t->get_serialize_size(); char *buf = static_cast(allocator.alloc(size)); diff --git a/src/storage/tx/ob_dup_table_base.cpp b/src/storage/tx/ob_dup_table_base.cpp index 1f327b12be..0205294a9b 100755 --- a/src/storage/tx/ob_dup_table_base.cpp +++ b/src/storage/tx/ob_dup_table_base.cpp @@ -513,7 +513,7 @@ int ObDupTableLSCheckpoint::online() // OB_SERIALIZE_MEMBER(ObDupTableLogBlockHeader, position_, remain_length_); OB_SERIALIZE_MEMBER(DupTableLogEntryHeader, entry_type_); -OB_SERIALIZE_MEMBER(DupTableStatLog, lease_addr_cnt_, readable_cnt_, all_tablet_set_cnt_); +OB_SERIALIZE_MEMBER(DupTableStatLog, lease_addr_cnt_, leader_readable_cnt_, all_tablet_set_cnt_); // OB_SERIALIZE_MEMBER(ObLSDupTabletsMgr, max_submitted_tablet_change_ts_); @@ -754,8 +754,8 @@ int ObDupTableLogOperator::sync_log_succ_(const bool for_replay) } } } - if (stat_log_.readable_cnt_ > 0 && logging_readable_cnt > 0 - && stat_log_.readable_cnt_ == logging_readable_cnt) { + if (stat_log_.leader_readable_cnt_ > 0 && logging_readable_cnt > 0 + && stat_log_.leader_readable_cnt_ == logging_readable_cnt) { contain_all_readable = true; } if (!contain_all_readable) { @@ -804,7 +804,7 @@ int ObDupTableLogOperator::prepare_serialize_log_entry_(int64_t &max_ser_size, DupTableStatLog max_stat_log; max_stat_log.lease_addr_cnt_ = INT64_MAX; - max_stat_log.readable_cnt_ = INT64_MAX; + max_stat_log.leader_readable_cnt_ = INT64_MAX; max_stat_log.all_tablet_set_cnt_ = INT64_MAX; if (big_segment_buf_.is_active()) { @@ -913,7 +913,7 @@ int ObDupTableLogOperator::serialize_log_entry_(const int64_t max_ser_size, case DupTableLogEntryType::DuptableStatLog: { DupTableStatLog stat_log; stat_log.lease_addr_cnt_ = logging_lease_addrs_.count(); - stat_log.readable_cnt_ = tablet_mgr_ptr_->get_readable_tablet_set_count(); + stat_log.leader_readable_cnt_ = tablet_mgr_ptr_->get_readable_tablet_set_count(); stat_log.all_tablet_set_cnt_ = tablet_mgr_ptr_->get_all_tablet_set_count(); if (OB_FAIL(stat_log.serialize(big_segment_buf_.get_serialize_buf(), big_segment_buf_.get_serialize_buf_len(), data_pos))) { diff --git a/src/storage/tx/ob_dup_table_base.h b/src/storage/tx/ob_dup_table_base.h index 62a5e7b5c0..da034cae94 100644 --- a/src/storage/tx/ob_dup_table_base.h +++ b/src/storage/tx/ob_dup_table_base.h @@ -901,11 +901,11 @@ struct DupTableLogEntryHeader struct DupTableStatLog { int64_t lease_addr_cnt_; - int64_t readable_cnt_; + int64_t leader_readable_cnt_; int64_t all_tablet_set_cnt_; int64_t logging_readable_cnt_; - TO_STRING_KV(K(lease_addr_cnt_), K(readable_cnt_), K(all_tablet_set_cnt_), + TO_STRING_KV(K(lease_addr_cnt_), K(leader_readable_cnt_), K(all_tablet_set_cnt_), K(logging_readable_cnt_)); DupTableStatLog() { reset(); } @@ -913,7 +913,7 @@ struct DupTableStatLog void reset() { lease_addr_cnt_ = -1; - readable_cnt_ = -1; + leader_readable_cnt_ = -1; all_tablet_set_cnt_ = -1; logging_readable_cnt_ = -1; } @@ -979,6 +979,7 @@ private: private: void after_submit_log(const bool submit_result, const bool for_replay); + #define LOG_OPERATOR_INIT_CHECK \ if (OB_SUCC(ret)) { \ if (OB_ISNULL(log_handler_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablet_mgr_ptr_)) { \ diff --git a/src/storage/tx/ob_dup_table_dump.cpp b/src/storage/tx/ob_dup_table_dump.cpp new file mode 100644 index 0000000000..4c586baa1d --- /dev/null +++ b/src/storage/tx/ob_dup_table_dump.cpp @@ -0,0 +1,277 @@ +// Copyright (c) 2021 OceanBase +// OceanBase is licensed under Mulan PubL v2. +// You can use this software according to the terms and conditions of the Mulan PubL v2. +// You may obtain a copy of Mulan PubL v2 at: +// http://license.coscl.org.cn/MulanPubL-2.0 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PubL v2 for more details. + +#include "storage/tx/ob_dup_table_dump.h" + +namespace oceanbase +{ +namespace transaction +{ + +void ObDupTableLogDumpIterator::reset() +{ + str_arg_ = nullptr; + iter_buf_ = nullptr; + iter_buf_len_ = 0; + iter_buf_pos_ = 0; + + big_segment_.reset(); + + dup_tablet_map_.reset(); + dup_table_lease_map_.destroy(); + stat_log_.reset(); +} + +bool ObDupTableLogDumpIterator::is_inited() +{ + bool inited = false; + inited = OB_NOT_NULL(str_arg_) && OB_NOT_NULL(iter_buf_) && big_segment_.is_active() + && dup_tablet_map_.created() && dup_table_lease_map_.created(); + return inited; +} + +int ObDupTableLogDumpIterator::init_with_log_buf(const char *buf, + const int64_t buf_len, + share::ObAdminMutatorStringArg *str_arg_ptr) +{ + int ret = OB_SUCCESS; + + logservice::ObLogBaseHeader base_header; + int64_t replay_pos = 0; + + if (OB_ISNULL(buf) || buf_len < 0 || OB_ISNULL(str_arg_ptr)) { + ret = OB_INVALID_ARGUMENT; + DUP_TABLE_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_len), KP(str_arg_ptr)); + } else if (OB_FAIL(big_segment_.collect_one_part(buf, buf_len, replay_pos))) { + if (OB_ITER_END == ret) { + // need clear big_segment after collected all part for replay + ret = OB_ERR_UNEXPECTED; + DUP_TABLE_LOG(WARN, "Log entry is completed, can not merge new block", K(ret), + K(big_segment_)); + } else if (OB_START_LOG_CURSOR_INVALID == ret) { + DUP_TABLE_LOG(INFO, "start replay from the middle of a big log entry", K(ret), + K(big_segment_)); + } else if (big_segment_.is_completed()) { + ret = OB_ITER_END; + } + } else if (!dup_tablet_map_.created() + && OB_FAIL(dup_tablet_map_.create(OB_SYS_TENANT_ID, 1024))) { + DUP_TABLE_LOG(WARN, "create dup tablet map failed", K(ret), KPC(this), + K(dup_tablet_map_.created())); + } else if (!dup_table_lease_map_.created() + && OB_FAIL(dup_table_lease_map_.create(64, "DupTableDump"))) { + DUP_TABLE_LOG(WARN, "create dup table lease map failed", K(ret), K(this), + K(dup_table_lease_map_.created())); + } else { + str_arg_ = str_arg_ptr; + dup_tablet_map_.clear(); + dup_table_lease_map_.clear(); + } + + return ret; +} + +int ObDupTableLogDumpIterator::dump_dup_table_log() +{ + int ret = OB_SUCCESS; + + if (OB_SUCC(ret)) { + if (!big_segment_.is_completed()) { + ret = OB_STATE_NOT_MATCH; + DUP_TABLE_LOG(WARN, "need collect more parts of log entry", K(ret), K(big_segment_)); + } else { + int64_t deser_pos = big_segment_.get_deserialize_buf_pos(); + const int64_t deser_buf_len = big_segment_.get_deserialize_buf_len(); + + if (OB_NOT_NULL(str_arg_)) { + share::ObAdminLogNormalDumper normal_writer; + str_arg_->writer_ptr_ = &normal_writer; + str_arg_->writer_ptr_->dump_key("###"); + str_arg_->writer_ptr_->start_object(); + } + + while (OB_SUCC(ret) && deser_pos < deser_buf_len) { + DupTableLogEntryHeader entry_header; + int64_t log_entry_size = 0; + int64_t after_header_pos = 0; + stat_log_.reset(); + if (OB_FAIL(entry_header.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len, + deser_pos))) { + + DUP_TABLE_LOG(WARN, "deserialize entry header failed", K(ret), K(deser_pos), + K(big_segment_), K(entry_header)); + } else if (OB_FAIL(serialization::decode_i64(big_segment_.get_deserialize_buf(), + deser_buf_len, deser_pos, &log_entry_size))) { + DUP_TABLE_LOG(WARN, "decode log entry size failed", K(ret), K(log_entry_size), + K(big_segment_)); + } else if (OB_FALSE_IT(after_header_pos = deser_pos)) { + // do nothing + } else { + switch (entry_header.entry_type_) { + case DupTableLogEntryType::TabletChangeLog: { + if (OB_FAIL(iter_tablet_log_(deser_pos + log_entry_size, deser_pos))) { + DUP_TABLE_LOG(WARN, "iter dup table tablet log failed", K(ret), K(deser_pos), + K(log_entry_size)); + } + break; + } + case DupTableLogEntryType::LeaseListLog: { + if (OB_FAIL(iter_lease_log_(deser_pos + log_entry_size, deser_pos))) { + DUP_TABLE_LOG(WARN, "iter dup table lease log failed", K(ret), K(deser_pos), + K(log_entry_size)); + } + break; + } + case DupTableLogEntryType::DuptableStatLog: { + if (OB_FAIL(iter_stat_log_(deser_pos + log_entry_size, deser_pos))) { + DUP_TABLE_LOG(WARN, "iter dup table stat log failed", K(ret), K(deser_pos), + K(log_entry_size)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + DUP_TABLE_LOG(WARN, "unexpected log entry type", K(ret), K(entry_header), K(deser_pos)); + break; + } + } + } + } + if (OB_SUCC(ret)) { + iter_buf_pos_ = deser_pos; + if (OB_NOT_NULL(str_arg_)) { + str_arg_->writer_ptr_->end_object(); + } + } + } + } + + return ret; +} + +int ObDupTableLogDumpIterator::iter_stat_log_(const int64_t deser_buf_len, int64_t &deser_pos) +{ + int ret = OB_SUCCESS; + int64_t tmp_pos = deser_pos; + + if (OB_FAIL(stat_log_.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len, tmp_pos))) { + DUP_TABLE_LOG(WARN, "deserialize stat log failed", K(ret), K(deser_pos), K(deser_buf_len), + K(stat_log_)); + } else if (OB_ISNULL(str_arg_)) { + ret = OB_INVALID_ARGUMENT; + DUP_TABLE_LOG(WARN, "invalid ObAdminMutatorStringArg", K(ret), KPC(this)); + } else { + str_arg_->writer_ptr_->dump_key("##"); + str_arg_->writer_ptr_->dump_string(to_cstring(stat_log_)); + } + + if (OB_SUCC(ret)) { + deser_pos = tmp_pos; + } + + return ret; +} + +int ObDupTableLogDumpIterator::iter_tablet_log_(const int64_t deser_buf_len, int64_t &deser_pos) +{ + int ret = OB_SUCCESS; + int64_t tmp_pos = deser_pos; + + if (OB_ISNULL(str_arg_)) { + ret = OB_INVALID_ARGUMENT; + DUP_TABLE_LOG(WARN, "invalid ObAdminMutatorStringArg", K(ret), KPC(this)); + } else { + str_arg_->writer_ptr_->dump_key("##"); + str_arg_->writer_ptr_->start_object(); + + while (OB_SUCC(ret) && tmp_pos < deser_buf_len) { + dup_tablet_map_.reuse(); + DupTabletLogBody dup_log_body(dup_tablet_map_); + if (OB_FAIL(dup_log_body.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len, + tmp_pos))) { + DUP_TABLE_LOG(WARN, "deserialize dup log body failed", K(ret), K(dup_log_body)); + } else { + str_arg_->writer_ptr_->dump_key("DupTabletSet"); + + str_arg_->writer_ptr_->start_object(); + + str_arg_->writer_ptr_->dump_key("#(TabletSetAttribute)"); + str_arg_->writer_ptr_->dump_string(to_cstring(dup_tablet_map_.get_RO_attribute())); + str_arg_->writer_ptr_->dump_key("#(RelatedTabletSetAttribute)"); + str_arg_->writer_ptr_->dump_string(to_cstring(dup_tablet_map_.get_RO_related_attribute())); + str_arg_->writer_ptr_->dump_key("#(TabletCount)"); + str_arg_->writer_ptr_->dump_string(to_cstring(dup_tablet_map_.size())); + str_arg_->writer_ptr_->dump_key("#(TabletIdList)"); + str_arg_->writer_ptr_->start_object(); + for (DupTabletChangeMap::const_iterator iter = dup_tablet_map_.begin(); + iter != dup_tablet_map_.end(); iter++) { + str_arg_->writer_ptr_->dump_string(to_cstring(iter->first)); + } + str_arg_->writer_ptr_->end_object(); + + str_arg_->writer_ptr_->end_object(); + } + } + + str_arg_->writer_ptr_->end_object(); + } + if (OB_SUCC(ret)) { + deser_pos = tmp_pos; + } + return ret; +} + +int ObDupTableLogDumpIterator::iter_lease_log_(const int64_t deser_buf_len, int64_t &deser_pos) +{ + int ret = OB_SUCCESS; + int64_t tmp_pos = deser_pos; + + if (OB_ISNULL(str_arg_)) { + ret = OB_INVALID_ARGUMENT; + DUP_TABLE_LOG(WARN, "invalid ObAdminMutatorStringArg", K(ret), KPC(this)); + } else { + str_arg_->writer_ptr_->dump_key("##"); + str_arg_->writer_ptr_->start_object(); + + while (OB_SUCC(ret) && tmp_pos < deser_buf_len) { + + DupTableLeaseLogHeader lease_log_header; + DupTableLeaderLeaseInfo leader_lease_info; + DupTableDurableLeaseLogBody durable_lease_log_body(leader_lease_info.confirmed_lease_info_); + if (OB_FAIL(lease_log_header.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len, + tmp_pos))) { + DUP_TABLE_LOG(WARN, "deserialize lease log header failed", K(ret), K(lease_log_header), + K(tmp_pos), K(deser_buf_len)); + } else if (lease_log_header.is_durable_lease_log()) { + if (OB_FAIL(durable_lease_log_body.deserialize(big_segment_.get_deserialize_buf(), + deser_buf_len, tmp_pos))) { + DUP_TABLE_LOG(WARN, "deserialize leader lease info failed", K(ret)); + } else { + str_arg_->writer_ptr_->start_object(); + str_arg_->writer_ptr_->dump_key("#(LeaseKey)"); + str_arg_->writer_ptr_->dump_string(to_cstring(lease_log_header)); + str_arg_->writer_ptr_->dump_key("#(ConfirmedLeaseInfo)"); + str_arg_->writer_ptr_->dump_string(to_cstring(leader_lease_info.confirmed_lease_info_)); + str_arg_->writer_ptr_->end_object(); + } + } + } + + str_arg_->writer_ptr_->end_object(); + } + + if (OB_SUCC(ret)) { + deser_pos = tmp_pos; + } + return ret; +} + +} // namespace transaction +} // namespace oceanbase diff --git a/src/storage/tx/ob_dup_table_dump.h b/src/storage/tx/ob_dup_table_dump.h new file mode 100644 index 0000000000..33b9922771 --- /dev/null +++ b/src/storage/tx/ob_dup_table_dump.h @@ -0,0 +1,69 @@ +// Copyright (c) 2021 OceanBase +// OceanBase is licensed under Mulan PubL v2. +// You can use this software according to the terms and conditions of the Mulan PubL v2. +// You may obtain a copy of Mulan PubL v2 at: +// http://license.coscl.org.cn/MulanPubL-2.0 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PubL v2 for more details. + +#ifndef OCEANBASE_TRANSACTION_DUP_TABLE_DUMP_H +#define OCEANBASE_TRANSACTION_DUP_TABLE_DUMP_H + +#include "src/share/ob_admin_dump_helper.h" +#include "storage/tx/ob_dup_table_lease.h" +#include "storage/tx/ob_dup_table_tablets.h" + +namespace oceanbase +{ +namespace transaction +{ + +class ObDupTableLogDumpIterator +{ +public: + ObDupTableLogDumpIterator() + : str_arg_(nullptr), iter_buf_(nullptr), iter_buf_len_(0), dup_tablet_map_(1), + dup_table_lease_map_() + {} + void reset(); + + bool is_inited(); + int init_with_log_buf(const char *buf, + const int64_t buf_len, + share::ObAdminMutatorStringArg *str_arg_ptr); + + int dump_dup_table_log(); + int64_t get_iter_buf_pos() { return iter_buf_pos_; } + + TO_STRING_KV(KP(str_arg_), + KP(iter_buf_), + K(iter_buf_len_), + K(dup_tablet_map_.size()), + K(dup_table_lease_map_.size())); + +private: + int iter_stat_log_(const int64_t deser_buf_len, int64_t &deser_pos); + int iter_tablet_log_(const int64_t deser_buf_len, int64_t &deser_pos); + int iter_lease_log_(const int64_t deser_buf_len, int64_t &deser_pos); + +private: + share::ObAdminMutatorStringArg *str_arg_; + + const char *iter_buf_; + int64_t iter_buf_len_; + int64_t iter_buf_pos_; + + ObTxBigSegmentBuf big_segment_; + + DupTabletChangeMap dup_tablet_map_; + DupTableLeaderLeaseMap dup_table_lease_map_; + DupTableStatLog stat_log_; +}; + +} // namespace transaction + +} // namespace oceanbase + +#endif diff --git a/src/storage/tx/ob_dup_table_tablets.h b/src/storage/tx/ob_dup_table_tablets.h index 6712dfea4a..ac057d1932 100644 --- a/src/storage/tx/ob_dup_table_tablets.h +++ b/src/storage/tx/ob_dup_table_tablets.h @@ -441,6 +441,9 @@ public: } int64_t get_last_gc_scan_ts() { return last_gc_scan_ts_; } + const DupTabletSetAttribute & get_RO_attribute() const {return dup_set_attr_;} + const RelatedSetAttribute & get_RO_related_attribute() const {return related_set_attr_;} + bool is_logging() const { return dup_set_attr_.change_status_.check_logging(); } void set_logging() { dup_set_attr_.change_status_.set_logging(); } void clean_logging() { dup_set_attr_.change_status_.clean_logging(); } diff --git a/src/storage/tx/ob_standby_timestamp_service.cpp b/src/storage/tx/ob_standby_timestamp_service.cpp index 7eb015a65b..696cdd5e32 100644 --- a/src/storage/tx/ob_standby_timestamp_service.cpp +++ b/src/storage/tx/ob_standby_timestamp_service.cpp @@ -96,15 +96,17 @@ void ObStandbyTimestampService::wait() void ObStandbyTimestampService::destroy() { - inited_ = false; - tenant_id_ = OB_INVALID_ID; - //TODO(SCN):zhaoxing last_id should be uint64_t - last_id_ = OB_INVALID_VERSION; - epoch_ = OB_INVALID_TIMESTAMP; - switch_to_leader_ts_ = OB_INVALID_TIMESTAMP; - TG_DESTROY(tg_id_); - rpc_.destroy(); - TRANS_LOG(INFO, "standby timestamp service destroy", K_(tenant_id)); + if (inited_) { + inited_ = false; + tenant_id_ = OB_INVALID_ID; + //TODO(SCN):zhaoxing last_id should be uint64_t + last_id_ = OB_INVALID_VERSION; + epoch_ = OB_INVALID_TIMESTAMP; + switch_to_leader_ts_ = OB_INVALID_TIMESTAMP; + TG_DESTROY(tg_id_); + rpc_.destroy(); + TRANS_LOG(INFO, "standby timestamp service destroy", K_(tenant_id)); + } } int ObStandbyTimestampService::query_and_update_last_id() diff --git a/src/storage/tx/ob_trans_define_v4.cpp b/src/storage/tx/ob_trans_define_v4.cpp index 1af7bde761..38b489852f 100644 --- a/src/storage/tx/ob_trans_define_v4.cpp +++ b/src/storage/tx/ob_trans_define_v4.cpp @@ -364,7 +364,7 @@ inline void ObTxDesc::FLAG::switch_to_idle_() ObTxDesc::FLAG ObTxDesc::FLAG::update_with(const ObTxDesc::FLAG &flag) { - auto n = flag; + ObTxDesc::FLAG n = flag; #define KEEP_(x) n.x = x LST_DO(KEEP_, (;), SHADOW_, REPLICA_, TRACING_, INTERRUPTED_, RELEASED_, BLOCK_); #undef KEEP_ @@ -550,7 +550,7 @@ bool ObTxDesc::contain_savepoint(const ObString &sp) { bool hit = false; ARRAY_FOREACH_X(savepoints_, i, cnt, !hit) { - auto &it = savepoints_[cnt - 1 - i]; + ObTxSavePoint &it = savepoints_[cnt - 1 - i]; if (it.is_savepoint() && it.name_ == sp) { hit = true; } @@ -564,7 +564,7 @@ int ObTxDesc::update_part_(ObTxPart &a, const bool append) int ret = OB_SUCCESS; bool hit = false; ARRAY_FOREACH_NORET(parts_, i) { - auto &p = parts_[i]; + ObTxPart &p = parts_[i]; if (p.id_ == a.id_) { hit = true; if (p.epoch_ == ObTxPart::EPOCH_DEAD) { @@ -639,7 +639,7 @@ int ObTxDesc::update_parts(const share::ObLSArray &list) { int ret = OB_SUCCESS, tmp_ret = ret; ARRAY_FOREACH_NORET(list, i) { - auto &it = list[i]; + const ObLSID &it = list[i]; ObTxPart n; n.id_ = it; n.epoch_ = ObTxPart::EPOCH_UNKNOWN; @@ -679,7 +679,7 @@ int ObTxDesc::update_parts_(const ObTxPartList &list) int ret = OB_SUCCESS; ARRAY_FOREACH(list, i) { bool hit = false; - auto &a = list[i]; + const ObTxPart &a = list[i]; ret = update_part_(const_cast(a)); } return ret; @@ -705,7 +705,7 @@ int ObTxDesc::get_inc_exec_info(ObTxExecResult &exec_info) ObSpinLockGuard guard(lock_); if (exec_info_reap_ts_ >= 0) { ARRAY_FOREACH(parts_, i) { - auto &p = parts_[i]; + ObTxPart &p = parts_[i]; if (p.last_touch_ts_ > exec_info_reap_ts_ && OB_FAIL(exec_info.parts_.push_back(p))) { TRANS_LOG(WARN, "push fail", K(ret), K(p), KPC(this), K(exec_info)); @@ -809,8 +809,8 @@ bool ObTxDesc::execute_commit_cb() */ ATOMIC_LOAD_ACQ((int*)&state_); if (is_tx_end() || is_xa_terminate_state_()) { - auto tx_id = tx_id_; - auto cb = commit_cb_; + ObTransID tx_id = tx_id_; + ObITxCallback *cb = commit_cb_; int ret = OB_SUCCESS; if (OB_NOT_NULL(commit_cb_) && acq_commit_cb_lock_if_need_()) { if (OB_NOT_NULL(commit_cb_)) { @@ -1406,7 +1406,7 @@ int ObTxDescMgr::add(ObTxDesc &tx_desc) int ObTxDescMgr::add_with_txid(const ObTransID &tx_id, ObTxDesc &tx_desc) { int ret = OB_SUCCESS; - auto desc_tx_id = tx_desc.get_tx_id(); + ObTransID desc_tx_id = tx_desc.get_tx_id(); if (!inited_) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "ObTxDescMgr not inited", K(ret)); @@ -1430,7 +1430,7 @@ int ObTxDescMgr::add_with_txid(const ObTransID &tx_id, ObTxDesc &tx_desc) if (OB_FAIL(ret) && !desc_tx_id.is_valid()) { tx_desc.reset_tx_id(); } if (OB_SUCC(ret) && tx_desc.flags_.SHADOW_) { tx_desc.flags_.SHADOW_ = false; } } - TRANS_LOG(INFO, "txDescMgr.register trans with txid", K(ret), K(tx_id), + TRANS_LOG(TRACE, "txDescMgr.register trans with txid", K(ret), K(tx_id), K(map_.alloc_cnt())); return ret; } @@ -1449,7 +1449,7 @@ int ObTxDescMgr::get(const ObTransID &tx_id, ObTxDesc *&tx_desc) void ObTxDescMgr::revert(ObTxDesc &tx) { int ret = OB_SUCCESS; - auto tx_id = tx.get_tx_id(); + ObTransID tx_id = tx.get_tx_id(); OV(inited_, OB_NOT_INIT); if (OB_SUCC(ret)) { map_.revert(&tx); @@ -1564,13 +1564,13 @@ ObTxSEQ ObTxDesc::get_tx_seq(int64_t seq_abs) const ObTxSEQ ObTxDesc::get_and_inc_tx_seq(int16_t branch, int N) const { UNUSED(branch); - auto seq = ObSequence::get_and_inc_max_seq_no(N); + int64_t seq = ObSequence::get_and_inc_max_seq_no(N); return ObTxSEQ::mk_v0(seq); } ObTxSEQ ObTxDesc::inc_and_get_tx_seq(int16_t branch) const { UNUSED(branch); - auto seq = ObSequence::inc_and_get_max_seq_no(); + int64_t seq = ObSequence::inc_and_get_max_seq_no(); return ObTxSEQ::mk_v0(seq); } } // transaction diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 9f88175b0a..a743d1cdd4 100755 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -7213,10 +7213,9 @@ int ObPartTransCtx::check_status_() * purpose: * 1) verify transaction ctx is *writable* * 2) acquire memtable ctx's ref - * 3) remember data_scn + * 3) alloc data_scn */ -int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, - const ObTxSEQ data_scn) +int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, ObTxSEQ &data_scn) { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); @@ -7225,14 +7224,18 @@ int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, ret = OB_TRANS_SQL_SEQUENCE_ILLEGAL; TRANS_LOG(WARN, "stale access operation", K(ret), K_(tx_desc.op_sn), K_(last_op_sn), KPC(this), K(tx_desc)); - } else if (FALSE_IT(++pending_write_)) { - } else if (FALSE_IT(last_scn_ = MAX(data_scn, last_scn_))) { - } else if (!first_scn_.is_valid() && FALSE_IT(first_scn_ = last_scn_)) { - } else if (tx_desc.op_sn_ != last_op_sn_) { - last_op_sn_ = tx_desc.op_sn_; - } - - if (OB_SUCC(ret)) { + } else { + if (!data_scn.is_valid()) { + data_scn = tx_desc.inc_and_get_tx_seq(0); + } + ++pending_write_; + last_scn_ = MAX(data_scn, last_scn_); + if (!first_scn_.is_valid()) { + first_scn_ = last_scn_; + } + if (tx_desc.op_sn_ != last_op_sn_) { + last_op_sn_ = tx_desc.op_sn_; + } mt_ctx_.inc_ref(); mt_ctx_.acquire_callback_list(); } @@ -7318,14 +7321,13 @@ int ObPartTransCtx::rollback_to_savepoint(const int64_t op_sn, K(trans_id_), K(ls_id_), K(busy_cbs_.get_size())); } else if (op_sn < last_op_sn_) { ret = OB_TRANS_SQL_SEQUENCE_ILLEGAL; - } else if (op_sn > last_op_sn_ && last_scn_ <= to_scn) { - last_op_sn_ = op_sn; - TRANS_LOG(INFO, "rollback succeed trivially", K(op_sn), K(to_scn), K_(last_scn)); - } else if (op_sn > last_op_sn_ && pending_write_ > 0) { + } else if (FALSE_IT(last_op_sn_ = op_sn)) { + } else if (pending_write_ > 0) { ret = OB_NEED_RETRY; TRANS_LOG(WARN, "has pending write, rollback blocked", K(ret), K(pending_write_), KPC(this)); - } else if (FALSE_IT(last_op_sn_ = op_sn)) { + } else if (last_scn_ <= to_scn) { + TRANS_LOG(INFO, "rollback succeed trivially", K(op_sn), K(to_scn), K_(last_scn)); } else if (OB_FAIL(rollback_to_savepoint_(from_scn, to_scn))) { TRANS_LOG(WARN, "rollback_to_savepoint fail", K(ret), K(from_scn), K(to_scn), K(op_sn), KPC(this)); @@ -7947,7 +7949,9 @@ int ObPartTransCtx::build_and_post_ask_state_msg_(const SCN &snapshot) { int ret = OB_SUCCESS; if (is_root()) { - handle_trans_ask_state_(snapshot); + if (!exec_info_.participants_.empty()) { + handle_trans_ask_state_(snapshot); + } } else { ObAskStateMsg msg; msg.snapshot_ = snapshot; @@ -8010,11 +8014,30 @@ int ObPartTransCtx::handle_trans_ask_state(const SCN &snapshot, ObAskStateRespMs if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObPartTransCtx not inited"); ret = OB_NOT_INIT; + } else if (exec_info_.participants_.empty()) { + // if coord not know any participants(before replay commit info log), + // fill self state to resp + ObStateInfo state_info; + state_info.ls_id_ = ls_id_; + state_info.state_ = exec_info_.state_; + state_info.snapshot_version_ = snapshot; + if (ObTxState::INIT == state_info.state_) { + if (OB_FAIL(get_ls_replica_readable_scn_(state_info.ls_id_, state_info.version_))) { + TRANS_LOG(WARN, "get replica readable scn failed", K(ret), K(state_info), K(snapshot)); + } else if (OB_FAIL(resp.state_info_array_.push_back(state_info))) { + TRANS_LOG(WARN, "push back state info to resp msg failed", K(ret), K(snapshot), KPC(this)); + } + } else { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(ERROR, "coord should not in other state befroe replay commit info log", K(ret), + KPC(this)); + } + TRANS_LOG(INFO, "coord not know any participants", K(ret), K(state_info), KPC(this)); } else { handle_trans_ask_state_(snapshot); - } - if (OB_SUCC(ret) && OB_FAIL(resp.state_info_array_.assign(state_info_array_))) { - TRANS_LOG(WARN, "build ObAskStateRespMsg fail", K(ret), K(snapshot), KPC(this)); + if (OB_FAIL(resp.state_info_array_.assign(state_info_array_))) { + TRANS_LOG(WARN, "build ObAskStateRespMsg fail", K(ret), K(snapshot), KPC(this)); + } } TRANS_LOG(INFO, "handle ask state msg", K(ret), K(snapshot), KPC(this)); return ret; @@ -8138,7 +8161,7 @@ int ObPartTransCtx::handle_trans_collect_state(ObStateInfo &state_info, const SC // The state is still init or redo complete at the second check, do nothing need_loop = false; } else if (OB_FAIL(get_ls_replica_readable_scn_(state_info.ls_id_, state_info.version_))) { - TRANS_LOG(WARN, "double check fail", K(ret), K(state_info), K(snapshot)); + TRANS_LOG(WARN, "get replica readable scn failed", K(ret), K(state_info), K(snapshot)); } else if (snapshot <= state_info.version_) { need_loop = true; } else { @@ -8168,7 +8191,7 @@ int ObPartTransCtx::handle_trans_collect_state(ObStateInfo &state_info, const SC } } while(need_loop); } - TRANS_LOG(INFO, "handle trans collect state", K(ret), K(state_info), KPC(this)); + TRANS_LOG(INFO, "handle trans collect state", K(ret), K(state_info), K(snapshot), KPC(this)); return ret; } diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index cb5a47073c..3a24a5966b 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -733,8 +733,7 @@ public: * @data_seq: the sequence_no of current access * new created data will marked with this seq no */ - int start_access(const ObTxDesc &tx_desc, - const ObTxSEQ data_seq); + int start_access(const ObTxDesc &tx_desc, ObTxSEQ &data_seq); /* * end_access - end of txn protected resources access */ diff --git a/src/storage/tx/ob_trans_rpc.cpp b/src/storage/tx/ob_trans_rpc.cpp index bf0a15caf7..fc46078451 100644 --- a/src/storage/tx/ob_trans_rpc.cpp +++ b/src/storage/tx/ob_trans_rpc.cpp @@ -30,7 +30,7 @@ using namespace share; namespace obrpc { OB_SERIALIZE_MEMBER(ObTransRpcResult, status_, send_timestamp_, private_data_); -OB_SERIALIZE_MEMBER(ObTxRpcRollbackSPResult, status_, send_timestamp_, addr_, born_epoch_); +OB_SERIALIZE_MEMBER(ObTxRpcRollbackSPResult, status_, send_timestamp_, addr_, born_epoch_, ignore_); bool need_refresh_location_cache_(const int ret) { @@ -72,12 +72,14 @@ int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, const int64_t epoch, const transaction::ObTransID &tx_id, const int status, - const ObAddr &addr, const int64_t request_id, const ObTxRpcRollbackSPResult &result) { + if (result.ignore_) { + return OB_SUCCESS; + } return MTL(ObTransService *)->handle_sp_rollback_resp(receiver_ls_id, - epoch, tx_id, status, addr, request_id, result); + epoch, tx_id, status, request_id, result.born_epoch_, result.addr_); } void ObTransRpcResult::reset() @@ -140,6 +142,7 @@ TX_Process(Commit, handle_trans_commit_request); TX_Process(CommitResp, handle_trans_commit_response); TX_Process(Abort, handle_trans_abort_request); TX_Process(RollbackSP, handle_sp_rollback_request); +TX_Process(RollbackSPResp, handle_sp_rollback_response); TX_Process(Keepalive, handle_trans_keepalive); TX_Process(KeepaliveResp, handle_trans_keepalive_response); TX_Process(AskState, handle_trans_ask_state); @@ -180,6 +183,8 @@ int ObTransRpc::init(ObTransService *trans_service, TRANS_LOG(WARN, "transaction callback init error", KR(ret)); } else if (OB_SUCCESS != (ret = tx_rollback_sp_cb_.init())) { TRANS_LOG(WARN, "transaction callback init error", KR(ret)); + } else if (OB_SUCCESS != (ret = tx_rollback_sp_resp_cb_.init())) { + TRANS_LOG(WARN, "transaction callback init error", KR(ret)); } else if (OB_SUCCESS != (ret = tx_keepalive_cb_.init())) { TRANS_LOG(WARN, "transaction callback init error", KR(ret)); } else if (OB_SUCCESS != (ret = tx_keepalive_resp_cb_.init())) { @@ -316,6 +321,12 @@ int ObTransRpc::post_(const ObAddr &server, ObTxMsg &msg) post_rollback_sp_msg(static_cast(msg), &tx_rollback_sp_cb_); break; } + case ROLLBACK_SAVEPOINT_RESP: + { + ret = rpc_proxy_.to(server).by(tenant_id).timeout(GCONF._ob_trans_rpc_timeout). + post_rollback_sp_resp_msg(static_cast(msg), &tx_rollback_sp_resp_cb_); + break; + } case KEEPALIVE: { ret = rpc_proxy_.to(server).by(tenant_id).timeout(GCONF._ob_trans_rpc_timeout). diff --git a/src/storage/tx/ob_trans_rpc.h b/src/storage/tx/ob_trans_rpc.h index ef6c2bdb3c..d8b78a80fd 100644 --- a/src/storage/tx/ob_trans_rpc.h +++ b/src/storage/tx/ob_trans_rpc.h @@ -70,13 +70,17 @@ struct ObTxRpcRollbackSPResult { OB_UNIS_VERSION(1); public: + ObTxRpcRollbackSPResult(): ignore_(false) {} int status_; int64_t send_timestamp_; int64_t born_epoch_; ObAddr addr_; + // rollback response has changed to use ObTxRollbackSPRespMsg + // use this field to indicate handler ignore handle by this msg + bool ignore_; public: int get_status() const { return status_; } - TO_STRING_KV(K_(status), K_(send_timestamp), K_(born_epoch), K_(addr)); + TO_STRING_KV(K_(status), K_(send_timestamp), K_(born_epoch), K_(addr), K_(ignore)); }; class ObTransRpcProxy : public obrpc::ObRpcProxy @@ -88,6 +92,7 @@ public: RPC_AP(PR3 post_commit_resp_msg, OB_TX_COMMIT_RESP, (transaction::ObTxCommitRespMsg), ObTransRpcResult); RPC_AP(PR3 post_abort_msg, OB_TX_ABORT, (transaction::ObTxAbortMsg), ObTransRpcResult); RPC_AP(PR3 post_rollback_sp_msg, OB_TX_ROLLBACK_SAVEPOINT, (transaction::ObTxRollbackSPMsg), ObTxRpcRollbackSPResult); + RPC_AP(PR3 post_rollback_sp_resp_msg, OB_TX_ROLLBACK_SAVEPOINT_RESP, (transaction::ObTxRollbackSPRespMsg), ObTransRpcResult); RPC_AP(PR3 post_keep_alive_msg, OB_TX_KEEPALIVE, (transaction::ObTxKeepaliveMsg), ObTransRpcResult); RPC_AP(PR3 post_keep_alive_resp_msg, OB_TX_KEEPALIVE_RESP, (transaction::ObTxKeepaliveRespMsg), ObTransRpcResult); // for standby @@ -125,6 +130,7 @@ TX_P_(Commit, OB_TX_COMMIT); TX_P_(CommitResp, OB_TX_COMMIT_RESP); TX_P_(Abort, OB_TX_ABORT); TX_P_(RollbackSP, OB_TX_ROLLBACK_SAVEPOINT); +TX_P_(RollbackSPResp, OB_TX_ROLLBACK_SAVEPOINT_RESP); TX_P_(Keepalive, OB_TX_KEEPALIVE); TX_P_(KeepaliveResp, OB_TX_KEEPALIVE_RESP); //for standby @@ -227,7 +233,6 @@ int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, const int64_t epoch, const transaction::ObTransID &tx_id, const int status, - const ObAddr &addr, const int64_t request_id, const ObTxRpcRollbackSPResult &result); template @@ -276,7 +281,7 @@ int ObTxRPCCB::process() } } } - if (OB_SUCCESS != ret || OB_SUCCESS != status) { + if (OB_SUCCESS != ret || (OB_SUCCESS != status && status != -1)) { TRANS_LOG(WARN, "trx rpc callback", K(ret), K(status), K(dst), K(result)); } return ret; @@ -301,7 +306,6 @@ int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, const int64_t epoch, const transaction::ObTransID &tx_id, const int status, - const ObAddr &addr, const int64_t request_id, const ObTxRpcRollbackSPResult &result); template<> @@ -311,7 +315,6 @@ inline int ObTxRPCCB::handle_tx_msg_cb_(const int stat epoch_, trans_id_, status, - dst, request_id_, result_); } @@ -437,6 +440,7 @@ private: obrpc::ObTxRPCCB tx_commit_resp_cb_; obrpc::ObTxRPCCB tx_abort_cb_; obrpc::ObTxRPCCB tx_rollback_sp_cb_; + obrpc::ObTxRPCCB tx_rollback_sp_resp_cb_; obrpc::ObTxRPCCB tx_keepalive_cb_; obrpc::ObTxRPCCB tx_keepalive_resp_cb_; obrpc::ObTxRPCCB tx_ask_state_cb_; diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index ee781e84d7..448eea6b7d 100755 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -356,7 +356,7 @@ int ObTransService::handle_tx_commit_timeout(ObTxDesc &tx, const int64_t delay) // remember tx_id because tx maybe cleanout and reused // in this function's following steps. tx.lock_.lock(); - auto tx_id = tx.tx_id_; + ObTransID tx_id = tx.tx_id_; int64_t now = ObClockGenerator::getClock(); bool cb_executed = false; if (!tx.commit_task_.is_registered()){ @@ -613,7 +613,7 @@ void ObTransService::invalid_registered_snapshot_(ObTxDesc &tx) { int ret = OB_SUCCESS; ARRAY_FOREACH(tx.savepoints_, i) { - auto &it = tx.savepoints_[i]; + ObTxSavePoint &it = tx.savepoints_[i]; if (it.is_snapshot()) { it.rollback(); } @@ -624,7 +624,7 @@ void ObTransService::registered_snapshot_clear_part_(ObTxDesc &tx) { int ret = OB_SUCCESS; ARRAY_FOREACH(tx.savepoints_, i) { - auto &p = tx.savepoints_[i]; + ObTxSavePoint &p = tx.savepoints_[i]; if (p.is_snapshot() && p.snapshot_->valid_) { p.snapshot_->parts_.reset(); } @@ -956,7 +956,7 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, ObStoreCtx &store_ctx) { int ret = OB_SUCCESS; - auto ls_id = store_ctx.ls_id_; + ObLSID ls_id = store_ctx.ls_id_; if (!ls_id.is_valid() || !snapshot.valid_) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid ls_id or invalid snapshot store_ctx", K(ret), K(snapshot), K(store_ctx), K(lbt())); @@ -974,7 +974,7 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, } bool check_readable_ok = false; - auto snap_tx_id = snapshot.core_.tx_id_; + ObTransID snap_tx_id = snapshot.core_.tx_id_; ObPartTransCtx *tx_ctx = NULL; if (OB_SUCC(ret) && snap_tx_id.is_valid()) { // inner tx read, we verify txCtx's status @@ -1079,7 +1079,7 @@ int ObTransService::get_write_store_ctx(ObTxDesc &tx, int ret = OB_SUCCESS; const share::ObLSID &ls_id = store_ctx.ls_id_; ObPartTransCtx *tx_ctx = NULL; - const ObTxSEQ data_scn = spec_seq_no.is_valid() ? spec_seq_no : tx.inc_and_get_tx_seq(0); + ObTxSEQ data_scn = spec_seq_no; // for LOB aux table, spec_seq_no is valid ObTxSnapshot snap = snapshot.core_; ObTxTableGuard tx_table_guard; bool access_started = false; @@ -1281,8 +1281,8 @@ void ObTransService::fetch_cflict_tx_ids_from_mem_ctx_to_desc_(ObMvccAccessCtx & int ObTransService::revert_store_ctx(storage::ObStoreCtx &store_ctx) { int ret = OB_SUCCESS; - auto &acc_ctx = store_ctx.mvcc_acc_ctx_; - auto *tx_ctx = acc_ctx.tx_ctx_; + ObMvccAccessCtx &acc_ctx = store_ctx.mvcc_acc_ctx_; + ObPartTransCtx *tx_ctx = acc_ctx.tx_ctx_; if (acc_ctx.is_read()) { if (OB_NOT_NULL(tx_ctx)) { acc_ctx.tx_ctx_ = NULL; @@ -1591,15 +1591,15 @@ int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, const bool can_elr = MTL_IS_PRIMARY_TENANT() ? true : false; if (OB_FAIL(tx_ctx_mgr_.get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { TRANS_LOG(WARN, "get ls_tx_ctx_mgr fail", K(ret), K(ls_id)); + } else if (OB_FAIL(ls_tx_ctx_mgr->get_ls_log_adapter()->get_role(leader, epoch))) { + TRANS_LOG(WARN, "get replica role fail", K(ret), K(ls_id)); + } else if (!leader) { + ret = OB_NOT_MASTER; } else if (!ls_tx_ctx_mgr->in_leader_serving_state()) { ret = OB_NOT_MASTER; // XXX In standby cluster mode, the failure to call acquire_local_snapshot_ is an // normal situation, no error log needs to be printed // TRANS_LOG(WARN, "check ls tx service leader serving state fail", K(ret), K(ls_id), K(ret)); - } else if (OB_FAIL(ls_tx_ctx_mgr->get_ls_log_adapter()->get_role(leader, epoch))) { - TRANS_LOG(WARN, "get replica role fail", K(ret), K(ls_id)); - } else if (!leader) { - ret = OB_NOT_MASTER; } if (OB_NOT_MASTER == ret && is_read_only) { @@ -1653,7 +1653,6 @@ int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, // } - if(OB_FAIL(ret)) { //do nothing } else if (FALSE_IT(snapshot0 = tx_version_mgr_.get_max_commit_ts(can_elr))) { @@ -1676,7 +1675,7 @@ int ObTransService::sync_acquire_global_snapshot_(ObTxDesc &tx, int64_t &uncertain_bound) { int ret = OB_SUCCESS; - auto op_sn = tx.op_sn_; + uint64_t op_sn = tx.op_sn_; tx.flags_.BLOCK_ = true; tx.lock_.unlock(); ret = acquire_global_snapshot__(expire_ts, @@ -1767,7 +1766,7 @@ int ObTransService::batch_post_rollback_savepoint_msg_(ObTxDesc &tx, post_succ_num = 0; const ObTxDesc *msg_tx_ptr = msg.tx_ptr_; ARRAY_FOREACH_NORET(list, idx) { - auto &p = list.at(idx); + const ObTxLSEpochPair &p = list.at(idx); msg.receiver_ = p.left_; msg.epoch_ = p.right_; if (msg.epoch_ > 0) { @@ -2006,10 +2005,29 @@ int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, msg.savepoint_, ctx_born_epoch, msg.tx_ptr_); + if (msg.use_async_resp()) { + ObTxRollbackSPRespMsg resp; + resp.cluster_version_ = msg.cluster_version_; + resp.tenant_id_ = msg.tenant_id_; + resp.sender_addr_ = self_; + resp.sender_ = msg.receiver_; + resp.receiver_ = msg.sender_; + resp.cluster_id_ = msg.cluster_id_; + resp.tx_id_ = msg.tx_id_; + resp.request_id_ = msg.request_id_; + resp.ret_ = ret; + resp.orig_epoch_ = msg.epoch_, + resp.epoch_ = ctx_born_epoch; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(rpc_->post_msg(msg.sender_addr_, resp))) { + TRANS_LOG(WARN, "pos rollback sp resp fail", K(tmp_ret), K(resp)); + } + } result.status_ = ret; result.addr_ = self_; result.born_epoch_ = ctx_born_epoch; result.send_timestamp_ = msg.get_timestamp(); + result.ignore_ = msg.use_async_resp(); #ifndef NDEBUG TRANS_LOG(INFO, "handle savepoint rollback request", K(ret), K(msg), K(result)); #else @@ -2020,6 +2038,21 @@ int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, return ret; } +int ObTransService::handle_sp_rollback_response(ObTxRollbackSPRespMsg &msg, + obrpc::ObTransRpcResult &result) +{ + int ret = OB_SUCCESS; + ret = handle_sp_rollback_resp(msg.sender_, + msg.orig_epoch_, + msg.tx_id_, + msg.ret_, + msg.request_id_, + msg.epoch_, + msg.sender_addr_); + result.reset(); + result.init(ret, msg.get_timestamp()); + return ret; +} int ObTransService::check_ls_status_(const share::ObLSID &ls_id, bool &leader) { int ret = OB_SUCCESS; @@ -2154,44 +2187,44 @@ void ObTransService::on_sp_rollback_succ_(const ObTxLSEpochPair &part, } int ObTransService::handle_sp_rollback_resp(const share::ObLSID &ls_id, - const int64_t epoch, + const int64_t orig_epoch, const transaction::ObTransID &tx_id, const int status, - const ObAddr &addr, const int64_t request_id, - const obrpc::ObTxRpcRollbackSPResult &result) + const int64_t ret_epoch, + const ObAddr &ret_addr) { int ret = OB_SUCCESS; ObTxDesc *tx = NULL; if (OB_FAIL(tx_desc_mgr_.get(tx_id, tx))) { TRANS_LOG(WARN, "get trans_desc fail", K(ret), K(tx_id)); - } else if (tx->op_sn_ > request_id || tx->tx_id_ != tx_id) { // fast fail + } else if (tx->op_sn_ > request_id || tx->tx_id_ != tx_id || tx->state_ != ObTxDesc::State::ROLLBACK_SAVEPOINT) { // fast fail TRANS_LOG(WARN, "receive stale rollback response message", - K(addr), K(status), K(request_id), K(result), K(tx_id), K(tx->tx_id_), K(tx->op_sn_)); + K(status), K(request_id), K(ret_epoch), K(ret_addr), K(tx_id), K(tx->tx_id_), K(tx->op_sn_)); + } else if (status == OB_TRANS_RPC_TIMEOUT || common_retryable_error_(status)) { + TRANS_LOG(WARN, "rollback savepoint on ls return an retryable error", K(status), K(ls_id), K(tx_id), K(request_id)); + } else if (OB_FAIL(tx->lock_.lock(10_ms))) { + TRANS_LOG(WARN, "lock fail", K(ret), K(ls_id), K(tx_id), K(request_id), K(status)); } else { - ObSpinLockGuard guard(tx->lock_); if (tx->state_ != ObTxDesc::State::ROLLBACK_SAVEPOINT) { - TRANS_LOG(WARN, "receive stale rollback response message", - K(addr), K(status), K(request_id), K(result), KPC(tx)); + TRANS_LOG(WARN, "receive stale rollback response message", K(status), K(request_id), KPC(tx)); } else if (tx->tx_id_ != tx_id || tx->op_sn_ > request_id) { - TRANS_LOG(WARN, "receive old rpc result msg", - K(ret), K_(tx->op_sn), K(request_id), K(tx_id), K(tx->tx_id_)); - } else if (status == OB_TRANS_RPC_TIMEOUT || common_retryable_error_(status)) { - TRANS_LOG(WARN, "rollback savepoint on ls return an retryable error", - K(status), K(ls_id), K(tx_id), K(addr)); + TRANS_LOG(WARN, "receive old rpc result msg", K(ret), K_(tx->op_sn), K(request_id), K(tx_id), K(tx->tx_id_)); } else if (status == OB_SUCCESS) { - ObTxLSEpochPair pair(ls_id, epoch); - (void)on_sp_rollback_succ_(pair, *tx, result.born_epoch_, result.addr_); + ObTxLSEpochPair pair(ls_id, orig_epoch); + (void)on_sp_rollback_succ_(pair, *tx, ret_epoch, ret_addr); if (tx->brpc_mask_set_.is_all_mask()) { tx->rpc_cond_.notify(OB_SUCCESS); } } else { // other failure // notify waiter, cause the savepoint rollback fail TRANS_LOG(WARN, "rollback_sp response an error", K(status), - K(tx_id), K(tx->tx_id_), K(addr), - K(request_id), K(ls_id), K(result)); + K(tx_id), K(tx->tx_id_), K(ret_epoch), + K(request_id), K(ls_id), K(ret_addr)); tx->rpc_cond_.notify(status); } + + tx->lock_.unlock(); } if (OB_NOT_NULL(tx)) { tx_desc_mgr_.revert(*tx); @@ -2208,7 +2241,7 @@ int ObTransService::handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const int64_t request_id, const SCN &private_data) { - auto start_ts = ObClockGenerator::getClock(); + int64_t start_ts = ObClockGenerator::getClock(); int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_running_)) { ret = OB_NOT_RUNNING; @@ -2227,7 +2260,7 @@ int ObTransService::handle_trans_msg_callback(const share::ObLSID &sender_ls_id, case OB_NOT_MASTER: case OB_SUCCESS: break; default: - auto commit_version = private_data; + share::SCN commit_version = private_data; if (OB_FAIL(handle_tx_commit_result(tx_id, status, commit_version))) { TRANS_LOG(WARN, "handle tx commit fail", K(ret), K(tx_id)); } @@ -2260,7 +2293,7 @@ int ObTransService::handle_trans_msg_callback(const share::ObLSID &sender_ls_id, } } } - auto elapsed_ts = ObClockGenerator::getClock() - start_ts; + int64_t elapsed_ts = ObClockGenerator::getClock() - start_ts; #ifndef NDEBUG TRANS_LOG(INFO, "handle trans msg callback", K(ret), K(elapsed_ts), K(tx_id), K(sender_ls_id), K(receiver_ls_id), @@ -2736,7 +2769,7 @@ int ObTransService::handle_timeout_for_xa(ObTxDesc &tx, const int64_t delay) int ret = OB_SUCCESS; int64_t now = ObClockGenerator::getClock(); bool cb_executed = false; - auto tx_id = tx.tx_id_; + ObTransID tx_id = tx.tx_id_; if (OB_FAIL(tx.lock_.lock(5000000))) { TRANS_LOG(WARN, "failed to acquire lock in specified time", K(tx)); // FIXME: how to handle it without lock protection diff --git a/src/storage/tx/ob_trans_service_v4.h b/src/storage/tx/ob_trans_service_v4.h index df7fa1bc22..45d1d3699d 100644 --- a/src/storage/tx/ob_trans_service_v4.h +++ b/src/storage/tx/ob_trans_service_v4.h @@ -94,13 +94,14 @@ int handle_trans_commit_request(ObTxCommitMsg &commit_req, obrpc::ObTransRpcResu int handle_trans_commit_response(ObTxCommitRespMsg &commit_resp, obrpc::ObTransRpcResult &result); int handle_trans_abort_request(ObTxAbortMsg &abort_req, obrpc::ObTransRpcResult &result); int handle_sp_rollback_request(ObTxRollbackSPMsg &sp_rollbacck_req, obrpc::ObTxRpcRollbackSPResult &result); +int handle_sp_rollback_response(ObTxRollbackSPRespMsg &sp_rollbacck_resp, obrpc::ObTransRpcResult &result); int handle_sp_rollback_resp(const share::ObLSID &ls_id, - const int64_t epoch, + const int64_t orig_epoch, const transaction::ObTransID &tx_id, const int status, - const ObAddr &addr, const int64_t request_id, - const obrpc::ObTxRpcRollbackSPResult &result); + const int64_t ret_epoch, + const ObAddr &ret_addr); int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const share::ObLSID &receiver_ls_id, const ObTransID &tx_id, diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index 483bdf2750..bb4b548c97 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -193,7 +193,7 @@ int ObTransService::reuse_tx(ObTxDesc &tx) } } // it is safe to operate tx without lock when not shared - auto session_id = tx.sess_id_; + uint32_t session_id = tx.sess_id_; tx.reset(); init_tx_(tx, session_id); } @@ -260,7 +260,7 @@ int ObTransService::start_tx(ObTxDesc &tx, const ObTxParam &tx_param, const ObTr tx.active_ts_ = ObClockGenerator::getClock(); tx.timeout_us_ = tx_param.timeout_us_; tx.lock_timeout_us_ = tx_param.lock_timeout_us_; - auto a = tx.timeout_us_ + tx.active_ts_; + int64_t a = tx.timeout_us_ + tx.active_ts_; tx.expire_ts_ = a < 0 ? INT64_MAX : a; // start tx need reacquire snapshot tx.snapshot_version_.reset(); @@ -373,7 +373,7 @@ namespace { int ObTransService::commit_tx(ObTxDesc &tx, const int64_t expire_ts, const ObString *trace_info) { int ret = OB_SUCCESS; - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); SyncTxCommitCb cb; if (OB_SUCC(submit_commit_tx(tx, expire_ts, cb, trace_info))) { int result = 0; @@ -388,7 +388,7 @@ int ObTransService::commit_tx(ObTxDesc &tx, const int64_t expire_ts, const ObStr ret = result; } } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; #ifndef NDEBUG TRANS_LOG(INFO, "sync commit tx", K(ret), K(tx), K(expire_ts)); #else @@ -497,7 +497,7 @@ int ObTransService::submit_commit_tx(ObTxDesc &tx, if (OB_SUCC(ret) && ( tx.state_ == ObTxDesc::State::ACTIVE || tx.state_ == ObTxDesc::State::IMPLICIT_ACTIVE)) { - auto state0 = tx.state_; + ObTxDesc::State state0 = tx.state_; tx.state_ = ObTxDesc::State::IN_TERMINATE; // record trace_info if (OB_NOT_NULL(trace_info) && @@ -817,7 +817,7 @@ int ObTransService::release_snapshot(ObTxDesc &tx) int ObTransService::register_tx_snapshot_verify(ObTxReadSnapshot &snapshot) { int ret = OB_SUCCESS; - const auto &tx_id = snapshot.core_.tx_id_; + const ObTransID &tx_id = snapshot.core_.tx_id_; if (tx_id.is_valid()) { ObTxDesc *tx = NULL; if (OB_SUCC(tx_desc_mgr_.get(tx_id, tx))) { @@ -854,7 +854,7 @@ int ObTransService::register_tx_snapshot_verify(ObTxReadSnapshot &snapshot) void ObTransService::unregister_tx_snapshot_verify(ObTxReadSnapshot &snapshot) { int ret = OB_SUCCESS; - const auto &tx_id = snapshot.core_.tx_id_; + const ObTransID &tx_id = snapshot.core_.tx_id_; if (tx_id.is_valid()) { ObTxDesc *tx = NULL; if (OB_SUCC(tx_desc_mgr_.get(tx_id, tx))) { @@ -1014,7 +1014,7 @@ int ObTransService::rollback_to_local_implicit_savepoint_(ObTxDesc &tx, { int ret = OB_SUCCESS; ObTxPartRefList parts; - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); if (OB_FAIL(find_parts_after_sp_(tx, parts, savepoint))) { TRANS_LOG(WARN, "find rollback parts fail", K(ret), K(savepoint), K(tx)); } else { @@ -1035,7 +1035,7 @@ int ObTransService::rollback_to_local_implicit_savepoint_(ObTxDesc &tx, } } } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; #ifndef NDEBUG TRANS_LOG(INFO, "rollback local implicit savepoint", K(ret), K(savepoint)); #else @@ -1059,7 +1059,7 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, const share::ObLSArray *extra_touched_ls) { int ret = OB_SUCCESS; - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); tx.inc_op_sn(); bool reset_tx = false, normal_rollback = false; // merge extra touched ls @@ -1118,6 +1118,8 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, if (OB_FAIL(ret)) { TRANS_LOG(WARN, "rollback savepoint fail, abort tx", K(ret), K(savepoint), KP(extra_touched_ls), K(parts), K(tx)); + // advance op_sequence to reject further rollback resp messsages + tx.inc_op_sn(); abort_tx_(tx, ObTxAbortCause::SAVEPOINT_ROLLBACK_FAIL); } else { /* @@ -1141,7 +1143,7 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, TRANS_LOG(WARN, "switch tx to idle fail", K(ret), K(tx)); } } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; #ifndef NDEBUG TRANS_LOG(INFO, "rollback to implicit savepoint", K(ret), K(savepoint), K(elapsed_us), K(tx)); #else @@ -1169,7 +1171,7 @@ int ObTransService::ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, int ret = OB_SUCCESS; int64_t retry_cnt = 0; bool blockable = expire_ts > 0; - const auto from_scn = savepoint.clone_with_seq(ObSequence::inc_and_get_max_seq_no()); + const ObTxSEQ from_scn = savepoint.clone_with_seq(ObSequence::inc_and_get_max_seq_no()); do { ret = part_ctx->rollback_to_savepoint(op_sn, from_scn, savepoint); if (OB_NEED_RETRY == ret && blockable) { @@ -1207,7 +1209,7 @@ int ObTransService::create_explicit_savepoint(ObTxDesc &tx, int ret = OB_SUCCESS; ObSpinLockGuard guard(tx.lock_); tx.inc_op_sn(); - const auto scn = tx.inc_and_get_tx_seq(0); + const ObTxSEQ scn = tx.inc_and_get_tx_seq(0); ObTxSavePoint sp; if (OB_SUCC(sp.init(scn, savepoint, session_id, user_create))) { if (OB_FAIL(tx.savepoints_.push_back(sp))) { @@ -1250,7 +1252,7 @@ int ObTransService::rollback_to_explicit_savepoint(ObTxDesc &tx, const uint32_t session_id) { int ret = OB_SUCCESS; - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); ObTxSEQ sp_scn; ObSpinLockGuard guard(tx.lock_); if (OB_SUCC(tx_sanity_check_(tx))) { @@ -1295,7 +1297,7 @@ int ObTransService::rollback_to_explicit_savepoint(ObTxDesc &tx, } } tx.state_change_flags_.EXTRA_CHANGED_ = true; - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx.get_tlog(); REC_TRANS_TRACE_EXT(&tlog, rollback_explicit_savepoint, OB_Y(ret), OB_ID(id), savepoint, @@ -1353,7 +1355,7 @@ int ObTransService::create_stash_savepoint(ObTxDesc &tx, const ObString &name) int ret = OB_SUCCESS; ObSpinLockGuard guard(tx.lock_); tx.inc_op_sn(); - const auto seq_no = tx.inc_and_get_tx_seq(0); + const ObTxSEQ seq_no = tx.inc_and_get_tx_seq(0); ObTxSavePoint sp; if (OB_SUCC(sp.init(seq_no, name, 0, false, true))) { if (OB_FAIL(tx.savepoints_.push_back(sp))) { @@ -1495,14 +1497,13 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, } } if (OB_SUCC(ret) && OB_NOT_NULL(ctx)) { + ctx_born_epoch = ctx->epoch_; if (verify_epoch > 0 && ctx->epoch_ != verify_epoch) { ret = OB_TRANS_CTX_NOT_EXIST; TRANS_LOG(WARN, "current ctx illegal, born epoch not match", K(ret), K(ls), K(tx_id), K(verify_epoch), KPC(ctx)); } else if(OB_FAIL(ls_sync_rollback_savepoint__(ctx, savepoint, op_sn, expire_ts))){ TRANS_LOG(WARN, "LS rollback to savepoint fail", K(ret), K(tx_id), K(ls), K(op_sn), K(savepoint), KPC(ctx)); - } else if (verify_epoch <= 0) { - ctx_born_epoch = ctx->epoch_; } } if (OB_NOT_NULL(ctx)) { @@ -1574,7 +1575,7 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, } if (OB_SUCC(ret)) { // setup state before release lock - auto save_state = tx.state_; + ObTxDesc::State save_state = tx.state_; tx.state_ = ObTxDesc::State::ROLLBACK_SAVEPOINT; tx.flags_.BLOCK_ = true; // release lock before blocking @@ -1597,7 +1598,7 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, tmp_tx_desc->~ObTxDesc(); ob_free(tmp_tx_desc); } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; TRANS_LOG(INFO, "rollback savepoint slowpath", K(ret), K_(tx.tx_id), K(start_ts), K(retries), K(savepoint), K(expire_ts), K(tx), K(parts.count())); @@ -1624,14 +1625,14 @@ inline int ObTransService::sync_rollback_savepoint__(ObTxDesc &tx, expire_ts = std::max(ObTimeUtility::current_time() + MIN_WAIT_TIME, expire_ts); while (OB_SUCC(ret)) { int64_t retry_intval = std::min(min_retry_intval * (1 + retries), max_retry_intval); - auto waittime = std::min(expire_ts - ObTimeUtility::current_time(), retry_intval); + int64_t waittime = std::min(expire_ts - ObTimeUtility::current_time(), retry_intval); if (waittime <=0) { ret = OB_TIMEOUT; TRANS_LOG(WARN, "tx rpc wait result timeout", K(ret), K(expire_ts), K(retries)); } else { ObSEArray remain; mask_set.get_not_mask(remain); - auto remain_cnt = remain.count(); + int64_t remain_cnt = remain.count(); TRANS_LOG(DEBUG, "unmasked parts", K(remain), K(tx), K(retries)); // post msg to participants if (remain_cnt > 0) { diff --git a/src/storage/tx/ob_tx_data_functor.cpp b/src/storage/tx/ob_tx_data_functor.cpp index 8368aeb175..6234b956e6 100644 --- a/src/storage/tx/ob_tx_data_functor.cpp +++ b/src/storage/tx/ob_tx_data_functor.cpp @@ -320,8 +320,8 @@ int LockForReadFunctor::operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx { int ret = OB_ERR_SHARED_LOCK_CONFLICT; const int64_t MAX_SLEEP_US = 1000; - auto &acc_ctx = lock_for_read_arg_.mvcc_acc_ctx_; - auto lock_expire_ts = acc_ctx.eval_lock_expire_ts(); + ObMvccAccessCtx &acc_ctx = lock_for_read_arg_.mvcc_acc_ctx_; + int64_t lock_expire_ts = acc_ctx.eval_lock_expire_ts(); // check lock_for_read blocked or not every 1ms * 100 = 100ms int64_t retry_cnt = 0; const int64_t MAX_RETRY_CNT = 100; diff --git a/src/storage/tx/ob_tx_free_route.cpp b/src/storage/tx/ob_tx_free_route.cpp index 5e7a44852c..53506b4898 100644 --- a/src/storage/tx/ob_tx_free_route.cpp +++ b/src/storage/tx/ob_tx_free_route.cpp @@ -51,7 +51,7 @@ void ObTxnFreeRouteCtx::init_before_handle_request(ObTxDesc *tx) audit_record_.proxy_flag_ = is_proxy_support_; if (OB_NOT_NULL(tx)) { if (tx->flags_.DEFER_ABORT_) { - auto txs = MTL_WITH_CHECK_TENANT(ObTransService*, tx->tenant_id_); + ObTransService *txs = MTL_WITH_CHECK_TENANT(ObTransService*, tx->tenant_id_); if (OB_ISNULL(txs)) { int ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "[tx free route] MTL(txs) is null", K(ret), K(tx->tenant_id_)); @@ -155,7 +155,7 @@ inline int ObTxnFreeRouteCtx::state_update_verify_by_version(const TxnFreeRouteS TRANS_LOG(ERROR, "the state is stale", K(ret)); } dup_sync = false; - auto &sync_info = state_sync_infos_[state]; + const StateSyncInfo &sync_info = state_sync_infos_[state]; if (sync_info.last_version_ > version) { // stale ret = OB_ERR_UNEXPECTED; @@ -289,7 +289,7 @@ static int encode_header_(const ObTxnFreeRouteCtx &ctx, char* buf, const int64_t { int ret = OB_SUCCESS; TxStateHeader header; - auto &tx_id = ctx.get_prev_tx_id().is_valid() ? ctx.get_prev_tx_id() : ctx.get_tx_id(); + ObTransID tx_id = ctx.get_prev_tx_id().is_valid() ? ctx.get_prev_tx_id() : ctx.get_tx_id(); if (OB_FAIL(OB_E(EventTable::EN_TX_FREE_ROUTE_ENCODE_STATE_ERROR, ctx.get_session_id()) OB_SUCCESS)) { TRANS_LOG(ERROR, "inject failure", K(ret), K(ctx)); } else if (!tx_id.is_valid()) { @@ -337,7 +337,7 @@ int ObTransService::txn_free_route__handle_tx_exist_(const ObTransID &tx_id, ObT } else if (!tmp_tx->is_xa_trans()) { // some session hold this txn already, close the session and release this txn // then continue with retry - auto assoc_sess_id = tmp_tx->assoc_sess_id_; + uint32_t assoc_sess_id = tmp_tx->assoc_sess_id_; TRANS_LOG(WARN, "tx found associate with other session, will kill the session", K(assoc_sess_id), K(tx_id)); if (OB_FAIL(txn_free_route__kill_session_(assoc_sess_id))) { @@ -386,9 +386,9 @@ int ObTransService::txn_free_route__update_static_state(const uint32_t session_i { int ret = OB_SUCCESS; bool need_add_tx = false; - auto &audit_record = ctx.audit_record_; + ObTxnFreeRouteAuditRecord &audit_record = ctx.audit_record_; audit_record.upd_static_ = true; - auto before_tx_id = OB_NOT_NULL(tx) ? tx->tx_id_ : ObTransID(); + ObTransID before_tx_id = OB_NOT_NULL(tx) ? tx->tx_id_ : ObTransID(); TXN_FREE_ROUTE_PROCESS_HEADER(TxnFreeRouteState::STATIC); if (OB_FAIL(ret)) { } else if (header.flag_.is_tx_terminated()) { @@ -430,7 +430,7 @@ int ObTransService::txn_free_route__update_static_state(const uint32_t session_i } } if (OB_SUCC(ret)) { - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); ObSpinLockGuard guard(tx->lock_); if (OB_FAIL(tx->decode_static_state(buf, len, pos))) { // unretryable @@ -450,7 +450,7 @@ int ObTransService::txn_free_route__update_static_state(const uint32_t session_i } else if (FALSE_IT(tx->flags_.SHADOW_ = tx->is_xa_trans() && tx->addr_ != self_)) { // mark as SHADOW_ for XA's temporary node, exclude XA orig node } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx->get_tlog(); REC_TRANS_TRACE_EXT(&tlog, tx_free_route_update_static, OB_Y(ret), OB_ID(txid), header.tx_id_.get_id(), @@ -502,7 +502,7 @@ int ObTransService::txn_free_route__update_dynamic_state(const uint32_t session_ int64_t &pos) { int ret = OB_SUCCESS; - auto &audit_record = ctx.audit_record_; + ObTxnFreeRouteAuditRecord &audit_record = ctx.audit_record_; audit_record.upd_dyn_ = true; int64_t logic_clock = 0; TXN_FREE_ROUTE_PROCESS_HEADER(TxnFreeRouteState::DYNAMIC); @@ -520,7 +520,7 @@ int ObTransService::txn_free_route__update_dynamic_state(const uint32_t session_ ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "tx should not be null", K(ret), K(session_id)); } else { - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); ObSpinLockGuard guard(tx->lock_); if (!tx->tx_id_.is_valid()) { // bug, dynamic state exist, txn should be active @@ -533,7 +533,7 @@ int ObTransService::txn_free_route__update_dynamic_state(const uint32_t session_ } else if (OB_FAIL(tx->decode_dynamic_state(buf, len, pos))) { TRANS_LOG(ERROR, "decode dynamic state fail", K(ret)); } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx->get_tlog(); REC_TRANS_TRACE_EXT(&tlog, tx_free_route_update_dynamic, OB_Y(ret), OB_ID(time_used), elapsed_us, @@ -561,7 +561,7 @@ int ObTransService::txn_free_route__update_parts_state(const uint32_t session_id int64_t &pos) { int ret = OB_SUCCESS; - auto &audit_record = ctx.audit_record_; + ObTxnFreeRouteAuditRecord &audit_record = ctx.audit_record_; audit_record.upd_parts_ = true; TXN_FREE_ROUTE_PROCESS_HEADER(TxnFreeRouteState::PARTICIPANT); if (OB_FAIL(ret)) { @@ -583,7 +583,7 @@ int ObTransService::txn_free_route__update_parts_state(const uint32_t session_id ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "tx should not be null", K(ret), K(session_id)); } else { - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); ObSpinLockGuard guard(tx->lock_); if (!tx->tx_id_.is_valid()) { // bug, dynamic state exist, txn should be active @@ -592,7 +592,7 @@ int ObTransService::txn_free_route__update_parts_state(const uint32_t session_id } else if (OB_FAIL(tx->decode_parts_state(buf, len, pos))) { TRANS_LOG(WARN, "decode participants fail", K(ret)); } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx->get_tlog(); REC_TRANS_TRACE_EXT(&tlog, tx_free_route_update_participants, OB_Y(ret), OB_ID(txid), header.tx_id_.get_id(), @@ -619,7 +619,7 @@ int ObTransService::txn_free_route__update_extra_state(const uint32_t session_id { int ret = OB_SUCCESS; int64_t logic_clock = 0; - auto &audit_record = ctx.audit_record_; + ObTxnFreeRouteAuditRecord &audit_record = ctx.audit_record_; audit_record.upd_extra_ = true; TXN_FREE_ROUTE_PROCESS_HEADER(TxnFreeRouteState::EXTRA); if (OB_FAIL(ret)) { @@ -642,7 +642,7 @@ int ObTransService::txn_free_route__update_extra_state(const uint32_t session_id } else { bool add_tx = OB_ISNULL(tx); bool replace_tx = OB_NOT_NULL(tx) && tx->tx_id_ != header.tx_id_; - auto before_tx_id = OB_NOT_NULL(tx) ? tx->tx_id_ : ObTransID(); + ObTransID before_tx_id = OB_NOT_NULL(tx) ? tx->tx_id_ : ObTransID(); audit_record.replace_tx_ = replace_tx; audit_record.alloc_tx_ = add_tx; if (OB_FAIL(decode_i64(buf, len, pos, &logic_clock))) { @@ -664,7 +664,7 @@ int ObTransService::txn_free_route__update_extra_state(const uint32_t session_id } } if (OB_SUCC(ret)) { - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); ObSpinLockGuard guard(tx->lock_); if (OB_FAIL(tx->decode_extra_state(buf, len, pos))) { TRANS_LOG(ERROR, "decode extra fail", K(ret)); @@ -676,7 +676,7 @@ int ObTransService::txn_free_route__update_extra_state(const uint32_t session_id release_tx(*tx); tx = NULL; } else { - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx->get_tlog(); REC_TRANS_TRACE_EXT(&tlog, tx_free_route_update_extra, OB_Y(ret), OB_ID(txid), header.tx_id_.get_id(), @@ -850,7 +850,7 @@ int ObTransService::calc_txn_free_route(ObTxDesc *tx, ObTxnFreeRouteCtx &ctx) // // if niether of these flag was setted, the detailed changed txn state is exist as the content of // normal state, otherwise it's empty - auto &audit_record = ctx.audit_record_; + ObTxnFreeRouteAuditRecord &audit_record = ctx.audit_record_; if (OB_NOT_NULL(tx)) { tx->lock_.lock(); } @@ -1078,7 +1078,7 @@ bool ObTransService::need_fallback_(ObTxDesc &tx, int64_t &total_size) int ObTransService::push_tx_state_to_remote_(ObTxDesc &tx, const ObAddr &txn_addr) { int ret = OB_SUCCESS; - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); ObTxFreeRoutePushState state; state.tenant_id_ = tenant_id_; int64_t len = 0; @@ -1121,7 +1121,7 @@ int ObTransService::push_tx_state_to_remote_(ObTxDesc &tx, const ObAddr &txn_add TRANS_LOG(INFO, "[tx free route] push txn state success", K(txn_addr), K(tx)); } } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx.get_tlog(); REC_TRANS_TRACE_EXT(&tlog, tx_free_route_send_state, OB_Y(ret), OB_ID(time_used), elapsed_us, @@ -1146,7 +1146,7 @@ int ObTransService::tx_free_route_handle_push_state(const ObTxFreeRoutePushState ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "tx is null", K(ret)); } else { - auto start_ts = ObTimeUtility::current_time(); + int64_t start_ts = ObTimeUtility::current_time(); const char *buf = state.buf_.ptr(); int64_t buf_len = state.buf_.length(); int64_t static_len = state.dynamic_offset_; @@ -1193,7 +1193,7 @@ int ObTransService::tx_free_route_handle_push_state(const ObTxFreeRoutePushState ret = COVER_SUCC(tmp_ret); } } - auto elapsed_us = ObTimeUtility::current_time() - start_ts; + int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx->get_tlog(); REC_TRANS_TRACE_EXT(&tlog, tx_free_route_recv_state, OB_Y(ret), OB_ID(time_used), elapsed_us, diff --git a/src/storage/tx/ob_tx_free_route_rpc.cpp b/src/storage/tx/ob_tx_free_route_rpc.cpp index 543b5226e4..f86e740eb1 100644 --- a/src/storage/tx/ob_tx_free_route_rpc.cpp +++ b/src/storage/tx/ob_tx_free_route_rpc.cpp @@ -39,7 +39,7 @@ int ObTxFreeRouteCheckAliveP::process() session->get_query_lock().unlock(); } } - auto txs = MTL(transaction::ObTransService*); + transaction::ObTransService *txs = MTL(transaction::ObTransService*); if (OB_ISNULL(txs)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "can not get trans service", KR(ret)); @@ -100,8 +100,8 @@ int ObTxFreeRouteCheckAliveRespP::release_session_tx_() } else if (OB_FAIL(session->try_lock_thread_data())) { session->unlock_query(); } else { - auto &ctx = session->get_txn_free_route_ctx(); - auto &tx_desc = session->get_tx_desc(); + transaction::ObTxnFreeRouteCtx &ctx = session->get_txn_free_route_ctx(); + transaction::ObTxDesc *&tx_desc = session->get_tx_desc(); if (ctx.get_local_version() != arg_.request_id_) { TRANS_LOG(INFO, "skip handle checkAliveResp, staled", K(arg_), K(ctx.get_local_version())); } else if (OB_NOT_NULL(tx_desc) && tx_desc->get_tx_id() == arg_.tx_id_) { @@ -137,7 +137,7 @@ int ObTxFreeRoutePushStateP::process() int ret = OB_SUCCESS; transaction::ObTxFreeRoutePushState &tx_state = arg_; transaction::ObTxFreeRoutePushStateResp &resp = result_; - auto txs = MTL(transaction::ObTransService*); + transaction::ObTransService *txs = MTL(transaction::ObTransService*); if (OB_ISNULL(txs)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "fail to get trans service", K(ret)); diff --git a/src/storage/tx/ob_tx_msg.cpp b/src/storage/tx/ob_tx_msg.cpp index 68f5cfdd96..76dbb36738 100644 --- a/src/storage/tx/ob_tx_msg.cpp +++ b/src/storage/tx/ob_tx_msg.cpp @@ -126,6 +126,7 @@ OB_TX_MSG_SERDE(ObAskStateMsg, ObTxMsg, snapshot_); OB_TX_MSG_SERDE(ObAskStateRespMsg, ObTxMsg, state_info_array_); OB_TX_MSG_SERDE(ObCollectStateMsg, ObTxMsg, snapshot_); OB_TX_MSG_SERDE(ObCollectStateRespMsg, ObTxMsg, state_info_); +OB_SERIALIZE_MEMBER((ObTxRollbackSPRespMsg, ObTxMsg), ret_, orig_epoch_); OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) { @@ -138,6 +139,7 @@ OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) } else { OB_UNIS_ADD_LEN(false); } + OB_UNIS_ADD_LEN(flag_); return len; } @@ -152,6 +154,7 @@ OB_DEF_SERIALIZE(ObTxRollbackSPMsg) } else { OB_UNIS_ENCODE(false); } + OB_UNIS_ENCODE(flag_); } return ret; } @@ -173,6 +176,7 @@ OB_DEF_DESERIALIZE(ObTxRollbackSPMsg) tx_ptr_ = tmp; } } + OB_UNIS_DECODE(flag_); } return ret; } diff --git a/src/storage/tx/ob_tx_msg.h b/src/storage/tx/ob_tx_msg.h index 7cc56f57ee..78c0052d19 100644 --- a/src/storage/tx/ob_tx_msg.h +++ b/src/storage/tx/ob_tx_msg.h @@ -59,6 +59,8 @@ namespace transaction ASK_STATE_RESP = 64, COLLECT_STATE = 65, COLLECT_STATE_RESP = 66, + // rollback savepoint resp + ROLLBACK_SAVEPOINT_RESP = 67, /* for txn free route */ TX_FREE_ROUTE_PUSH_STATE = 80, TX_FREE_ROUTE_CHECK_ALIVE = 81, @@ -240,7 +242,8 @@ namespace transaction op_sn_(-1), //todo:后续branch_id使用方式确定后，需要相应修改 branch_id_(-1), - tx_ptr_(NULL) + tx_ptr_(NULL), + flag_(USE_ASYNC_RESP) {} ~ObTxRollbackSPMsg() { if (OB_NOT_NULL(tx_ptr_)) { @@ -254,13 +257,32 @@ namespace transaction //todo:后期设计中操作编号是否等于branch_id int64_t branch_id_; const ObTxDesc *tx_ptr_; + uint8_t flag_; + bool use_async_resp() const { return (flag_ & USE_ASYNC_RESP) !=0; } + const static uint8_t USE_ASYNC_RESP = 0x01; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, - K_(savepoint), K_(op_sn), K_(branch_id), + K_(savepoint), K_(op_sn), K_(branch_id), K_(flag), KP_(tx_ptr)); OB_UNIS_VERSION(1); }; + struct ObTxRollbackSPRespMsg : public ObTxMsg { + ObTxRollbackSPRespMsg() : + ObTxMsg(ROLLBACK_SAVEPOINT_RESP), + ret_(-1), + orig_epoch_(0) + {} + ~ObTxRollbackSPRespMsg() { + ret_ = -1; + orig_epoch_ = 0; + } + int ret_; + int64_t orig_epoch_; + INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret), K_(orig_epoch)); + OB_UNIS_VERSION(1); + }; + struct ObTxKeepaliveMsg : public ObTxMsg { ObTxKeepaliveMsg() : ObTxMsg(KEEPALIVE) diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index b3cc9c8047..d787b4589f 100755 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -350,7 +350,7 @@ int ObAccessService::get_write_store_ctx_guard_( ret = OB_ERR_UNEXPECTED; LOG_ERROR("ls should not be null", K(ret), K(ls_id), K_(tenant_id)); } else { - auto &ctx = ctx_guard.get_store_ctx(); + ObStoreCtx &ctx = ctx_guard.get_store_ctx(); ctx.ls_ = ls; ctx.timeout_ = timeout; if (OB_FAIL(ls->get_write_store_ctx(tx_desc, snapshot, write_flag, ctx, spec_seq_no))) { @@ -400,7 +400,7 @@ int ObAccessService::get_source_ls_tx_table_guard_(ObStoreCtxGuard &ctx_guard) ret = OB_ERR_UNEXPECTED; LOG_WARN("transfer_scn or source ls tx_table_guard is invalid", K(ret), K(src_tx_table_guard), K(user_data)); } else { - auto &ctx = ctx_guard.get_store_ctx(); + ObStoreCtx &ctx = ctx_guard.get_store_ctx(); ctx.mvcc_acc_ctx_.set_src_tx_table_guard(src_tx_table_guard); ctx.mvcc_acc_ctx_.set_transfer_scn(user_data.transfer_scn_); LOG_DEBUG("succ get src tx table guard", K(ret), K(src_ls->get_ls_id()), K(src_tx_table_guard), K(user_data)); @@ -463,7 +463,7 @@ int ObAccessService::check_read_allowed_( ret = OB_ERR_UNEXPECTED; LOG_ERROR("ls should not be null", K(ret), K(ls_id), K_(tenant_id)); } else { - auto &ctx = ctx_guard.get_store_ctx(); + ObStoreCtx &ctx = ctx_guard.get_store_ctx(); ctx.ls_ = ls; ctx.timeout_ = scan_param.timeout_; ctx.tablet_id_ = tablet_id; @@ -576,7 +576,7 @@ int ObAccessService::check_write_allowed_( LOG_WARN("failed to check replica allow to read", K(ret), K(tablet_id)); } else { // TODO: this may confuse user, because of txn timeout won't notify user proactively - auto lock_expired_ts = MIN(dml_param.timeout_, tx_desc.get_expire_ts()); + int64_t lock_expired_ts = MIN(dml_param.timeout_, tx_desc.get_expire_ts()); if (OB_FAIL(get_lock_id(tablet_id, lock_id))) { LOG_WARN("get lock id failed", K(ret), K(tablet_id)); } else if (OB_FAIL(lock_param.set(lock_id, diff --git a/tools/deploy/mysql_test/test_suite/expr/t/expr_floor.test b/tools/deploy/mysql_test/test_suite/expr/t/expr_floor.test index f4562b4a73..e08b558d41 100644 --- a/tools/deploy/mysql_test/test_suite/expr/t/expr_floor.test +++ b/tools/deploy/mysql_test/test_suite/expr/t/expr_floor.test @@ -1,7 +1,7 @@ --disable_query_log set @@session.explicit_defaults_for_timestamp=off; --enable_query_log -#### owner: yuchen.wyc +#### owner: xiaoyi.xy #### owner group: sql3 #### description: 测试地板函数（向下取整函数，高斯函数） #/--disableabortonerror/ diff --git a/tools/deploy/mysql_test/test_suite/expr/t/expr_instr.test b/tools/deploy/mysql_test/test_suite/expr/t/expr_instr.test index 25ad02c46e..bf3d7aa0fb 100644 --- a/tools/deploy/mysql_test/test_suite/expr/t/expr_instr.test +++ b/tools/deploy/mysql_test/test_suite/expr/t/expr_instr.test @@ -1,7 +1,7 @@ --disable_query_log set @@session.explicit_defaults_for_timestamp=off; --enable_query_log -#### owner: yuchen.wyc +#### owner: jim.wjh #### owner group: sql3 #### description: 测试instr函数 # tags: expr, datatype diff --git a/tools/deploy/mysql_test/test_suite/expr/t/expr_locate.test b/tools/deploy/mysql_test/test_suite/expr/t/expr_locate.test index 7f26fcecea..c43e774f6f 100644 --- a/tools/deploy/mysql_test/test_suite/expr/t/expr_locate.test +++ b/tools/deploy/mysql_test/test_suite/expr/t/expr_locate.test @@ -1,7 +1,7 @@ --disable_query_log set @@session.explicit_defaults_for_timestamp=off; --enable_query_log -#### owner: yuchen.wyc +#### owner: xiaoyi.xy #### owner group: sql3 #### description: 测试locate函数 # tags: expr, datatype diff --git a/tools/deploy/mysql_test/test_suite/expr/t/expr_nseq.test b/tools/deploy/mysql_test/test_suite/expr/t/expr_nseq.test index 00afbe6c79..6b88b49b60 100644 --- a/tools/deploy/mysql_test/test_suite/expr/t/expr_nseq.test +++ b/tools/deploy/mysql_test/test_suite/expr/t/expr_nseq.test @@ -1,7 +1,7 @@ --disable_query_log set @@session.explicit_defaults_for_timestamp=off; --enable_query_log -#### owner: yuchen.wyc +#### owner: xiaoyi.xy #### owner group: sql3 #### description: 测试<＝>运算符 --disable_warnings diff --git a/tools/deploy/mysql_test/test_suite/expr/t/expr_position.test b/tools/deploy/mysql_test/test_suite/expr/t/expr_position.test index a3fb43d1e8..9af36916a8 100644 --- a/tools/deploy/mysql_test/test_suite/expr/t/expr_position.test +++ b/tools/deploy/mysql_test/test_suite/expr/t/expr_position.test @@ -1,7 +1,7 @@ --disable_query_log set @@session.explicit_defaults_for_timestamp=off; --enable_query_log -#### owner: yuchen.wyc +#### owner: jim.wjh #### owner group: sql3 #### description: 测试position函数 --echo ================ expression position ================ diff --git a/tools/deploy/mysql_test/test_suite/expr/t/func_equal.test b/tools/deploy/mysql_test/test_suite/expr/t/func_equal.test index 5783d391fb..7d9f4f86de 100644 --- a/tools/deploy/mysql_test/test_suite/expr/t/func_equal.test +++ b/tools/deploy/mysql_test/test_suite/expr/t/func_equal.test @@ -1,7 +1,7 @@ --disable_query_log set @@session.explicit_defaults_for_timestamp=off; --enable_query_log -#### owner: yuchen.wyc +#### owner: peihan.dph #### owner group: sql3 #### description: 测试相等比较运算符 # Initialise diff --git a/tools/deploy/mysql_test/test_suite/geometry/t/import_default_srs_data_mysql.inc b/tools/deploy/mysql_test/test_suite/geometry/t/import_default_srs_data_mysql.inc index 9e2a67066e..f721a406a8 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/t/import_default_srs_data_mysql.inc +++ b/tools/deploy/mysql_test/test_suite/geometry/t/import_default_srs_data_mysql.inc @@ -13,12 +13,12 @@ let $c1 = query_get_value(select count(*) as cnt from oceanbase.__all_spatial_re #alter system change tenant mysql ; #sleep 2; -if ($c1 != 5152) +if ($c1 != 5160) { --source mysql_test/test_suite/geometry/t/default_srs_data_mysql.sql + sleep 30; } -sleep 30; # TODO@dazhi: use mysql tenant #disconnect conn_admin; #connection default; diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index caf2d6bda0..108c5bb4dc 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -316,6 +316,7 @@ _minor_compaction_amplification_factor _min_malloc_sample_interval _mvcc_gc_using_min_txn_snapshot _nested_loop_join_enabled +_obkv_feature_mode _ob_ddl_timeout _ob_elr_fast_freeze_threshold _ob_enable_direct_load @@ -378,7 +379,7 @@ _transfer_start_rpc_timeout _transfer_start_trans_timeout _tx_result_retention _upgrade_stage -_wait_interval_after_truncate +_wait_interval_after_parallel_ddl _with_subquery _xa_gc_interval _xa_gc_timeout diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_nvl.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_nvl.result index ac13a64dc9..0eaecb47dc 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_nvl.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_nvl.result @@ -95,7 +95,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -136,7 +136,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -177,7 +177,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -218,7 +218,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -259,7 +259,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -300,7 +300,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -341,7 +341,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -382,7 +382,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -423,7 +423,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -464,7 +464,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -505,7 +505,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -546,7 +546,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -587,7 +587,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -628,7 +628,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -669,7 +669,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -710,7 +710,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -751,7 +751,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -792,7 +792,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -833,7 +833,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -874,7 +874,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -915,7 +915,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -956,7 +956,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c1, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c1, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -997,7 +997,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1038,7 +1038,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1079,7 +1079,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1120,7 +1120,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1161,7 +1161,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1202,7 +1202,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1243,7 +1243,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1284,7 +1284,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1325,7 +1325,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1366,7 +1366,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1407,7 +1407,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1448,7 +1448,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1489,7 +1489,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1530,7 +1530,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1571,7 +1571,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1612,7 +1612,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1653,7 +1653,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1694,7 +1694,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1735,7 +1735,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1776,7 +1776,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1817,7 +1817,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1858,7 +1858,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c2, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c2, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1899,7 +1899,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1940,7 +1940,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -1981,7 +1981,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2022,7 +2022,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2063,7 +2063,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2104,7 +2104,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2145,7 +2145,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2186,7 +2186,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2227,7 +2227,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2268,7 +2268,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2309,7 +2309,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2350,7 +2350,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2391,7 +2391,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2432,7 +2432,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2473,7 +2473,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2514,7 +2514,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2555,7 +2555,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2596,7 +2596,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2637,7 +2637,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2678,7 +2678,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2719,7 +2719,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2760,7 +2760,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c3, BIGINT(4, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c3, BIGINT(4, 0))]), filter(nil), rowset=16 access([t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2801,7 +2801,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2842,7 +2842,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2883,7 +2883,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2924,7 +2924,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2965,7 +2965,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3006,7 +3006,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3047,7 +3047,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3088,7 +3088,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3129,7 +3129,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3170,7 +3170,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3211,7 +3211,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3252,7 +3252,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3293,7 +3293,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3334,7 +3334,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3375,7 +3375,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3416,7 +3416,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3457,7 +3457,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3498,7 +3498,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3539,7 +3539,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3580,7 +3580,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3621,7 +3621,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3662,7 +3662,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c4, BIGINT UNSIGNED(3, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c4, BIGINT UNSIGNED(3, 0))]), filter(nil), rowset=16 access([t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3703,7 +3703,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3744,7 +3744,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3785,7 +3785,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3826,7 +3826,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3867,7 +3867,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3908,7 +3908,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3949,7 +3949,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3990,7 +3990,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4031,7 +4031,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4072,7 +4072,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4113,7 +4113,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4154,7 +4154,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4195,7 +4195,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4236,7 +4236,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4277,7 +4277,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4318,7 +4318,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4359,7 +4359,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4400,7 +4400,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4441,7 +4441,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4482,7 +4482,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4523,7 +4523,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4564,7 +4564,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c5, BIGINT(6, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c5, BIGINT(6, 0))]), filter(nil), rowset=16 access([t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4605,7 +4605,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4646,7 +4646,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4687,7 +4687,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4728,7 +4728,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4769,7 +4769,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4810,7 +4810,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4851,7 +4851,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4892,7 +4892,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4933,7 +4933,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -4974,7 +4974,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5015,7 +5015,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5056,7 +5056,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5097,7 +5097,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5138,7 +5138,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5179,7 +5179,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5220,7 +5220,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5261,7 +5261,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5302,7 +5302,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5343,7 +5343,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5384,7 +5384,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5425,7 +5425,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5466,7 +5466,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c6, BIGINT UNSIGNED(5, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c6, BIGINT UNSIGNED(5, 0))]), filter(nil), rowset=16 access([t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5507,7 +5507,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5548,7 +5548,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5589,7 +5589,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5630,7 +5630,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5671,7 +5671,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5712,7 +5712,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5753,7 +5753,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5794,7 +5794,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5835,7 +5835,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5876,7 +5876,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5917,7 +5917,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5958,7 +5958,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5999,7 +5999,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6040,7 +6040,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6081,7 +6081,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6122,7 +6122,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6163,7 +6163,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6204,7 +6204,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6245,7 +6245,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6286,7 +6286,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6327,7 +6327,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6368,7 +6368,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c7, BIGINT(9, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c7, BIGINT(9, 0))]), filter(nil), rowset=16 access([t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6409,7 +6409,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6450,7 +6450,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6491,7 +6491,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6532,7 +6532,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6573,7 +6573,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6614,7 +6614,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6655,7 +6655,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6696,7 +6696,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6737,7 +6737,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6778,7 +6778,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6819,7 +6819,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6860,7 +6860,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6901,7 +6901,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6942,7 +6942,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6983,7 +6983,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7024,7 +7024,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7065,7 +7065,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7106,7 +7106,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7147,7 +7147,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7188,7 +7188,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7229,7 +7229,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7270,7 +7270,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c8, BIGINT UNSIGNED(8, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c8, BIGINT UNSIGNED(8, 0))]), filter(nil), rowset=16 access([t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7311,7 +7311,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7352,7 +7352,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7393,7 +7393,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7434,7 +7434,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7475,7 +7475,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7516,7 +7516,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7557,7 +7557,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7598,7 +7598,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7639,7 +7639,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7680,7 +7680,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7721,7 +7721,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7762,7 +7762,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7803,7 +7803,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7844,7 +7844,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7885,7 +7885,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7926,7 +7926,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7967,7 +7967,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8008,7 +8008,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8049,7 +8049,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8090,7 +8090,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8131,7 +8131,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8172,7 +8172,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c9, BIGINT(11, 0)), BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c9, BIGINT(11, 0))]), filter(nil), rowset=16 access([t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8213,7 +8213,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8254,7 +8254,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8295,7 +8295,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8336,7 +8336,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8377,7 +8377,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8418,7 +8418,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8459,7 +8459,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8500,7 +8500,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8541,7 +8541,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8582,7 +8582,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8623,7 +8623,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8664,7 +8664,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8705,7 +8705,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8746,7 +8746,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8787,7 +8787,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8828,7 +8828,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8869,7 +8869,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8910,7 +8910,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8951,7 +8951,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8992,7 +8992,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9033,7 +9033,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9074,7 +9074,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(cast(t1.c10, BIGINT UNSIGNED(10, 0)), BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([cast(t1.c10, BIGINT UNSIGNED(10, 0))]), filter(nil), rowset=16 access([t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9115,7 +9115,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9156,7 +9156,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9197,7 +9197,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9238,7 +9238,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9279,7 +9279,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9320,7 +9320,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9361,7 +9361,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9402,7 +9402,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9443,7 +9443,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9484,7 +9484,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9525,7 +9525,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9566,7 +9566,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9607,7 +9607,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9648,7 +9648,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9689,7 +9689,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9730,7 +9730,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9771,7 +9771,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9812,7 +9812,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9853,7 +9853,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9894,7 +9894,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9935,7 +9935,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9976,7 +9976,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c11, BIGINT(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c11]), filter(nil), rowset=16 access([t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10017,7 +10017,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10058,7 +10058,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10099,7 +10099,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10140,7 +10140,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10181,7 +10181,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10222,7 +10222,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10263,7 +10263,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10304,7 +10304,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10345,7 +10345,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10386,7 +10386,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10427,7 +10427,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10468,7 +10468,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10509,7 +10509,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10550,7 +10550,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10591,7 +10591,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10632,7 +10632,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10673,7 +10673,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10714,7 +10714,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10755,7 +10755,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10796,7 +10796,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10837,7 +10837,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10878,7 +10878,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([cast(t1.c12, BIGINT UNSIGNED(0, 0))]), filter(nil), rowset=16 + 0 - output([t1.c12]), filter(nil), rowset=16 access([t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true diff --git a/tools/docker/standalone/Dockerfile b/tools/docker/standalone/Dockerfile index 3e0c42e47a..0687df0dc0 100644 --- a/tools/docker/standalone/Dockerfile +++ b/tools/docker/standalone/Dockerfile @@ -1,16 +1,18 @@ -FROM centos:centos7 +FROM openanolis/anolisos:8.8 +# docker build --build-arg VERSION={VERSION} . ARG VERSION -RUN yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo && \ +RUN yum install -y yum-utils && \ + yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo && \ + sed -i 's/$releasever/7/' /etc/yum.repos.d/OceanBase.repo && \ yum install -y ob-deploy obclient ob-sysbench libaio && \ rm -rf /usr/obd/mirror/remote/* && \ - rm -rf /u01/mysql /u01/obclient/bin/mysqld* /u01/obclient/bin/aria* /u01/obclient/bin/maria* && \ yum clean all -RUN mkdir /root/pkg && \ +RUN mkdir -p /root/pkg && \ cd /root/pkg && \ - yum install --downloadonly --downloaddir=. oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 obagent && \ + yum install -y --downloadonly --downloaddir=. oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 obagent && \ rm -rf /usr/obd/mirror/remote/* && \ yum clean all diff --git a/tools/docker/standalone/boot/obagent.yaml b/tools/docker/standalone/boot/obagent.yaml index 6dc012ebe9..c782c21b53 100644 --- a/tools/docker/standalone/boot/obagent.yaml +++ b/tools/docker/standalone/boot/obagent.yaml @@ -10,9 +10,9 @@ obagent: # The working directory for obagent. obagent is started under this directory. This is a required field. home_path: /root/obagent # The port that pulls and manages the metrics. The default port number is 8088. - server_port: 8088 + monagent_http_port: 8088 # Debug port for pprof. The default port number is 8089. - pprof_port: 8089 + mgragent_http_port: 8089 # Log level. The default value is INFO. log_level: INFO # Log path. The default value is log/monagent.log. diff --git a/tools/ob_admin/io_bench/bench_io.sh b/tools/ob_admin/io_bench/bench_io.sh index aefa5447a4..9ed4fc9ada 100755 --- a/tools/ob_admin/io_bench/bench_io.sh +++ b/tools/ob_admin/io_bench/bench_io.sh @@ -9,7 +9,7 @@ echo "bench_dir=$bench_dir, file_size=$file_size, output_dir=$output_dir" # prepare bench file bench_file_name=$bench_dir/bench_chunk -prepare_cmd="fio -filename=$bench_file_name -filesize=$file_size -rw=randread -bs=1M -runtime=1 -name=prepare --sync=fsync 2>&1 > /dev/null" +prepare_cmd="fio -filename=$bench_file_name -filesize=$file_size -rw=randread -bs=1M -runtime=1 -name=prepare --fsync=1 2>&1 > /dev/null" echo "prepare bench file" echo " prepare_cmd: $prepare_cmd" echo "" diff --git a/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp b/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp index d55a3e839d..47220ad723 100644 --- a/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp +++ b/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp @@ -22,6 +22,7 @@ #include "storage/tx/ob_id_service.h" #include "storage/memtable/ob_memtable_mutator.h" #include "storage/tx/ob_keep_alive_ls_handler.h" +#include "storage/tx/ob_dup_table_dump.h" #include "logservice/ob_log_base_header.h" #include "logservice/ob_garbage_collector.h" #include "logservice/data_dictionary/ob_data_dict_iterator.h" // ObDataDictIterator @@ -578,6 +579,23 @@ int ObAdminParserLogEntry::parse_medium_log_() return ret; } +int ObAdminParserLogEntry::parse_dup_table_log_() +{ + int ret = OB_SUCCESS; + + oceanbase::transaction::ObDupTableLogDumpIterator dup_table_log_dump_iter; + + if (OB_FAIL(dup_table_log_dump_iter.init_with_log_buf(buf_ + pos_, buf_len_, &str_arg_))) { + LOG_WARN("fail to init dup table log dump_iter", K(ret)); + } else if (OB_FAIL(dup_table_log_dump_iter.dump_dup_table_log())) { + LOG_WARN("fail to dump dup table log", K(ret)); + } else { + pos_ += dup_table_log_dump_iter.get_iter_buf_pos(); + } + + return ret; +} + int ObAdminParserLogEntry::parse_different_entry_type_(const logservice::ObLogBaseHeader &header) { int ret = OB_SUCCESS; @@ -656,6 +674,10 @@ int ObAdminParserLogEntry::parse_different_entry_type_(const logservice::ObLogBa ret = parse_medium_log_(); break; } + case oceanbase::logservice::ObLogBaseType::DUP_TABLE_LOG_BASE_TYPE: { + ret = parse_dup_table_log_(); + break; + } default: { fprintf(stdout, " Unknown Base Log Type : %d\n", header.get_log_type()); diff --git a/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.h b/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.h index a82c6a6b7f..4c6011a1b9 100644 --- a/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.h +++ b/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.h @@ -64,6 +64,7 @@ private: int parse_data_dict_log_(); int parse_reserved_snapshot_log_(); int parse_medium_log_(); + int parse_dup_table_log_(); //log type belong to trans_service int parse_trans_redo_log_(transaction::ObTxLogBlock &tx_log_block, diff --git a/unittest/observer/CMakeLists.txt b/unittest/observer/CMakeLists.txt index 39ff2f2303..fc0e64f742 100644 --- a/unittest/observer/CMakeLists.txt +++ b/unittest/observer/CMakeLists.txt @@ -5,6 +5,7 @@ storage_unittest(test_create_executor table/test_create_executor.cpp) storage_unittest(test_table_aggregation table/test_table_aggregation.cpp) storage_unittest(test_table_sess_pool table/test_table_sess_pool.cpp) storage_unittest(test_ingress_bw_alloc_manager net/test_ingress_bw_alloc_manager.cpp) +ob_unittest(test_obkv_config tableapi/test_obkv_config.cpp) ob_unittest(test_uniq_task_queue) add_subdirectory(rpc EXCLUDE_FROM_ALL) diff --git a/unittest/observer/table/test_create_executor.cpp b/unittest/observer/table/test_create_executor.cpp index 2594815c70..cc5686ca03 100644 --- a/unittest/observer/table/test_create_executor.cpp +++ b/unittest/observer/table/test_create_executor.cpp @@ -166,7 +166,8 @@ void TestCreateExecutor::fake_ctx_init_common(ObTableCtx &fake_ctx, ObTableSchem g_sess_node_val.is_inited_ = true; g_sess_node_val.sess_info_.test_init(0, 0, 0, NULL); g_sess_node_val.sess_info_.load_all_sys_vars(schema_guard_); - fake_ctx.init_phy_plan_ctx(); + fake_ctx.init_physical_plan_ctx(0, 1); + ASSERT_EQ(OB_SUCCESS, fake_ctx.construct_column_items()); } TEST_F(TestCreateExecutor, scan) @@ -296,7 +297,7 @@ TEST_F(TestCreateExecutor, update) ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); fake_ctx.set_expr_info(&fake_expr_info); - ASSERT_EQ(6, fake_ctx.get_all_exprs().get_expr_array().count()); + ASSERT_EQ(4, fake_ctx.get_all_exprs().get_expr_array().count()); ObTableApiSpec *root_spec = nullptr; ObTableApiExecutor *executor = nullptr; // generate update spec tree @@ -343,7 +344,7 @@ TEST_F(TestCreateExecutor, insertup) ASSERT_EQ(OB_SUCCESS, fake_ctx.init_insert_up()); ASSERT_EQ(OB_SUCCESS, ObTableExprCgService::generate_exprs(fake_ctx, allocator_, fake_expr_info)); fake_ctx.set_expr_info(&fake_expr_info); - ASSERT_EQ(6, fake_ctx.get_all_exprs().get_expr_array().count()); + ASSERT_EQ(4, fake_ctx.get_all_exprs().get_expr_array().count()); ObTableApiSpec *root_spec = nullptr; ObTableApiExecutor *executor = nullptr; diff --git a/unittest/observer/tableapi/test_obkv_config.cpp b/unittest/observer/tableapi/test_obkv_config.cpp new file mode 100644 index 0000000000..d36df7bfd8 --- /dev/null +++ b/unittest/observer/tableapi/test_obkv_config.cpp @@ -0,0 +1,156 @@ +#include "share/table/ob_table_config_util.h" +#include "lib/time/ob_time_utility.h" +#include "share/parameter/ob_parameter_macro.h" +#include + +namespace oceanbase +{ +using namespace share; +namespace common +{ +ObConfigContainer l_container; +static ObConfigContainer *local_container() +{ + return &l_container; +} +class TestObKvConfig : public ::testing::Test +{ +public: +#undef OB_CLUSTER_PARAMETER +#define OB_CLUSTER_PARAMETER(args...) args +DEF_MODE_WITH_PARSER(_obkv_feature_mode, OB_CLUSTER_PARAMETER, "", common::ObKvFeatureModeParser, + "_obkv_feature_mode is a option list to control specified OBKV features on/off.", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +#undef OB_CLUSTER_PARAMETER +}; + +TEST_F(TestObKvConfig, test_process_item) +{ + ObString key; + ObString value; + char str1[100] = {"ttl=off"}; + ASSERT_EQ(OB_SUCCESS, ObModeConfigParserUitl::parse_item_to_kv(str1, key, value)); + ASSERT_EQ(key, ObString("ttl")); + ASSERT_EQ(value, ObString("off")); + + char str2[100] = {" ttl = off "}; + ASSERT_EQ(OB_SUCCESS, ObModeConfigParserUitl::parse_item_to_kv(str2, key, value)); + ASSERT_EQ(key, ObString("ttl")); + ASSERT_EQ(value, ObString("off")); + + char str3[100] = {" rerouting =off "}; + ASSERT_EQ(OB_SUCCESS, ObModeConfigParserUitl::parse_item_to_kv(str3, key, value)); + ASSERT_EQ(key, ObString("rerouting")); + ASSERT_EQ(value, ObString("off")); + + char str4[100] = {"ttl=on, rerouting=off"}; + common::ObSEArray, 8> kv_list; + ASSERT_EQ(OB_SUCCESS, ObModeConfigParserUitl::get_kv_list(str4, kv_list)); + ASSERT_EQ(2, kv_list.count()); + ASSERT_EQ(kv_list.at(0).first, ObString("ttl")); + ASSERT_EQ(kv_list.at(0).second, ObString("on")); + ASSERT_EQ(kv_list.at(1).first, ObString("rerouting")); + ASSERT_EQ(kv_list.at(1).second, ObString("off")); +} + + +TEST_F(TestObKvConfig, test_parse) +{ + ObKvFeatureModeParser parser; + uint8_t arr[32]; + ASSERT_EQ(true, parser.parse("ttl=on, rerouting=off", arr, 32)); + ASSERT_EQ(arr[0], 0b00001001); + + MEMSET(arr, 0, 32); + ASSERT_EQ(true, parser.parse("ttl=on, ttl=off, rerouting=on", arr, 32)); + ASSERT_EQ(arr[0], 0b0000110); + + MEMSET(arr, 0, 32); + ASSERT_EQ(true, parser.parse("ttl=on, ttl=off", arr, 32)); + ASSERT_EQ(arr[0], 0b00000010); + + MEMSET(arr, 0, 32); + ASSERT_EQ(true, parser.parse("ttl=on, hotkey=on, rerouting=on", arr, 32)); + ASSERT_EQ(arr[0], 0b00010101); + + MEMSET(arr, 0, 32); + ASSERT_EQ(false, parser.parse("ttl=on, ttt=off", arr, 32)); + + MEMSET(arr, 0, 32); + ASSERT_EQ(false, parser.parse("ttl=on, rerouting=default", arr, 32)); + + MEMSET(arr, 0, 32); + ASSERT_EQ(true, parser.parse("", arr, 32)); + ASSERT_EQ(arr[0], 0); + + MEMSET(arr, 0, 32); + ASSERT_EQ(false, parser.parse("ttl=on, ttt=off, ", arr, 32)); + + MEMSET(arr, 0, 32); + ASSERT_EQ(false, parser.parse("ttl=on, ttt=off,", arr, 32)); +} + +TEST_F(TestObKvConfig, test_obkv_feature_mode) +{ + const uint8_t *value = _obkv_feature_mode; + ASSERT_NE((void*)NULL, (void*)value); + ASSERT_EQ(value[0], 0); + ASSERT_EQ(true, _obkv_feature_mode.set_value("ttl=off, rerouting=on, hotkey=on")); + const uint8_t *value1 = _obkv_feature_mode.get(); + ASSERT_NE((void*)NULL, (void*)value1); + ASSERT_EQ(value1[0], 0b00010110); + // bad case + ASSERT_EQ(false, _obkv_feature_mode.set_value("hotkey=off=")); + ASSERT_EQ(false, _obkv_feature_mode.set_value("=off")); + ASSERT_EQ(false, _obkv_feature_mode.set_value(",")); + ASSERT_EQ(false, _obkv_feature_mode.set_value(" , ")); + ASSERT_EQ(false, _obkv_feature_mode.set_value(" ,hotkey=off")); + ASSERT_EQ(false, _obkv_feature_mode.set_value(",hotkey=off")); + ASSERT_EQ(false, _obkv_feature_mode.set_value("ttl=off,, hotkey=off")); + ASSERT_EQ(false, _obkv_feature_mode.set_value("ttl=off, , hotkey=off")); + ASSERT_EQ(false, _obkv_feature_mode.set_value("ttl=off, rerouting=on, ")); + ASSERT_EQ(false, _obkv_feature_mode.set_value("ttl=off, rerouting=on,")); +} + +TEST_F(TestObKvConfig, testObKVFeatureMode) +{ + // rerouting=off, ttl=on + uint8_t values[1] = {0b001001}; + ObKVFeatureMode kv_mode(values); + ASSERT_EQ(true, kv_mode.is_ttl_enable()); + ASSERT_EQ(false, kv_mode.is_rerouting_enable()); + // default + ObKVFeatureMode kv_mode_1; + ASSERT_EQ(false, kv_mode_1.is_valid()); + kv_mode_1.set_ttl_mode(0b10); + kv_mode_1.set_rerouting_mode(0b01); + kv_mode_1.set_hotkey_mode(0b01); + ASSERT_EQ(true, kv_mode_1.is_valid()); + ASSERT_EQ(false, kv_mode_1.is_ttl_enable()); + ASSERT_EQ(true, kv_mode_1.is_rerouting_enable()); + ASSERT_EQ(true, kv_mode_1.is_hotkey_enable()); + kv_mode_1.set_value(0b11101001); + ASSERT_EQ(true, kv_mode_1.is_valid()); + ASSERT_EQ(true, kv_mode_1.is_ttl_enable()); + ASSERT_EQ(false, kv_mode_1.is_rerouting_enable()); + ASSERT_EQ(false, kv_mode_1.is_hotkey_enable()); + // bad case + kv_mode_1.set_ttl_mode(0b11); + ASSERT_EQ(false, kv_mode_1.is_valid()); + kv_mode_1.set_hotkey_mode(0b11); + ASSERT_EQ(false, kv_mode_1.is_valid()); + kv_mode_1.set_rerouting_mode(0b11); + ASSERT_EQ(false, kv_mode_1.is_valid()); + kv_mode_1.set_value(0b00111111); + ASSERT_EQ(false, kv_mode_1.is_valid()); +} + +} +} + +int main(int argc, char **argv) +{ + oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/unittest/sql/engine/basic/test_chunk_datum_store.cpp b/unittest/sql/engine/basic/test_chunk_datum_store.cpp index 5db0c010b7..45968429f8 100644 --- a/unittest/sql/engine/basic/test_chunk_datum_store.cpp +++ b/unittest/sql/engine/basic/test_chunk_datum_store.cpp @@ -90,6 +90,7 @@ class TestChunkDatumStore : public blocksstable::TestDataFilePrepare public: TestChunkDatumStore() : blocksstable::TestDataFilePrepare(&getter, "TestDisk_chunk_datum_store", 2<<20, 5000), + rs_("TEST"), plan_ctx_(alloc_), exec_ctx_(alloc_), eval_ctx_(exec_ctx_) @@ -340,7 +341,7 @@ public: void test_time(int64_t block_size, int64_t rows) { ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20); - ObChunkDatumStore rs(&alloc); + ObChunkDatumStore rs("TEST", &alloc); int64_t v = 0; int64_t i; int64_t begin = ObTimeUtil::current_time(); @@ -572,7 +573,7 @@ TEST_F(TestChunkDatumStore, multi_iter) int total = 100; int64_t i = 0; int64_t j = 0; - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ret = rs.init(1 << 20, tenant_id_, ctx_id_, label_); ASSERT_EQ(OB_SUCCESS, ret); @@ -614,7 +615,7 @@ TEST_F(TestChunkDatumStore, multi_iter) TEST_F(TestChunkDatumStore, basic2) { int ret = OB_SUCCESS; - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Iterator it; //mem limit 5M ret = rs.init(5L << 20, tenant_id_, ctx_id_, label_); @@ -636,7 +637,7 @@ TEST_F(TestChunkDatumStore, test_copy_row) { int ret = OB_SUCCESS; int64_t rows = 1000; - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Iterator it; const ObChunkDatumStore::StoredRow *sr; LOG_WARN("starting mem_perf test: append rows", K(rows)); @@ -652,7 +653,7 @@ TEST_F(TestChunkDatumStore, mem_perf) { int ret = OB_SUCCESS; int64_t rows = 2000000; - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Iterator it; LOG_WARN("starting mem_perf test: append rows", K(rows)); int64_t begin = ObTimeUtil::current_time(); @@ -677,7 +678,7 @@ TEST_F(TestChunkDatumStore, disk) int64_t round = 500; int64_t rows = round * 10000; LOG_INFO("starting write disk test: append rows", K(rows)); - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Iterator it; ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_)); ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id()); @@ -715,7 +716,7 @@ TEST_F(TestChunkDatumStore, disk_with_chunk) int64_t cnt = 10000; int64_t rows = round * cnt; LOG_INFO("starting write disk test: append rows", K(rows)); - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Iterator it; ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_)); ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id()); @@ -783,7 +784,7 @@ TEST_F(TestChunkDatumStore, test_add_block) { int ret = OB_SUCCESS; //send - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Block *block; ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20); @@ -815,7 +816,7 @@ TEST_F(TestChunkDatumStore, test_add_block) memcpy(mem2, mem, block->get_buffer()->data_size()); //recv - ObChunkDatumStore rs2; + ObChunkDatumStore rs2("TEST"); ObChunkDatumStore::Block *block2 = reinterpret_cast(mem2); ret = rs2.init(0, tenant_id_, ctx_id_, label_); ASSERT_EQ(OB_SUCCESS, ret); @@ -841,7 +842,7 @@ TEST_F(TestChunkDatumStore, row_with_extend_size) int64_t round = 500; int64_t rows = round * 10000; LOG_INFO("starting write disk test: append rows", K(rows)); - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Iterator it; ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_, true, 8)); LOG_INFO("starting basic test: append 3000 rows"); @@ -889,7 +890,7 @@ TEST_F(TestChunkDatumStore, test_only_disk_data) int64_t cnt = 10000; int64_t rows = round * cnt; LOG_INFO("starting write disk test: append rows", K(rows)); - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id()); ObChunkDatumStore::Iterator it; ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_)); @@ -916,7 +917,7 @@ TEST_F(TestChunkDatumStore, test_only_disk_data1) int64_t cnt = 10000; int64_t rows = round * cnt; LOG_INFO("starting write disk test: append rows", K(rows)); - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id()); ObChunkDatumStore::Iterator it; ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_)); @@ -945,7 +946,7 @@ TEST_F(TestChunkDatumStore, test_append_block) { int ret = OB_SUCCESS; //send - ObChunkDatumStore rs; + ObChunkDatumStore rs("TEST"); ObChunkDatumStore::Block *block; ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20); @@ -977,7 +978,7 @@ TEST_F(TestChunkDatumStore, test_append_block) memcpy(mem2, mem, block->get_buffer()->data_size()); //recv - ObChunkDatumStore rs2; + ObChunkDatumStore rs2("TEST"); rs2.alloc_dir_id(); ObChunkDatumStore::Block *block2 = reinterpret_cast(mem2); ret = rs2.init(0, tenant_id_, ctx_id_, label_); diff --git a/unittest/storage/mockcontainer/mock_ob_iterator.cpp b/unittest/storage/mockcontainer/mock_ob_iterator.cpp index 97c0e542b1..420f817753 100644 --- a/unittest/storage/mockcontainer/mock_ob_iterator.cpp +++ b/unittest/storage/mockcontainer/mock_ob_iterator.cpp @@ -859,7 +859,7 @@ int ObMockIteratorBuilder::static_init() && OB_SUCCESS != (ret = str_to_flag_.create( cal_next_prime(FLAG_NUM * 2), ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memeory"); + STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_flag_.set_refactored( ObString::make_string("RF_ROW_DOES_NOT_EXIST"), ObDmlFlag::DF_NOT_EXIST) @@ -894,7 +894,7 @@ int ObMockIteratorBuilder::static_init() && OB_SUCCESS != (ret = str_to_dml_.create( cal_next_prime(DML_NUM * 2), ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memeory"); + STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_dml_.set_refactored( ObString::make_string("INSERT"), DF_INSERT) || OB_SUCCESS != str_to_dml_.set_refactored( @@ -913,7 +913,7 @@ int ObMockIteratorBuilder::static_init() && OB_SUCCESS != (ret = str_to_base_.create( cal_next_prime(BASE_NUM * 2), ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memeory"); + STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_base_.set_refactored( ObString::make_string("TRUE"), true) || OB_SUCCESS != str_to_base_.set_refactored( @@ -928,7 +928,7 @@ int ObMockIteratorBuilder::static_init() && OB_SUCCESS != (ret = str_to_is_get_.create( cal_next_prime(BASE_NUM * 2), ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memeory"); + STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_is_get_.set_refactored( ObString::make_string("TRUE"), true) || OB_SUCCESS != str_to_is_get_.set_refactored( @@ -943,7 +943,7 @@ int ObMockIteratorBuilder::static_init() && OB_SUCCESS != (ret = str_to_trans_id_.create( cal_next_prime(BASE_NUM * 3), ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memeory"); + STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_trans_id_.set_refactored( ObString::make_string("trans_id_1"), trans_id_list_[0]) || OB_SUCCESS != str_to_trans_id_.set_refactored( @@ -981,7 +981,7 @@ int ObMockIteratorBuilder::static_init() if (ret == OB_SUCCESS && OB_SUCCESS != (ret = str_to_multi_version_row_flag_.create( cal_next_prime(MULTI_VERSION_ROW_FLAG_NUM * 2), ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memeory"); + STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_multi_version_row_flag_.set_refactored(ObString::make_string("N"), 0) || OB_SUCCESS != str_to_multi_version_row_flag_.set_refactored(ObString::make_string("C"), 8) || OB_SUCCESS != str_to_multi_version_row_flag_.set_refactored(ObString::make_string("L"), 32) diff --git a/unittest/storage/test_tablet_dumped_medium_info.cpp b/unittest/storage/test_tablet_dumped_medium_info.cpp index 2eb727330f..e1ecfae4ce 100644 --- a/unittest/storage/test_tablet_dumped_medium_info.cpp +++ b/unittest/storage/test_tablet_dumped_medium_info.cpp @@ -94,7 +94,7 @@ TEST_F(TestTabletDumpedMediumInfo, overlap) compaction::ObMediumCompactionInfo *info = nullptr; ObTabletDumpedMediumInfo input_medium_info1; - ret = input_medium_info1.init(allocator); + ret = input_medium_info1.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info1, allocator, 5, 4, true, info); @@ -102,7 +102,7 @@ TEST_F(TestTabletDumpedMediumInfo, overlap) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 7, 6, true, info); ObTabletDumpedMediumInfo input_medium_info2; - ret = input_medium_info2.init(allocator); + ret = input_medium_info2.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 2, 1, true, info); @@ -112,7 +112,7 @@ TEST_F(TestTabletDumpedMediumInfo, overlap) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 6, 5, true, info); ObTabletDumpedMediumInfo result; - ret = result.init(allocator); + ret = result.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); const int64_t finish_medium_scn = 3; @@ -136,7 +136,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap) compaction::ObMediumCompactionInfo *info = nullptr; ObTabletDumpedMediumInfo input_medium_info1; - ret = input_medium_info1.init(allocator); + ret = input_medium_info1.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info1, allocator, 5, 4, true, info); @@ -144,7 +144,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 7, 6, true, info); ObTabletDumpedMediumInfo input_medium_info2; - ret = input_medium_info2.init(allocator); + ret = input_medium_info2.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 2, 1, true, info); @@ -152,7 +152,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap) { ObTabletDumpedMediumInfo result; - ret = result.init(allocator); + ret = result.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); const int64_t finish_medium_scn = 1; @@ -172,7 +172,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap) APPEND_MEDIUM_INFO(input_medium_info2, allocator, 4, 3, true, info); { ObTabletDumpedMediumInfo result; - ret = result.init(allocator); + ret = result.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); const int64_t finish_medium_scn = 1; @@ -198,7 +198,7 @@ TEST_F(TestTabletDumpedMediumInfo, overlap_and_filter) compaction::ObMediumCompactionInfo *info = nullptr; ObTabletDumpedMediumInfo input_medium_info1; - ret = input_medium_info1.init(allocator); + ret = input_medium_info1.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info1, allocator, 5, 4, true, info); @@ -206,7 +206,7 @@ TEST_F(TestTabletDumpedMediumInfo, overlap_and_filter) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 7, 6, true, info); ObTabletDumpedMediumInfo input_medium_info2; - ret = input_medium_info2.init(allocator); + ret = input_medium_info2.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 2, 1, true, info); @@ -216,7 +216,7 @@ TEST_F(TestTabletDumpedMediumInfo, overlap_and_filter) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 6, 5, true, info); ObTabletDumpedMediumInfo result; - ret = result.init(allocator); + ret = result.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); const int64_t finish_medium_scn = 6; @@ -236,7 +236,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap_and_filter) compaction::ObMediumCompactionInfo *info = nullptr; ObTabletDumpedMediumInfo input_medium_info1; - ret = input_medium_info1.init(allocator); + ret = input_medium_info1.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info1, allocator, 5, 4, true, info); @@ -244,7 +244,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap_and_filter) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 7, 6, true, info); ObTabletDumpedMediumInfo input_medium_info2; - ret = input_medium_info2.init(allocator); + ret = input_medium_info2.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 2, 1, true, info); @@ -252,7 +252,7 @@ TEST_F(TestTabletDumpedMediumInfo, no_overlap_and_filter) APPEND_MEDIUM_INFO(input_medium_info2, allocator, 4, 3, true, info); ObTabletDumpedMediumInfo result; - ret = result.init(allocator); + ret = result.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); const int64_t finish_medium_scn = 7; @@ -271,21 +271,21 @@ TEST_F(TestTabletDumpedMediumInfo, standby_cluster) compaction::ObMediumCompactionInfo *info = nullptr; ObTabletDumpedMediumInfo input_medium_info1; - ret = input_medium_info1.init(allocator); + ret = input_medium_info1.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info1, allocator, 7, 5, false, info); APPEND_MEDIUM_INFO(input_medium_info1, allocator, 9, 8, false, info); ObTabletDumpedMediumInfo input_medium_info2; - ret = input_medium_info2.init(allocator); + ret = input_medium_info2.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 2, 1, false, info); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 4, 2, false, info); ObTabletDumpedMediumInfo result; - ret = result.init(allocator); + ret = result.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); const int64_t finish_medium_scn = 3; @@ -307,7 +307,7 @@ TEST_F(TestTabletDumpedMediumInfo, mds_table_dump) compaction::ObMediumCompactionInfo *info = nullptr; ObTabletDumpedMediumInfo input_medium_info1; - ret = input_medium_info1.init(allocator); + ret = input_medium_info1.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); ObTabletComplexAddr mds_table_data; mds_table_data.ptr_ = &input_medium_info1; @@ -316,7 +316,7 @@ TEST_F(TestTabletDumpedMediumInfo, mds_table_dump) APPEND_MEDIUM_INFO(input_medium_info1, allocator, 9, 7, false, info); ObTabletDumpedMediumInfo input_medium_info2; - ret = input_medium_info2.init(allocator); + ret = input_medium_info2.init_for_first_creation(allocator); ASSERT_EQ(OB_SUCCESS, ret); ObTabletComplexAddr base_data; base_data.ptr_ = &input_medium_info2; @@ -326,19 +326,17 @@ TEST_F(TestTabletDumpedMediumInfo, mds_table_dump) APPEND_MEDIUM_INFO(input_medium_info2, allocator, 6, 4, false, info); APPEND_MEDIUM_INFO(input_medium_info2, allocator, 7, 6, false, info); - ObTabletDumpedMediumInfo result; - ret = result.init(allocator); ASSERT_EQ(OB_SUCCESS, ret); ObTabletComplexAddr result_data; - result_data.ptr_ = &result; + ObTabletDumpedMediumInfo *&result = result_data.ptr_; const int64_t finish_medium_scn = 4; - ret = ObTabletMdsData::fuse_mds_dump_node(allocator, finish_medium_scn, mds_table_data, base_data, result_data); + ret = ObTabletMdsData::init_single_complex_addr(allocator, mds_table_data, base_data, finish_medium_scn, result_data); ASSERT_EQ(OB_SUCCESS, ret); - ASSERT_EQ(3, result.medium_info_list_.count()); - ASSERT_EQ(6, result.medium_info_list_.at(0)->medium_snapshot_); - ASSERT_EQ(7, result.medium_info_list_.at(1)->medium_snapshot_); - ASSERT_EQ(9, result.medium_info_list_.at(2)->medium_snapshot_); + ASSERT_EQ(3, result->medium_info_list_.count()); + ASSERT_EQ(6, result->medium_info_list_.at(0)->medium_snapshot_); + ASSERT_EQ(7, result->medium_info_list_.at(1)->medium_snapshot_); + ASSERT_EQ(9, result->medium_info_list_.at(2)->medium_snapshot_); } } // namespace unittest } // namespace oceanbase diff --git a/unittest/storage/test_tablet_helper.h b/unittest/storage/test_tablet_helper.h index 9283d28a98..2e15ffa9db 100644 --- a/unittest/storage/test_tablet_helper.h +++ b/unittest/storage/test_tablet_helper.h @@ -19,14 +19,15 @@ #include "lib/ob_errno.h" #include "lib/oblog/ob_log_module.h" #include "share/ob_rpc_struct.h" +#include "share/scn.h" #include "storage/tx/ob_trans_define.h" #include "storage/ls/ob_ls_tablet_service.h" #include "storage/ls/ob_ls.h" -#include "share/scn.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" +#include "storage/tablet/ob_tablet_create_delete_helper.h" +#include "storage/tablet/ob_tablet_obj_load_helper.h" #include "storage/tablet/ob_tablet_slog_helper.h" #include "storage/init_basic_struct.h" -#include "storage/tablet/ob_tablet_create_delete_helper.h" namespace oceanbase { @@ -112,12 +113,9 @@ inline int TestTabletHelper::create_tablet( ObTabletCreateSSTableParam param; prepare_sstable_param(tablet_id, table_schema, param); - void *buff = nullptr; ObSSTable *sstable = nullptr; - if (OB_ISNULL(buff = allocator.alloc(sizeof(ObSSTable)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to allocate memory", K(ret)); - } else if (FALSE_IT(sstable = new (buff) ObSSTable())) { + if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, sstable))) { + STORAGE_LOG(WARN, "fail to alloc and new", K(ret)); } else if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, *sstable))) { STORAGE_LOG(WARN, "failed to acquire sstable", K(ret)); } else if (OB_FAIL(ObSSTableMergeRes::fill_column_checksum_for_empty_major(param.column_cnt_, param.column_checksums_))) { @@ -145,13 +143,17 @@ inline int TestTabletHelper::create_tablet( if (tablet_status == ObTabletStatus::Status::DELETED) { data.delete_commit_scn_ = SCN::base_scn(); } + + ObTabletComplexAddr &tablet_status_committed_kv = tablet_handle.get_obj()->mds_data_.tablet_status_.committed_kv_; const int64_t data_serialize_size = data.get_serialize_size(); int64_t pos = 0; char *buf = static_cast(t3m->full_tablet_creator_.get_allocator().alloc(data_serialize_size)); if (OB_FAIL(data.serialize(buf, data_serialize_size, pos))) { STORAGE_LOG(WARN, "data serialize failed", K(ret), K(data_serialize_size), K(pos)); + } else if (ObTabletObjLoadHelper::alloc_and_new(*tablet_handle.get_allocator(), tablet_status_committed_kv.ptr_)) { + STORAGE_LOG(WARN, "failed to alloc and new", K(ret)); } else { - tablet_handle.get_obj()->mds_data_.tablet_status_.committed_kv_.get_ptr()->v_.user_data_.assign(buf, data_serialize_size); + tablet_status_committed_kv.ptr_->v_.user_data_.assign(buf, data_serialize_size); } } diff --git a/unittest/storage/tx/it/tx_node.cpp b/unittest/storage/tx/it/tx_node.cpp index dff4170709..48cdfa3a79 100644 --- a/unittest/storage/tx/it/tx_node.cpp +++ b/unittest/storage/tx/it/tx_node.cpp @@ -311,13 +311,7 @@ int ObTxNode::recv_msg_callback_(TxMsgCallbackMsg &msg) int ret = OB_SUCCESS; switch(msg.type_) { case TxMsgCallbackMsg::SAVEPOINT_ROLLBACK: - OZ(txs_.handle_sp_rollback_resp(msg.receiver_ls_id_, - msg.epoch_, - msg.tx_id_, - msg.sp_rollback_result_.status_, - msg.receiver_addr_, - msg.request_id_, - msg.sp_rollback_result_)); + // ignore, has changed to use async resp msg break; case TxMsgCallbackMsg::NORMAL: OZ(txs_.handle_trans_msg_callback(msg.sender_ls_id_, @@ -402,6 +396,7 @@ int ObTxNode::handle_msg_(MsgPack *pkt) TX_MSG_HANDLER__(TX_ABORT, ObTxAbortMsg, handle_trans_abort_request); TX_MSG_HANDLER__(KEEPALIVE, ObTxKeepaliveMsg, handle_trans_keepalive); TX_MSG_HANDLER__(KEEPALIVE_RESP, ObTxKeepaliveRespMsg, handle_trans_keepalive_response); + TX_MSG_HANDLER__(ROLLBACK_SAVEPOINT_RESP, ObTxRollbackSPRespMsg, handle_sp_rollback_response); #undef TX_MSG_HANDLER__ case TX_FREE_ROUTE_CHECK_ALIVE: { @@ -444,7 +439,6 @@ int ObTxNode::handle_msg_(MsgPack *pkt) OZ(msg.deserialize(buf, size, pos)); TRANS_LOG(TRACE, "handle_msg", K(msg), KPC(this)); OZ(txs_.handle_sp_rollback_request(msg, rslt), msg); - OZ(fake_rpc_.send_msg_callback(sender, msg, rslt), msg); break; } case TX_2PC_PREPARE_REQ: