hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE]4_1_sql_feature

Browse Source 
Co-authored-by: leslieyuchen <leslieyuchen@gmail.com>
Co-authored-by: Charles0429 <xiezhenjiang@gmail.com>
Co-authored-by: raywill <hustos@gmail.com>
This commit is contained in:
obdev
2023-01-28 16:01:26 +08:00
committed by ob-robot
parent 3080f2b66f
commit 2d19a9d8f5
846 changed files with 161957 additions and 116661 deletions
Download Patch File Download Diff File Expand all files Collapse all files

480
src/sql/das/ob_das_scan_op.cpp
View File

@ -15,7 +15,6 @@
#include "sql/das/ob_das_extra_data.h"
#include "sql/das/ob_das_spatial_index_lookup_op.h"
#include "sql/engine/table/ob_table_scan_op.h"
#include "sql/engine/table/ob_table_lookup_op.h"
#include "sql/engine/px/ob_px_util.h"
#include "sql/engine/ob_des_exec_context.h"
#include "storage/access/ob_table_scan_iterator.h"
@ -642,7 +641,7 @@ int ObDASScanResult::get_next_row(ObNewRow *&row)
int ObDASScanResult::get_next_row()
{
int ret = OB_SUCCESS;
if (OB_FAIL(result_iter_.get_next_row_skip_const<false>(*eval_ctx_, *output_exprs_))) {
if (OB_FAIL(result_iter_.get_next_row<false>(*eval_ctx_, *output_exprs_))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from result iterator failed", K(ret));
} else if (extra_result_ != nullptr) {
@ -663,10 +662,10 @@ int ObDASScanResult::get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(need_check_output_datum_)) {
ret = result_iter_.get_next_batch_skip_const<true>(*output_exprs_, *eval_ctx_,
ret = result_iter_.get_next_batch<true>(*output_exprs_, *eval_ctx_,
capacity, count);
} else {
ret = result_iter_.get_next_batch_skip_const<false>(*output_exprs_, *eval_ctx_,
ret = result_iter_.get_next_batch<false>(*output_exprs_, *eval_ctx_,
capacity, count);
}
if (OB_FAIL(ret)) {
@ -788,283 +787,64 @@ int ObLocalIndexLookupOp::get_next_row(ObNewRow *&row)
int ObLocalIndexLookupOp::get_next_row()
{
int ret = OB_SUCCESS;
bool got_next_row = false;
int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_LOOKUP_BATCH_ROW_COUNT);
int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? simulate_batch_row_cnt : DEFAULT_BATCH_ROW_COUNT;
LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt));
do {
switch (state_) {
case INDEX_SCAN: {
lookup_rowkey_cnt_ = 0;
if (is_group_scan_) {
//See the comment in the ObLocalIndexLookupOp::get_next_rows
set_index_group_cnt(get_lookup_group_cnt());
ret = set_rowkey_scan_group(get_lookup_group_cnt() - 1);
if (OB_SUCCESS != ret) {
LOG_WARN("set_rowkey_scan_group fail",K(get_lookup_group_cnt() - 1),K(ret));
if (OB_ITER_END == ret) {
ret = OB_ERR_UNEXPECTED;
}
}
}
int64_t start_group_idx = get_index_group_cnt() - 1;
while (OB_SUCC(ret) && lookup_rowkey_cnt_ < default_row_batch_cnt) {
index_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
if (OB_FAIL(rowkey_iter_->get_next_row())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from index scan failed", K(ret));
} else if (is_group_scan_) {
// switch to next index group
if (OB_FAIL(switch_rowkey_scan_group())) {
if (OB_ITER_END != ret) {
LOG_WARN("rescan index operator failed", K(ret));
} else {
LOG_DEBUG("switch group end",
K(get_index_group_cnt()), K(lookup_rowkey_cnt_), KP(this));
}
} else {
inc_index_group_cnt();
LOG_DEBUG("switch to next index batch to fetch rowkey",
K(get_index_group_cnt()), K(lookup_rowkey_cnt_), KP(this));
}
}
} else if (OB_FAIL(process_data_table_rowkey())) {
LOG_WARN("process data table rowkey with das failed", K(ret));
} else {
++lookup_rowkey_cnt_;
}
}
if (OB_SUCC(ret) || OB_ITER_END == ret) {
state_ = DO_LOOKUP;
index_end_ = (OB_ITER_END == ret);
ret = OB_SUCCESS;
if (is_group_scan_) {
OZ(init_group_range(start_group_idx, get_index_group_cnt()));
}
}
break;
if (OB_FAIL(ObIndexLookupOpImpl::get_next_row())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from index table lookup failed", K(ret));
} else {
LOG_DEBUG("get next row from index table lookup", K(ret));
}
case DO_LOOKUP: {
lookup_row_cnt_ = 0;
if (OB_FAIL(do_index_lookup())) {
LOG_WARN("do index lookup failed", K(ret));
} else {
state_ = OUTPUT_ROWS;
}
break;
}
case OUTPUT_ROWS: {
lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
if (scan_param_.key_ranges_.empty()) {
ret= OB_ITER_END;
state_ = FINISHED;
} else if (OB_FAIL(lookup_iter_->get_next_row())) {
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
if (need_next_index_batch()) {
reset_lookup_state(false);
index_end_ = false;
state_ = INDEX_SCAN;
} else {
state_ = FINISHED;
}
} else {
LOG_WARN("look up get next row failed", K(ret));
}
} else {
got_next_row = true;
++lookup_row_cnt_;
}
break;
}
case FINISHED: {
if (OB_SUCC(ret) || OB_ITER_END == ret) {
ret = OB_ITER_END;
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected state", K(state_));
}
}
} while (!got_next_row && OB_SUCC(ret));
}
return ret;
}
int ObLocalIndexLookupOp::get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
bool got_next_row = false;
int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_LOOKUP_BATCH_ROW_COUNT);
int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? simulate_batch_row_cnt : DEFAULT_BATCH_ROW_COUNT;
LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt));
if (OB_FAIL(ObIndexLookupOpImpl::get_next_rows(count, capacity))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next rows from index table lookup failed", K(ret));
} else {
LOG_DEBUG("get next rows from index table lookup ", K(ret));
}
}
return ret;
}
int ObLocalIndexLookupOp::get_next_row_from_index_table()
{
int ret = OB_SUCCESS;
bool got_row = false;
do {
switch (state_) {
case INDEX_SCAN: {
int64_t rowkey_count = 0;
lookup_rowkey_cnt_ = 0;
if (is_group_scan_) {
//Do the group scan jump read.
//Now we support jump read in GroupScan iter.
//Some of row read from index maybe jump.
//We need to sync index_group_cnt with lookup_group_cnt.
//Because in the rescan we manipulate the lookup_group_cnt.
set_index_group_cnt(get_lookup_group_cnt());
ret = set_rowkey_scan_group(get_lookup_group_cnt() - 1);
if (OB_SUCCESS != ret) {
LOG_WARN("set_rowkey_scan_group fail",K(get_lookup_group_cnt() - 1),K(ret));
if (OB_ITER_END == ret) {
ret = OB_ERR_UNEXPECTED;
}
}
}
int64_t start_group_idx = get_index_group_cnt() - 1;
while (OB_SUCC(ret) && lookup_rowkey_cnt_ < default_row_batch_cnt) {
int64_t batch_size = min(capacity, default_row_batch_cnt - lookup_rowkey_cnt_);
index_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
ret = rowkey_iter_->get_next_rows(rowkey_count, batch_size);
if (OB_ITER_END == ret && rowkey_count > 0) {
ret = OB_SUCCESS;
}
if (OB_UNLIKELY(OB_SUCCESS != ret)) {
if (OB_FAIL(rowkey_iter_->get_next_row())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from index scan failed", K(ret));
} else if (is_group_scan()) {
// switch to next index group
if (OB_FAIL(switch_rowkey_scan_group())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next batch from index scan failed", K(ret));
} else if (is_group_scan_) {
//switch to next index iterator, call child's rescan
if (OB_FAIL(switch_rowkey_scan_group())) {
if (OB_ITER_END != ret) {
LOG_WARN("rescan index operator failed", K(ret));
}
} else {
inc_index_group_cnt();
LOG_DEBUG("switch to next index batch to fetch rowkey", K(get_index_group_cnt()), K(lookup_rowkey_cnt_));
}
}
} else if (OB_FAIL(process_data_table_rowkeys(rowkey_count))) {
LOG_WARN("process data table rowkey with das failed", K(ret));
} else {
lookup_rowkey_cnt_ += rowkey_count;
}
}
if (OB_SUCC(ret) || OB_ITER_END == ret) {
state_ = DO_LOOKUP;
index_end_ = (OB_ITER_END == ret);
ret = OB_SUCCESS;
if (is_group_scan_) {
OZ(init_group_range(start_group_idx, get_index_group_cnt()));
}
}
LOG_DEBUG("index scan end", K(state_), K(index_end_),
K(start_group_idx), K(get_index_group_cnt()), K(ret));
break;
}
case DO_LOOKUP: {
lookup_row_cnt_ = 0;
if (OB_FAIL(do_index_lookup())) {
LOG_WARN("do index lookup failed", K(ret));
} else {
LOG_DEBUG("do index lookup end", K(get_index_group_cnt()), K(ret));
state_ = OUTPUT_ROWS;
}
break;
}
case OUTPUT_ROWS: {
LOG_DEBUG("local index lookup output rows", K(lookup_row_cnt_), K(get_index_group_cnt()),
K(get_lookup_group_cnt()), K(lookup_rowkey_cnt_));
lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
if (scan_param_.key_ranges_.empty()) {
ret = OB_ITER_END;
state_ = FINISHED;
} else {
ret = lookup_iter_->get_next_rows(count, capacity);
if (OB_ITER_END == ret && count > 0) {
ret = OB_SUCCESS;
}
if (OB_UNLIKELY(OB_SUCCESS != ret)) {
if (OB_ITER_END == ret) {
if (OB_FAIL(check_lookup_row_cnt())) {
LOG_WARN("check lookup row cnt failed", K(ret));
} else if (need_next_index_batch()) {
reset_lookup_state(false);
index_end_ = false;
state_ = INDEX_SCAN;
ret = OB_SUCCESS;
} else {
state_ = FINISHED;
}
LOG_WARN("rescan index operator failed", K(ret));
} else {
LOG_WARN("lookup get next row failed", K(ret));
LOG_DEBUG("switch group end",K(get_index_group_cnt()), K(lookup_rowkey_cnt_), KP(this));
}
} else {
got_next_row = true;
lookup_row_cnt_ += count;
inc_index_group_cnt();
LOG_DEBUG("switch to next index batch to fetch rowkey",K(get_index_group_cnt()), K(lookup_rowkey_cnt_), KP(this));
}
}
break;
} else {
got_row = true;
}
case FINISHED: {
if (OB_SUCC(ret) || OB_ITER_END == ret) {
ret = OB_ITER_END;
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected state", K(state_));
}
}
} while (!got_next_row && OB_SUCC(ret));
} while (OB_SUCC(ret)&& !got_row);
return ret;
}
int ObLocalIndexLookupOp::check_lookup_row_cnt()
{
int ret = OB_SUCCESS;
//In group scan the jump read may happend, so the lookup_group_cnt and lookup_rowkey_cnt_ mismatch.
if (GCONF.enable_defensive_check()
&& !is_group_scan_
&& lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) {
if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)
&& get_index_group_cnt() == get_lookup_group_cnt()) {
ret = OB_ERR_DEFENSIVE_CHECK;
ObString func_name = ObString::make_string("check_lookup_row_cnt");
LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr());
LOG_ERROR("Fatal Error!!! Catch a defensive error!",
K(ret), K_(lookup_rowkey_cnt), K_(lookup_row_cnt),
"index_group_cnt", get_index_group_cnt(),
"lookup_group_cnt", get_lookup_group_cnt(),
"scan_range", scan_param_.key_ranges_,
KPC_(lookup_ctdef), KPC_(lookup_rtdef));
}
}
return ret;
}
OB_INLINE int ObLocalIndexLookupOp::process_data_table_rowkeys(int64_t count)
{
int ret = OB_SUCCESS;
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*index_rtdef_->eval_ctx_);
batch_info_guard.set_batch_size(count);
for (auto i = 0; OB_SUCC(ret) && i < count; i++) {
batch_info_guard.set_batch_idx(i);
if (OB_FAIL(process_data_table_rowkey())) {
LOG_WARN("Failed to process_data_table_rowkey", K(ret), K(i));
}
}
return ret;
}
OB_INLINE int ObLocalIndexLookupOp::process_data_table_rowkey()
int ObLocalIndexLookupOp::process_data_table_rowkey()
{
int ret = OB_SUCCESS;
// for group scan lookup, das result output of index
// contain rowkey and group_idx_expr, so when build rowkey range,
// need remove group_idx_expr
int64_t rowkey_cnt = is_group_scan_ ? index_ctdef_->result_output_.count() - 1
int64_t rowkey_cnt = is_group_scan() ? index_ctdef_->result_output_.count() - 1
: index_ctdef_->result_output_.count();
ObObj *obj_ptr = nullptr;
void *buf = nullptr;
@ -1103,6 +883,21 @@ OB_INLINE int ObLocalIndexLookupOp::process_data_table_rowkey()
return ret;
}
int ObLocalIndexLookupOp::process_data_table_rowkeys(const int64_t size, const ObBitVector *skip)
{
int ret = OB_SUCCESS;
UNUSED(skip);
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*index_rtdef_->eval_ctx_);
batch_info_guard.set_batch_size(size);
for (auto i = 0; OB_SUCC(ret) && i < size; i++) {
batch_info_guard.set_batch_idx(i);
if (OB_FAIL(process_data_table_rowkey())) {
LOG_WARN("Failed to process_data_table_rowkey", K(ret), K(i));
}
}
return ret;
}
int ObLocalIndexLookupOp::do_index_lookup()
{
int ret = OB_SUCCESS;
@ -1134,6 +929,151 @@ int ObLocalIndexLookupOp::do_index_lookup()
return ret;
}
int ObLocalIndexLookupOp::get_next_row_from_data_table()
{
int ret = OB_SUCCESS;
do_clear_evaluated_flag();
if (scan_param_.key_ranges_.empty()) {
ret= OB_ITER_END;
state_ = FINISHED;
} else if (OB_FAIL(lookup_iter_->get_next_row())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from data table failed", K(ret));
} else {
LOG_DEBUG("get next row from data table ", K(ret));
}
}
return ret;
}
int ObLocalIndexLookupOp::get_next_rows_from_data_table(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
LOG_DEBUG("local index lookup output rows", K(lookup_row_cnt_), K(get_index_group_cnt()), K(get_lookup_group_cnt()), K(lookup_rowkey_cnt_));
lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
if (scan_param_.key_ranges_.empty()) {
ret = OB_ITER_END;
state_ = FINISHED;
} else {
ret = lookup_iter_->get_next_rows(count, capacity);
if (OB_ITER_END == ret && count > 0) {
ret = OB_SUCCESS;
}
}
return ret;
}
int ObLocalIndexLookupOp::process_next_index_batch_for_row()
{
int ret = OB_SUCCESS;
if (need_next_index_batch()) {
reset_lookup_state(false);
index_end_ = false;
state_ = INDEX_SCAN;
} else {
state_ = FINISHED;
}
return ret;
}
int ObLocalIndexLookupOp::process_next_index_batch_for_rows(int64_t &count)
{
int ret = OB_SUCCESS;
if (OB_FAIL(check_lookup_row_cnt())) {
LOG_WARN("check lookup row cnt failed", K(ret));
} else if (need_next_index_batch()) {
reset_lookup_state(false);
index_end_ = false;
state_ = INDEX_SCAN;
ret = OB_SUCCESS;
} else {
state_ = FINISHED;
}
return ret;
}
bool ObLocalIndexLookupOp::need_next_index_batch() const
{
return !index_end_;
}
int ObLocalIndexLookupOp::check_lookup_row_cnt()
{
int ret = OB_SUCCESS;
//In group scan the jump read may happend, so the lookup_group_cnt and lookup_rowkey_cnt_ mismatch.
if (GCONF.enable_defensive_check()
&& !is_group_scan_
&& lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) {
if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)
&& get_index_group_cnt() == get_lookup_group_cnt()) {
ret = OB_ERR_DEFENSIVE_CHECK;
ObString func_name = ObString::make_string("check_lookup_row_cnt");
LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr());
LOG_ERROR("Fatal Error!!! Catch a defensive error!",
K(ret), K_(lookup_rowkey_cnt), K_(lookup_row_cnt),
"index_group_cnt", get_index_group_cnt(),
"lookup_group_cnt", get_lookup_group_cnt(),
"scan_range", scan_param_.key_ranges_,
KPC_(lookup_ctdef), KPC_(lookup_rtdef));
}
}
return ret;
}
int ObLocalIndexLookupOp::do_index_table_scan_for_rows(const int64_t max_row_cnt,
const int64_t start_group_idx,
const int64_t default_row_batch_cnt)
{
int ret = OB_SUCCESS;
int64_t rowkey_count = 0;
while (OB_SUCC(ret) && lookup_rowkey_cnt_ < default_row_batch_cnt) {
int64_t batch_size = min(max_row_cnt, default_row_batch_cnt - lookup_rowkey_cnt_);
do_clear_evaluated_flag();
ret = rowkey_iter_->get_next_rows(rowkey_count, batch_size);
if (OB_ITER_END == ret && rowkey_count > 0) {
ret = OB_SUCCESS;
}
if (OB_UNLIKELY(OB_SUCCESS != ret)) {
if (OB_ITER_END != ret) {
LOG_WARN("get next batch from index scan failed", K(ret));
} else if (is_group_scan()) {
//switch to next index iterator, call child's rescan
if (OB_FAIL(switch_rowkey_scan_group())) {
if (OB_ITER_END != ret) {
LOG_WARN("rescan index operator failed", K(ret));
}
} else {
inc_index_group_cnt();
LOG_DEBUG("switch to next index batch to fetch rowkey", K(get_index_group_cnt()), K(lookup_rowkey_cnt_));
}
}
} else if (OB_FAIL(process_data_table_rowkeys(rowkey_count, nullptr/*skip*/))) {
LOG_WARN("process data table rowkey with das failed", K(ret));
} else {
lookup_rowkey_cnt_ += rowkey_count;
}
}
if (OB_SUCC(ret) || OB_ITER_END == ret) {
state_ = DO_LOOKUP;
index_end_ = (OB_ITER_END == ret);
ret = OB_SUCCESS;
if (is_group_scan()) {
OZ(init_group_range(start_group_idx, get_index_group_cnt()));
}
}
LOG_DEBUG("index scan end", K(state_), K(index_end_),K(start_group_idx), K(get_index_group_cnt()), K(ret));
return ret;
}
void ObLocalIndexLookupOp::update_state_in_output_rows_state(int64_t &count)
{
lookup_row_cnt_ += count;
}
void ObLocalIndexLookupOp::update_states_in_finish_state()
{ }
OB_INLINE ObITabletScan &ObLocalIndexLookupOp::get_tsc_service()
{
return is_virtual_table(lookup_ctdef_->ref_table_id_) ?
@ -1209,6 +1149,9 @@ int ObLocalIndexLookupOp::reset_lookup_state(bool need_switch_param)
int ret = OB_SUCCESS;
state_ = INDEX_SCAN;
index_end_ = false;
lookup_rtdef_->stmt_allocator_.set_alloc(index_rtdef_->stmt_allocator_.get_alloc());
// Keep lookup_rtdef_->stmt_allocator_.alloc_ consistent with index_rtdef_->stmt_allocator_.alloc_
// to avoid memory expansion
if (lookup_iter_ != nullptr) {
scan_param_.need_switch_param_ = need_switch_param;
scan_param_.key_ranges_.reuse();
@ -1241,9 +1184,26 @@ int ObLocalIndexLookupOp::revert_iter()
return ret;
}
bool ObLocalIndexLookupOp::need_next_index_batch() const
int ObLocalIndexLookupOp::switch_index_table_and_rowkey_group_id()
{
return !index_end_;
int ret = OB_SUCCESS;
if (is_group_scan_) {
//Do the group scan jump read.
//Now we support jump read in GroupScan iter.
//Some of row read from index maybe jump.
//We need to sync index_group_cnt with lookup_group_cnt.
//Because in the rescan we manipulate the lookup_group_cnt.
set_index_group_cnt(get_lookup_group_cnt());
ret = set_rowkey_scan_group(get_lookup_group_cnt() - 1);
if (OB_SUCCESS != ret) {
LOG_WARN("set_rowkey_scan_group fail",K(get_lookup_group_cnt() - 1),K(ret));
if (OB_ITER_END == ret) {
ret = OB_ERR_UNEXPECTED;
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase