hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix recover external table bug

Browse Source
This commit is contained in:
SevenJ-swj
2023-12-12 12:22:30 +00:00
committed by ob-robot
parent 6c4338e939
commit 3c06df8d70
3 changed files with 46 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/share/external_table/ob_external_table_utils.cpp
View File

@ -23,6 +23,7 @@
#include "sql/ob_sql_utils.h"
#include "sql/rewrite/ob_query_range.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "deps/oblib/src/lib/net/ob_addr.h"
namespace oceanbase
{
@ -420,5 +421,25 @@ int ObExternalTableUtils::calc_assigned_files_to_sqcs(
return ret;
}
int ObExternalTableUtils::filter_files_in_locations(common::ObIArray<share::ObExternalFileInfo> &files,
common::ObIArray<common::ObAddr> &locations,
common::ObIArray<share::ObExternalFileInfo> &res)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < files.count(); i++) {
ObExternalFileInfo &table_info = files.at(i);
bool found = false;
for (int64_t j = 0; OB_SUCC(ret) && !found && j < locations.count(); j++) {
if (table_info.file_addr_ == locations.at(j)) {
found = true;
}
}
if (OB_SUCC(ret) && found) {
ret = res.push_back(table_info);
}
}
return ret;
}
} // namespace share
} // namespace oceanbase

6
src/share/external_table/ob_external_table_utils.h
View File

@ -24,6 +24,7 @@ namespace common
{
class ObObj;
class ObNewRange;
class ObAddr;
}
namespace sql
@ -98,6 +99,10 @@ class ObExternalTableUtils {
const common::ObIArray<ObExternalFileInfo> &files,
common::ObIArray<int64_t> &assigned_idx,
int64_t sqc_count);
static int filter_files_in_locations(common::ObIArray<share::ObExternalFileInfo> &files,
common::ObIArray<common::ObAddr> &locations,
common::ObIArray<share::ObExternalFileInfo> &res);
private:
static bool is_left_edge(const common::ObObj &value);
static bool is_right_edge(const common::ObObj &value);
@ -109,6 +114,7 @@ class ObExternalTableUtils {
const int64_t last_lineno,
common::ObIAllocator &allocator,
common::ObNewRange &new_range);
};
}
}

27
src/sql/engine/px/ob_px_util.cpp
View File

@ -136,6 +136,7 @@ int ObPXServerAddrUtil::get_external_table_loc(
uint64_t tenant_id = OB_INVALID_ID;
bool is_external_files_on_disk = false;
ObIArray<ObExternalFileInfo> &ext_file_urls = dfo.get_external_table_files();
ObSEArray<ObAddr, 16> all_locations;
ObQueryRangeArray ranges;
if (OB_ISNULL(local_loc = DAS_CTX(ctx).get_table_loc_by_id(table_id, ref_table_id))) {
ret = OB_ERR_UNEXPECTED;
@ -143,9 +144,15 @@ int ObPXServerAddrUtil::get_external_table_loc(
} else if (OB_ISNULL(ctx.get_my_session())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session is null", K(ret));
} else if (OB_ISNULL(GCTX.location_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null location service", K(ret));
} else {
tenant_id = ctx.get_my_session()->get_effective_tenant_id();
is_external_files_on_disk = local_loc->loc_meta_->is_external_files_on_disk_;
if (OB_FAIL(GCTX.location_service_->external_table_get(tenant_id, ref_table_id, all_locations))) {
LOG_WARN("fail to get external table location", K(ret));
}
}
if (OB_SUCC(ret) && ext_file_urls.empty()) {
// TODO EXTARNAL TABLE
@ -153,14 +160,23 @@ int ObPXServerAddrUtil::get_external_table_loc(
// ret = OB_NOT_SUPPORTED;
// LOG_WARN("Has dynamic params in external table or empty range is not supported", K(ret),
// K(pre_query_range.has_exec_param()), K(pre_query_range.get_column_count()));
ObArray<ObExternalFileInfo> tmp_ext_file_urls;
if (OB_FAIL(ObSQLUtils::extract_pre_query_range(
pre_query_range, ctx.get_allocator(), ctx, ranges,
ObBasicSessionInfo::create_dtc_params(ctx.get_my_session())))) {
LOG_WARN("failed to extract external file fiter", K(ret));
} else if (OB_FAIL(ObExternalTableFileManager::get_instance().get_external_files(
tenant_id, ref_table_id, is_external_files_on_disk,
ctx.get_allocator(), ext_file_urls, ranges.empty() ? NULL : &ranges))) {
ctx.get_allocator(), tmp_ext_file_urls, ranges.empty() ? NULL : &ranges))) {
LOG_WARN("fail to get external files", K(ret));
} else if (OB_FAIL(ObExternalTableUtils::filter_files_in_locations(tmp_ext_file_urls,
all_locations,
ext_file_urls))) {
//For recovered cluster, the file addr may not in the cluster. Then igore it.
LOG_WARN("filter files in location failed", K(ret));
}
if (OB_FAIL(ret)) {
} else if (ext_file_urls.empty()) {
const char* dummy_file_name = "#######DUMMY_FILE#######";
ObExternalFileInfo dummy_file;
@ -191,22 +207,17 @@ int ObPXServerAddrUtil::get_external_table_loc(
}
}
} else {
ObSEArray<ObAddr, 16> all_locations;
int64_t expected_location_cnt = std::min(dfo.get_dop(), dfo.get_external_table_files().count());
if (1 == expected_location_cnt) {
if (OB_FAIL(target_locations.push_back(GCTX.self_addr()))) {
LOG_WARN("fail to push push back", K(ret));
}
} else {
if (OB_FAIL(GCTX.location_service_->external_table_get(tenant_id, ref_table_id, all_locations))) {
LOG_WARN("fail to get external table location", K(ret));
} else if (expected_location_cnt >= all_locations.count() ?
} else if (expected_location_cnt >= all_locations.count() ?
OB_FAIL(target_locations.assign(all_locations))
: OB_FAIL(ObPXServerAddrUtil::do_random_dfo_distribution(all_locations,
expected_location_cnt,
target_locations))) {
LOG_WARN("fail to calc random dfo distribution", K(ret), K(all_locations), K(expected_location_cnt));
}
LOG_WARN("fail to calc random dfo distribution", K(ret), K(all_locations), K(expected_location_cnt));
}
}
LOG_TRACE("calc external table location", K(target_locations));