hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[LogService] Support the ls delay GC

Browse Source
This commit is contained in:
obdev
2023-07-11 19:18:02 +00:00
committed by ob-robot
parent 0472c5cc9a
commit ddb3abbab6
7 changed files with 121 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/logservice/libobcdc/src/ob_log_sequencer1.cpp
View File

@ -8,8 +8,6 @@
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*
* Sequencer: sequence trans.
*/
#define USING_LOG_PREFIX OBLOG_SEQUENCER
@ -382,7 +380,7 @@ int ObLogSequencer::handle_to_be_sequenced_trans_(TrxSortElem &trx_sort_elem,
}
}
LOG_DEBUG("handle_to_be_sequenced_trans_ end", KR(ret), K(trans_id));
LOG_TRACE("handle_to_be_sequenced_trans_ end", KR(ret), K(trans_id), K(trx_sort_elem));
}
return ret;

2
src/logservice/libobcdc/src/ob_log_sequencer1.h
View File

@ -8,8 +8,6 @@
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*
* Sequencer: sequence trans.
*/
#ifndef OCEANBASE_LIBOBCDC_SEQUENCER_H__

1
src/logservice/logrouteservice/ob_log_systable_queryer.h
View File

@ -45,6 +45,7 @@ public:
void destroy();
public:
// SELECT SVR_IP, SVR_PORT, ROLE, BEGIN_LSN, END_LSN FROM GV$OB_LOG_STAT
int get_ls_log_info(
const uint64_t tenant_id,
const share::ObLSID &ls_id,

110
src/logservice/ob_garbage_collector.cpp
View File

@ -15,6 +15,7 @@
#include "ob_log_handler.h"
#include "ob_log_service.h"
#include "ob_switch_leader_adapter.h"
#include "common_util/ob_log_time_utils.h"
#include "archiveservice/ob_archive_service.h"
#include "share/scn.h"
#include "rpc/obrpc/ob_rpc_net_handler.h"
@ -617,11 +618,10 @@ void ObGCHandler::try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_st
} else if (OB_FAIL(check_if_tenant_in_archive_(tenant_in_archive))) {
CLOG_LOG(WARN, "check_if_tenant_in_archive_ failed", K(ret), K(ls_id), K(gc_state));
} else if (! tenant_in_archive) {
if (OB_FAIL(ls_->set_gc_state(LSGCState::WAIT_GC))) {
CLOG_LOG(WARN, "set_gc_state failed", K(ls_id), K(gc_state), K(ret));
if (OB_FAIL(try_check_and_set_wait_gc_when_log_archive_is_off_(gc_state, readable_scn, offline_scn, ls_status))) {
CLOG_LOG(WARN, "try_check_and_set_wait_gc_when_log_archive_is_off_ failed", K(ret), K(ls_id), K(gc_state),
K(readable_scn), K(offline_scn), K(ls_status));
}
ls_status = ObGarbageCollector::LSStatus::LS_NEED_DELETE_ENTRY;
CLOG_LOG(INFO, "try_check_and_set_wait_gc_ success", K(ls_id), K(gc_state), K(offline_scn), K(readable_scn));
} else if (OB_FAIL(archive_service->get_ls_archive_progress(ls_id, lsn, scn, force_wait, ignore))){
CLOG_LOG(WARN, "get_ls_archive_progress failed", K(ls_id), K(gc_state), K(offline_scn), K(ret));
} else if (ignore) {
@ -641,6 +641,96 @@ void ObGCHandler::try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_st
}
}
int ObGCHandler::try_check_and_set_wait_gc_when_log_archive_is_off_(
const LSGCState &gc_state,
const share::SCN &readable_scn,
const share::SCN &offline_scn,
ObGarbageCollector::LSStatus &ls_status)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(ls_)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(WARN, "ls_ is nullptr", KR(ret));
} else {
const uint64_t tenant_id = MTL_ID();
int tmp_ret = OB_SUCCESS;
bool is_tenant_dropping_or_dropped = false;
if (OB_SUCCESS != (tmp_ret = check_if_tenant_is_dropping_or_dropped_(tenant_id, is_tenant_dropping_or_dropped))) {
CLOG_LOG(WARN, "check_if_tenant_has_been_dropped_ failed", K(tmp_ret), K(tenant_id), K(ls_id));
} else if (is_tenant_dropping_or_dropped) {
// The LS delay deletion mechanism will no longer take effect when the tenant is dropped.
if (OB_FAIL(ls_->set_gc_state(LSGCState::WAIT_GC))) {
CLOG_LOG(WARN, "set_gc_state failed", K(ls_id), K(gc_state), K(ret));
}
ls_status = ObGarbageCollector::LSStatus::LS_NEED_DELETE_ENTRY;
CLOG_LOG(INFO, "Tenant is dropped and the log stream can be removed, try_check_and_set_wait_gc_ success",
K(tenant_id), K(ls_id), K(gc_state), K(offline_scn), K(readable_scn));
} else {
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
// The LS delay deletion mechanism will take effect when the tenant is not dropped.
if (! tenant_config.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "tenant_config is not valid", K(ret), K(tenant_id));
} else if (OB_UNLIKELY(OB_INVALID_TIMESTAMP == gc_start_ts_)) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "gc_start_ts_ is not valid", KR(ret), K(ls_), K(gc_start_ts_));
} else {
const int64_t ls_gc_delay_time = tenant_config->ls_gc_delay_time;
const int64_t current_time_us = common::ObTimeUtility::current_time();
if ((current_time_us - gc_start_ts_) >= ls_gc_delay_time) {
if (OB_FAIL(ls_->set_gc_state(LSGCState::WAIT_GC))) {
CLOG_LOG(WARN, "set_gc_state failed", K(ls_id), K(gc_state), K(ret));
}
ls_status = ObGarbageCollector::LSStatus::LS_NEED_DELETE_ENTRY;
CLOG_LOG(INFO, "The log stream can be removed, try_check_and_set_wait_gc_ success",
K(ls_id), K(gc_state), K(offline_scn), K(readable_scn), K(ls_gc_delay_time));
} else {
CLOG_LOG(INFO, "The log stream requires delayed gc", K(ls_id),
K(ls_gc_delay_time), K(gc_start_ts_),
"gc_start_ts", TS_TO_STR(gc_start_ts_),
"current_time", TS_TO_STR(current_time_us));
}
}
}
}
return ret;
}
int ObGCHandler::check_if_tenant_is_dropping_or_dropped_(const uint64_t tenant_id,
bool &is_tenant_dropping_or_dropped)
{
int ret = OB_SUCCESS;
schema::ObMultiVersionSchemaService *schema_service = GCTX.schema_service_;
schema::ObSchemaGetterGuard guard;
is_tenant_dropping_or_dropped = false;
const ObTenantSchema *tenant_schema = nullptr;
if (OB_ISNULL(schema_service)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(WARN, "schema_service is nullptr", KR(ret));
} else if (OB_FAIL(schema_service->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
CLOG_LOG(WARN, "fail to get schema guard", KR(ret), K(tenant_id));
} else if (OB_FAIL(guard.get_tenant_info(tenant_id, tenant_schema))) {
CLOG_LOG(WARN, "get tenant info failed", KR(ret), K(tenant_id));
} else if (OB_ISNULL(tenant_schema)) {
// Double check the tenant status to avoid any potential problems in the schema module.
if (OB_FAIL(guard.check_if_tenant_has_been_dropped(tenant_id, is_tenant_dropping_or_dropped))) {
CLOG_LOG(WARN, "fail to check if tenant has been dropped", KR(ret), K(tenant_id));
} else {
CLOG_LOG(INFO, "tenant info is nullptr, check the tenant status",
K(tenant_id), K(is_tenant_dropping_or_dropped));
}
} else {
is_tenant_dropping_or_dropped = tenant_schema->is_dropping();
}
return ret;
}
int ObGCHandler::get_tenant_readable_scn_(SCN &readable_scn)
{
int ret = OB_SUCCESS;
@ -817,7 +907,11 @@ void ObGCHandler::handle_gc_ls_dropping_(const ObGarbageCollector::LSStatus &ls_
ObRole role;
ObLSID ls_id = ls_->get_ls_id();
LSGCState gc_state = INVALID_LS_GC_STATE;
gc_start_ts_ = ObTimeUtility::current_time();
// If gc_start_ts_ is an invalid value, it is necessary to get the current time again to avoid a situation
// where gc_start_ts_ remains an invalid value after ObServer restart, which may affect the GC logic.
if (OB_INVALID_TIMESTAMP == gc_start_ts_) {
gc_start_ts_ = ObTimeUtility::current_time();
}
if (OB_FAIL(get_palf_role_(role))) {
CLOG_LOG(WARN, "get_palf_role_ failed", K(ls_id));
} else if (ObRole::LEADER != role) {
@ -849,6 +943,12 @@ void ObGCHandler::handle_gc_ls_offline_(ObGarbageCollector::LSStatus &ls_status)
ObRole role;
ObLSID ls_id = ls_->get_ls_id();
LSGCState gc_state = INVALID_LS_GC_STATE;
// If gc_start_ts_ is an invalid value, it is necessary to get the current time again to avoid a situation
// where gc_start_ts_ remains an invalid value after ObServer restart, which may affect the GC logic.
if (OB_INVALID_TIMESTAMP == gc_start_ts_) {
gc_start_ts_ = ObTimeUtility::current_time();
}
if (OB_FAIL(get_palf_role_(role))) {
CLOG_LOG(WARN, "get_palf_role_ failed", K(ls_id));
} else if (ObRole::LEADER != role) {

7
src/logservice/ob_garbage_collector.h
View File

@ -299,6 +299,13 @@ private:
bool is_ls_offline_finished_(const LSGCState &state);
bool is_tablet_clear_(const ObGarbageCollector::LSStatus &ls_status);
void try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_status);
int try_check_and_set_wait_gc_when_log_archive_is_off_(
const LSGCState &gc_state,
const share::SCN &readable_scn,
const share::SCN &offline_scn,
ObGarbageCollector::LSStatus &ls_status);
int check_if_tenant_is_dropping_or_dropped_(const uint64_t tenant_id,
bool &is_tenant_dropping_or_dropped);
int get_tenant_readable_scn_(share::SCN &readable_scn);
int check_if_tenant_in_archive_(bool &in_archive);
void submit_log_(const ObGCLSLOGType log_type);

6
src/share/parameter/ob_parameter_seed.ipp
View File

@ -558,6 +558,12 @@ DEF_TIME(log_storage_warning_tolerance_time, OB_CLUSTER_PARAMETER, "5s",
"Range: [1s,300s]",
ObParameterAttr(Section::LOGSERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_TIME(ls_gc_delay_time, OB_TENANT_PARAMETER, "1h",
"[0s,)",
"The max delay time for ls gc when log archive is off. The default value is 3600s. Range: [0s, +∞). "
"The ls delay deletion mechanism will no longer take effect when the tenant is dropped.",
ObParameterAttr(Section::LOGSERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_TIME(standby_db_fetch_log_rpc_timeout, OB_TENANT_PARAMETER, "15s",
"[2s,)",
"The threshold for detecting the RPC timeout for the standby tenant to fetch log from the log restore source tenant. "

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -127,6 +127,7 @@ log_restore_concurrency
log_storage_warning_tolerance_time
log_transport_compress_all
log_transport_compress_func
ls_gc_delay_time
ls_meta_table_check_interval
major_compact_trigger
major_freeze_duty_time