hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
a8f362279648de947e79f361f7f63e8fc6ccb498
oceanbase/src/rootserver/ob_partition_group_coordinator.cpp
oceanbase-admin cea7de1475 init push
2021-05-31 22:56:52 +08:00

1042 lines
43 KiB
C++
Raw Blame History

/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_partition_group_coordinator.h"
#include "lib/container/ob_array.h"
#include "lib/container/ob_array_iterator.h"
#include "lib/container/ob_se_array.h"
#include "lib/container/ob_se_array_iterator.h"
#include "observer/ob_server_struct.h"
#include "share/ob_multi_cluster_util.h"
#include "share/partition_table/ob_partition_table_operator.h"
#include "ob_rebalance_task_mgr.h"
#include "ob_root_utils.h"
#include "ob_locality_checker.h"
using namespace oceanbase::common;
using namespace oceanbase::rootserver;
using namespace oceanbase::share;
ObPartitionGroupCoordinator::ObPartitionGroupCoordinator()
: inited_(false),
task_mgr_(NULL),
tenant_stat_(NULL),
check_stop_provider_(NULL),
two_paxos_zones_(),
logonly_units_()
{}
int ObPartitionGroupCoordinator::init(
ObRebalanceTaskMgr& task_mgr, TenantBalanceStat& tenant_stat, share::ObCheckStopProvider& check_stop_provider)
{
int ret = OB_SUCCESS;
if (inited_) {
ret = OB_INIT_TWICE;
} else {
task_mgr_ = &task_mgr;
tenant_stat_ = &tenant_stat;
check_stop_provider_ = &check_stop_provider;
two_paxos_zones_.reset();
logonly_units_.reset();
inited_ = true;
}
return ret;
}
int ObPartitionGroupCoordinator::prepare()
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
for (int64_t i = 0; i < ts.all_zone_paxos_info_.count() && OB_SUCC(ret); i++) {
const ZonePaxosInfo& zone_paxos_info = ts.all_zone_paxos_info_.at(i);
if (zone_paxos_info.full_replica_num_ == 1 && zone_paxos_info.logonly_replica_num_ == 1) {
if (OB_FAIL(two_paxos_zones_.push_back(zone_paxos_info.zone_))) {
LOG_WARN("fail to push back", K(ret), K(zone_paxos_info));
}
} else {
}
}
for (int64_t i = 0; i < ts.all_zone_unit_.count() && OB_SUCC(ret); i++) {
const ZoneUnit& zu = ts.all_zone_unit_.at(i);
for (int64_t j = 0; j < zu.all_unit_.count() && OB_SUCC(ret); j++) {
UnitStat* u = zu.all_unit_.at(j);
if (REPLICA_TYPE_LOGONLY == u->info_.unit_.replica_type_) {
if (OB_FAIL(logonly_units_.push_back(u))) {
LOG_WARN("fail to push back", K(ret));
}
}
}
}
return ret;
}
int ObPartitionGroupCoordinator::normal_pg_member_coordinate(
common::ObArray<Partition*>::iterator& p, Partition& primary_partition, int64_t& task_cnt)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
bool primary_partition_migrated = false;
int64_t tmp_task_cnt = 0;
if (!primary_partition.can_balance()) {
LOG_INFO("primary partition has no leader", K(primary_partition));
} else if (!(*p)->is_valid_quorum()) {
LOG_WARN("quorum not report yet, just skip", K(ret), KPC(*p));
} else if (OB_FAIL(try_coordinate_primary_by_unit(primary_partition, primary_partition_migrated, tmp_task_cnt))) {
LOG_WARN("fail to coordiante logonly replica", K(ret), K(primary_partition));
} else if (primary_partition_migrated) {
task_cnt += tmp_task_cnt;
} else if (!can_do_coordinate(primary_partition)) {
LOG_WARN("can't do coordinate", K(ret), K(primary_partition));
} else {
do {
int64_t tmp_task_cnt = 0;
__typeof__(p) next = p + 1;
if (next >= ts.sorted_partition_.end() || (*next)->partition_idx_ != (*p)->partition_idx_ ||
(*next)->is_primary()) {
break;
}
p = next;
if (!(*p)->is_valid_quorum()) {
LOG_WARN("quorum not report yet, just skip", K(ret), KPC(*p));
} else if (OB_FAIL(
coordinate_partition_member(primary_partition, *(*p), primary_partition_migrated, tmp_task_cnt))) {
LOG_WARN("coordinate partition member failed", K(ret), K(primary_partition), "partition", *(*p));
} else {
task_cnt += tmp_task_cnt;
if (primary_partition_migrated) {
break;
}
}
} while (OB_SUCC(ret));
}
return ret;
}
// aligned step:
// 1. Using the first partition of pg as a template, called the primary partition,
// first solve the alignment of the primary partition;
// 1.1 try_coordinate_primary_by_unit: Ensure that the replica of type L is stored on the logonly unit,
// and if there is a logonly unit, the logonly replica must be on the logonlg unit;
// 1.2 In the case where the primary partition is already aligned, solve the alignment of the remaining partitions
// 2.1 migrate_primary_partition: For each replica in pg,
// check whether the destination OBS can migrate in occasionally when aligning,
// if not, try to migrate back
// 2.2 Solve the alignment of each type by type without moving back
// 2.2.1 There are two cases of paxos copy alignment.
// One is that there are two replicas of paxos in the zone;
// the other is that there is only one replica of paxos in the zone;
// 2.2.2 Solve alignment issues for other replica types
int ObPartitionGroupCoordinator::coordinate_pg_member(int64_t& task_cnt)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
task_cnt = 0;
two_paxos_zones_.reset();
logonly_units_.reset();
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ts.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), "tenant stat valid", ts.is_valid());
} else if (OB_FAIL(prepare())) {
LOG_WARN("fail to prepare", K(ret));
} else {
FOREACH_X(p, ts.sorted_partition_, OB_SUCCESS == ret)
{
if (OB_FAIL(check_stop())) {
LOG_WARN("balancer stop", K(ret));
break;
} else {
if ((*p)->is_primary() && task_cnt > ObRebalanceTaskMgr::ONCE_ADD_TASK_CNT) {
break;
}
}
if ((*p)->is_primary()) {
const uint64_t partition_entity_id = (*p)->table_id_;
Partition& primary_partition = *(*p);
if (OB_FAIL(normal_pg_member_coordinate(p, primary_partition, task_cnt))) {
LOG_WARN("fail to do normal pg member coordinate", K(ret));
}
}
}
}
return ret;
}
int ObPartitionGroupCoordinator::try_coordinate_primary_by_unit(
const Partition& primary_partition, bool& primary_partition_migrated, int64_t& task_count)
{
int ret = OB_SUCCESS;
task_count = 0;
TenantBalanceStat& ts = *tenant_stat_;
primary_partition_migrated = false;
bool can_add_task = false;
for (int64_t i = 0; i < logonly_units_.count() && OB_SUCC(ret) && !primary_partition_migrated; i++) {
const Replica* replica_on_logonly = NULL;
const Replica* logonly_replica = NULL;
FOR_BEGIN_END_E(ppr, primary_partition, ts.all_replica_, OB_SUCC(ret))
{
if (!ppr->is_in_service()) {
// nothing todo
} else if (ppr->zone_ != logonly_units_.at(i)->info_.unit_.zone_) {
// nothing todo
} else {
if (REPLICA_TYPE_LOGONLY == ppr->replica_type_) {
logonly_replica = ppr;
}
if (REPLICA_TYPE_LOGONLY == ppr->unit_->info_.unit_.replica_type_) {
replica_on_logonly = ppr;
}
}
} // end FOR_BEGIN_END_E(ppr
// Check for alignment
if (OB_FAIL(ret)) {
// nothing todo
} else if (OB_ISNULL(logonly_replica)) {
// Lack of logonly replica; waiting to make up the replica
if (!OB_ISNULL(replica_on_logonly)) {
// Try to move the replica on the logonly unit
LOG_WARN("need coordiante because non_logonly replica in logonly unit",
K(primary_partition),
K(*replica_on_logonly));
primary_partition_migrated = true;
Replica dest;
if (OB_FAIL(get_random_dest(primary_partition, *replica_on_logonly, dest))) {
LOG_WARN("fail to get random dest", K(ret));
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("fail to get random dest", K(ret));
}
} else if (OB_FAIL(do_migrate_replica(primary_partition, *replica_on_logonly, dest, can_add_task))) {
LOG_WARN("fail to migrate replica",
K(ret),
"partition_id",
primary_partition.partition_id_,
"table_id",
primary_partition.table_id_,
"replica",
*replica_on_logonly);
} else if (can_add_task) {
task_count++;
}
}
} else if (OB_ISNULL(replica_on_logonly)) {
primary_partition_migrated = true;
Replica dest_replica;
dest_replica.zone_ = logonly_units_.at(i)->info_.unit_.zone_;
dest_replica.server_ = logonly_units_.at(i)->server_;
dest_replica.unit_ = logonly_units_.at(i);
LOG_WARN("need coordiante becase logonly_replica not in logonly unit",
K(primary_partition),
K(*logonly_replica),
K(dest_replica));
// Empty unit, logonly replica needs to be migrated
if (OB_FAIL(do_migrate_replica(primary_partition, *logonly_replica, dest_replica, can_add_task))) {
LOG_WARN("fail to migreate replica", K(ret));
} else if (can_add_task) {
task_count++;
}
} else if (logonly_replica != replica_on_logonly) {
LOG_WARN("need coordiante becase logonly_replica not in logonly unit",
K(primary_partition),
K(*replica_on_logonly),
K(*logonly_replica));
primary_partition_migrated = true;
Replica dest;
// Try to move away first, if the migration fails, need to delete first and then migrate
if (OB_FAIL(get_random_dest(primary_partition, *replica_on_logonly, dest))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("fail to get random dest", K(ret));
} else {
// Priority to keep the replica of F
if (REPLICA_TYPE_FULL == replica_on_logonly->replica_type_) {
if (OB_FAIL(try_remove_and_migrate(
primary_partition, *replica_on_logonly, *logonly_replica, *logonly_replica, can_add_task))) {
LOG_WARN("fail to remove and migrate", K(ret));
} else if (can_add_task) {
task_count++;
}
} else if (OB_FAIL(try_remove_and_migrate(primary_partition,
*logonly_replica,
*replica_on_logonly,
*replica_on_logonly,
can_add_task))) {
LOG_WARN("fail to remove and migrate", K(ret));
} else if (can_add_task) {
task_count++;
}
}
} else if (OB_FAIL(do_migrate_replica(primary_partition, *replica_on_logonly, dest, can_add_task))) {
LOG_WARN("fail to do migrare replica", K(ret), K(primary_partition), K(*replica_on_logonly), K(dest));
} else if (can_add_task) {
task_count++;
}
} else {
// nothing todo
}
} // end for (int64_t i = 0; i < logonly_zones
return ret;
}
int ObPartitionGroupCoordinator::coordinate_partition_member(
const Partition& primary_partition, Partition& partition, bool& primary_partition_migrated, int64_t& task_count)
{
int ret = OB_SUCCESS;
task_count = 0;
TenantBalanceStat& ts = *tenant_stat_;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ts.is_valid() || OB_INVALID_ID == primary_partition.table_id_ || OB_INVALID_ID == partition.table_id_ ||
primary_partition.tablegroup_id_ != partition.tablegroup_id_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), "tenant stat valid", ts.is_valid(), K(primary_partition), K(partition));
}
// ppr : primary partition replica
primary_partition_migrated = false;
bool skip_coordinate_paxos_replica = false;
int64_t tmp_task_count = 0;
// 1. Align the replica of the paxos type first
// 2. Then align replicas of non-paxos types
if (OB_SUCC(ret)) {
// Check whether ppr needs to be migrated back.
// Need to iterate all the partitions until a suitable location is found for migration back
if (OB_FAIL(migrate_primary_partition(primary_partition, partition, primary_partition_migrated, tmp_task_count))) {
LOG_WARN("migrate primary partition", K(ret), K(primary_partition), K(partition));
} else if (primary_partition_migrated) {
task_count += tmp_task_count;
// nothing todo
} else if (!can_do_coordinate(partition)) {
LOG_WARN("fail to to coordinate", K(partition));
} else if (OB_FAIL(coordinate_paxos_replica(
primary_partition, partition, tmp_task_count, skip_coordinate_paxos_replica))) {
LOG_WARN("fail to process replica in same addr", K(ret));
} else if (0 == tmp_task_count // Do only one replica at a time
&& !skip_coordinate_paxos_replica) {
if (OB_FAIL(coordinate_non_paxos_replica(primary_partition, partition, tmp_task_count))) {
LOG_WARN("fail to process replica in same zone", K(ret), K(primary_partition), K(partition));
} else {
task_count += tmp_task_count;
}
} else {
task_count += tmp_task_count;
}
}
return ret;
}
// Try to find an available machine on the same server/zone in the partition for migration;
// Check whether the primary partition needs to find the correct unit to migrate.
// In addition, if there is a block, need to migrate back or find the correct unit to migrate.
int ObPartitionGroupCoordinator::migrate_primary_partition(
const Partition& primary_partition, const Partition& partition, bool& primary_partition_migrated, int64_t& task_cnt)
{
int ret = OB_SUCCESS;
Replica dest;
primary_partition_migrated = false;
task_cnt = 0;
TenantBalanceStat& ts = *tenant_stat_;
bool can_add_task = false;
FOR_BEGIN_END_E(ppr, primary_partition, ts.all_replica_, OB_SUCC(ret) && !primary_partition_migrated)
{
if (!ppr->is_in_service()) {
// nothing todo
} else if (!ppr->server_->need_rebalance()) {
// nothing todo
} else {
primary_partition_migrated = true;
if (OB_FAIL(get_migrate_dest(primary_partition, *ppr, partition, dest))) {
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("fail to get migrate dest addr", K(ret), K(primary_partition), K(*ppr), K(partition));
}
} else if (OB_FAIL(do_migrate_replica(primary_partition, *ppr, dest, can_add_task))) {
LOG_WARN("fail to build migrate task", KR(ret));
} else if (can_add_task) {
task_cnt++;
LOG_INFO("migrate primary partition back", K(ret), K(primary_partition), K(dest));
}
}
}
return ret;
}
// Only handle the alignment of the paxos replica of the zone containing two replicas of paxos
int ObPartitionGroupCoordinator::coordinate_paxos_replica(const Partition& primary_partition,
const Partition& partition, int64_t& task_count, bool& skip_coordinate_paxos_replica)
{
int ret = OB_SUCCESS;
task_count = 0;
skip_coordinate_paxos_replica = false;
if (OB_FAIL(ret)) {
// nothing todo
} else if (two_paxos_zones_.count() <= 0) {
// nothing todo
} else if (OB_FAIL(coordinate_specific_paxos_replica(
primary_partition, partition, task_count, skip_coordinate_paxos_replica))) {
LOG_WARN("fail to coornate sepcific paxos replica", K(ret), K(two_paxos_zones_));
}
if (OB_FAIL(ret) || task_count > 0 || skip_coordinate_paxos_replica) {
// nothing todo
} else if (OB_FAIL(coordinate_normal_paxos_replica(
primary_partition, partition, task_count, skip_coordinate_paxos_replica))) {
LOG_WARN("fail to coordinate paxos replica", K(ret));
}
return ret;
}
// If there are two replicas of paxos on the zone, first check whether the replica of F is aligned.
// If the replica of F is not aligned, it needs to be migrated for alignment,
// or it needs to be deleted and then migrated for alignment
// Secondly, under the premise that the replica of F is aligned, the alignment of the replica of L is performed
int ObPartitionGroupCoordinator::coordinate_specific_paxos_replica(const Partition& primary_partition,
const Partition& partition, int64_t& task_count, bool& skip_coordinate_paxos_replica)
{
int ret = OB_SUCCESS;
skip_coordinate_paxos_replica = false;
task_count = 0;
TenantBalanceStat& ts = *tenant_stat_;
bool can_add_task = false;
for (int64_t i = 0; i < two_paxos_zones_.count() && OB_SUCC(ret) && task_count == 0; i++) {
const ObZone& zone = two_paxos_zones_.at(i);
Replica* ppr_full_replica = NULL;
Replica* ppr_logonly_replica = NULL;
FOR_BEGIN_END_E(ppr, primary_partition, ts.all_replica_, OB_SUCC(ret) && 0 == task_count)
{
if (!ppr->is_in_service()) {
// not a normal replica
} else if (!ppr->is_valid_paxos_member() || ppr->zone_ != zone) {
// nothing todo
} else if (REPLICA_TYPE_LOGONLY == ppr->replica_type_) {
if (REPLICA_TYPE_LOGONLY != ppr->unit_->info_.unit_.replica_type_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("primary partition not coordinate now", K(ret), K(*ppr), "unit", ppr->unit_->info_);
} else if (OB_ISNULL(ppr_logonly_replica)) {
ppr_logonly_replica = ppr;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("there are two logonly replica in zone", K(ret), K(zone), K(primary_partition));
}
} else if (REPLICA_TYPE_FULL == ppr->replica_type_) {
if (REPLICA_TYPE_LOGONLY == ppr->unit_->info_.unit_.replica_type_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("primary partition not coordinate now", K(ret), K(*ppr), "unit", ppr->unit_->info_);
} else if (OB_ISNULL(ppr_full_replica)) {
ppr_full_replica = ppr;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("there are two full replica in zone", K(ret), K(zone), K(primary_partition));
}
}
} // end FOR_BEGIN_END_E
Replica* pr_full_replica = NULL;
Replica* pr_logonly_replica = NULL;
Replica* pr_barrier_full = NULL;
Replica* pr_barrier_logonly = NULL;
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, OB_SUCC(ret) && 0 == task_count)
{
if (!pr->is_in_service()) {
// not a normal replica
} else if (!pr->is_valid_paxos_member() || pr->zone_ != zone) {
// nothing todo
} else if (REPLICA_TYPE_LOGONLY == pr->replica_type_) {
if (OB_ISNULL(pr_logonly_replica)) {
pr_logonly_replica = pr;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("there are two logonly replica in zone", K(ret), K(zone), K(partition));
}
} else if (REPLICA_TYPE_FULL == pr->replica_type_) {
if (OB_ISNULL(pr_full_replica)) {
pr_full_replica = pr;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("there are two full replica in zone", K(ret), K(zone), K(partition));
}
}
if (!OB_ISNULL(ppr_full_replica) && pr->server_->server_ == ppr_full_replica->server_->server_) {
pr_barrier_full = pr;
}
if (!OB_ISNULL(ppr_logonly_replica) && pr->server_->server_ == ppr_logonly_replica->server_->server_) {
pr_barrier_logonly = pr;
}
} // end FOR_BEGIN_END_E(pr, partition
// Start to check whether it needs to be migrated or deleted
// First check whether the replica of F needs coordinate
// Second, check whether the replica of L needs coordinate
if (OB_FAIL(ret)) {
// nothing todo
} else if (OB_ISNULL(ppr_full_replica) || OB_ISNULL(pr_full_replica)) {
// nothing todo
} else if (ppr_full_replica->server_->server_ == pr_full_replica->server_->server_) {
// nothing todo
} else {
// need to migrate
if (OB_ISNULL(pr_barrier_full)) {
skip_coordinate_paxos_replica = true;
// try migrate
if (OB_FAIL(do_migrate_replica(partition, *pr_full_replica, *ppr_full_replica, can_add_task))) {
LOG_WARN("fail to do migrate replica", K(ret));
} else if (can_add_task) {
task_count++;
}
} else if (OB_FAIL(try_remove_and_migrate(
partition, *pr_full_replica, *pr_barrier_full, *ppr_full_replica, can_add_task))) {
LOG_WARN("fail to delete and migrate", K(ret));
skip_coordinate_paxos_replica = true;
} else if (can_add_task) {
skip_coordinate_paxos_replica = true;
task_count++;
}
} // end check FULL replica
// begin to check logonly replica
if (OB_FAIL(ret) || skip_coordinate_paxos_replica) {
// nothing todo
} else if (OB_ISNULL(ppr_logonly_replica) || OB_ISNULL(pr_logonly_replica)) {
// nothing todo
} else if (ppr_logonly_replica->server_->server_ == pr_logonly_replica->server_->server_) {
// nothing todo
} else {
// need to migrate
if (OB_ISNULL(pr_barrier_logonly)) {
skip_coordinate_paxos_replica = true;
// try migrate
if (OB_FAIL(do_migrate_replica(partition, *pr_logonly_replica, *ppr_logonly_replica, can_add_task))) {
LOG_WARN("fail to do migrate replica", K(ret));
} else if (can_add_task) {
task_count++;
}
} else if (OB_FAIL(try_remove_and_migrate(
partition, *pr_logonly_replica, *pr_barrier_logonly, *ppr_logonly_replica, can_add_task))) {
LOG_WARN("fail to delete and migrate", K(ret));
skip_coordinate_paxos_replica = true;
} else if (can_add_task) {
skip_coordinate_paxos_replica = true;
task_count++;
}
} // end check logonly replica
} // end for (int64_t i = 0; i < two_paxos_zones_.count()
return ret;
}
int ObPartitionGroupCoordinator::coordinate_normal_paxos_replica(const Partition& primary_partition,
const Partition& partition, int64_t& task_count, bool& skip_coordinate_paxos_replica)
{
int ret = OB_SUCCESS;
task_count = 0;
skip_coordinate_paxos_replica = false;
TenantBalanceStat& ts = *tenant_stat_;
bool can_add_task = false;
// Handling of paxos type replicas separately
FOR_BEGIN_END_E(ppr, primary_partition, ts.all_replica_, OB_SUCC(ret) && 0 == task_count)
{
if (!ppr->is_valid_paxos_member()) {
continue;
} else if (!ppr->server_->can_migrate_in() || !ppr->unit_->in_pool_) {
skip_coordinate_paxos_replica = true;
continue;
} else if (has_exist_in_array(two_paxos_zones_, ppr->zone_)) {
// nothing todo
} else {
Replica* paxos_replica = NULL;
Replica* same_addr = NULL;
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, OB_SUCC(ret) && 0 == task_count)
{
if (!pr->is_in_service()) {
continue;
}
if (pr->zone_ == ppr->zone_ && pr->is_valid_paxos_member() && pr->server_->can_migrate_in()) {
paxos_replica = &(*pr);
}
if (pr->server_->server_ == ppr->server_->server_) {
same_addr = &(*pr);
}
} // FOR_BEGIN_END_E(pr
if (OB_SUCC(ret) && 0 == task_count) {
if (NULL == paxos_replica) {
// Missing replica of paxos; waiting for additional replicas of other disaster recovery processes
// nothing to do
} else if (NULL == same_addr) {
skip_coordinate_paxos_replica = true;
// There is an unaligned replica of paxos type, which needs to be migrated;
//(it may also need to do type conversion, but we only do one operation in one step)
if (OB_FAIL(do_migrate_replica(partition, *paxos_replica, *ppr, can_add_task))) {
LOG_WARN("fail to migrate replica", K(ret), K(partition), "target", *paxos_replica, "dest replica", *ppr);
} else if (can_add_task) {
task_count++;
}
} else if (same_addr == paxos_replica) {
// An aligned replica of the paxos type exists, but type conversion may be required
if (ppr->replica_type_ == paxos_replica->replica_type_) {
// nothing todo
} else {
// Type conversion can be moved outside to do. Avoid comparing locality
}
} else {
skip_coordinate_paxos_replica = true;
// There are two replicas, for example, ppr is located on machine A,
// but partition R is located on A, and F is located on machine B
// Need to delete first, then migrate
if (OB_FAIL(do_remove_and_migrate(partition, *paxos_replica, *ppr, can_add_task))) {
LOG_WARN(
"fail to do remove and migrate", K(ret), K(partition), "target", *paxos_replica, "dest replica", *ppr);
} else if (can_add_task) {
task_count++;
}
}
}
}
}
return ret;
}
int ObPartitionGroupCoordinator::coordinate_non_paxos_replica(
const Partition& primary_partition, Partition& partition, int64_t& task_count)
{
int ret = OB_SUCCESS;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
}
ObArray<Replica*> ppr_coordinated_replica; // Record the aligned ppr
ObArray<Replica*> pr_coordinated_replica; // Record the aligned pr
task_count = 0;
//(1) On the same OBS, see if there is a replica, if there are and the types are the same, do nothing,
// otherwise perform type conversion
//(2) In the same zone, check to see if there is no replica of the same type, and if there is one, perform a
// migration.
// Otherwise, do a type conversion first.(Need to exclude aligned replicas).
if (OB_SUCC(ret)) {
if (OB_FAIL(process_non_paxos_replica_in_same_addr(
primary_partition, partition, ppr_coordinated_replica, pr_coordinated_replica, task_count))) {
LOG_WARN("fail to process replica in same addr", K(ret));
} else if (task_count == 0 // Do only one replica at a time
&& OB_FAIL(process_non_paxos_replica_in_same_zone(
primary_partition, partition, ppr_coordinated_replica, pr_coordinated_replica, task_count))) {
LOG_WARN("fail to process replica in same zone", K(ret), K(primary_partition), K(partition));
}
}
return ret;
}
int ObPartitionGroupCoordinator::process_non_paxos_replica_in_same_zone(const Partition& primary_partition,
Partition& partition, ObIArray<Replica*>& ppr_coordinated_replica, ObIArray<Replica*>& pr_coordinated_replica,
int64_t& task_count)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
bool can_add_task = false;
FOR_BEGIN_END_E(ppr, primary_partition, ts.all_replica_, OB_SUCC(ret) && 0 == task_count)
{
if (!ppr->is_in_service() || !ppr->server_->active_ || !ppr->server_->online_ || !ppr->unit_->in_pool_ ||
ppr->server_->server_ != ppr->unit_->info_.unit_.server_) {
continue;
} else if (ObReplicaTypeCheck::is_paxos_replica_V2(ppr->replica_type_)) {
continue;
} else if (has_exist_in_array(ppr_coordinated_replica, &(*ppr))) {
continue;
}
bool found_addr = false;
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, !found_addr && OB_SUCCESS == ret)
{
if (!pr->is_in_service() || has_exist_in_array(pr_coordinated_replica, &(*pr))) {
continue;
} else if (ObReplicaTypeCheck::is_paxos_replica_V2(pr->replica_type_)) {
continue;
} else if (pr->zone_ == ppr->zone_) {
// need to migrate;
found_addr = true;
if (OB_FAIL(do_migrate_replica(partition, *pr, *ppr, can_add_task))) {
LOG_WARN("fail to do migrate", K(ret), K(partition), K(primary_partition));
} else if (OB_FAIL(pr_coordinated_replica.push_back(&(*pr)))) {
LOG_WARN("fail to push back", K(ret));
} else if (can_add_task) {
task_count++;
}
}
}
}
return ret;
}
int ObPartitionGroupCoordinator::process_non_paxos_replica_in_same_addr(const Partition& primary_partition,
Partition& partition, ObIArray<Replica*>& ppr_coordinated_replica, ObIArray<Replica*>& pr_coordinated_replica,
int64_t& task_count)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
FOR_BEGIN_END_E(ppr, primary_partition, ts.all_replica_, OB_SUCC(ret) && 0 == task_count)
{
if (!ppr->is_in_service() || !ppr->server_->can_migrate_in() || !ppr->unit_->in_pool_ ||
ppr->server_->server_ != ppr->unit_->info_.unit_.server_) {
continue;
} else if (ObReplicaTypeCheck::is_paxos_replica_V2(ppr->replica_type_)) {
continue;
}
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, OB_SUCCESS == ret)
{
if (pr->is_in_service() && ppr->server_->server_ == pr->server_->server_) {
if (pr->replica_type_ == ppr->replica_type_) {
// nothing todo
} else if (ObReplicaTypeCheck::is_paxos_replica_V2(pr->replica_type_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to be here", K(ret), K(primary_partition), K(*pr), K(partition));
} else {
// Don't do type conversion here
// For the time being, the situation where there are three types of replicas in a zone will not be processed
// If there are three types of replicas in a zone,
// except for the paxos type, the other two types should also be aligned.
// At this time, the problem of type conversion may be involved.
}
if (OB_SUCC(ret)) {
if (OB_FAIL(pr_coordinated_replica.push_back(pr))) {
LOG_WARN("fail to push back coordinated replica", K(ret), K(*pr));
} else if (OB_FAIL(ppr_coordinated_replica.push_back(&(*ppr)))) {
LOG_WARN("fail to push back", K(ret));
}
}
} // if server=server
} // end FOR_BEGIN_END_E(pr, partition
} // end FOR_BEGIN_END_E(ppr, primary_partition
return ret;
}
// 14x new deployment method: F/L@zone
//(1) Check whether a zone has only one paxos type;
// if a zone has multiple replicas of paxos, it will affect the coordinate
// NOTE: In the pg coordinate stage, type conversion is not considered.
// In such a situation, skip all
//(2) If there is a unit migration operation, do not coordinate;
// wait for the unit migration to be completed
bool ObPartitionGroupCoordinator::can_do_coordinate(const Partition& partition)
{
bool bret = true;
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
if (!ts.is_valid()) {
bret = false;
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid tenant stat", K(ret));
} else {
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, OB_SUCC(ret) && bret)
{
if (!pr->is_in_service()) {
continue;
}
if (pr->server_->server_ != pr->unit_->info_.unit_.server_) {
bret = false;
LOG_INFO("can not do coordiante now, partition have unit balance task todo",
K(partition),
"unit",
pr->unit_->info_.unit_,
"replica_addr",
pr->server_->server_);
}
}
}
return bret;
;
}
// If the replica that you want to delete is an L replica,
// need to determine whether there is a lack of PAXOS replicas.
// If the paxos replica is not enough, you cannot delete
int ObPartitionGroupCoordinator::try_remove_and_migrate(const Partition& partition, const Replica& to_migrate_replica,
const Replica& to_delete_replica, const Replica& migrate_dest_in_ppr, bool& can_add_task)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
can_add_task = false;
if (!ts.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid tenant stat", K(ret));
} else if (ObReplicaTypeCheck::is_paxos_replica_V2(to_delete_replica.replica_type_)) {
if (partition.valid_member_cnt_ < partition.schema_replica_cnt_) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("already lost paxos replica, refuse to delete one", K(ret), K(partition));
}
}
if (OB_FAIL(ret)) {
// nothing todo
} else if (OB_FAIL(do_remove_replica(partition, to_delete_replica))) {
LOG_WARN("fail to remove replica", K(ret));
} else if (OB_FAIL(do_migrate_replica(partition, to_migrate_replica, migrate_dest_in_ppr, can_add_task))) {
LOG_WARN("fail to migrate replica", K(ret));
}
return ret;
}
// partition: objects to be deleted and migrated;
// First find the replica on the machine where the ppr in the partition is located, and delete it;
// then migrate the replica to the machine where the ppr is located;
int ObPartitionGroupCoordinator::do_remove_and_migrate(
const Partition& partition, const Replica& replica, const Replica& ppr, bool& can_add_task)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
can_add_task = false;
if (!ts.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid tenant stat", K(ret));
} else {
// find replica in ppr->server in partition and remove it
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, OB_SUCC(ret))
{
if (pr->is_in_service() && pr->server_->server_ == ppr.server_->server_) {
if (ObReplicaTypeCheck::is_paxos_replica_V2(pr->replica_type_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN(
"should not remove paxos member in partition group", K(ret), K(partition), K(replica), K(ppr), K(*pr));
} else if (OB_FAIL(do_remove_replica(partition, *pr))) {
LOG_WARN("fail to remove replica", K(ret), K(partition), K(replica), K(ppr));
} else if (OB_FAIL(do_migrate_replica(partition, replica, ppr, can_add_task))) {
LOG_WARN("fail to migrate replica", K(ret), K(partition), K(replica), K(ppr));
}
break;
}
}
}
return ret;
}
int ObPartitionGroupCoordinator::do_remove_replica(const Partition& partition, const Replica& replica)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
ObPartitionKey key;
OnlineReplica remove_member;
remove_member.member_ = ObReplicaMember(replica.server_->server_, replica.member_time_us_, replica.replica_type_);
remove_member.zone_ = replica.zone_;
const char* comment = balancer::PG_COORDINATE_REMOVE_REPLICA;
if (!ts.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid tenant stat", K(ret));
} else if (OB_FAIL(ts.gen_partition_key(partition, key))) {
LOG_WARN("fail to gen partition key", K(ret), K(partition));
} else if (ObReplicaTypeCheck::is_paxos_replica_V2(replica.replica_type_)) {
ObRemoveMemberTask task;
ObRemoveMemberTaskInfo task_info;
common::ObArray<ObRemoveMemberTaskInfo> task_info_array;
int64_t quorum = 0;
if (replica.is_in_blacklist(ObRebalanceTaskType::MEMBER_CHANGE, replica.server_->server_, tenant_stat_)) {
ret = OB_REBALANCE_TASK_CANT_EXEC;
LOG_WARN("replica member change frequent failed, now in black list", K(replica));
} else if (OB_FAIL(ts.get_remove_replica_quorum_size(partition, replica.zone_, replica.replica_type_, quorum))) {
LOG_WARN("fail to get quorum size", K(ret));
} else if (OB_FAIL(task_info.build(remove_member, key, quorum, partition.get_quorum()))) {
LOG_WARN("fail to build remove member task info", K(ret));
} else if (OB_FAIL(task_info_array.push_back(task_info))) {
LOG_WARN("fail to push back", K(ret));
} else if (OB_FAIL(task.build(task_info_array, remove_member.member_.get_server(), comment))) {
LOG_WARN("fail to build remove member task", K(ret));
} else if (OB_FAIL(task_mgr_->add_task(task))) {
LOG_WARN("fail to add task", K(ret), K(replica));
} else {
LOG_INFO("add task to remove member for coordinate", K(replica), K(partition));
}
} else {
ObRemoveNonPaxosTask task;
ObRemoveNonPaxosTaskInfo task_info;
common::ObArray<ObRemoveNonPaxosTaskInfo> task_info_array;
if (replica.is_in_blacklist(
ObRebalanceTaskType::REMOVE_NON_PAXOS_REPLICA, replica.server_->server_, tenant_stat_)) {
ret = OB_REBALANCE_TASK_CANT_EXEC;
LOG_WARN("task in black list", K(replica));
} else if (OB_FAIL(task_info.build(remove_member, key))) {
LOG_WARN("fail to build remove non paxos replica task info", K(ret), K(key));
} else if (OB_FAIL(task_info_array.push_back(task_info))) {
LOG_WARN("fail to push back", K(ret));
} else if (OB_FAIL(task.build(task_info_array, remove_member.member_.get_server(), comment))) {
LOG_WARN("fail to build remove non paxos replcia", K(ret), K(partition), K(remove_member));
} else if (OB_FAIL(task_mgr_->add_task(task))) {
LOG_WARN("add task failed", K(ret), K(task));
} else {
LOG_INFO("add remove replica task for partition group coordinate", K(task));
}
}
return ret;
}
// Migrate replica to the machine where dest_replica is located
// Don't worry about the migration target is the leader.
// When executing, the leader will be moved first.
int ObPartitionGroupCoordinator::do_migrate_replica(
const Partition& partition, const Replica& replica, const Replica& dest_replica, bool& can_add_task)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
can_add_task = false;
if (!ts.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid tenant stat", K(ret));
} else if (!ts.has_leader_while_member_change(partition, replica.server_->server_)) {
// Ignore the error code,
// otherwise it will fail to initiate the migration replica task due to
// the inability to align the standby database for a single zone and multiple units,
// affecting the swtich_over operation
LOG_WARN("can not do migrate, may no leader", K(ret), K(partition), K(replica), K(dest_replica));
} else if (replica.is_in_blacklist(
ObRebalanceTaskType::MIGRATE_REPLICA, dest_replica.server_->server_, tenant_stat_)) {
ret = OB_REBALANCE_TASK_CANT_EXEC;
LOG_INFO("replica in black list, wait", K(replica), K(dest_replica));
} else {
ObMigrateReplicaTask task;
ObMigrateTaskInfo task_info;
common::ObArray<ObMigrateTaskInfo> task_info_array;
ObPartitionKey key;
ObReplicaMember data_source;
ObReplicaMember src = ObReplicaMember(
replica.server_->server_, replica.member_time_us_, replica.replica_type_, replica.get_memstore_percent());
const char* comment = balancer::PG_COORDINATE_MIGRATE;
OnlineReplica dst;
dst.member_ = ObReplicaMember(dest_replica.server_->server_,
ObTimeUtility::current_time(),
replica.replica_type_,
replica.get_memstore_percent());
dst.zone_ = dest_replica.zone_;
dst.member_.set_region(dest_replica.region_);
dst.unit_id_ = dest_replica.unit_->info_.unit_.unit_id_;
int64_t quorum = 0;
int64_t transmit_data_size = 0;
int64_t cluster_id = OB_INVALID_ID;
bool is_restore = partition.in_physical_restore();
const obrpc::MigrateMode migrate_mode = obrpc::MigrateMode::MT_LOCAL_FS_MODE;
if (OB_FAIL(ts.gen_partition_key(partition, key))) {
LOG_WARN("fail to gen partition key", K(ret), K(partition));
} else if (OB_FAIL(ts.choose_data_source(partition,
dst.member_,
src,
false,
data_source,
transmit_data_size,
ObRebalanceTaskType::MIGRATE_REPLICA,
cluster_id))) {
LOG_WARN("fail to choose data source", K(ret), K(partition), K(replica));
} else if (OB_FAIL(ts.get_migrate_replica_quorum_size(partition, quorum))) {
LOG_WARN("fail to get quorum size", K(ret));
} else if (FALSE_IT(task_info.set_transmit_data_size(transmit_data_size))) {
// nothing todo
} else if (OB_FAIL(task_info.build(migrate_mode, dst, key, src, data_source, quorum, is_restore))) {
LOG_WARN("fail to build migrate task info", K(ret));
} else if (OB_FAIL(task_info_array.push_back(task_info))) {
LOG_WARN("fail to push back", K(ret));
} else if (OB_FAIL(task.build(migrate_mode,
task_info_array,
dst.member_.get_server(),
ObRebalanceTaskPriority::LOW_PRI,
comment))) {
LOG_WARN("fail to build migrate task", K(ret), K(key), K(partition));
} else if (OB_FAIL(task_mgr_->add_task(task))) {
LOG_WARN("add task failed", K(ret), K(task));
} else {
can_add_task = true;
LOG_INFO("add migrate task for partition group coordinate", K(task));
}
}
return ret;
}
// Choose a migration destination for ppr
// First select the replica location of the same type in this zone; secondly,
// choose a replica location in this zone at will
int ObPartitionGroupCoordinator::get_migrate_dest(
const Partition& primary_partition, const Replica& ppr, const Partition& partition, Replica& dest)
{
int ret = OB_SUCCESS;
if (ppr.server_->can_migrate_in()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ppr));
} else {
Replica* same_type_replica = NULL;
Replica* same_zone_repilca = NULL;
TenantBalanceStat& ts = *tenant_stat_;
bool find = false;
FOR_BEGIN_END_E(pr, partition, ts.all_replica_, OB_SUCC(ret))
{
if (pr->is_in_service() && pr->zone_ == ppr.zone_) {
if (pr->server_->server_ == ppr.server_->server_) { // ppr->server has been blocked
// nothing todo
} else if (pr->unit_->info_.unit_.replica_type_ != ppr.unit_->info_.unit_.replica_type_) {
// If the unit types are not the same, can't migrate back and wait for the fault to be resolved
// nothing todo
} else if (pr->server_->can_migrate_in() && !ts.has_replica(primary_partition, pr->server_->server_)) {
same_zone_repilca = &(*pr);
if (pr->replica_type_ == ppr.replica_type_) {
same_type_replica = &(*pr);
}
}
}
} // end FOR_BEGIN_END_E(pr,
if (OB_SUCC(ret) && (same_type_replica != NULL || same_zone_repilca != NULL)) {
if (same_type_replica != NULL) {
dest = *same_type_replica;
find = true;
} else {
dest = *same_zone_repilca;
find = true;
}
}
// The above logic may not find a suitable location, need to find a unit to move in
if (OB_FAIL(ret) || find) {
// nothing todo
} else if (OB_FAIL(get_random_dest(primary_partition, ppr, dest))) {
LOG_WARN("fail to get random dest", K(ret));
}
}
return ret;
}
int ObPartitionGroupCoordinator::get_random_dest(const Partition& primary_partition, const Replica& ppr, Replica& dest)
{
int ret = OB_SUCCESS;
TenantBalanceStat& ts = *tenant_stat_;
bool find = false;
for (int64_t i = 0; i < ts.all_zone_unit_.count() && OB_SUCC(ret) && !find; i++) {
const ZoneUnit& zu = ts.all_zone_unit_.at(i);
if (zu.zone_ != ppr.zone_) {
// nothing todo
} else {
for (int64_t j = 0; j < zu.all_unit_.count() && OB_SUCC(ret); j++) {
UnitStat* us = zu.all_unit_.at(j);
if (!us->server_->can_migrate_in() || ts.has_replica(primary_partition, us->server_->server_) ||
(REPLICA_TYPE_LOGONLY == us->info_.unit_.replica_type_ &&
ppr.unit_->info_.unit_.replica_type_ != us->info_.unit_.replica_type_)) {
// nothing todo
} else {
find = true;
dest.unit_ = us;
dest.server_ = us->server_;
dest.zone_ = ppr.zone_;
dest.region_ = ppr.region_;
}
}
}
}
if (OB_SUCC(ret) && !find) {
ret = OB_ENTRY_NOT_EXIST;
LOG_WARN("fail to find dest for replica", K(primary_partition), K(ppr));
}
return ret;
}
Reference in New Issue View Git Blame Copy Permalink