database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Missing many files in last commit

Browse Source 
In last commit, a lot of files has been missed
This commit is contained in:
morningman
2018-10-31 16:19:21 +08:00
parent 5d3fc80067
commit 051aced48d
596 changed files with 16703 additions and 18103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
be/src/agent/agent_server.cpp
View File

@ -33,7 +33,6 @@
#include "gen_cpp/MasterService_types.h"
#include "gen_cpp/Status_types.h"
#include "olap/utils.h"
#include "olap/command_executor.h"
#include "runtime/exec_env.h"
#include "runtime/etl_job_mgr.h"
#include "util/debug_util.h"
@ -55,21 +54,17 @@ AgentServer::AgentServer(ExecEnv* exec_env,
_exec_env(exec_env),
_master_info(master_info),
_topic_subscriber(new TopicSubscriber()) {
// clean dpp download dir
_command_executor = new CommandExecutor();
vector<RootPathInfo> root_paths_info;
_command_executor->get_all_root_path_info(&root_paths_info, false);
for (auto root_path_info: root_paths_info) {
for (auto& path : exec_env->store_paths()) {
try {
string dpp_download_path_str = root_path_info.path + DPP_PREFIX;
string dpp_download_path_str = path.path + DPP_PREFIX;
boost::filesystem::path dpp_download_path(dpp_download_path_str);
if (boost::filesystem::exists(dpp_download_path)) {
boost::filesystem::remove_all(dpp_download_path);
}
} catch (...) {
OLAP_LOG_WARNING("boost exception when remove dpp download path. [path='%s']",
root_path_info.path.c_str());
LOG(WARNING) << "boost exception when remove dpp download path. path="
<< path.path;
}
}
@ -93,6 +88,18 @@ AgentServer::AgentServer(ExecEnv* exec_env,
TaskWorkerPool::TaskWorkerType::PUSH,
_exec_env,
master_info);
_publish_version_workers = new TaskWorkerPool(
TaskWorkerPool::TaskWorkerType::PUBLISH_VERSION,
_exec_env,
master_info);
_clear_alter_task_workers = new TaskWorkerPool(
TaskWorkerPool::TaskWorkerType::CLEAR_ALTER_TASK,
_exec_env,
master_info);
_clear_transaction_task_workers = new TaskWorkerPool(
TaskWorkerPool::TaskWorkerType::CLEAR_TRANSACTION_TASK,
exec_env,
master_info);
_delete_workers = new TaskWorkerPool(
TaskWorkerPool::TaskWorkerType::DELETE,
_exec_env,
@ -149,10 +156,17 @@ AgentServer::AgentServer(ExecEnv* exec_env,
TaskWorkerPool::TaskWorkerType::MOVE,
_exec_env,
master_info);
_recover_tablet_workers = new TaskWorkerPool(
TaskWorkerPool::TaskWorkerType::RECOVER_TABLET,
_exec_env,
master_info);
#ifndef BE_TEST
_create_table_workers->start();
_drop_table_workers->start();
_push_workers->start();
_publish_version_workers->start();
_clear_alter_task_workers->start();
_clear_transaction_task_workers->start();
_delete_workers->start();
_alter_table_workers->start();
_clone_workers->start();
@ -167,6 +181,7 @@ AgentServer::AgentServer(ExecEnv* exec_env,
_make_snapshot_workers->start();
_release_snapshot_workers->start();
_move_dir_workers->start();
_recover_tablet_workers->start();
// Add subscriber here and register listeners
TopicListener* user_resource_listener = new UserResourceListener(exec_env, master_info);
LOG(INFO) << "Register user resource listener";
@ -175,9 +190,6 @@ AgentServer::AgentServer(ExecEnv* exec_env,
}
AgentServer::~AgentServer() {
if (_command_executor != NULL) {
delete _command_executor;
}
if (_create_table_workers != NULL) {
delete _create_table_workers;
}
@ -187,6 +199,15 @@ AgentServer::~AgentServer() {
if (_push_workers != NULL) {
delete _push_workers;
}
if (_publish_version_workers != NULL) {
delete _publish_version_workers;
}
if (_clear_alter_task_workers != NULL) {
delete _clear_alter_task_workers;
}
if (_clear_transaction_task_workers != NULL) {
delete _clear_transaction_task_workers;
}
if (_delete_workers != NULL) {
delete _delete_workers;
}
@ -226,6 +247,9 @@ AgentServer::~AgentServer() {
if (_move_dir_workers!= NULL) {
delete _move_dir_workers;
}
if (_recover_tablet_workers != NULL) {
delete _recover_tablet_workers;
}
if (_release_snapshot_workers != NULL) {
delete _release_snapshot_workers;
}
@ -270,6 +294,7 @@ void AgentServer::submit_tasks(
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
case TTaskType::REALTIME_PUSH:
case TTaskType::PUSH:
if (task.__isset.push_req) {
if (task.push_req.push_type == TPushType::LOAD
@ -284,6 +309,27 @@ void AgentServer::submit_tasks(
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
case TTaskType::PUBLISH_VERSION:
if (task.__isset.publish_version_req) {
_publish_version_workers->submit_task(task);
} else {
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
case TTaskType::CLEAR_ALTER_TASK:
if (task.__isset.clear_alter_task_req) {
_clear_alter_task_workers->submit_task(task);
} else {
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
case TTaskType::CLEAR_TRANSACTION_TASK:
if (task.__isset.clear_transaction_task_req) {
_clear_transaction_task_workers->submit_task(task);
} else {
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
case TTaskType::ROLLUP:
case TTaskType::SCHEMA_CHANGE:
if (task.__isset.alter_tablet_req) {
@ -355,6 +401,13 @@ void AgentServer::submit_tasks(
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
case TTaskType::RECOVER_TABLET:
if (task.__isset.recover_tablet_req) {
_recover_tablet_workers->submit_task(task);
} else {
status_code = TStatusCode::ANALYSIS_ERROR;
}
break;
default:
status_code = TStatusCode::ANALYSIS_ERROR;
break;
@ -378,7 +431,7 @@ void AgentServer::make_snapshot(TAgentResult& return_value,
string snapshot_path;
OLAPStatus make_snapshot_status =
_command_executor->make_snapshot(snapshot_request, &snapshot_path);
_exec_env->olap_engine()->make_snapshot(snapshot_request, &snapshot_path);
if (make_snapshot_status != OLAP_SUCCESS) {
status_code = TStatusCode::RUNTIME_ERROR;
OLAP_LOG_WARNING("make_snapshot failed. tablet_id: %ld, schema_hash: %ld, status: %d",
@ -387,15 +440,17 @@ void AgentServer::make_snapshot(TAgentResult& return_value,
error_msgs.push_back("make_snapshot failed. status: " +
boost::lexical_cast<string>(make_snapshot_status));
} else {
OLAP_LOG_INFO("make_snapshot success. tablet_id: %ld, schema_hash: %ld, snapshot_path: %s",
snapshot_request.tablet_id, snapshot_request.schema_hash,
snapshot_path.c_str());
LOG(INFO) << "make_snapshot success. tablet_id: " << snapshot_request.tablet_id
<< " schema_hash: " << snapshot_request.schema_hash << " snapshot_path: " << snapshot_path;
return_value.__set_snapshot_path(snapshot_path);
}
status.__set_error_msgs(error_msgs);
status.__set_status_code(status_code);
return_value.__set_status(status);
if (snapshot_request.__isset.allow_incremental_clone) {
return_value.__set_allow_incremental_clone(snapshot_request.allow_incremental_clone);
}
}
void AgentServer::release_snapshot(TAgentResult& return_value, const std::string& snapshot_path) {
@ -403,16 +458,14 @@ void AgentServer::release_snapshot(TAgentResult& return_value, const std::string
TStatusCode::type status_code = TStatusCode::OK;
OLAPStatus release_snapshot_status =
_command_executor->release_snapshot(snapshot_path);
_exec_env->olap_engine()->release_snapshot(snapshot_path);
if (release_snapshot_status != OLAP_SUCCESS) {
status_code = TStatusCode::RUNTIME_ERROR;
OLAP_LOG_WARNING("release_snapshot failed. snapshot_path: %s, status: %d",
snapshot_path.c_str(), release_snapshot_status);
LOG(WARNING) << "release_snapshot failed. snapshot_path: " << snapshot_path << ", status: " << release_snapshot_status;
error_msgs.push_back("release_snapshot failed. status: " +
boost::lexical_cast<string>(release_snapshot_status));
} else {
OLAP_LOG_INFO("release_snapshot success. snapshot_path: %s, status: %d",
snapshot_path.c_str(), release_snapshot_status);
LOG(INFO) << "release_snapshot success. snapshot_path: " << snapshot_path << ", status: " << release_snapshot_status;
}
return_value.status.__set_error_msgs(error_msgs);