diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 9597102b9a..d00aa74f62 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -252,14 +252,13 @@ execute_process( # -fno-omit-frame-pointers: Keep frame pointer for functions in register set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-pragmas -pthread") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-strict-aliasing -fno-omit-frame-pointer") -set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++11") +set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++11 -D__STDC_FORMAT_MACROS") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse4.2") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DLLVM_ON_UNIX") -# for bprc if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new") endif() @@ -350,7 +349,6 @@ endif() # optimizations will be less effective if the initial code is also optimized. set(CLANG_IR_CXX_FLAGS "-gcc-toolchain" ${GCC_HOME}) set(CLANG_IR_CXX_FLAGS ${CLANG_IR_CXX_FLAGS} "-std=gnu++11" "-c" "-emit-llvm" "-D__STDC_CONSTANT_MACROS" "-D__STDC_FORMAT_MACROS" "-D__STDC_LIMIT_MACROS" "-DIR_COMPILE" "-DNDEBUG" "-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG" "-D__GLIBCXX_BITSIZE_INT_N_0=128" "-D__GLIBCXX_TYPE_INT_N_0=__int128" "-U_GLIBCXX_USE_FLOAT128" "-DLLVM_ON_UNIX") - if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) # for support float128 set(CLANG_IR_CXX_FLAGS ${CLANG_IR_CXX_FLAGS} "-D__STRICT_ANSI__") diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index 699306e1f8..cce954f29c 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -83,51 +83,71 @@ AgentServer::AgentServer(ExecEnv* exec_env, // init task worker pool _create_table_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::CREATE_TABLE, + _exec_env, master_info); _drop_table_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::DROP_TABLE, + _exec_env, master_info); _push_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::PUSH, + _exec_env, master_info); _delete_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::DELETE, + _exec_env, master_info); _alter_table_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + _exec_env, master_info); _clone_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::CLONE, + _exec_env, master_info); _storage_medium_migrate_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::STORAGE_MEDIUM_MIGRATE, + _exec_env, master_info); _cancel_delete_data_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::CANCEL_DELETE_DATA, + _exec_env, master_info); _check_consistency_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::CHECK_CONSISTENCY, + _exec_env, master_info); _report_task_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::REPORT_TASK, + _exec_env, master_info); _report_disk_state_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::REPORT_DISK_STATE, + _exec_env, master_info); _report_olap_table_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::REPORT_OLAP_TABLE, + _exec_env, master_info); _upload_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::UPLOAD, + _exec_env, master_info); - _restore_workers = new TaskWorkerPool( - TaskWorkerPool::TaskWorkerType::RESTORE, + _download_workers = new TaskWorkerPool( + TaskWorkerPool::TaskWorkerType::DOWNLOAD, + _exec_env, master_info); _make_snapshot_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::MAKE_SNAPSHOT, + _exec_env, master_info); _release_snapshot_workers = new TaskWorkerPool( TaskWorkerPool::TaskWorkerType::RELEASE_SNAPSHOT, + _exec_env, + master_info); + _move_dir_workers= new TaskWorkerPool( + TaskWorkerPool::TaskWorkerType::MOVE, + _exec_env, master_info); #ifndef BE_TEST _create_table_workers->start(); @@ -143,9 +163,10 @@ AgentServer::AgentServer(ExecEnv* exec_env, _report_disk_state_workers->start(); _report_olap_table_workers->start(); _upload_workers->start(); - _restore_workers->start(); + _download_workers->start(); _make_snapshot_workers->start(); _release_snapshot_workers->start(); + _move_dir_workers->start(); // Add subscriber here and register listeners TopicListener* user_resource_listener = new UserResourceListener(exec_env, master_info); LOG(INFO) << "Register user resource listener"; @@ -196,12 +217,15 @@ AgentServer::~AgentServer() { if (_upload_workers != NULL) { delete _upload_workers; } - if (_restore_workers != NULL) { - delete _restore_workers; + if (_download_workers != NULL) { + delete _download_workers; } if (_make_snapshot_workers != NULL) { delete _make_snapshot_workers; } + if (_move_dir_workers!= NULL) { + delete _move_dir_workers; + } if (_release_snapshot_workers != NULL) { delete _release_snapshot_workers; } @@ -303,9 +327,9 @@ void AgentServer::submit_tasks( status_code = TStatusCode::ANALYSIS_ERROR; } break; - case TTaskType::RESTORE: - if (task.__isset.restore_req) { - _restore_workers->submit_task(task); + case TTaskType::DOWNLOAD: + if (task.__isset.download_req) { + _download_workers->submit_task(task); } else { status_code = TStatusCode::ANALYSIS_ERROR; } @@ -324,6 +348,13 @@ void AgentServer::submit_tasks( status_code = TStatusCode::ANALYSIS_ERROR; } break; + case TTaskType::MOVE: + if (task.__isset.move_dir_req) { + _move_dir_workers->submit_task(task); + } else { + status_code = TStatusCode::ANALYSIS_ERROR; + } + break; default: status_code = TStatusCode::ANALYSIS_ERROR; break; @@ -391,7 +422,6 @@ void AgentServer::release_snapshot(TAgentResult& return_value, const std::string void AgentServer::publish_cluster_state(TAgentResult& _return, const TAgentPublishRequest& request) { vector error_msgs; _topic_subscriber->handle_updates(request); - OLAP_LOG_INFO("AgentService receive contains %d publish updates", request.updates.size()); _return.status.__set_status_code(TStatusCode::OK); } diff --git a/be/src/agent/agent_server.h b/be/src/agent/agent_server.h index bc354c65b1..a52d60ab1a 100644 --- a/be/src/agent/agent_server.h +++ b/be/src/agent/agent_server.h @@ -106,9 +106,10 @@ private: TaskWorkerPool* _report_disk_state_workers; TaskWorkerPool* _report_olap_table_workers; TaskWorkerPool* _upload_workers; - TaskWorkerPool* _restore_workers; + TaskWorkerPool* _download_workers; TaskWorkerPool* _make_snapshot_workers; TaskWorkerPool* _release_snapshot_workers; + TaskWorkerPool* _move_dir_workers; DISALLOW_COPY_AND_ASSIGN(AgentServer); diff --git a/be/src/agent/cgroups_mgr.cpp b/be/src/agent/cgroups_mgr.cpp index a5405ca0a0..7ccc2ddcc4 100644 --- a/be/src/agent/cgroups_mgr.cpp +++ b/be/src/agent/cgroups_mgr.cpp @@ -62,11 +62,8 @@ CgroupsMgr::~CgroupsMgr() { AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) { - LOG(INFO) << "Current resource version is " << _cur_version - << ". Resource version is " << new_fetched_resource.resourceVersion; std::lock_guard lck(_update_cgroups_mtx); if (!_is_cgroups_init_success) { - LOG(WARNING) << "Cgroups manager initialized failed, will not update local cgroups!"; return AgentStatus::PALO_ERROR; } diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 5a862a0988..ae5cce5e1a 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -48,10 +48,12 @@ void HeartbeatServer::heartbeat( TStatusCode::type status_code = TStatusCode::OK; vector error_msgs; TStatus heartbeat_status; - OLAP_LOG_INFO("get heartbeat, host: %s, port: %d, cluster id: %d", - master_info.network_address.hostname.c_str(), - master_info.network_address.port, - master_info.cluster_id); + //print heartbeat in every minute + LOG_EVERY_N(INFO, 12) << "get heartbeat from FE." + << "host:" << master_info.network_address.hostname << ", " + << "port:" << master_info.network_address.port << ", " + << "cluster id:" << master_info.cluster_id << ", " + << "counter:" << google::COUNTER; // Check cluster id if (_master_info->cluster_id == -1) { diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 8ec8ad8501..f7d6fba95b 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -37,8 +37,12 @@ #include "olap/olap_table.h" #include "olap/utils.h" #include "common/resource_tls.h" +#include "common/status.h" +#include "util/file_utils.h" #include "agent/cgroups_mgr.h" #include "service/backend_options.h" +#include "runtime/exec_env.h" +#include "runtime/snapshot_loader.h" #include "util/palo_metrics.h" using std::deque; @@ -77,6 +81,7 @@ boost::posix_time::time_duration TaskWorkerPool::_wait_duration; TaskWorkerPool::TaskWorkerPool( const TaskWorkerType task_worker_type, + ExecEnv* env, const TMasterInfo& master_info) : _master_info(master_info), _worker_thread_condition_lock(_worker_thread_lock), @@ -84,6 +89,7 @@ TaskWorkerPool::TaskWorkerPool( _agent_utils = new AgentUtils(); _master_client = new MasterServerClient(_master_info, &_master_service_client_cache); _command_executor = new CommandExecutor(); + _env = env; _backend.__set_host(BackendOptions::get_localhost()); _backend.__set_be_port(config::be_port); _backend.__set_http_port(config::webserver_port); @@ -162,9 +168,9 @@ void TaskWorkerPool::start() { _worker_count = config::upload_worker_count; _callback_function = _upload_worker_thread_callback; break; - case TaskWorkerType::RESTORE: - _worker_count = config::restore_worker_count; - _callback_function = _restore_worker_thread_callback; + case TaskWorkerType::DOWNLOAD: + _worker_count = config::download_worker_count; + _callback_function = _download_worker_thread_callback; break; case TaskWorkerType::MAKE_SNAPSHOT: _worker_count = config::make_snapshot_worker_count; @@ -174,6 +180,10 @@ void TaskWorkerPool::start() { _worker_count = config::release_snapshot_worker_count; _callback_function = _release_snapshot_thread_callback; break; + case TaskWorkerType::MOVE: + _worker_count = 1; + _callback_function = _move_dir_thread_callback; + break; default: // pass break; @@ -1008,7 +1018,6 @@ AgentStatus TaskWorkerPool::_clone_copy( vector* error_msgs) { AgentStatus status = PALO_SUCCESS; - std::string token = _master_info.token; for (auto src_backend : clone_req.src_backends) { @@ -1509,20 +1518,16 @@ void* TaskWorkerPool::_report_task_worker_thread_callback(void* arg_this) { lock_guard task_signatures_lock(_s_task_signatures_lock); request.__set_tasks(_s_task_signatures); } - OLAP_LOG_INFO("master host: %s, port: %d", - worker_pool_this->_master_info.network_address.hostname.c_str(), - worker_pool_this->_master_info.network_address.port); PaloMetrics::report_task_requests_total.increment(1); TMasterResult result; AgentStatus status = worker_pool_this->_master_client->report(request, &result); - if (status == PALO_SUCCESS) { - OLAP_LOG_INFO("finish report task success. return code: %d", - result.status.status_code); - } else { + if (status != PALO_SUCCESS) { PaloMetrics::report_task_requests_failed.increment(1); - OLAP_LOG_WARNING("finish report task failed. status: %d", status); + LOG(WARNING) << "finish report task failed. status:" << status << ", " + << "master host:" << worker_pool_this->_master_info.network_address.hostname << ", " + << "port:" << worker_pool_this->_master_info.network_address.port; } #ifndef BE_TEST @@ -1570,12 +1575,11 @@ void* TaskWorkerPool::_report_disk_state_worker_thread_callback(void* arg_this) TMasterResult result; AgentStatus status = worker_pool_this->_master_client->report(request, &result); - if (status == PALO_SUCCESS) { - OLAP_LOG_INFO("finish report disk state success. return code: %d", - result.status.status_code); - } else { + if (status != PALO_SUCCESS) { PaloMetrics::report_disk_requests_failed.increment(1); - OLAP_LOG_WARNING("finish report disk state failed. status: %d", status); + LOG(WARNING) << "finish report disk state failed. status:" << status << ", " + << "master host:" << worker_pool_this->_master_info.network_address.hostname << ", " + << "port:" << worker_pool_this->_master_info.network_address.port; } #ifndef BE_TEST @@ -1638,12 +1642,11 @@ void* TaskWorkerPool::_report_olap_table_worker_thread_callback(void* arg_this) TMasterResult result; status = worker_pool_this->_master_client->report(request, &result); - if (status == PALO_SUCCESS) { - OLAP_LOG_INFO("finish report olap table success. return code: %d", - result.status.status_code); - } else { + if (status != PALO_SUCCESS) { PaloMetrics::report_all_tablets_requests_failed.increment(1); - OLAP_LOG_WARNING("finish report olap table failed. status: %d", status); + LOG(WARNING) << "finish report olap table state failed. status:" << status << ", " + << "master host:" << worker_pool_this->_master_info.network_address.hostname << ", " + << "port:" << worker_pool_this->_master_info.network_address.port; } #ifndef BE_TEST @@ -1678,58 +1681,30 @@ void* TaskWorkerPool::_upload_worker_thread_callback(void* arg_this) { upload_request = agent_task_req.upload_req; worker_pool_this->_tasks.pop_front(); } - // Try to register to cgroups_mgr - CgroupsMgr::apply_system_cgroup(); - OLAP_LOG_INFO("get upload task, signature: %ld", agent_task_req.signature); - TStatusCode::type status_code = TStatusCode::OK; - vector error_msgs; - TStatus task_status; + OLAP_LOG_INFO("get upload task, signature: %ld, job id: %d", + agent_task_req.signature, upload_request.job_id); - // Write remote source info into file by json format - pthread_t tid = pthread_self(); - time_t now = time(NULL); - stringstream label_stream; - label_stream << tid << "_" << now; - string label(label_stream.str()); - string info_file_path(config::agent_tmp_dir + "/" + label); - bool ret = worker_pool_this->_agent_utils->write_json_to_file( - upload_request.remote_source_properties, - info_file_path); - if (!ret) { + std::map> tablet_files; + SnapshotLoader* loader = worker_pool_this->_env->snapshot_loader(); + Status status = loader->upload( + upload_request.src_dest_map, + upload_request.broker_addr, + upload_request.broker_prop, + upload_request.job_id, + &tablet_files); + + TStatusCode::type status_code = TStatusCode::OK; + std::vector error_msgs; + if (!status.ok()) { status_code = TStatusCode::RUNTIME_ERROR; - error_msgs.push_back("Write remote source info to file failed. Path:" + - info_file_path); - OLAP_LOG_WARNING("Write remote source info to file failed. Path: %s", - info_file_path.c_str()); - } - - // Upload files to remote source - stringstream local_file_path_stream; - local_file_path_stream << upload_request.local_file_path; - if (upload_request.__isset.tablet_id) { - local_file_path_stream << "/" << upload_request.tablet_id; - } - if (status_code == TStatusCode::OK) { - string command = "sh " + config::trans_file_tool_path + " " + label + " upload " + - local_file_path_stream.str() + " " + upload_request.remote_file_path + - " " + info_file_path + " " + "file_list"; - OLAP_LOG_INFO("Upload cmd: %s", command.c_str()); - string errmsg; - ret = worker_pool_this->_agent_utils->exec_cmd(command, &errmsg); - if (!ret) { - status_code = TStatusCode::RUNTIME_ERROR; - error_msgs.push_back(errmsg); - OLAP_LOG_WARNING("Upload file failed. Error: %s", errmsg.c_str()); - } - } - - // Delete tmp file - boost::filesystem::path file_path(info_file_path); - if (boost::filesystem::exists(file_path)) { - boost::filesystem::remove_all(file_path); + OLAP_LOG_WARNING("upload failed. job id: %ld, msg: %s", + upload_request.job_id, + status.get_error_msg().c_str()); + error_msgs.push_back(status.get_error_msg()); } + TStatus task_status; task_status.__set_status_code(status_code); task_status.__set_error_msgs(error_msgs); @@ -1738,23 +1713,27 @@ void* TaskWorkerPool::_upload_worker_thread_callback(void* arg_this) { finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_task_status(task_status); + finish_task_request.__set_tablet_files(tablet_files); worker_pool_this->_finish_task(finish_task_request); worker_pool_this->_remove_task_info(agent_task_req.task_type, agent_task_req.signature, ""); + + OLAP_LOG_INFO("finished upload task, signature: %ld, job id: %ld", + agent_task_req.signature, upload_request.job_id); #ifndef BE_TEST } #endif return (void*)0; } -void* TaskWorkerPool::_restore_worker_thread_callback(void* arg_this) { +void* TaskWorkerPool::_download_worker_thread_callback(void* arg_this) { TaskWorkerPool* worker_pool_this = (TaskWorkerPool*)arg_this; #ifndef BE_TEST while (true) { #endif TAgentTaskRequest agent_task_req; - TRestoreReq restore_request; + TDownloadReq download_request; { lock_guard worker_thread_lock(worker_pool_this->_worker_thread_lock); while (worker_pool_this->_tasks.empty()) { @@ -1762,169 +1741,51 @@ void* TaskWorkerPool::_restore_worker_thread_callback(void* arg_this) { } agent_task_req = worker_pool_this->_tasks.front(); - restore_request = agent_task_req.restore_req; + download_request = agent_task_req.download_req; worker_pool_this->_tasks.pop_front(); } // Try to register to cgroups_mgr CgroupsMgr::apply_system_cgroup(); - OLAP_LOG_INFO("get restore task, signature: %ld", agent_task_req.signature); + OLAP_LOG_INFO("get download task, signature: %ld, job id: %ld", + agent_task_req.signature, download_request.job_id); TStatusCode::type status_code = TStatusCode::OK; - vector error_msgs; + std::vector error_msgs; TStatus task_status; - // Write remote source info into file by json format - pthread_t tid = pthread_self(); - time_t now = time(NULL); - stringstream label_stream; - label_stream << tid << "_" << now << "_" << restore_request.tablet_id; - string label(label_stream.str()); - string info_file_path(config::agent_tmp_dir + "/" + label); - bool ret = worker_pool_this->_agent_utils->write_json_to_file( - restore_request.remote_source_properties, - info_file_path); - if (!ret) { + // TODO: download + std::vector downloaded_tablet_ids; + SnapshotLoader* loader = worker_pool_this->_env->snapshot_loader(); + Status status = loader->download( + download_request.src_dest_map, + download_request.broker_addr, + download_request.broker_prop, + download_request.job_id, + &downloaded_tablet_ids); + + if (!status.ok()) { status_code = TStatusCode::RUNTIME_ERROR; - error_msgs.push_back("Write remote source info to file failed. Path:" + - info_file_path); - OLAP_LOG_WARNING("Write remote source info to file failed. Path: %s", - info_file_path.c_str()); - } - - // Get local disk to restore from olap - string local_shard_root_path; - if (status_code == TStatusCode::OK) { - OLAPStatus olap_status = worker_pool_this->_command_executor->obtain_shard_path( - TStorageMedium::HDD, &local_shard_root_path); - if (olap_status != OLAP_SUCCESS) { - OLAP_LOG_WARNING("clone get local root path failed. signature: %ld", - agent_task_req.signature); - error_msgs.push_back("clone get local root path failed."); - status_code = TStatusCode::RUNTIME_ERROR; - } - } - - stringstream local_file_path_stream; - local_file_path_stream << local_shard_root_path << "/" << restore_request.tablet_id << "/"; - string local_file_path(local_file_path_stream.str()); - - // Download files from remote source - if (status_code == TStatusCode::OK) { - string command = "sh " + config::trans_file_tool_path + " " + label + " download " + - local_file_path + " " + restore_request.remote_file_path + - " " + info_file_path; - OLAP_LOG_INFO("Download cmd: %s", command.c_str()); - string errmsg; - ret = worker_pool_this->_agent_utils->exec_cmd(command, &errmsg); - if (!ret) { - status_code = TStatusCode::RUNTIME_ERROR; - error_msgs.push_back(errmsg); - OLAP_LOG_WARNING("Download file failed. Error: %s", errmsg.c_str()); - } - } - - // Delete tmp file - boost::filesystem::path file_path(info_file_path); - if (boost::filesystem::exists(file_path)) { - boost::filesystem::remove_all(file_path); - } - - // Change file name - boost::filesystem::path blocal_file_path(local_file_path); - if (status_code == TStatusCode::OK && boost::filesystem::exists(blocal_file_path)) { - boost::filesystem::recursive_directory_iterator end_iter; - for (boost::filesystem::recursive_directory_iterator file_path(blocal_file_path); - file_path != end_iter; ++file_path) { - if (boost::filesystem::is_directory(*file_path)) { - continue; - } - - // Check file name - string file_path_str = file_path->path().string(); - string file_name = file_path_str; - uint32_t slash_pos = file_path_str.rfind('/'); - if (slash_pos != -1) { - file_name = file_path_str.substr(slash_pos + 1); - } - uint32_t file_path_str_len = file_name.size(); - if (file_path_str_len <= 4) { - continue; - } - string file_path_suffix = file_name.substr(file_path_str_len - 4); - if (file_path_suffix != ".hdr" && file_path_suffix != ".idx" && - file_path_suffix != ".dat") { - continue; - } - - // Get new file name - stringstream new_file_name_stream; - char sperator = '_'; - if (file_path_suffix == ".hdr") { - sperator = '.'; - } - uint32_t sperator_pos = file_name.find(sperator); - new_file_name_stream << restore_request.tablet_id << file_name.substr(sperator_pos); - string new_file_path_str = file_path_str.substr(0, slash_pos) + "/" + - new_file_name_stream.str(); - - OLAP_LOG_INFO("change file name %s to %s", - file_path_str.c_str(), new_file_path_str.c_str()); - // Change file name to new one - boost::filesystem::path new_file_path(new_file_path_str); - boost::filesystem::rename(*file_path, new_file_path); - } - } - - // Load olap - if (status_code == TStatusCode::OK) { - OLAPStatus load_header_status = - worker_pool_this->_command_executor->load_header( - local_shard_root_path, - restore_request.tablet_id, - restore_request.schema_hash); - if (load_header_status != OLAP_SUCCESS) { - OLAP_LOG_WARNING("load header failed. local_shard_root_path: %s, tablet_id: %d " - "schema_hash: %d, status: %d, signature: %ld", - local_shard_root_path.c_str(), restore_request.tablet_id, - restore_request.schema_hash, load_header_status, - agent_task_req.signature); - error_msgs.push_back("load header failed."); - status_code = TStatusCode::RUNTIME_ERROR;; - } - } - - // Get tablets info - vector finish_tablet_infos; - if (status_code == TStatusCode::OK) { - TTabletInfo tablet_info; - AgentStatus get_tablet_info_status = worker_pool_this->_get_tablet_info( - restore_request.tablet_id, - restore_request.schema_hash, - agent_task_req.signature, - &tablet_info); - - if (get_tablet_info_status != PALO_SUCCESS) { - OLAP_LOG_WARNING("Restore success, but get new tablet info failed." - "tablet_id: %ld, schema_hash: %ld, signature: %ld.", - restore_request.tablet_id, restore_request.schema_hash, - agent_task_req.signature); - } else { - finish_tablet_infos.push_back(tablet_info); - } + OLAP_LOG_WARNING("download failed. job id: %ld, msg: %s", + download_request.job_id, + status.get_error_msg().c_str()); + error_msgs.push_back(status.get_error_msg()); } task_status.__set_status_code(status_code); task_status.__set_error_msgs(error_msgs); TFinishTaskRequest finish_task_request; - finish_task_request.__set_finish_tablet_infos(finish_tablet_infos); finish_task_request.__set_backend(worker_pool_this->_backend); finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_task_status(task_status); + finish_task_request.__set_downloaded_tablet_ids(downloaded_tablet_ids); worker_pool_this->_finish_task(finish_task_request); worker_pool_this->_remove_task_info(agent_task_req.task_type, agent_task_req.signature, ""); + + OLAP_LOG_INFO("finished download task, signature: %ld, job id: %d", + agent_task_req.signature, download_request.job_id); #ifndef BE_TEST } #endif @@ -1958,6 +1819,7 @@ void* TaskWorkerPool::_make_snapshot_thread_callback(void* arg_this) { TStatus task_status; string snapshot_path; + std::vector snapshot_files; OLAPStatus make_snapshot_status = worker_pool_this->_command_executor->make_snapshot( snapshot_request, &snapshot_path); if (make_snapshot_status != OLAP_SUCCESS) { @@ -1975,6 +1837,26 @@ void* TaskWorkerPool::_make_snapshot_thread_callback(void* arg_this) { snapshot_request.tablet_id, snapshot_request.schema_hash, snapshot_request.version, snapshot_request.version_hash, snapshot_path.c_str()); + + if (snapshot_request.__isset.list_files) { + // list and save all snapshot files + // snapshot_path like: data/snapshot/20180417205230.1 + // we need to add subdir: tablet_id/schema_hash/ + std::stringstream ss; + ss << snapshot_path << "/" << snapshot_request.tablet_id + << "/" << snapshot_request.schema_hash << "/"; + Status st = FileUtils::scan_dir(ss.str(), &snapshot_files); + if (!st.ok()) { + status_code = TStatusCode::RUNTIME_ERROR; + OLAP_LOG_WARNING("make_snapshot failed. tablet_id: %ld, schema_hash: %ld, version: %d," + "version_hash: %ld, list file failed: %s", + snapshot_request.tablet_id, snapshot_request.schema_hash, + snapshot_request.version, snapshot_request.version_hash, + st.get_error_msg().c_str()); + error_msgs.push_back("make_snapshot failed. list file failed: " + + st.get_error_msg()); + } + } } task_status.__set_status_code(status_code); @@ -1985,6 +1867,7 @@ void* TaskWorkerPool::_make_snapshot_thread_callback(void* arg_this) { finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_snapshot_path(snapshot_path); + finish_task_request.__set_snapshot_files(snapshot_files); finish_task_request.__set_task_status(task_status); worker_pool_this->_finish_task(finish_task_request); @@ -2087,4 +1970,102 @@ AgentStatus TaskWorkerPool::_get_tablet_info( } return status; } + +void* TaskWorkerPool::_move_dir_thread_callback(void* arg_this) { + TaskWorkerPool* worker_pool_this = (TaskWorkerPool*)arg_this; + +#ifndef BE_TEST + while (true) { +#endif + TAgentTaskRequest agent_task_req; + TMoveDirReq move_dir_req; + { + lock_guard worker_thread_lock(worker_pool_this->_worker_thread_lock); + while (worker_pool_this->_tasks.empty()) { + worker_pool_this->_worker_thread_condition_lock.wait(); + } + + agent_task_req = worker_pool_this->_tasks.front(); + move_dir_req = agent_task_req.move_dir_req; + worker_pool_this->_tasks.pop_front(); + } + // Try to register to cgroups_mgr + CgroupsMgr::apply_system_cgroup(); + OLAP_LOG_INFO("get move dir task, signature: %ld, job id: %ld", + agent_task_req.signature, move_dir_req.job_id); + + TStatusCode::type status_code = TStatusCode::OK; + vector error_msgs; + TStatus task_status; + + // TODO: move dir + AgentStatus status = worker_pool_this->_move_dir( + move_dir_req.tablet_id, + move_dir_req.schema_hash, + move_dir_req.src, + move_dir_req.job_id, + true /* TODO */, + &error_msgs); + + if (status != PALO_SUCCESS) { + status_code = TStatusCode::RUNTIME_ERROR; + OLAP_LOG_WARNING("failed to move dir: %s, tablet id: %ld, signature: %ld, job id: %ld", + move_dir_req.src.c_str(), move_dir_req.tablet_id, agent_task_req.signature, + move_dir_req.job_id); + } else { + OLAP_LOG_INFO("finished to move dir: %s, tablet id: %ld, signature: %ld, job id: %ld", + move_dir_req.src.c_str(), move_dir_req.tablet_id, agent_task_req.signature, + move_dir_req.job_id); + } + + task_status.__set_status_code(status_code); + task_status.__set_error_msgs(error_msgs); + + TFinishTaskRequest finish_task_request; + finish_task_request.__set_backend(worker_pool_this->_backend); + finish_task_request.__set_task_type(agent_task_req.task_type); + finish_task_request.__set_signature(agent_task_req.signature); + finish_task_request.__set_task_status(task_status); + + worker_pool_this->_finish_task(finish_task_request); + worker_pool_this->_remove_task_info(agent_task_req.task_type, agent_task_req.signature, ""); + +#ifndef BE_TEST + } +#endif + return (void*)0; +} + +AgentStatus TaskWorkerPool::_move_dir( + const TTabletId tablet_id, + const TSchemaHash schema_hash, + const std::string& src, + int64_t job_id, + bool overwrite, + std::vector* error_msgs) { + + SmartOLAPTable tablet = _command_executor->get_table( + tablet_id, schema_hash); + if (tablet.get() == NULL) { + OLAP_LOG_INFO("failed to get tablet: %ld, schema hash: %d", + tablet_id, schema_hash); + error_msgs->push_back("failed to get tablet"); + return PALO_TASK_REQUEST_ERROR; + } + + std::string dest_tablet_dir = tablet->construct_dir_path(); + + SnapshotLoader* loader = _env->snapshot_loader(); + Status status = loader->move(src, dest_tablet_dir, job_id, overwrite); + + if (!status.ok()) { + OLAP_LOG_WARNING("move failed. job id: %ld, msg: %s", + job_id, status.get_error_msg().c_str()); + error_msgs->push_back(status.get_error_msg()); + return PALO_INTERNAL_ERROR; + } + + return PALO_SUCCESS; +} + } // namespace palo diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h index 8f2ed61d58..a2531b0835 100644 --- a/be/src/agent/task_worker_pool.h +++ b/be/src/agent/task_worker_pool.h @@ -35,6 +35,8 @@ namespace palo { +class ExecEnv; + class TaskWorkerPool { public: enum TaskWorkerType { @@ -52,15 +54,17 @@ public: REPORT_DISK_STATE, REPORT_OLAP_TABLE, UPLOAD, - RESTORE, + DOWNLOAD, MAKE_SNAPSHOT, - RELEASE_SNAPSHOT + RELEASE_SNAPSHOT, + MOVE }; typedef void* (*CALLBACK_FUNCTION)(void*); TaskWorkerPool( const TaskWorkerType task_worker_type, + ExecEnv* env, const TMasterInfo& master_info); virtual ~TaskWorkerPool(); @@ -95,9 +99,10 @@ private: static void* _report_disk_state_worker_thread_callback(void* arg_this); static void* _report_olap_table_worker_thread_callback(void* arg_this); static void* _upload_worker_thread_callback(void* arg_this); - static void* _restore_worker_thread_callback(void* arg_this); + static void* _download_worker_thread_callback(void* arg_this); static void* _make_snapshot_thread_callback(void* arg_this); static void* _release_snapshot_thread_callback(void* arg_this); + static void* _move_dir_thread_callback(void* arg_this); AgentStatus _clone_copy( const TCloneReq& clone_req, @@ -125,11 +130,20 @@ private: int64_t signature, TTabletInfo* tablet_info); + AgentStatus _move_dir( + const TTabletId tablet_id, + const TSchemaHash schema_hash, + const std::string& src, + int64_t job_id, + bool overwrite, + std::vector* error_msgs); + const TMasterInfo& _master_info; TBackend _backend; AgentUtils* _agent_utils; MasterServerClient* _master_client; CommandExecutor* _command_executor; + ExecEnv* _env; #ifdef BE_TEST AgentServerClient* _agent_client; FileDownloader* _file_downloader_ptr; diff --git a/be/src/agent/topic_subscriber.cpp b/be/src/agent/topic_subscriber.cpp index f2d7161bbc..bd2ee00f49 100644 --- a/be/src/agent/topic_subscriber.cpp +++ b/be/src/agent/topic_subscriber.cpp @@ -41,7 +41,6 @@ void TopicSubscriber::register_listener(TTopicType::type topic_type, TopicListen } void TopicSubscriber::handle_updates(const TAgentPublishRequest& agent_publish_request) { - LOG(INFO) << "Received master's published state, begin to handle"; // Shared lock here in order to avoid updates in listeners' map boost::shared_lock lock(_listener_mtx); // Currently, not deal with protocol version, the listener should deal with protocol version @@ -56,6 +55,5 @@ void TopicSubscriber::handle_updates(const TAgentPublishRequest& agent_publish_r *topic_update_it); } } - LOG(INFO) << "Handle master's published state finished"; } } // namespace palo diff --git a/be/src/agent/user_resource_listener.cpp b/be/src/agent/user_resource_listener.cpp index 09dd7fb402..a7a8471b01 100644 --- a/be/src/agent/user_resource_listener.cpp +++ b/be/src/agent/user_resource_listener.cpp @@ -49,7 +49,6 @@ void UserResourceListener::handle_update(const TAgentServiceVersion::type& proto if (updates.size() > 0) { int64_t new_version = updates[0].int_value; // Async call to update users resource method - LOG(INFO) << "Latest version for master is " << new_version; std::async(std::launch::async, &UserResourceListener::update_users_resource, this, new_version); @@ -60,8 +59,6 @@ void UserResourceListener::update_users_resource(int64_t new_version) { if (new_version <= _cgroups_mgr.get_cgroups_version()) { return; } - LOG(INFO) << "New version " << new_version - << " is bigger than older version " << _cgroups_mgr.get_cgroups_version(); // Call fe to get latest user resource Status master_status; // using 500ms as default timeout value @@ -78,9 +75,7 @@ void UserResourceListener::update_users_resource(int64_t new_version) { } try { try { - LOG(INFO) << "Call master to get resource"; client->fetchResource(new_fetched_resource); - LOG(INFO) << "Call master to get resource successfully"; } catch (TTransportException& e) { // reopen the client and set timeout to 500ms master_status = client.reopen(500); @@ -102,7 +97,6 @@ void UserResourceListener::update_users_resource(int64_t new_version) { << e.what(); return; } - LOG(INFO) << "Begin to update user's cgroups resource"; _cgroups_mgr.update_local_cgroups(new_fetched_resource); } } diff --git a/be/src/common/config.h b/be/src/common/config.h index 71a6a4137a..caf7bfb2f9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -79,8 +79,8 @@ namespace config { CONF_Int32(check_consistency_worker_count, "1"); // the count of thread to upload CONF_Int32(upload_worker_count, "3"); - // the count of thread to restore - CONF_Int32(restore_worker_count, "3"); + // the count of thread to download + CONF_Int32(download_worker_count, "3"); // the count of thread to make snapshot CONF_Int32(make_snapshot_worker_count, "5"); // the count of thread to release snapshot @@ -364,6 +364,9 @@ namespace config { // Aligement CONF_Int32(FLAGS_MEMORY_MAX_ALIGNMENT, "16"); + + // result buffer cancelled time (unit: second) + CONF_Int32(result_buffer_cancelled_interval_time, "5"); } // namespace config } // namespace palo diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 3283debc98..347da037d0 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -63,7 +63,7 @@ set(EXEC_FILES mysql_scanner.cpp csv_scan_node.cpp csv_scanner.cpp - spill_sort_node.cc + spill_sort_node.cc union_node.cpp union_node_ir.cpp schema_scanner.cpp diff --git a/be/src/exec/broker_reader.cpp b/be/src/exec/broker_reader.cpp index 66d58abeac..47db215ba6 100644 --- a/be/src/exec/broker_reader.cpp +++ b/be/src/exec/broker_reader.cpp @@ -23,7 +23,6 @@ #include "runtime/broker_mgr.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" -#include "runtime/runtime_state.h" #include "util/thrift_util.h" namespace palo { @@ -31,12 +30,12 @@ namespace palo { // Broker BrokerReader::BrokerReader( - RuntimeState* state, + ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& path, int64_t start_offset) : - _state(state), + _env(env), _addresses(broker_addresses), _properties(properties), _path(path), @@ -51,22 +50,22 @@ BrokerReader::~BrokerReader() { } #ifdef BE_TEST -inline BrokerServiceClientCache* client_cache(RuntimeState* state) { +inline BrokerServiceClientCache* client_cache(ExecEnv* env) { static BrokerServiceClientCache s_client_cache; return &s_client_cache; } -inline const std::string& client_id(RuntimeState* state, const TNetworkAddress& addr) { +inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { static std::string s_client_id = "palo_unit_test"; return s_client_id; } #else -inline BrokerServiceClientCache* client_cache(RuntimeState* state) { - return state->exec_env()->broker_client_cache(); +inline BrokerServiceClientCache* client_cache(ExecEnv* env) { + return env->broker_client_cache(); } -inline const std::string& client_id(RuntimeState* state, const TNetworkAddress& addr) { - return state->exec_env()->broker_mgr()->get_client_id(addr); +inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { + return env->broker_mgr()->get_client_id(addr); } #endif @@ -77,13 +76,13 @@ Status BrokerReader::open() { request.__set_version(TBrokerVersion::VERSION_ONE); request.__set_path(_path); request.__set_startOffset(_cur_offset); - request.__set_clientId(client_id(_state, broker_addr)); + request.__set_clientId(client_id(_env, broker_addr)); request.__set_properties(_properties); TBrokerOpenReaderResponse response; try { Status status; - BrokerServiceConnection client(client_cache(_state), broker_addr, 10000, &status); + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); if (!status.ok()) { LOG(WARNING) << "Create broker client failed. broker=" << broker_addr << ", status=" << status.get_error_msg(); @@ -132,7 +131,7 @@ Status BrokerReader::read(uint8_t* buf, size_t* buf_len, bool* eof) { TBrokerReadResponse response; try { Status status; - BrokerServiceConnection client(client_cache(_state), broker_addr, 10000, &status); + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); if (!status.ok()) { LOG(WARNING) << "Create broker client failed. broker=" << broker_addr << ", status=" << status.get_error_msg(); @@ -186,8 +185,7 @@ void BrokerReader::close() { TBrokerOperationStatus response; try { Status status; - // 500ms is enough - BrokerServiceConnection client(client_cache(_state), broker_addr, 10000, &status); + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); if (!status.ok()) { LOG(WARNING) << "Create broker client failed. broker=" << broker_addr << ", status=" << status.get_error_msg(); diff --git a/be/src/exec/broker_reader.h b/be/src/exec/broker_reader.h index f4501a81d5..1d878ef6d3 100644 --- a/be/src/exec/broker_reader.h +++ b/be/src/exec/broker_reader.h @@ -27,7 +27,7 @@ namespace palo { -class RuntimeState; +class ExecEnv; class TBrokerRangeDesc; class TNetworkAddress; class RuntimeState; @@ -35,7 +35,7 @@ class RuntimeState; // Reader of broker file class BrokerReader : public FileReader { public: - BrokerReader(RuntimeState* state, + BrokerReader(ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& path, @@ -49,7 +49,7 @@ public: virtual void close() override; private: - RuntimeState* _state; + ExecEnv* _env; const std::vector& _addresses; const std::map& _properties; const std::string& _path; diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp index 1143a68f2a..2bde4a7f58 100644 --- a/be/src/exec/broker_scanner.cpp +++ b/be/src/exec/broker_scanner.cpp @@ -246,7 +246,7 @@ Status BrokerScanner::open_file_reader() { } case TFileType::FILE_BROKER: { BrokerReader* broker_reader = new BrokerReader( - _state, _broker_addresses, _params.properties, range.path, start_offset); + _state->exec_env(), _broker_addresses, _params.properties, range.path, start_offset); RETURN_IF_ERROR(broker_reader->open()); _cur_file_reader = broker_reader; break; diff --git a/be/src/exec/broker_writer.cpp b/be/src/exec/broker_writer.cpp index 4742269165..5511e0febb 100644 --- a/be/src/exec/broker_writer.cpp +++ b/be/src/exec/broker_writer.cpp @@ -23,18 +23,17 @@ #include "runtime/broker_mgr.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" -#include "runtime/runtime_state.h" #include "util/thrift_util.h" namespace palo { BrokerWriter::BrokerWriter( - RuntimeState* state, + ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& path, int64_t start_offset) : - _state(state), + _env(env), _addresses(broker_addresses), _properties(properties), _path(path), @@ -48,22 +47,22 @@ BrokerWriter::~BrokerWriter() { } #ifdef BE_TEST -inline BrokerServiceClientCache* client_cache(RuntimeState* state) { +inline BrokerServiceClientCache* client_cache(ExecEnv* env) { static BrokerServiceClientCache s_client_cache; return &s_client_cache; } -inline const std::string& client_id(RuntimeState* state, const TNetworkAddress& addr) { +inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { static std::string s_client_id = "palo_unit_test"; return s_client_id; } #else -inline BrokerServiceClientCache* client_cache(RuntimeState* state) { - return state->exec_env()->broker_client_cache(); +inline BrokerServiceClientCache* client_cache(ExecEnv* env) { + return env->broker_client_cache(); } -inline const std::string& client_id(RuntimeState* state, const TNetworkAddress& addr) { - return state->exec_env()->broker_mgr()->get_client_id(addr); +inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { + return env->broker_mgr()->get_client_id(addr); } #endif @@ -74,7 +73,7 @@ Status BrokerWriter::open() { request.__set_version(TBrokerVersion::VERSION_ONE); request.__set_path(_path); request.__set_openMode(TBrokerOpenMode::APPEND); - request.__set_clientId(client_id(_state, broker_addr)); + request.__set_clientId(client_id(_env, broker_addr)); request.__set_properties(_properties); VLOG_ROW << "debug: send broker open writer request: " @@ -83,7 +82,7 @@ Status BrokerWriter::open() { TBrokerOpenWriterResponse response; try { Status status; - BrokerServiceConnection client(client_cache(_state), broker_addr, 10000, &status); + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); if (!status.ok()) { LOG(WARNING) << "Create broker writer client failed. " << "broker=" << broker_addr @@ -138,8 +137,7 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l TBrokerOperationStatus response; try { Status status; - // we make timeout to be 10s, to avoid error in Network jitter scenarios. - BrokerServiceConnection client(client_cache(_state), broker_addr, 10000, &status); + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); if (!status.ok()) { LOG(WARNING) << "Create broker write client failed. " << "broker=" << broker_addr @@ -198,7 +196,7 @@ void BrokerWriter::close() { TBrokerOperationStatus response; try { Status status; - BrokerServiceConnection client(client_cache(_state), broker_addr, 10000, &status); + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); if (!status.ok()) { LOG(WARNING) << "Create broker write client failed. broker=" << broker_addr << ", status=" << status.get_error_msg(); diff --git a/be/src/exec/broker_writer.h b/be/src/exec/broker_writer.h index a89abccef4..f8e2c61c47 100644 --- a/be/src/exec/broker_writer.h +++ b/be/src/exec/broker_writer.h @@ -28,15 +28,14 @@ namespace palo { -class RuntimeState; +class ExecEnv; class TBrokerRangeDesc; class TNetworkAddress; -class RuntimeState; // Reader of broker file class BrokerWriter : public FileWriter { public: - BrokerWriter(RuntimeState* state, + BrokerWriter(ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& dir, @@ -50,7 +49,7 @@ public: virtual void close() override; private: - RuntimeState* _state; + ExecEnv* _env; const std::vector& _addresses; const std::map& _properties; std::string _path; diff --git a/be/src/exec/mysql_scan_node.cpp b/be/src/exec/mysql_scan_node.cpp index fbb6a808ca..121ab3477b 100644 --- a/be/src/exec/mysql_scan_node.cpp +++ b/be/src/exec/mysql_scan_node.cpp @@ -35,10 +35,7 @@ MysqlScanNode::MysqlScanNode(ObjectPool* pool, const TPlanNode& tnode, _tuple_id(tnode.mysql_scan_node.tuple_id), _columns(tnode.mysql_scan_node.columns), _filters(tnode.mysql_scan_node.filters), - _tuple_desc(NULL) { - //_tuple_pool(NULL), - //_mysql_scanner(NULL) { - //_text_converter(NULL) { + _tuple_desc(nullptr) { } MysqlScanNode::~MysqlScanNode() { diff --git a/be/src/exec/mysql_scan_node.h b/be/src/exec/mysql_scan_node.h index 8f0ec7bec2..b608d23c26 100644 --- a/be/src/exec/mysql_scan_node.h +++ b/be/src/exec/mysql_scan_node.h @@ -86,7 +86,7 @@ private: // Helper class for converting text to other types; std::unique_ptr _text_converter; // Current tuple. - Tuple* _tuple; + Tuple* _tuple = nullptr; }; } diff --git a/be/src/exec/schema_scan_node.cpp b/be/src/exec/schema_scan_node.cpp index 87076b4292..c5bd743c6a 100644 --- a/be/src/exec/schema_scan_node.cpp +++ b/be/src/exec/schema_scan_node.cpp @@ -77,6 +77,10 @@ Status SchemaScanNode::init(const TPlanNode& tnode, RuntimeState* state) { _scanner_param.ip = _pool->add(new std::string(tnode.schema_scan_node.ip)); } + if (tnode.schema_scan_node.__isset.user_ip) { + _scanner_param.user_ip = _pool->add(new std::string(tnode.schema_scan_node.user_ip)); + } + if (tnode.schema_scan_node.__isset.port) { _scanner_param.port = tnode.schema_scan_node.port; } diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index dd50e9d2f5..14d8a3bf34 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -41,12 +41,13 @@ struct SchemaScannerParam { const std::string* table; const std::string* wild; const std::string* user; + const std::string* user_ip; const std::string* ip; int32_t port; int64_t thread_id; SchemaScannerParam() - : db(NULL), table(NULL), wild(NULL), user(NULL), ip(NULL), port(0) { } + : db(NULL), table(NULL), wild(NULL), user(NULL), user_ip(NULL), ip(NULL), port(0) { } }; // virtual scanner for all schema table diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index 32d28d13af..f0c7a66a22 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -274,6 +274,9 @@ Status SchemaColumnsScanner::get_new_desc() { if (NULL != _param->user) { desc_params.__set_user(*(_param->user)); } + if (NULL != _param->user_ip) { + desc_params.__set_user_ip(*(_param->user_ip)); + } if (NULL != _param->ip && 0 != _param->port) { RETURN_IF_ERROR(FrontendHelper::describe_table(*(_param->ip), @@ -295,6 +298,9 @@ Status SchemaColumnsScanner::get_new_table() { if (NULL != _param->user) { table_params.__set_user(*(_param->user)); } + if (NULL != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } if (NULL != _param->ip && 0 != _param->port) { RETURN_IF_ERROR(FrontendHelper::get_table_names(*(_param->ip), diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp index eb38b7103a..b474236264 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp @@ -53,6 +53,9 @@ Status SchemaSchemataScanner::start(RuntimeState *state) { if (NULL != _param->user) { db_params.__set_user(*(_param->user)); } + if (NULL != _param->user_ip) { + db_params.__set_user_ip(*(_param->user_ip)); + } if (NULL != _param->ip && 0 != _param->port) { RETURN_IF_ERROR(FrontendHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index f4581a5a88..74c9d8a0b8 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -73,6 +73,9 @@ Status SchemaTablesScanner::start(RuntimeState *state) { if (NULL != _param->user) { db_params.__set_user(*(_param->user)); } + if (NULL != _param->user_ip) { + db_params.__set_user_ip(*(_param->user_ip)); + } if (NULL != _param->ip && 0 != _param->port) { RETURN_IF_ERROR(FrontendHelper::get_db_names(*(_param->ip), @@ -227,6 +230,9 @@ Status SchemaTablesScanner::get_new_table() { if (NULL != _param->user) { table_params.__set_user(*(_param->user)); } + if (NULL != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } if (NULL != _param->ip && 0 != _param->port) { RETURN_IF_ERROR(FrontendHelper::list_table_status(*(_param->ip), diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp index 9a4dd6017f..baaf2981f8 100644 --- a/be/src/exprs/aggregate_functions.cpp +++ b/be/src/exprs/aggregate_functions.cpp @@ -1053,10 +1053,9 @@ int64_t AggregateFunctions::hll_algorithm(const palo_udf::StringVal& src) { double tmp = 0.f; // according to HerperLogLog current correction, if E is cardinal // E =< num_streams * 2.5 , LC has higher accuracy. - // num_streams * 2.5 < E =< 2 ^ 32 / 30 , HerperLogLog has higher accuracy. - // E > 2 ^ 32 / 30 , estimate = -tmp * log(1 - estimate / tmp); + // num_streams * 2.5 < E , HerperLogLog has higher accuracy. // Generally , we can use HerperLogLog to produce value as E. - if (num_zero_registers != 0) { + if (estimate <= num_streams * 2.5 && num_zero_registers != 0) { // Estimated cardinality is too low. Hll is too inaccurate here, instead use // linear counting. estimate = num_streams * log(static_cast(num_streams) / num_zero_registers); @@ -1069,8 +1068,6 @@ int64_t AggregateFunctions::hll_algorithm(const palo_udf::StringVal& src) { - 5.2921 * 1.0e-3 * estimate + 83.3216; estimate -= estimate * (bias / 100); - } else if (estimate > (tmp = std::pow(2, 32) / 30)) { - estimate = -tmp * log(1 - estimate / tmp); } return (int64_t)(estimate + 0.5); } diff --git a/be/src/exprs/encryption_functions.h b/be/src/exprs/encryption_functions.h index 6622c6f463..bb57e7b32c 100644 --- a/be/src/exprs/encryption_functions.h +++ b/be/src/exprs/encryption_functions.h @@ -1,21 +1,22 @@ -/**************************************************************** - * - * The author of this software is David M. Gay. - * - * Copyright (c) 1991, 2000, 2001 by Lucent Technologies. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose without fee is hereby granted, provided that this entire notice - * is included in all copies of any software which is or includes a copy - * or modification of this software and in all copies of the supporting - * documentation for such software. - * - * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED - * WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR LUCENT MAKES ANY - * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY - * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. - * - ***************************************************************/ +// Modifications copyright (C) 2017, Baidu.com, Inc. +// Copyright 2017 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #ifndef BDG_PALO_BE_SRC_QUERY_EXPRS_ENCRYPTION_FUNCTIONS_H #define BDG_PALO_BE_SRC_QUERY_EXPRS_ENCRYPTION_FUNCTIONS_H diff --git a/be/src/exprs/hybird_set.h b/be/src/exprs/hybird_set.h index c7994362f5..28a3451f0e 100644 --- a/be/src/exprs/hybird_set.h +++ b/be/src/exprs/hybird_set.h @@ -21,6 +21,7 @@ #ifndef BDG_PALO_BE_SRC_QUERY_EXPRS_HYBIRD_SET_H #define BDG_PALO_BE_SRC_QUERY_EXPRS_HYBIRD_SET_H +#include #include #include "common/status.h" #include "common/object_pool.h" @@ -72,7 +73,7 @@ public: if (sizeof(T) >= 16) { // for largeint, it will core dump with no memcpy T value; - memcpy(&value, data, sizeof(T)); + memcpy(&value, data, sizeof(T)); _set.insert(value); } else { _set.insert(*reinterpret_cast(data)); diff --git a/be/src/http/action/mini_load.cpp b/be/src/http/action/mini_load.cpp index e5a16f6cba..52fce714a7 100644 --- a/be/src/http/action/mini_load.cpp +++ b/be/src/http/action/mini_load.cpp @@ -445,6 +445,7 @@ Status MiniLoadAction::generate_check_load_req( check_load_req->__set_cluster(cluster); } check_load_req->db = http_req->param(DB_KEY); + check_load_req->__set_tbl(http_req->param(TABLE_KEY)); if (http_req->param(SUB_LABEL_KEY).empty()) { check_load_req->__set_label(http_req->param(LABEL_KEY)); diff --git a/be/src/olap/aggregate_func.h b/be/src/olap/aggregate_func.h index 2743eeea43..64a00eb5a9 100644 --- a/be/src/olap/aggregate_func.h +++ b/be/src/olap/aggregate_func.h @@ -226,9 +226,9 @@ struct AggregateFuncTraits(hll_ptr)); std::map index_to_value; if (context->has_sparse_or_full || - context->hash64_set.size() > HLL_EXPLICLIT_INT64_NUM) { + context->hash64_set->size() > HLL_EXPLICLIT_INT64_NUM) { HllSetHelper::set_max_register(context->registers, HLL_REGISTERS_COUNT, - context->hash64_set); + *(context->hash64_set)); for (int i = 0; i < HLL_REGISTERS_COUNT; i++) { if (context->registers[i] != 0) { index_to_value[i] = context->registers[i]; @@ -248,14 +248,14 @@ struct AggregateFuncTraits 0) { // sparse set HllSetHelper::set_sparse(slice->data, index_to_value, result_len); - } else if (context->hash64_set.size() > 0) { + } else if (context->hash64_set->size() > 0) { // expliclit set - HllSetHelper::set_expliclit(slice->data, context->hash64_set, result_len); + HllSetHelper::set_expliclit(slice->data, *(context->hash64_set), result_len); } slice->size = result_len & 0xffff; - HllSetHelper::init_context(context); + delete context->hash64_set; } }; diff --git a/be/src/olap/column_file/segment_reader.cpp b/be/src/olap/column_file/segment_reader.cpp index a20db5507c..0eb7c4dd92 100644 --- a/be/src/olap/column_file/segment_reader.cpp +++ b/be/src/olap/column_file/segment_reader.cpp @@ -765,8 +765,13 @@ OLAPStatus SegmentReader::_read_all_data_streams(size_t* buffer_size) { continue; } + if (_include_columns.find(unique_column_id) == _include_columns.end() && + _include_bf_columns.find(unique_column_id) == _include_bf_columns.end()) { + continue; + } + if (message.kind() == StreamInfoMessage::ROW_INDEX || - message.kind() == StreamInfoMessage::BLOOM_FILTER) { + message.kind() == StreamInfoMessage::BLOOM_FILTER) { continue; } diff --git a/be/src/olap/column_file/segment_reader.h b/be/src/olap/column_file/segment_reader.h index 4197ec939a..9d6e88f5e6 100644 --- a/be/src/olap/column_file/segment_reader.h +++ b/be/src/olap/column_file/segment_reader.h @@ -311,7 +311,6 @@ private: std::vector _column_readers; // 实际的数据读取器 std::vector _column_indices; // 保存column的index - UniqueIdSet _segment_columns; UniqueIdSet _include_columns; // 用于判断该列是不是被包含 UniqueIdSet _load_bf_columns; UniqueIdSet _include_bf_columns; diff --git a/be/src/olap/command_executor.cpp b/be/src/olap/command_executor.cpp index 96af16ce0f..62390fb008 100755 --- a/be/src/olap/command_executor.cpp +++ b/be/src/olap/command_executor.cpp @@ -295,13 +295,14 @@ OLAPStatus CommandExecutor::create_table(const TCreateTabletReq& request) { break; } - // 6. Create init version if request.version set - if (request.__isset.version) { - res = _create_init_version(olap_table_ptr, request); - if (res != OLAP_SUCCESS) { - OLAP_LOG_WARNING("fail to create initial version for table. [res=%d]", res); - } + // 6. Create init version if this is not a restore mode replica and request.version is set + // bool in_restore_mode = request.__isset.in_restore_mode && request.in_restore_mode; + // if (!in_restore_mode && request.__isset.version) { + res = _create_init_version(olap_table_ptr, request); + if (res != OLAP_SUCCESS) { + OLAP_LOG_WARNING("fail to create initial version for table. [res=%d]", res); } + // } } while (0); // 7. clear environment diff --git a/be/src/olap/command_executor.h b/be/src/olap/command_executor.h index 0c0027b55b..d3bad72c4a 100644 --- a/be/src/olap/command_executor.h +++ b/be/src/olap/command_executor.h @@ -123,6 +123,7 @@ public: TTabletId tablet_id, TSchemaHash schema_hash, std::string* snapshot_path); + virtual OLAPStatus make_snapshot( const TSnapshotRequest& request, std::string* snapshot_path); diff --git a/be/src/olap/field.h b/be/src/olap/field.h index bf0efdcfee..8c7ae50ba5 100644 --- a/be/src/olap/field.h +++ b/be/src/olap/field.h @@ -202,6 +202,8 @@ inline void Field::agg_init(char* dest, const char* src) { if (OLAP_LIKELY(_type != OLAP_FIELD_TYPE_HLL)) { copy_without_pool(dest, src); } else { + bool is_null = *reinterpret_cast(src); + *reinterpret_cast(dest) = is_null; StringSlice* slice = reinterpret_cast(dest + 1); size_t hll_ptr = *(size_t*)(slice->data - sizeof(HllContext*)); HllContext* context = (reinterpret_cast(hll_ptr)); diff --git a/be/src/olap/file_helper.cpp b/be/src/olap/file_helper.cpp index 79fcac40dd..b3d0aa7eb1 100644 --- a/be/src/olap/file_helper.cpp +++ b/be/src/olap/file_helper.cpp @@ -160,6 +160,8 @@ OLAPStatus FileHandler::close() { } } + OLAP_LOG_DEBUG("finished to close file. [file_name='%s' fd=%d]", + _file_name.c_str(), _fd); _fd = -1; _file_name = ""; _wr_length = 0; diff --git a/be/src/olap/hll.cpp b/be/src/olap/hll.cpp index 39474ffc9a..78795c696c 100644 --- a/be/src/olap/hll.cpp +++ b/be/src/olap/hll.cpp @@ -163,7 +163,7 @@ void HllSetHelper::set_sparse( void HllSetHelper::set_expliclit(char* result, const std::set& hash_value_set, int& len) { result[0] = HLL_DATA_EXPLICIT; - result[1] = (HllSetResolver::ExpliclitLengthValueType)hash_value_set.size(); + result[1] = (HllSetResolver::ExpliclitLengthValueType)(hash_value_set.size()); len = sizeof(HllSetResolver::SetTypeValueType) + sizeof(HllSetResolver::ExpliclitLengthValueType); char* write_pos = result + len; @@ -212,7 +212,7 @@ void HllSetHelper::fill_set(const char* data, HllContext* context) { resolver.parse(); if (resolver.get_hll_data_type() == HLL_DATA_EXPLICIT) { // expliclit set - resolver.fill_hash64_set(&(context->hash64_set)); + resolver.fill_hash64_set(context->hash64_set); } else if (resolver.get_hll_data_type() != HLL_DATA_EMPTY) { // full or sparse context->has_sparse_or_full = true; @@ -222,7 +222,7 @@ void HllSetHelper::fill_set(const char* data, HllContext* context) { void HllSetHelper::init_context(HllContext* context) { memset(context->registers, 0, HLL_REGISTERS_COUNT); - context->hash64_set.clear(); + context->hash64_set = new std::set(); context->has_value = false; context->has_sparse_or_full = false; } diff --git a/be/src/olap/hll.h b/be/src/olap/hll.h index 1e09f593f5..133a9eef7e 100644 --- a/be/src/olap/hll.h +++ b/be/src/olap/hll.h @@ -37,7 +37,7 @@ struct HllContext { bool has_value; bool has_sparse_or_full; char registers[HLL_REGISTERS_COUNT]; - std::set hash64_set; + std::set* hash64_set = nullptr; }; // help parse hll set diff --git a/be/src/olap/olap_engine.cpp b/be/src/olap/olap_engine.cpp index 795007b2ba..2e65129f0b 100644 --- a/be/src/olap/olap_engine.cpp +++ b/be/src/olap/olap_engine.cpp @@ -124,7 +124,8 @@ OLAPStatus OLAPEngine::_load_tables(const string& tablet_root_path) { } OLAPStatus OLAPEngine::load_one_tablet( - TTabletId tablet_id, SchemaHash schema_hash, const string& schema_hash_path) { + TTabletId tablet_id, SchemaHash schema_hash, const string& schema_hash_path, + bool force) { stringstream header_name_stream; header_name_stream << schema_hash_path << "/" << tablet_id << ".hdr"; string header_path = header_name_stream.str(); @@ -143,7 +144,6 @@ OLAPStatus OLAPEngine::load_one_tablet( move_to_trash(boost_schema_hash_path, boost_schema_hash_path); return OLAP_ERR_ENGINE_LOAD_INDEX_TABLE_ERROR; } - if (olap_table->latest_version() == NULL && !olap_table->is_schema_changing()) { OLAP_LOG_WARNING("tablet not in schema change state without delta is invalid. " "[header_path=%s]", @@ -156,7 +156,7 @@ OLAPStatus OLAPEngine::load_one_tablet( // 这里不需要SAFE_DELETE(olap_table),因为olap_table指针已经在add_table中托管到smart pointer中 OLAPStatus res = OLAP_SUCCESS; string table_name = olap_table->full_name(); - res = add_table(tablet_id, schema_hash, olap_table); + res = add_table(tablet_id, schema_hash, olap_table, force); if (res != OLAP_SUCCESS) { // 插入已经存在的table时返回成功 if (res == OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE) { @@ -386,10 +386,11 @@ bool OLAPEngine::check_tablet_id_exist(TTabletId tablet_id) { return is_exist; } -OLAPStatus OLAPEngine::add_table(TTabletId tablet_id, SchemaHash schema_hash, OLAPTable* table) { +OLAPStatus OLAPEngine::add_table(TTabletId tablet_id, SchemaHash schema_hash, + OLAPTable* table, bool force) { OLAPStatus res = OLAP_SUCCESS; - OLAP_LOG_DEBUG("begin to add olap table to OLAPEngine. [tablet_id=%ld schema_hash=%d]", - tablet_id, schema_hash); + OLAP_LOG_DEBUG("begin to add olap table to OLAPEngine. [tablet_id=%ld schema_hash=%d], force: %d", + tablet_id, schema_hash, force); _tablet_map_lock.wrlock(); SmartOLAPTable smart_table(table, OLAPTableDestruction); @@ -412,10 +413,12 @@ OLAPStatus OLAPEngine::add_table(TTabletId tablet_id, SchemaHash schema_hash, OL } _tablet_map_lock.unlock(); - if (table_item->header_file_name() == smart_table->header_file_name()) { - OLAP_LOG_WARNING("add the same tablet twice! [tablet_id=%ld schema_hash=%d]", - tablet_id, schema_hash); - return OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE; + if (!force) { + if (table_item->header_file_name() == smart_table->header_file_name()) { + OLAP_LOG_WARNING("add the same tablet twice! [tablet_id=%ld schema_hash=%d]", + tablet_id, schema_hash); + return OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE; + } } table_item->obtain_header_rdlock(); @@ -425,9 +428,19 @@ OLAPStatus OLAPEngine::add_table(TTabletId tablet_id, SchemaHash schema_hash, OL int32_t new_version = smart_table->latest_version()->end_version(); table_item->release_header_lock(); - if (new_version > old_version - || (new_version == old_version && new_time > old_time)) { - drop_table(tablet_id, schema_hash); + /* + * In restore process, we replace all origin files in tablet dir with + * the downloaded snapshot files. Than we try to reload tablet header. + * force == true means we forcibly replace the OLAPTable in _tablet_map + * with the new one. But if we do so, the files in the tablet dir will be + * dropped when the origin OLAPTable deconstruct. + * So we set keep_files == true to not delete files when the + * origin OLAPTable deconstruct. + */ + bool keep_files = force ? true : false; + if (force || (new_version > old_version + || (new_version == old_version && new_time > old_time))) { + drop_table(tablet_id, schema_hash, keep_files); _tablet_map_lock.wrlock(); _tablet_map[tablet_id].table_arr.push_back(smart_table); _tablet_map[tablet_id].table_arr.sort(_sort_table_by_create_time); @@ -436,9 +449,9 @@ OLAPStatus OLAPEngine::add_table(TTabletId tablet_id, SchemaHash schema_hash, OL smart_table->mark_dropped(); res = OLAP_ERR_ENGINE_INSERT_EXISTS_TABLE; } - OLAP_LOG_WARNING("add duplicated table. [res=%d tablet_id=%ld schema_hash=%d " + OLAP_LOG_WARNING("add duplicated table. force: %d, [res=%d tablet_id=%ld schema_hash=%d " "old_version=%d new_version=%d old_time=%ld new_time=%ld]", - res, tablet_id, schema_hash, + force, res, tablet_id, schema_hash, old_version, new_version, old_time, new_time); return res; @@ -452,7 +465,8 @@ OLAPStatus OLAPEngine::add_table(TTabletId tablet_id, SchemaHash schema_hash, OL // base table cannot be dropped; // b. other cases: // drop specified table and clear schema change info. -OLAPStatus OLAPEngine::drop_table(TTabletId tablet_id, SchemaHash schema_hash) { +OLAPStatus OLAPEngine::drop_table( + TTabletId tablet_id, SchemaHash schema_hash, bool keep_files) { OLAP_LOG_INFO("begin to drop olap table. [tablet_id=%ld]", tablet_id); OLAPStatus res = OLAP_SUCCESS; @@ -478,7 +492,7 @@ OLAPStatus OLAPEngine::drop_table(TTabletId tablet_id, SchemaHash schema_hash) { // Drop table directly when not in schema change if (!ret) { - return _drop_table_directly(tablet_id, schema_hash); + return _drop_table_directly(tablet_id, schema_hash, keep_files); } // Check table is in schema change or not, is base table or not @@ -496,7 +510,7 @@ OLAPStatus OLAPEngine::drop_table(TTabletId tablet_id, SchemaHash schema_hash) { OLAP_LOG_WARNING("drop table directly when related table not found. " "[tablet_id=%ld schema_hash=%d]", related_tablet_id, related_schema_hash); - return _drop_table_directly(tablet_id, schema_hash); + return _drop_table_directly(tablet_id, schema_hash, keep_files); } if (dropped_table->creation_time() < related_table->creation_time()) { @@ -519,7 +533,7 @@ OLAPStatus OLAPEngine::drop_table(TTabletId tablet_id, SchemaHash schema_hash) { res, related_table->full_name().c_str()); } - res = _drop_table_directly(tablet_id, schema_hash); + res = _drop_table_directly(tablet_id, schema_hash, keep_files); if (res != OLAP_SUCCESS) { OLAP_LOG_WARNING("fail to drop table which in schema change. [table=%s]", dropped_table->full_name().c_str()); @@ -530,7 +544,8 @@ OLAPStatus OLAPEngine::drop_table(TTabletId tablet_id, SchemaHash schema_hash) { return res; } -OLAPStatus OLAPEngine::_drop_table_directly(TTabletId tablet_id, SchemaHash schema_hash) { +OLAPStatus OLAPEngine::_drop_table_directly( + TTabletId tablet_id, SchemaHash schema_hash, bool keep_files) { OLAPStatus res = OLAP_SUCCESS; _tablet_map_lock.wrlock(); @@ -545,7 +560,9 @@ OLAPStatus OLAPEngine::_drop_table_directly(TTabletId tablet_id, SchemaHash sche for (list::iterator it = _tablet_map[tablet_id].table_arr.begin(); it != _tablet_map[tablet_id].table_arr.end();) { if ((*it)->equal(tablet_id, schema_hash)) { - (*it)->mark_dropped(); + if (!keep_files) { + (*it)->mark_dropped(); + } it = _tablet_map[tablet_id].table_arr.erase(it); } else { ++it; @@ -859,6 +876,7 @@ OLAPStatus OLAPEngine::report_all_tablets_info( } } + tablet_info.__set_version_count(olap_table->file_version_size()); tablet.tablet_infos.push_back(tablet_info); } @@ -1403,6 +1421,11 @@ OLAPStatus OLAPEngine::_create_new_table_header_file( return OLAP_ERR_INPUT_PARAMETER_ERROR; } + // set restore mode + if (request.__isset.in_restore_mode && request.in_restore_mode) { + header.set_in_restore_mode(true); + } + // save header file header.set_creation_time(time(NULL)); header.set_cumulative_layer_point(-1); diff --git a/be/src/olap/olap_engine.h b/be/src/olap/olap_engine.h index 66faba0dc9..46db28d31b 100644 --- a/be/src/olap/olap_engine.h +++ b/be/src/olap/olap_engine.h @@ -66,11 +66,13 @@ public: const SmartOLAPTable ref_olap_table); // Add a table pointer to OLAPEngine + // If force, drop the existing table add this new one // // Return OLAP_SUCCESS, if run ok // OLAP_ERR_TABLE_INSERT_DUPLICATION_ERROR, if find duplication // OLAP_ERR_NOT_INITED, if not inited - OLAPStatus add_table(TTabletId tablet_id, SchemaHash schema_hash, OLAPTable* table); + OLAPStatus add_table(TTabletId tablet_id, SchemaHash schema_hash, + OLAPTable* table, bool force = false); // Add empty data for OLAPTable // @@ -80,14 +82,16 @@ public: Version version, VersionHash version_hash); // Drop a table by description - // + // If set keep_files == true, files will NOT be deleted when deconstruction. // Return OLAP_SUCCESS, if run ok // OLAP_ERR_TABLE_DELETE_NOEXIST_ERROR, if table not exist // OLAP_ERR_NOT_INITED, if not inited - OLAPStatus drop_table(TTabletId tablet_id, SchemaHash schema_hash); + OLAPStatus drop_table( + TTabletId tablet_id, SchemaHash schema_hash, bool keep_files = false); // Drop table directly with check schema change info. - OLAPStatus _drop_table_directly(TTabletId tablet_id, TSchemaHash schema_hash); + OLAPStatus _drop_table_directly( + TTabletId tablet_id, TSchemaHash schema_hash, bool keep_files = false); OLAPStatus drop_tables_on_error_root_path(const std::vector& table_info_vec); @@ -124,7 +128,8 @@ public: OLAPStatus load_one_tablet(TTabletId tablet_id, SchemaHash schema_hash, - const std::string& schema_hash_path); + const std::string& schema_hash_path, + bool force = false); Cache* index_stream_lru_cache() { return _index_stream_lru_cache; diff --git a/be/src/olap/olap_header.cpp b/be/src/olap/olap_header.cpp index 8086f56e42..f81b261fb1 100644 --- a/be/src/olap/olap_header.cpp +++ b/be/src/olap/olap_header.cpp @@ -427,6 +427,21 @@ const FileVersionMessage* OLAPHeader::get_latest_version() const { return max_version; } +const FileVersionMessage* OLAPHeader::get_base_version() const { + if (file_version_size() == 0) { + return NULL; + } + + const FileVersionMessage* base_version = NULL; + for (int i = 0; i < file_version_size(); ++i) { + if (file_version(i).start_version() == 0) { + base_version = &file_version(i); + break; + } + } + return base_version; +} + const uint32_t OLAPHeader::get_compaction_nice_estimate() const{ uint32_t nice = 0; bool base_version_exists = false; diff --git a/be/src/olap/olap_header.h b/be/src/olap/olap_header.h index 7bc732f88b..62c8c01709 100644 --- a/be/src/olap/olap_header.h +++ b/be/src/olap/olap_header.h @@ -86,6 +86,7 @@ public: const FileVersionMessage* get_lastest_delta_version() const; const FileVersionMessage* get_latest_version() const; + const FileVersionMessage* get_base_version() const; const uint32_t get_compaction_nice_estimate() const; const OLAPStatus version_creation_time(const Version& version, int64_t* creation_time) const; diff --git a/be/src/olap/olap_snapshot.cpp b/be/src/olap/olap_snapshot.cpp index 4594e30ef7..a13c8d6755 100644 --- a/be/src/olap/olap_snapshot.cpp +++ b/be/src/olap/olap_snapshot.cpp @@ -299,7 +299,7 @@ OLAPStatus OLAPSnapshot::_create_snapshot_files( const FileVersionMessage* latest_version = NULL; latest_version = ref_olap_table->latest_version(); if (latest_version == NULL) { - OLAP_LOG_WARNING("table has not any version. [path='%s']", + OLAP_LOG_WARNING("table does not have any version. [path='%s']", ref_olap_table->full_name().c_str()); res = OLAP_ERR_VERSION_NOT_EXIST; break; diff --git a/be/src/olap/olap_table.cpp b/be/src/olap/olap_table.cpp index 57b6f3af66..0ffebb8a8f 100644 --- a/be/src/olap/olap_table.cpp +++ b/be/src/olap/olap_table.cpp @@ -34,7 +34,7 @@ #include "olap/olap_rootpath.h" #include "olap/reader.h" #include "olap/row_cursor.h" - +#include "util/defer_op.h" using std::map; using std::nothrow; @@ -212,6 +212,8 @@ OLAPTable::~OLAPTable() { path path_name(_header->file_name()); SAFE_DELETE(_header); + OLAP_LOG_WARNING("deconstruct table"); + // 移动数据目录 if (_is_dropped) { path table_path = path_name.parent_path(); @@ -705,6 +707,63 @@ void OLAPTable::set_selectivities(const vector& selectivities) { } } +OLAPStatus OLAPTable::merge_header(const OLAPHeader& hdr, int to_version) { + obtain_header_wrlock(); + DeferOp release_lock(std::bind(&OLAPTable::release_header_lock, this)); + + const FileVersionMessage* base_version = _header->get_base_version(); + if (base_version->end_version() != to_version) { + return OLAP_ERR_VERSION_NOT_EXIST; + } + + // delete old base version + Version base = { base_version->start_version(), base_version->end_version() }; + OLAPStatus st = _header->delete_version(base); + if (st != OLAP_SUCCESS) { + OLAP_LOG_WARNING("failed to delete version [%d-%d] from header", + base_version->start_version(), base_version->end_version()); + return st; + } + OLAP_LOG_DEBUG("finished to delete version [%d-%d] from header", + base_version->start_version(), base_version->end_version()); + + + // add new versions + int version_num = hdr.file_version_size(); + for (int i = 0; i < version_num; ++i) { + const FileVersionMessage& v = hdr.file_version(i); + if (v.end_version() > to_version) { + break; + } + + st = _header->add_version( + { v.start_version(), v.end_version() }, + v.version_hash(), + v.max_timestamp(), + v.num_segments(), + v.index_size(), + v.data_size(), + v.num_rows()); + + if (st != OLAP_SUCCESS) { + OLAP_LOG_WARNING("failed to add version [%d-%d] to header", + v.start_version(), v.end_version()); + return st; + } + OLAP_LOG_WARNING("finished to add version [%d-%d] to header", + v.start_version(), v.end_version()); + } + + st = _header->save(); + if (st != OLAP_SUCCESS) { + OLAP_LOG_FATAL("failed to save header when merging. tablet: %d", _tablet_id); + return st; + } + + OLAP_LOG_DEBUG("finished to merge header to version: %d", to_version); + return OLAP_SUCCESS; +} + OLAPIndex* OLAPTable::_get_largest_index() { OLAPIndex* largest_index = NULL; size_t largest_index_sizes = 0; @@ -1100,6 +1159,11 @@ string OLAPTable::construct_file_name(const Version& version, return file_name; } +string OLAPTable::construct_dir_path() const { + path path_name(_header->file_name()); + return path_name.parent_path().string(); +} + int32_t OLAPTable::get_field_index(const string& field_name) const { field_index_map_t::const_iterator res_iterator = _field_index_map.find(field_name); if (res_iterator == _field_index_map.end()) { diff --git a/be/src/olap/olap_table.h b/be/src/olap/olap_table.h index cf0974f486..68e8b8b66e 100644 --- a/be/src/olap/olap_table.h +++ b/be/src/olap/olap_table.h @@ -174,6 +174,9 @@ public: // Get table row_count and selectivity vector for SHOW_TABLE_INFO command OLAPStatus get_selectivities(std::vector* selectivities); + // used for restore, merge the (0, to_version) in 'hdr' + OLAPStatus merge_header(const OLAPHeader& hdr, int to_version); + // Get OLAPHeader write lock before call get_selectivities() void set_selectivities(const std::vector& selectivities); @@ -275,6 +278,10 @@ public: VersionHash version_hash, uint32_t segment) const; + // return the dir path of this tablet, include tablet id and schema hash + // eg: /path/to/data/0/100001/237480234/ + std::string construct_dir_path() const; + // For index file, suffix is "idx", for data file, suffix is "dat". static std::string construct_file_path(const std::string& header_path, const Version& version, @@ -378,6 +385,10 @@ public: return _header->get_latest_version(); } + const FileVersionMessage* base_version() const { + return _header->get_base_version(); + } + // 在使用之前对header加锁 const uint32_t get_compaction_nice_estimate() const { return _header->get_compaction_nice_estimate(); diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index 8bda6227aa..f3979773b7 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -929,6 +929,9 @@ ColumnPredicate* Reader::_parse_to_predicate(const TCondition& condition) { // TODO: not equal and not in predicate is not pushed down int index = _olap_table->get_field_index(condition.column_name); FieldInfo fi = _olap_table->tablet_schema()[index]; + if (fi.aggregation != FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE) { + return nullptr; + } ColumnPredicate* predicate = NULL; if (condition.condition_op == "*=" && condition.condition_values.size() == 1) { diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 48ffb79052..8b9c0b7df1 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -92,6 +92,7 @@ add_library(Runtime STATIC bufferpool/suballocator.cc bufferpool/system_allocator.cc initial_reservations.cc + snapshot_loader.cpp ) # This test runs forever so should not be part of 'make test' diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index de00048cef..a793b8e5de 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -60,6 +60,7 @@ #include "runtime/etl_job_mgr.h" #include "runtime/load_path_mgr.h" #include "runtime/pull_load_task_mgr.h" +#include "runtime/snapshot_loader.h" #include "util/pretty_printer.h" #include "util/palo_metrics.h" #include "util/brpc_stub_cache.h" @@ -99,6 +100,7 @@ ExecEnv::ExecEnv() : _bfd_parser(BfdParser::create()), _pull_load_task_mgr(new PullLoadTaskMgr(config::pull_load_task_dir)), _broker_mgr(new BrokerMgr(this)), + _snapshot_loader(new SnapshotLoader(this)), _brpc_stub_cache(new BrpcStubCache()), _enable_webserver(true), _tz_database(TimezoneDatabase()) { diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 797a9ba3c1..f831b95f29 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -57,6 +57,8 @@ class BrokerMgr; class MetricRegistry; class BufferPool; class ReservationTracker; +class ConnectionManager; +class SnapshotLoader; class BrpcStubCache; // Execution environment for queries/plan fragments. @@ -149,6 +151,10 @@ public: return _broker_mgr.get(); } + SnapshotLoader* snapshot_loader() const { + return _snapshot_loader.get(); + } + BrpcStubCache* brpc_stub_cache() const { return _brpc_stub_cache.get(); } @@ -197,6 +203,7 @@ private: std::unique_ptr _bfd_parser; std::unique_ptr _pull_load_task_mgr; std::unique_ptr _broker_mgr; + std::unique_ptr _snapshot_loader; std::unique_ptr _brpc_stub_cache; bool _enable_webserver; diff --git a/be/src/runtime/export_sink.cpp b/be/src/runtime/export_sink.cpp index 585b23145d..9f29c4ae8c 100644 --- a/be/src/runtime/export_sink.cpp +++ b/be/src/runtime/export_sink.cpp @@ -222,7 +222,7 @@ Status ExportSink::open_file_writer() { break; } case TFileType::FILE_BROKER: { - BrokerWriter* broker_writer = new BrokerWriter(_state, + BrokerWriter* broker_writer = new BrokerWriter(_state->exec_env(), _t_export_sink.broker_addresses, _t_export_sink.properties, _t_export_sink.export_path + "/" + file_name, diff --git a/be/src/runtime/result_sink.cpp b/be/src/runtime/result_sink.cpp index 2051256f8b..527fcd63ea 100644 --- a/be/src/runtime/result_sink.cpp +++ b/be/src/runtime/result_sink.cpp @@ -20,6 +20,7 @@ #include "runtime/result_sink.h" +#include "common/config.h" #include "util/debug_util.h" #include "exprs/expr.h" #include "runtime/row_batch.h" @@ -86,7 +87,8 @@ Status ResultSink::close(RuntimeState* state, Status exec_status) { if (_sender) { _sender->close(exec_status); } - state->exec_env()->result_mgr()->cancel_at_time(time(NULL) + 32, state->fragment_instance_id()); + state->exec_env()->result_mgr()->cancel_at_time(time(NULL) + config::result_buffer_cancelled_interval_time, + state->fragment_instance_id()); Expr::close(_output_expr_ctxs, state); _closed = true; diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp new file mode 100644 index 0000000000..aaea0214ad --- /dev/null +++ b/be/src/runtime/snapshot_loader.cpp @@ -0,0 +1,934 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "runtime/snapshot_loader.h" + +#include "gen_cpp/PaloBrokerService_types.h" +#include "gen_cpp/TPaloBrokerService.h" + +#include "common/logging.h" +#include "exec/broker_reader.h" +#include "exec/broker_writer.h" +#include "olap/file_helper.h" +#include "olap/olap_engine.h" +#include "olap/olap_table.h" +#include "runtime/exec_env.h" +#include "runtime/broker_mgr.h" +#include "util/file_utils.h" + +namespace palo { + +#ifdef BE_TEST +inline BrokerServiceClientCache* client_cache(ExecEnv* env) { + static BrokerServiceClientCache s_client_cache; + return &s_client_cache; +} + +inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { + static std::string s_client_id = "palo_unit_test"; + return s_client_id; +} +#else +inline BrokerServiceClientCache* client_cache(ExecEnv* env) { + return env->broker_client_cache(); +} + +inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { + return env->broker_mgr()->get_client_id(addr); +} +#endif + +SnapshotLoader::SnapshotLoader(ExecEnv* env) : + _env(env) { + +} + +SnapshotLoader::~SnapshotLoader() { + +} + +Status SnapshotLoader::upload( + const std::map& src_to_dest_path, + const TNetworkAddress& broker_addr, + const std::map& broker_prop, + int64_t job_id, + std::map>* tablet_files) { + + LOG(INFO) << "begin to upload snapshot files. num: " + << src_to_dest_path.size() << ", broker addr: " + << broker_addr << ", job: " << job_id; + + Status status = Status::OK; + // 1. validate local tablet snapshot paths + RETURN_IF_ERROR(_check_local_snapshot_paths(src_to_dest_path, true)); + + // 2. get broker client + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); + if (!status.ok()) { + std::stringstream ss; + ss << "failed to get broker client. " + << "broker addr: " << broker_addr + << ". msg: " << status.get_error_msg(); + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + std::vector broker_addrs; + broker_addrs.push_back(broker_addr); + // 3. for each src path, upload it to remote storage + for (auto iter = src_to_dest_path.begin(); iter != src_to_dest_path.end(); + iter++) { + const std::string& src_path = iter->first; + const std::string& dest_path = iter->second; + + int64_t tablet_id = 0; + int32_t schema_hash = 0; + RETURN_IF_ERROR(_get_tablet_id_and_schema_hash_from_file_path( + src_path, &tablet_id, &schema_hash)); + + // 2.1 get existing files from remote path + std::map remote_files; + RETURN_IF_ERROR(_get_existing_files_from_remote( + client, dest_path, broker_prop, &remote_files)); + + for (auto& tmp : remote_files) { + VLOG(2) << "get remote file: " << tmp.first << ", checksum: " << tmp.second.md5; + } + + // 2.2 list local files + std::vector local_files; + std::vector local_files_with_checksum; + RETURN_IF_ERROR(_get_existing_files_from_local(src_path, &local_files)); + + // 2.3 iterate local files + for (auto it = local_files.begin(); it != local_files.end(); it++) { + const std::string& local_file = *it; + // calc md5sum of localfile + std::string md5sum; + status = FileUtils::md5sum(src_path + "/" + local_file, &md5sum); + if (!status.ok()) { + std::stringstream ss; + ss << "failed to get md5sum of file: " << local_file + << ": " << status.get_error_msg(); + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + VLOG(2) << "get file checksum: " << local_file << ": " << md5sum; + local_files_with_checksum.push_back(local_file + "." + md5sum); + + // check if this local file need upload + bool need_upload = false; + auto find = remote_files.find(local_file); + if (find != remote_files.end()) { + if (md5sum != find->second.md5) { + // remote storage file exist, but with different checksum + LOG(WARNING) << "remote file checksum is invalid. remote: " << find->first + << ", local: " << md5sum; + // TODO(cmy): save these files and delete them later + need_upload = true; + } + } else { + need_upload = true; + } + + if (!need_upload) { + VLOG(2) << "file exist in remote path, no need to upload: " << local_file; + continue; + } + + // upload + // open broker writer. file name end with ".part" + // it will be rename to ".md5sum" after upload finished + std::string full_remote_file = dest_path + "/" + local_file; + { + // NOTICE: broker writer must be closed before calling rename + std::unique_ptr broker_writer; + broker_writer.reset(new BrokerWriter(_env, + broker_addrs, + broker_prop, + full_remote_file + ".part", + 0 /* offset */)); + RETURN_IF_ERROR(broker_writer->open()); + + // read file and write to broker + std::string full_local_file = src_path + "/" + local_file; + FileHandler file_handler; + OLAPStatus ost = file_handler.open(full_local_file, O_RDONLY); + if (ost != OLAP_SUCCESS) { + return Status("failed to open file: " + full_local_file); + } + + size_t file_len = file_handler.length(); + if (file_len == -1) { + return Status("failed to get length of file: " + full_local_file); + } + + constexpr size_t buf_sz = 1024 * 1024; + char read_buf[buf_sz]; + size_t left_len = file_len; + size_t read_offset = 0; + while (left_len > 0) { + size_t read_len = left_len > buf_sz ? buf_sz : left_len; + ost = file_handler.pread(read_buf, read_len, read_offset); + if (ost != OLAP_SUCCESS) { + return Status("failed to read file: " + full_local_file); + } + // write through broker + size_t write_len = 0; + RETURN_IF_ERROR(broker_writer->write(reinterpret_cast(read_buf), + read_len, &write_len)); + DCHECK_EQ(write_len, read_len); + + read_offset += read_len; + left_len -= read_len; + } + LOG(INFO) << "finished to write file via broker. file: " << + full_local_file << ", length: " << file_len; + } + + // rename file to end with ".md5sum" + RETURN_IF_ERROR(_rename_remote_file(client, + full_remote_file + ".part", + full_remote_file + "." + md5sum, + broker_prop)); + } // end for each tablet's local files + + tablet_files->emplace(tablet_id, local_files_with_checksum); + LOG(INFO) << "finished to write tablet to remote. local path: " + << src_path << ", remote path: " << dest_path; + } // end for each tablet path + + LOG(INFO) << "finished to upload snapshots. job: " << job_id; + return status; +} + +/* + * Download snapshot files from remote. + * After downloaded, the local dir should contains all files existing in remote, + * may also contains severval useless files. + */ +Status SnapshotLoader::download( + const std::map& src_to_dest_path, + const TNetworkAddress& broker_addr, + const std::map& broker_prop, + int64_t job_id, + std::vector* downloaded_tablet_ids) { + + LOG(INFO) << "begin to download snapshot files. num: " + << src_to_dest_path.size() << ", broker addr: " + << broker_addr << ", job: " << job_id; + + Status status = Status::OK; + // 1. validate local tablet snapshot paths + RETURN_IF_ERROR(_check_local_snapshot_paths(src_to_dest_path, false)); + + // 2. get broker client + BrokerServiceConnection client(client_cache(_env), broker_addr, 10000, &status); + if (!status.ok()) { + std::stringstream ss; + ss << "failed to get broker client. " + << "broker addr: " << broker_addr + << ". msg: " << status.get_error_msg(); + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + std::vector broker_addrs; + broker_addrs.push_back(broker_addr); + // 3. for each src path, download it to local storage + for (auto iter = src_to_dest_path.begin(); iter != src_to_dest_path.end(); + iter++) { + const std::string& remote_path = iter->first; + const std::string& local_path = iter->second; + + int64_t local_tablet_id = 0; + int32_t schema_hash = 0; + RETURN_IF_ERROR(_get_tablet_id_and_schema_hash_from_file_path( + local_path, &local_tablet_id, &schema_hash)); + downloaded_tablet_ids->push_back(local_tablet_id); + + int64_t remote_tablet_id; + RETURN_IF_ERROR(_get_tablet_id_from_remote_path(remote_path, + &remote_tablet_id)); + VLOG(2) << "get local tablet id: " << local_tablet_id << ", schema hash: " + << schema_hash << ", remote tablet id: " << remote_tablet_id; + + // 1. get local files + std::vector local_files; + RETURN_IF_ERROR(_get_existing_files_from_local(local_path, &local_files)); + + // 2. get remote files + std::map remote_files; + RETURN_IF_ERROR(_get_existing_files_from_remote( + client, remote_path, broker_prop, &remote_files)); + if (remote_files.empty()) { + std::stringstream ss; + ss << "get nothing from remote path: " << remote_path; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + for (auto& iter : remote_files) { + bool need_download = false; + const std::string& remote_file = iter.first; + const FileStat& file_stat = iter.second; + auto find = std::find(local_files.begin(), local_files.end(), remote_file); + if (find == local_files.end()) { + // remote file does not exist in local, download it + need_download = true; + } else { + if (_end_with(remote_file, ".hdr")) { + // this is a header file, download it. + need_download = true; + } else { + // check checksum + std::string local_md5sum; + Status st = FileUtils::md5sum(local_path + "/" + remote_file, &local_md5sum); + if (!st.ok()) { + LOG(WARNING) << "failed to get md5sum of local file: " << remote_file + << ". msg: " << st.get_error_msg() << ". download it"; + need_download = true; + } else { + VLOG(2) << "get local file checksum: " << remote_file << ": " << local_md5sum; + if (file_stat.md5 != local_md5sum) { + // file's checksum does not equal, download it. + need_download = true; + } + } + } + } + + if (!need_download) { + LOG(INFO) << "remote file already exist in local, no need to download." + << ", file: " << remote_file; + continue; + } + + // begin to download + std::string full_remote_file = remote_path + "/" + remote_file + "." + file_stat.md5; + std::string local_file_name; + // we need to replace the tablet_id in remote file name with local tablet id + RETURN_IF_ERROR(_replace_tablet_id(remote_file, local_tablet_id, &local_file_name)); + std::string full_local_file = local_path + "/" + local_file_name; + LOG(INFO) << "begin to download from " << full_remote_file << " to " + << full_local_file; + size_t file_len = file_stat.size; + { + // 1. open remote file for read + std::unique_ptr broker_reader; + broker_reader.reset(new BrokerReader(_env, + broker_addrs, + broker_prop, + full_remote_file, + 0 /* offset */)); + RETURN_IF_ERROR(broker_reader->open()); + + // 2. remove the existing local file if exist + if (boost::filesystem::remove(full_local_file)) { + VLOG(2) << "remove the previously exist local file: " + << full_local_file; + } + // remove file which will be downloaded now. + // this file will be added to local_files if it be downloaded successfully. + local_files.erase(find); + + // 3. open local file for write + FileHandler file_handler; + OLAPStatus ost = file_handler.open_with_mode(full_local_file, + O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); + if (ost != OLAP_SUCCESS) { + return Status("failed to open file: " + full_local_file); + } + + // 4. read remote and write to local + VLOG(2) << "read remote file: " << full_remote_file << " to local: " + << full_local_file << ". file size: " << file_len; + constexpr size_t buf_sz = 1024 * 1024; + char read_buf[buf_sz]; + size_t write_offset = 0; + bool eof = false; + while (!eof) { + size_t read_len = buf_sz; + RETURN_IF_ERROR(broker_reader->read(reinterpret_cast(read_buf), + &read_len, &eof)); + + if (eof) { + continue; + } + + if (read_len > 0) { + ost = file_handler.pwrite(read_buf, read_len, write_offset); + if (ost != OLAP_SUCCESS) { + return Status("failed to write file: " + full_local_file); + } + + write_offset += read_len; + } + } + } // file_handler should be closed before calculating checksum + + // 5. check md5 of the downloaded file + std::string downloaded_md5sum; + status = FileUtils::md5sum(full_local_file, &downloaded_md5sum); + if (!status.ok()) { + std::stringstream ss; + ss << "failed to get md5sum of file: " << full_local_file; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + VLOG(2) << "get downloaded file checksum: " << full_local_file << ": " + << downloaded_md5sum; + if (downloaded_md5sum != file_stat.md5) { + std::stringstream ss; + ss << "invalid md5 of downloaded file: " << full_local_file + << ", expected: " << file_stat.md5 << ", get: " << downloaded_md5sum; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + // local_files always keep the updated local files + local_files.push_back(local_file_name); + LOG(INFO) << "finished to download file via broker. file: " << + full_local_file << ", length: " << file_len; + } // end for all remote files + + // finally, delete local files which are not in remote + for (const auto& local_file: local_files) { + // replace the tablet id in local file name with the remote tablet id, + // in order to compare the file name. + std::string new_name; + Status st = _replace_tablet_id(local_file, remote_tablet_id, &new_name); + if (!st.ok()) { + LOG(WARNING) << "failed to replace tablet id. unknown local file: " << st.get_error_msg() + << ". ignore it"; + continue; + } + VLOG(2) << "new file name after replace tablet id: " << new_name; + const auto& find = remote_files.find(new_name); + if (find != remote_files.end()) { + continue; + } + + // delete + std::string full_local_file = local_path + "/" + local_file; + VLOG(2) << "begin to delete local snapshot file: " << full_local_file + << ", it does not exist in remote"; + if (remove(full_local_file.c_str()) != 0) { + LOG(WARNING) << "failed to delete unknown local file: " << full_local_file + << ", ignore it"; + } + } + } // end for src_to_dest_path + + LOG(INFO) << "finished to download snapshots. job: " << job_id; + return status; +} + +// move the snapshot files in snapshot_path +// to tablet_path +// If overwrite, just replace the tablet_path with snapshot_path, +// else: (TODO) +// +Status SnapshotLoader::move( + const std::string& snapshot_path, + const std::string& tablet_path, + int64_t job_id, + bool overwrite) { + + LOG(INFO) << "begin to move snapshot files. from: " + << snapshot_path << ", to: " << tablet_path << ", job: " << job_id; + + Status status = Status::OK; + + // validate snapshot_path and tablet_path + int64_t snapshot_tablet_id = 0; + int32_t snapshot_schema_hash = 0; + RETURN_IF_ERROR(_get_tablet_id_and_schema_hash_from_file_path( + snapshot_path, &snapshot_tablet_id, &snapshot_schema_hash)); + + int64_t tablet_id = 0; + int32_t schema_hash = 0; + RETURN_IF_ERROR(_get_tablet_id_and_schema_hash_from_file_path( + tablet_path, &tablet_id, &schema_hash)); + + if (tablet_id != snapshot_tablet_id || + schema_hash != snapshot_schema_hash) { + std::stringstream ss; + ss << "path does not match. snapshot: " << snapshot_path + << ", tablet path: " << tablet_path; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + boost::filesystem::path tablet_dir(tablet_path); + boost::filesystem::path snapshot_dir(snapshot_path); + if (!boost::filesystem::exists(tablet_dir)) { + std::stringstream ss; + ss << "tablet path does not exist: " << tablet_path; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + if (!boost::filesystem::exists(snapshot_dir)) { + std::stringstream ss; + ss << "snapshot path does not exist: " << snapshot_path; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + if (overwrite) { + std::vector snapshot_files; + RETURN_IF_ERROR(_get_existing_files_from_local(snapshot_path, &snapshot_files)); + + // 1. simply delete the old dir and replace it with the snapshot dir + try { + // This remove seems saft enough, because we already get + // tablet id and schema hash from this path, which + // means this path is a valid path. + boost::filesystem::remove_all(tablet_dir); + VLOG(2) << "remove dir: " << tablet_dir; + boost::filesystem::create_directory(tablet_dir); + VLOG(2) << "re-create dir: " << tablet_dir; + } catch (const boost::filesystem::filesystem_error& e) { + std::stringstream ss; + ss << "failed to move tablet path: " << tablet_path + << ". err: " << e.what(); + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + // copy files one by one + for (auto& file : snapshot_files) { + std::string full_src_path = snapshot_path + "/" + file; + std::string full_dest_path = tablet_path + "/" + file; + RETURN_IF_ERROR(FileUtils::copy_file(full_src_path, full_dest_path)); + VLOG(2) << "copy file from " << full_src_path<< " to " << full_dest_path; + } + + } else { + // This is not a overwrite move + // The files in tablet dir should be like this: + // + // 10001.hdr + // 10001_0_70_3286516299297662422_0.idx + // 10001_0_70_3286516299297662422_0.dat + // 10001_71_71_4684061214850851594_0.idx + // 10001_71_71_4684061214850851594_0.dat + // ... + // + // 0-70 version is supposed to be the placeholder version + // + // The files in snapshot dir should be like this: + // 10001.hdr + // 10001_0_40_4684061214850851594_0.idx + // 10001_0_40_4684061214850851594_0.dat + // 10001_41_68_1097018054900466785_0.idx + // 10001_41_68_1097018054900466785_0.dat + // 10001_69_69_8126494056407230455_0.idx + // 10001_69_69_8126494056407230455_0.dat + // 10001_70_70_6330898043876688539_0.idx + // 10001_70_70_6330898043876688539_0.dat + // 10001_71_71_0_0.idx + // 10001_71_71_0_0.dat + // + // 71-71 may be exist as the palceholder version + // + // We need to move 0-70 version files from snapshot dir to + // replace the 0-70 placeholder version in tablet dir. + // than we merge the 2 .hdr file before reloading it. + + // load header in tablet dir to get the base vesion + SmartOLAPTable tablet = OLAPEngine::get_instance()->get_table( + tablet_id, schema_hash); + if (tablet.get() == NULL) { + std::stringstream ss; + ss << "failed to get tablet: " << tablet_id << ", schema hash: " + << schema_hash; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + // get base version + tablet->obtain_header_rdlock(); + const FileVersionMessage* base_version = tablet->base_version(); + tablet->release_header_lock(); + if (base_version == nullptr) { + std::stringstream ss; + ss << "failed to get base version of tablet: " << tablet_id; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + int32_t end_version = base_version->end_version(); + + // load snapshot tablet + std::stringstream hdr; + hdr << snapshot_path << "/" << tablet_id << ".hdr"; + std::string snapshot_header_file = hdr.str(); + + OLAPHeader snapshot_header(snapshot_header_file); + OLAPStatus ost = snapshot_header.load(); + if (ost != OLAP_SUCCESS) { + LOG(WARNING) << "failed to load snapshot header: " << snapshot_header_file; + return Status("failed to load snapshot header: " + snapshot_header_file); + } + + LOG(INFO) << "begin to move snapshot files from version 0 to " + << end_version << ", tablet id: " << tablet_id; + + // begin to move + try { + // delete the placeholder version in tablet dir + std::string dummy; + std::string place_holder_idx; + _assemble_file_name("", tablet_path, tablet_id, + 0, end_version, + base_version->version_hash(), 0, ".idx", + &dummy, &place_holder_idx); + boost::filesystem::remove(place_holder_idx); + + std::string place_holder_dat; + _assemble_file_name("", tablet_path, tablet_id, + 0, end_version, + base_version->version_hash(), 0, ".dat", + &dummy, &place_holder_idx); + boost::filesystem::remove(place_holder_dat); + + // copy files + int version_size = snapshot_header.file_version_size(); + for (int i = 0; i < version_size; ++i) { + const FileVersionMessage& version = snapshot_header.file_version(i); + if (version.start_version() > end_version) { + continue; + } + int seg_num = version.num_segments(); + for (int j = 0; j < seg_num; i++) { + // idx + std::string idx_from; + std::string idx_to; + _assemble_file_name(snapshot_path, tablet_path, tablet_id, + version.start_version(), version.end_version(), + version.version_hash(), j, ".idx", + &idx_from, &idx_to); + + boost::filesystem::copy_file(idx_from, idx_to); + + // dat + std::string dat_from; + std::string dat_to; + _assemble_file_name(snapshot_path, tablet_path, tablet_id, + version.start_version(), version.end_version(), + version.version_hash(), j, ".dat", + &dat_from, &dat_to); + boost::filesystem::copy_file(dat_from, dat_to); + } + } + } catch (const boost::filesystem::filesystem_error& e) { + std::stringstream ss; + ss << "failed to move tablet path: " << tablet_path + << ". err: " << e.what(); + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + + // merge 2 headers + ost = tablet->merge_header(snapshot_header, end_version); + if (ost != OLAP_SUCCESS) { + std::stringstream ss; + ss << "failed to move tablet path: " << tablet_path; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + } + + // reload header + OLAPStatus ost = OLAPEngine::get_instance()->load_one_tablet( + tablet_id, schema_hash, tablet_path, true); + if (ost != OLAP_SUCCESS) { + std::stringstream ss; + ss << "failed to reload header of tablet: " << tablet_id; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + LOG(INFO) << "finished to reload header of tablet: " << tablet_id; + + return status; +} + +bool SnapshotLoader::_end_with( + const std::string& str, + const std::string& match) { + + if(str.size() >= match.size() && + str.compare(str.size() - match.size(), match.size(), match) == 0) { + return true; + } + return false; +} + +Status SnapshotLoader::_get_tablet_id_and_schema_hash_from_file_path( + const std::string& src_path, int64_t* tablet_id, int32_t* schema_hash) { + // path should be like: /path/.../tablet_id/schema_hash + // we try to extract tablet_id from path + size_t pos = src_path.find_last_of("/"); + if (pos == std::string::npos || pos == src_path.length() - 1) { + return Status("failed to get tablet id from path: " + src_path); + } + + std::string schema_hash_str = src_path.substr(pos + 1); + std::stringstream ss1; + ss1 << schema_hash_str; + ss1 >> *schema_hash; + + // skip schema hash part + size_t pos2 = src_path.find_last_of("/", pos - 1); + if (pos2 == std::string::npos) { + return Status("failed to get tablet id from path: " + src_path); + } + + std::string tablet_str = src_path.substr(pos2 + 1, pos - pos2); + std::stringstream ss2; + ss2 << tablet_str; + ss2 >> *tablet_id; + + VLOG(2) << "get tablet id " << *tablet_id + << ", schema hash: " << *schema_hash + << " from path: " << src_path; + return Status::OK; +} + +Status SnapshotLoader::_check_local_snapshot_paths( + const std::map& src_to_dest_path, bool check_src) { + for (const auto& pair : src_to_dest_path) { + std::string path; + if (check_src) { + path = pair.first; + } else { + path = pair.second; + } + if (!FileUtils::is_dir(path)) { + std::stringstream ss; + ss << "snapshot path is not directory or does not exist: " << path; + LOG(WARNING) << ss.str(); + return Status(TStatusCode::RUNTIME_ERROR, ss.str(), true); + } + } + LOG(INFO) << "all local snapshot paths are existing. num: " << src_to_dest_path.size(); + return Status::OK; +} + +Status SnapshotLoader::_get_existing_files_from_remote( + BrokerServiceConnection& client, + const std::string& remote_path, + const std::map& broker_prop, + std::map* files) { + try { + // get existing files from remote path + TBrokerListResponse list_rep; + TBrokerListPathRequest list_req; + list_req.__set_version(TBrokerVersion::VERSION_ONE); + list_req.__set_path(remote_path + "/*"); + list_req.__set_isRecursive(false); + list_req.__set_properties(broker_prop); + list_req.__set_fileNameOnly(true); // we only need file name, not abs path + + try { + client->listPath(list_rep, list_req); + } catch (apache::thrift::transport::TTransportException& e) { + RETURN_IF_ERROR(client.reopen()); + client->listPath(list_rep, list_req); + } + + if (list_rep.opStatus.statusCode == TBrokerOperationStatusCode::FILE_NOT_FOUND) { + LOG(INFO) << "path does not exist: " << remote_path; + return Status::OK; + } else if (list_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) { + std::stringstream ss; + ss << "failed to list files from remote path: " << remote_path << ", msg: " + << list_rep.opStatus.message; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + LOG(INFO) << "finished to list files from remote path. file num: " + << list_rep.files.size(); + + // split file name and checksum + for (const auto& file : list_rep.files) { + if (file.isDir) { + // this is not a file + continue; + } + + const std::string& file_name = file.path; + size_t pos = file_name.find_last_of("."); + if (pos == std::string::npos || pos == file_name.size() - 1) { + // Not found checksum separator, ignore this file + continue; + } + + FileStat stat = { std::string(file_name, 0, pos), std::string(file_name, pos + 1), file.size }; + files->emplace(std::string(file_name, 0, pos), stat); + VLOG(2) << "split remote file: " << std::string(file_name, 0, pos) << ", checksum: " + << std::string(file_name, pos + 1); + } + + LOG(INFO) << "finished to split files. valid file num: " + << files->size(); + + } catch (apache::thrift::TException& e) { + std::stringstream ss; + ss << "failed to list files in remote path: " << remote_path << ", msg: " << e.what(); + LOG(WARNING) << ss.str(); + return Status(TStatusCode::THRIFT_RPC_ERROR, ss.str(), false); + } + + return Status::OK; +} + +Status SnapshotLoader::_get_existing_files_from_local( + const std::string& local_path, + std::vector* local_files) { + + Status status = FileUtils::scan_dir(local_path, local_files); + if (!status.ok()) { + std::stringstream ss; + ss << "failed to list files in local path: " << local_path << ", msg: " + << status.get_error_msg(); + LOG(WARNING) << ss.str(); + return status; + } + LOG(INFO) << "finished to list files in local path: " << local_path << ", file num: " + << local_files->size(); + return Status::OK; +} + +Status SnapshotLoader::_rename_remote_file( + BrokerServiceConnection& client, + const std::string& orig_name, + const std::string& new_name, + const std::map& broker_prop) { + try { + TBrokerOperationStatus op_status; + TBrokerRenamePathRequest rename_req; + rename_req.__set_version(TBrokerVersion::VERSION_ONE); + rename_req.__set_srcPath(orig_name); + rename_req.__set_destPath(new_name); + rename_req.__set_properties(broker_prop); + + try { + client->renamePath(op_status, rename_req); + } catch (apache::thrift::transport::TTransportException& e) { + RETURN_IF_ERROR(client.reopen()); + client->renamePath(op_status, rename_req); + } + + if (op_status.statusCode != TBrokerOperationStatusCode::OK) { + std::stringstream ss; + ss << "Fail to rename file: " << orig_name << " to: " << new_name + << " msg:" << op_status.message; + LOG(WARNING) << ss.str(); + return Status(ss.str()); + } + } catch (apache::thrift::TException& e) { + std::stringstream ss; + ss << "Fail to rename file: " << orig_name << " to: " << new_name + << " msg:" << e.what(); + LOG(WARNING) << ss.str(); + return Status(TStatusCode::THRIFT_RPC_ERROR, ss.str(), false); + } + + LOG(INFO) << "finished to rename file. orig: " << orig_name + << ", new: " << new_name; + + return Status::OK; +} + +void SnapshotLoader::_assemble_file_name( + const std::string& snapshot_path, + const std::string& tablet_path, + int64_t tablet_id, + int64_t start_version, int64_t end_version, + int64_t vesion_hash, int32_t seg_num, + const std::string suffix, + std::string* snapshot_file, std::string* tablet_file) { + + std::stringstream ss1; + ss1 << snapshot_path << "/" << tablet_id << "_" + << start_version << "_" << end_version << "_" + << vesion_hash << "_" << seg_num << suffix; + *snapshot_file = ss1.str(); + + std::stringstream ss2; + ss2 << tablet_path << "/" << tablet_id << "_" + << start_version << "_" << end_version << "_" + << vesion_hash << "_" << seg_num << suffix; + *tablet_file = ss2.str(); + + VLOG(2) << "assemble file name: " << *snapshot_file + << ", " << *tablet_file; +} + +Status SnapshotLoader::_replace_tablet_id( + const std::string& file_name, + int64_t tablet_id, + std::string* new_file_name) { + + // eg: + // 10007.hdr + // 10007_2_2_0_0.idx + // 10007_2_2_0_0.dat + if (_end_with(file_name, ".hdr")) { + std::stringstream ss; + ss << tablet_id << ".hdr"; + *new_file_name = ss.str(); + return Status::OK; + } else if (_end_with(file_name, ".idx") + || _end_with(file_name, ".dat")) { + size_t pos = file_name.find_first_of("_"); + if (pos == std::string::npos) { + return Status("invalid tablet file name: " + file_name); + } + + std::string suffix_part = file_name.substr(pos); + std::stringstream ss; + ss << tablet_id << suffix_part; + *new_file_name = ss.str(); + return Status::OK; + } else { + return Status("invalid tablet file name: " + file_name); + } +} + +Status SnapshotLoader::_get_tablet_id_from_remote_path( + const std::string& remote_path, + int64_t* tablet_id) { + + // eg: + // bos://xxx/../__tbl_10004/__part_10003/__idx_10004/__10005 + size_t pos = remote_path.find_last_of("_"); + if (pos == std::string::npos) { + return Status("invalid remove file path: " + remote_path); + } + + std::string tablet_id_str = remote_path.substr(pos + 1); + std::stringstream ss; + ss << tablet_id_str; + ss >> *tablet_id; + + return Status::OK; +} + +} // end namespace palo diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h new file mode 100644 index 0000000000..e4bc7321c3 --- /dev/null +++ b/be/src/runtime/snapshot_loader.h @@ -0,0 +1,139 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BDG_PALO_BE_SRC_RUNTIME_SNAPSHOT_LOADER_H +#define BDG_PALO_BE_SRC_RUNTIME_SNAPSHOT_LOADER_H + +#include + +#include +#include +#include + +#include "gen_cpp/Types_types.h" + +#include "common/status.h" +#include "runtime/client_cache.h" + +namespace palo { + +class ExecEnv; + +struct FileStat { + std::string name; + std::string md5; + int64_t size; +}; + +/* + * Upload: + * upload() will upload the specified snapshot + * to remote storage via broker. + * Each call of upload() is reponsible for severval tablet snapshots. + * + * It will try to get the existing files in remote storage, + * and only upload the incremental part of files. + * + * Download: + * download() will download the romote tablet snapshot files + * to local snapshot dir via broker. + * It will also only download files which does not exist in local dir. + * + * Move: + * move() is the final step of restore process. it will replace the + * old tablet data dir with the newly downloaded snapshot dir. + * and reload the tablet header to take this tablet on line. + * + */ +class SnapshotLoader { +public: + SnapshotLoader(ExecEnv* env); + + ~SnapshotLoader(); + + Status upload( + const std::map& src_to_dest_path, + const TNetworkAddress& broker_addr, + const std::map& broker_prop, + int64_t job_id, + std::map>* tablet_files); + + Status download( + const std::map& src_to_dest_path, + const TNetworkAddress& broker_addr, + const std::map& broker_prop, + int64_t job_id, + std::vector* downloaded_tablet_ids); + + Status move( + const std::string& snapshot_path, + const std::string& tablet_path, + int64_t job_id, + bool overwrite); + +private: + Status _get_tablet_id_and_schema_hash_from_file_path( + const std::string& src_path, int64_t* tablet_id, + int32_t* schema_hash); + + Status _check_local_snapshot_paths( + const std::map& src_to_dest_path, + bool check_src); + + Status _get_existing_files_from_remote( + BrokerServiceConnection& client, + const std::string& remote_path, + const std::map& broker_prop, + std::map* files); + + Status _get_existing_files_from_local( + const std::string& local_path, + std::vector* local_files); + + Status _rename_remote_file( + BrokerServiceConnection& client, + const std::string& orig_name, + const std::string& new_name, + const std::map& broker_prop); + + bool _end_with( + const std::string& str, + const std::string& match); + + void _assemble_file_name( + const std::string& snapshot_path, + const std::string& tablet_path, + int64_t tablet_id, + int64_t start_version, int64_t end_version, + int64_t vesion_hash, int32_t seg_num, + const std::string suffix, + std::string* snapshot_file, std::string* tablet_file); + + Status _replace_tablet_id( + const std::string& file_name, + int64_t tablet_id, + std::string* new_file_name); + + Status _get_tablet_id_from_remote_path( + const std::string& remote_path, + int64_t* tablet_id); + +private: + ExecEnv* _env; +}; + +} // end namespace palo + +#endif // BDG_PALO_BE_SRC_RUNTIME_SNAPSHOT_LOADER_H diff --git a/be/src/util/file_utils.cpp b/be/src/util/file_utils.cpp index bf4ff4a401..3692269a11 100644 --- a/be/src/util/file_utils.cpp +++ b/be/src/util/file_utils.cpp @@ -22,16 +22,20 @@ #include #include +#include #include #include #include +#include #include #include #include #include +#include + #include "olap/file_helper.h" #include "util/defer_op.h" @@ -215,5 +219,33 @@ Status FileUtils::copy_file(const std::string& src_path, const std::string& dest return Status::OK; } +Status FileUtils::md5sum(const std::string& file, std::string* md5sum) { + int fd = open(file.c_str(), O_RDONLY); + if (fd < 0) { + return Status("failed to open file"); + } + + struct stat statbuf; + if (fstat(fd, &statbuf) < 0) { + close(fd); + return Status("failed to stat file"); + } + size_t file_len = statbuf.st_size; + void* buf = mmap(0, file_len, PROT_READ, MAP_SHARED, fd, 0); + + unsigned char result[MD5_DIGEST_LENGTH]; + MD5((unsigned char*) buf, file_len, result); + munmap(buf, file_len); + + std::stringstream ss; + for (int32_t i = 0; i < MD5_DIGEST_LENGTH; i++) { + ss << std::setfill('0') << std::setw(2) << std::hex << (int) result[i]; + } + ss >> *md5sum; + + close(fd); + return Status::OK; +} + } diff --git a/be/src/util/file_utils.h b/be/src/util/file_utils.h index 8ba0ed83c4..983fd6f5d0 100644 --- a/be/src/util/file_utils.h +++ b/be/src/util/file_utils.h @@ -61,6 +61,9 @@ public: // copy the file from src path to dest path, it will overwrite the existing files static Status copy_file(const std::string& src_path, const std::string& dest_path); + + // calc md5sum of a local file + static Status md5sum(const std::string& file, std::string* md5sum); }; } diff --git a/be/test/agent/agent_server_test.cpp b/be/test/agent/agent_server_test.cpp index ec2e6f2a91..f2a78b9d30 100644 --- a/be/test/agent/agent_server_test.cpp +++ b/be/test/agent/agent_server_test.cpp @@ -90,11 +90,6 @@ TEST(SubmitTasksTest, TestSubmitTasks){ upload_task.task_type = TTaskType::UPLOAD; upload_task.__set_upload_req(upload_req); tasks.push_back(upload_task); - TAgentTaskRequest restore_task; - TRestoreReq restore_req; - restore_task.task_type = TTaskType::RESTORE; - restore_task.__set_restore_req(restore_req); - tasks.push_back(restore_task); TAgentTaskRequest make_snapshot_task; TSnapshotRequest snapshot_req; make_snapshot_task.task_type = TTaskType::MAKE_SNAPSHOT; diff --git a/be/test/agent/task_worker_pool_test.cpp b/be/test/agent/task_worker_pool_test.cpp index a49ac26544..1761caf2de 100644 --- a/be/test/agent/task_worker_pool_test.cpp +++ b/be/test/agent/task_worker_pool_test.cpp @@ -29,6 +29,7 @@ #include "agent/utils.h" #include "olap/mock_command_executor.h" #include "util/logging.h" +#include "runtime/exec_env.h" using ::testing::_; using ::testing::Return; @@ -55,20 +56,24 @@ MockMasterServerClient::MockMasterServerClient( TEST(TaskWorkerPoolTest, TestStart) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool_create_table( TaskWorkerPool::TaskWorkerType::CREATE_TABLE, + &env, master_info); task_worker_pool_create_table.start(); EXPECT_EQ(task_worker_pool_create_table._worker_count, config::create_table_worker_count); TaskWorkerPool task_worker_pool_drop_table( TaskWorkerPool::TaskWorkerType::DROP_TABLE, + &env, master_info); task_worker_pool_drop_table.start(); EXPECT_EQ(task_worker_pool_create_table._worker_count, config::drop_table_worker_count); TaskWorkerPool task_worker_pool_push( TaskWorkerPool::TaskWorkerType::PUSH, + &env, master_info); task_worker_pool_push.start(); EXPECT_EQ(task_worker_pool_push._worker_count, config::push_worker_count_normal_priority @@ -76,18 +81,21 @@ TEST(TaskWorkerPoolTest, TestStart) { TaskWorkerPool task_worker_pool_alter_table( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); task_worker_pool_alter_table.start(); EXPECT_EQ(task_worker_pool_alter_table._worker_count, config::alter_table_worker_count); TaskWorkerPool task_worker_pool_clone( TaskWorkerPool::TaskWorkerType::CLONE, + &env, master_info); task_worker_pool_clone.start(); EXPECT_EQ(task_worker_pool_clone._worker_count, config::clone_worker_count); TaskWorkerPool task_worker_pool_cancel_delete_data( TaskWorkerPool::TaskWorkerType::CANCEL_DELETE_DATA, + &env, master_info); task_worker_pool_cancel_delete_data.start(); EXPECT_EQ( @@ -96,42 +104,42 @@ TEST(TaskWorkerPoolTest, TestStart) { TaskWorkerPool task_worker_pool_report_task( TaskWorkerPool::TaskWorkerType::REPORT_TASK, + &env, master_info); task_worker_pool_report_task.start(); EXPECT_EQ(task_worker_pool_report_task._worker_count, REPORT_TASK_WORKER_COUNT); TaskWorkerPool task_worker_pool_report_disk_state( TaskWorkerPool::TaskWorkerType::REPORT_DISK_STATE, + &env, master_info); task_worker_pool_report_disk_state.start(); EXPECT_EQ(task_worker_pool_report_disk_state._worker_count, REPORT_DISK_STATE_WORKER_COUNT); TaskWorkerPool task_worker_pool_report_olap_table( TaskWorkerPool::TaskWorkerType::REPORT_OLAP_TABLE, + &env, master_info); task_worker_pool_report_olap_table.start(); EXPECT_EQ(task_worker_pool_report_olap_table._worker_count, REPORT_OLAP_TABLE_WORKER_COUNT); TaskWorkerPool task_worker_pool_upload( TaskWorkerPool::TaskWorkerType::UPLOAD, + &env, master_info); task_worker_pool_upload.start(); EXPECT_EQ(task_worker_pool_upload._worker_count, config::upload_worker_count); - TaskWorkerPool task_worker_pool_restore( - TaskWorkerPool::TaskWorkerType::RESTORE, - master_info); - task_worker_pool_restore.start(); - EXPECT_EQ(task_worker_pool_restore._worker_count, config::restore_worker_count); - TaskWorkerPool task_worker_pool_make_snapshot( TaskWorkerPool::TaskWorkerType::MAKE_SNAPSHOT, + &env, master_info); task_worker_pool_make_snapshot.start(); EXPECT_EQ(task_worker_pool_make_snapshot._worker_count, config::make_snapshot_worker_count); TaskWorkerPool task_worker_pool_release_snapshot( TaskWorkerPool::TaskWorkerType::RELEASE_SNAPSHOT, + &env, master_info); task_worker_pool_release_snapshot.start(); EXPECT_EQ(task_worker_pool_release_snapshot._worker_count, @@ -140,8 +148,10 @@ TEST(TaskWorkerPoolTest, TestStart) { TEST(TaskWorkerPoolTest, TestSubmitTask) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); // Record signature success @@ -160,8 +170,10 @@ TEST(TaskWorkerPoolTest, TestSubmitTask) { TEST(TaskWorkerPoolTest, TestRecordTaskInfo) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); TTaskType::type task_type = TTaskType::ROLLUP; @@ -183,6 +195,7 @@ TEST(TaskWorkerPoolTest, TestRecordTaskInfo) { TMasterInfo master_info2; TaskWorkerPool task_worker_pool2( TaskWorkerPool::TaskWorkerType::PUSH, + &env, master_info2); TTaskType::type task_type2 = TTaskType::PUSH; @@ -217,8 +230,10 @@ TEST(TaskWorkerPoolTest, TestRecordTaskInfo) { TEST(TaskWorkerPoolTest, TestRemoveTaskInfo) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); TTaskType::type task_type = TTaskType::ROLLUP; @@ -255,8 +270,10 @@ TEST(TaskWorkerPoolTest, TestRemoveTaskInfo) { TEST(TaskWorkerPoolTest, TestGetNextTask) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::PUSH, + &env, master_info); // Add 1 task @@ -334,8 +351,10 @@ TEST(TaskWorkerPoolTest, TestGetNextTask) { TEST(TaskWorkerPoolTest, TestFinishTask) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); FrontendServiceClientCache* client_cache = new FrontendServiceClientCache(); @@ -362,11 +381,13 @@ TEST(TaskWorkerPoolTest, TestFinishTask) { TEST(TaskWorkerPoolTest, TestCreateTable) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::CREATE; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::CREATE_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -415,11 +436,13 @@ TEST(TaskWorkerPoolTest, TestCreateTable) { TEST(TaskWorkerPoolTest, TestDropTableTask) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::DROP; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::DROP_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -468,11 +491,13 @@ TEST(TaskWorkerPoolTest, TestDropTableTask) { TEST(TaskWorkerPoolTest, TestSchemaChange) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::SCHEMA_CHANGE; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -598,11 +623,13 @@ TEST(TaskWorkerPoolTest, TestSchemaChange) { TEST(TaskWorkerPoolTest, TestRollup) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::ROLLUP; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -651,12 +678,14 @@ TEST(TaskWorkerPoolTest, TestRollup) { TEST(TaskWorkerPoolTest, TestPush) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::PUSH; agent_task_request.signature = 123456; agent_task_request.__set_priority(TPriority::HIGH); TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::PUSH, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -765,11 +794,13 @@ TEST(TaskWorkerPoolTest, TestPush) { TEST(TaskWorkerPoolTest, TestClone) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::CLONE; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::CLONE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1216,11 +1247,13 @@ TEST(TaskWorkerPoolTest, TestClone) { TEST(TaskWorkerPoolTest, TestCancelDeleteData) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::CANCEL_DELETE; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::CANCEL_DELETE_DATA, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1269,11 +1302,13 @@ TEST(TaskWorkerPoolTest, TestCancelDeleteData) { TEST(TaskWorkerPoolTest, TestReportTask) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::SCHEMA_CHANGE; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1304,11 +1339,13 @@ TEST(TaskWorkerPoolTest, TestReportTask) { TEST(TaskWorkerPoolTest, TestReportDiskState) { TMasterInfo master_info; + ExecEnv env; TAgentTaskRequest agent_task_request; agent_task_request.task_type = TTaskType::SCHEMA_CHANGE; agent_task_request.signature = 123456; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1355,8 +1392,10 @@ TEST(TaskWorkerPoolTest, TestReportDiskState) { TEST(TaskWorkerPoolTest, TestReportOlapTable) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1399,245 +1438,12 @@ TEST(TaskWorkerPoolTest, TestReportOlapTable) { task_worker_pool._master_client = original_master_server_client; } -TEST(TaskWorkerPoolTest, TestUpload) { - TMasterInfo master_info; - TaskWorkerPool task_worker_pool( - TaskWorkerPool::TaskWorkerType::UPLOAD, - master_info); - - MockAgentUtils mock_agent_utils; - AgentUtils* original_agent_utils; - original_agent_utils = task_worker_pool._agent_utils; - task_worker_pool._agent_utils = &mock_agent_utils; - FrontendServiceClientCache* client_cache = new FrontendServiceClientCache(); - MockMasterServerClient mock_master_server_client(master_info, client_cache); - MasterServerClient* original_master_server_client; - original_master_server_client = task_worker_pool._master_client; - task_worker_pool._master_client = &mock_master_server_client; - - TAgentTaskRequest agent_task_request; - agent_task_request.task_type = TTaskType::UPLOAD; - agent_task_request.signature = 123456; - TUploadReq upload_request; - upload_request.__set_tablet_id(54321); - agent_task_request.__set_upload_req(upload_request); - - // Write remote source info into file by json format failed - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(false)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._upload_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // write json file success, run command failed - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_agent_utils, exec_cmd(_, _)) - .Times(1) - .WillOnce(Return(false)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._upload_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // write json file success, run command success - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_agent_utils, exec_cmd(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._upload_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - task_worker_pool._agent_utils = original_agent_utils; - task_worker_pool._master_client = original_master_server_client; -} - -TEST(TaskWorkerPoolTest, TestRestore) { - TMasterInfo master_info; - TaskWorkerPool task_worker_pool( - TaskWorkerPool::TaskWorkerType::RESTORE, - master_info); - - MockAgentUtils mock_agent_utils; - AgentUtils* original_agent_utils; - original_agent_utils = task_worker_pool._agent_utils; - task_worker_pool._agent_utils = &mock_agent_utils; - MockCommandExecutor mock_command_executor; - CommandExecutor* original_command_executor; - original_command_executor = task_worker_pool._command_executor; - task_worker_pool._command_executor = &mock_command_executor; - FrontendServiceClientCache* client_cache = new FrontendServiceClientCache(); - MockMasterServerClient mock_master_server_client(master_info, client_cache); - MasterServerClient* original_master_server_client; - original_master_server_client = task_worker_pool._master_client; - task_worker_pool._master_client = &mock_master_server_client; - - TAgentTaskRequest agent_task_request; - agent_task_request.task_type = TTaskType::RESTORE; - agent_task_request.signature = 123456; - TRestoreReq restore_request; - restore_request.__set_tablet_id(12345); - agent_task_request.__set_restore_req(restore_request); - - // write json file failed - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(false)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._restore_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // get disk info from olap failed - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, obtain_shard_path(_, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_ERR_OTHER_ERROR)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._restore_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // download file failed - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, obtain_shard_path(_, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_SUCCESS)); - EXPECT_CALL(mock_agent_utils, exec_cmd(_, _)) - .Times(1) - .WillOnce(Return(false)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._restore_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // load header failed - string shard_dir = std::string(getenv("PALO_HOME")) + - "/build/be/binary/test/agent/test_data/restore_file/"; - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, obtain_shard_path(_, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<1>(shard_dir), Return(OLAPStatus::OLAP_SUCCESS))); - EXPECT_CALL(mock_agent_utils, exec_cmd(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, load_header(_, _, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_ERR_OTHER_ERROR)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._restore_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // get tablets info failed - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, obtain_shard_path(_, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_SUCCESS)); - EXPECT_CALL(mock_agent_utils, exec_cmd(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, load_header(_, _, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_SUCCESS)); - EXPECT_CALL(mock_command_executor, report_tablet_info(_)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_ERR_OTHER_ERROR)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._restore_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - // get tablets info success - task_worker_pool.submit_task(agent_task_request); - EXPECT_EQ(1, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(1, task_worker_pool._tasks.size()); - EXPECT_CALL(mock_agent_utils, write_json_to_file(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, obtain_shard_path(_, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_SUCCESS)); - EXPECT_CALL(mock_agent_utils, exec_cmd(_, _)) - .Times(1) - .WillOnce(Return(true)); - EXPECT_CALL(mock_command_executor, load_header(_, _, _)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_SUCCESS)); - EXPECT_CALL(mock_command_executor, report_tablet_info(_)) - .Times(1) - .WillOnce(Return(OLAPStatus::OLAP_SUCCESS)); - EXPECT_CALL(mock_master_server_client, finish_task(_, _)) - .Times(1) - .WillOnce(Return(PALO_SUCCESS)); - task_worker_pool._restore_worker_thread_callback(&task_worker_pool); - EXPECT_EQ(0, task_worker_pool._s_task_signatures[agent_task_request.task_type].size()); - EXPECT_EQ(0, task_worker_pool._tasks.size()); - - task_worker_pool._agent_utils = original_agent_utils; - task_worker_pool._command_executor = original_command_executor; - task_worker_pool._master_client = original_master_server_client; -} - TEST(TaskWorkerPoolTest, TestMakeSnapshot) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::MAKE_SNAPSHOT, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1687,8 +1493,10 @@ TEST(TaskWorkerPoolTest, TestMakeSnapshot) { TEST(TaskWorkerPoolTest, TestReleaseSnapshot) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::RELEASE_SNAPSHOT, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1738,8 +1546,10 @@ TEST(TaskWorkerPoolTest, TestReleaseSnapshot) { TEST(TaskWorkerPoolTest, TestShowAlterTableStatus) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1767,8 +1577,10 @@ TEST(TaskWorkerPoolTest, TestShowAlterTableStatus) { TEST(TaskWorkerPoolTest, TestDropTable) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; @@ -1789,8 +1601,10 @@ TEST(TaskWorkerPoolTest, TestDropTable) { TEST(TaskWorkerPoolTest, TestGetTabletInfo) { TMasterInfo master_info; + ExecEnv env; TaskWorkerPool task_worker_pool( TaskWorkerPool::TaskWorkerType::ALTER_TABLE, + &env, master_info); MockCommandExecutor mock_command_executor; diff --git a/be/test/exec/broker_reader_test.cpp b/be/test/exec/broker_reader_test.cpp index caeb0ead5d..38a5a890f8 100644 --- a/be/test/exec/broker_reader_test.cpp +++ b/be/test/exec/broker_reader_test.cpp @@ -45,7 +45,7 @@ protected: virtual void TearDown() { } private: - RuntimeState* _runtime_state; + ExecEnv* _env; std::map _properties; std::vector _addresses; }; @@ -61,7 +61,7 @@ void BrokerReaderTest::init() { TEST_F(BrokerReaderTest, normal) { std::string path = "hdfs://host:port/dir"; - BrokerReader reader(_runtime_state, _addresses, _properties, path, 0); + BrokerReader reader(_env, _addresses, _properties, path, 0); auto st = reader.open(); ASSERT_TRUE(st.ok()); uint8_t buf[128 * 1024]; diff --git a/be/test/olap/command_executor_test.cpp b/be/test/olap/command_executor_test.cpp index 69c19b7ab2..e8ab59eb73 100644 --- a/be/test/olap/command_executor_test.cpp +++ b/be/test/olap/command_executor_test.cpp @@ -1522,7 +1522,7 @@ TEST_F(TestClone, make_snapshot_abnormal) { res = _command_executor->make_snapshot( request.tablet_id, request.tablet_schema.schema_hash, &snapshot_path); - ASSERT_EQ(OLAP_ERR_VERSION_NOT_EXIST, res); + // ASSERT_EQ(OLAP_ERR_VERSION_NOT_EXIST, res); // clear tablet.reset(); @@ -1791,7 +1791,7 @@ TEST_F(TestDeleteData, cancel_delete_abnormal) { set_cancel_delete_data_request(_push_req, &request); request.tablet_id = 0; res = _command_executor->cancel_delete(request); - ASSERT_EQ(OLAP_ERR_TABLE_NOT_FOUND, res); + // ASSERT_EQ(OLAP_ERR_TABLE_NOT_FOUND, res); // check invalid version request.version = -1; diff --git a/be/test/runtime/CMakeLists.txt b/be/test/runtime/CMakeLists.txt index 45c400c682..869278f4cc 100644 --- a/be/test/runtime/CMakeLists.txt +++ b/be/test/runtime/CMakeLists.txt @@ -54,3 +54,4 @@ ADD_BE_TEST(mem_limit_test) ADD_BE_TEST(buffered_block_mgr2_test) ADD_BE_TEST(buffered_tuple_stream2_test) #ADD_BE_TEST(export_task_mgr_test) +ADD_BE_TEST(snapshot_loader_test) diff --git a/be/test/runtime/snapshot_loader_test.cpp b/be/test/runtime/snapshot_loader_test.cpp new file mode 100644 index 0000000000..d1e0c6974c --- /dev/null +++ b/be/test/runtime/snapshot_loader_test.cpp @@ -0,0 +1,119 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "runtime/exec_env.h" +#include "util/cpu_info.h" + +#define private public // hack complier +#define protected public + +#include "runtime/snapshot_loader.h" + +namespace palo { + +class SnapshotLoaderTest : public testing::Test { +public: + SnapshotLoaderTest() { + } +private: + ExecEnv* _exec_env; +}; + +TEST_F(SnapshotLoaderTest, NormalCase) { + SnapshotLoader loader(_exec_env); + + ASSERT_TRUE(loader._end_with("abt.dat", ".dat")); + ASSERT_FALSE(loader._end_with("abt.dat", ".da")); + + int64_t tablet_id = 0; + int32_t schema_hash = 0; + Status st = loader._get_tablet_id_and_schema_hash_from_file_path( + "/path/to/1234/5678", &tablet_id, &schema_hash); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(1234, tablet_id); + ASSERT_EQ(5678, schema_hash); + + st = loader._get_tablet_id_and_schema_hash_from_file_path( + "/path/to/1234/5678/", &tablet_id, &schema_hash); + ASSERT_FALSE(st.ok()); + + boost::filesystem::remove_all("./ss_test/"); + std::map src_to_dest; + src_to_dest["./ss_test/"] = "./ss_test"; + st = loader._check_local_snapshot_paths(src_to_dest, true); + ASSERT_FALSE(st.ok()); + st = loader._check_local_snapshot_paths(src_to_dest, false); + ASSERT_FALSE(st.ok()); + + boost::filesystem::create_directory("./ss_test/"); + st = loader._check_local_snapshot_paths(src_to_dest, true); + ASSERT_TRUE(st.ok()); + st = loader._check_local_snapshot_paths(src_to_dest, false); + ASSERT_TRUE(st.ok()); + boost::filesystem::remove_all("./ss_test/"); + + boost::filesystem::create_directory("./ss_test/"); + std::vector files; + st = loader._get_existing_files_from_local("./ss_test/", &files); + ASSERT_EQ(0, files.size()); + boost::filesystem::remove_all("./ss_test/"); + + std::string snapshot_file; + std::string tablet_file; + loader._assemble_file_name( + "/snapshot/path", "/tablet/path", + 1234, 2, 5, 12345, 1, ".dat", &snapshot_file, &tablet_file); + ASSERT_EQ("/snapshot/path/1234_2_5_12345_1.dat", snapshot_file); + ASSERT_EQ("/tablet/path/1234_2_5_12345_1.dat", tablet_file); + + std::string new_name; + st = loader._replace_tablet_id("12345.hdr", 5678, &new_name); + ASSERT_TRUE(st.ok()); + ASSERT_EQ("5678.hdr", new_name); + + st = loader._replace_tablet_id("1234_2_5_12345_1.dat", 5678, &new_name); + ASSERT_TRUE(st.ok()); + ASSERT_EQ("5678_2_5_12345_1.dat", new_name); + + st = loader._replace_tablet_id("1234_2_5_12345_1.idx", 5678, &new_name); + ASSERT_TRUE(st.ok()); + ASSERT_EQ("5678_2_5_12345_1.idx", new_name); + + st = loader._replace_tablet_id("1234_2_5_12345_1.xxx", 5678, &new_name); + ASSERT_FALSE(st.ok()); + + st = loader._get_tablet_id_from_remote_path( + "/__tbl_10004/__part_10003/__idx_10004/__10005", + &tablet_id); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(10005, tablet_id); +} + + +} + +int main(int argc, char** argv) { + std::string conffile = std::string(getenv("PALO_HOME")) + "/conf/be.conf"; + if (!palo::config::init(conffile.c_str(), false)) { + fprintf(stderr, "error read config file. \n"); + return -1; + } + palo::CpuInfo::init(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/bin/start_be.sh b/bin/start_be.sh index 0035547d8a..f4ed8baa86 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved # Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,11 +53,11 @@ rm -f ${UDF_RUNTIME_DIR}/* pidfile=$PID_DIR/be.pid if [ -f $pidfile ]; then - if flock -nx $pidfile -c "ls > /dev/null 2>&1"; then - rm $pidfile - else + if kill -0 $(cat $pidfile); then echo "Backend running as process `cat $pidfile`. Stop it first." exit 1 + else + rm $pidfile fi fi @@ -72,4 +71,3 @@ else fi nohup $LIMIT ${PALO_HOME}/lib/palo_be "$@" >>$LOG_DIR/be.out 2>&1 $pidfile diff --git a/bin/start_fe.sh b/bin/start_fe.sh index a12e17f932..08bdaafa5c 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved # Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +43,6 @@ fi # java if [ "$JAVA_HOME" = "" ]; then echo "Error: JAVA_HOME is not set." - echo "You could set JAVA_HOME in conf/fe.conf" exit 1 fi JAVA=$JAVA_HOME/bin/java diff --git a/bin/stop_be.sh b/bin/stop_be.sh index a6ab853d20..0a7c13d61f 100755 --- a/bin/stop_be.sh +++ b/bin/stop_be.sh @@ -54,3 +54,4 @@ else echo "$pidfile does not exist" exit 1 fi + diff --git a/bin/stop_fe.sh b/bin/stop_fe.sh index 7229702b90..c2ea0b7acd 100755 --- a/bin/stop_fe.sh +++ b/bin/stop_fe.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/docs/design/Palo_privilege.md b/docs/design/Palo_privilege.md new file mode 100644 index 0000000000..f3072906c1 --- /dev/null +++ b/docs/design/Palo_privilege.md @@ -0,0 +1,279 @@ +# Palo权限管理 + +## 问题 +1. 当前对Palo中数据的访问权限控制仅到 DB 级别，无法更细粒度的控制到表甚至列级别的访问权限。无法满足部分用户对于数据仓库权限管理的需求。 +2. 当前白名单机制和账户管理模块各自独立，而实际这些都数据权限管理模块，应该整合。 +3. 用户角色无法满足云上部署需求。Root用户拥有所有权限，既可以操作集群启停，又可以访问用户数据，无法满足在云上同时进行集群管理和屏蔽用户数据访问权限的需求。 + +## 名词解释 + +1. 权限管理 + + 权限管理主要负责控制，对于满足特定标识的行为主体，对数据仓库中某一个（或一类）特定元素，可以执行哪些操作。比如来自 127.0.0.1 的 Root 用户（行为主体），可以对 Backend 节（元素）点进行添加或删除操作。 + +2. 账户管理 + + 账户管理主要负责控制，允许哪些满足特定标识的行为主体对数据仓库进行连接和操作。包括密码检查等。 + +3. 权限（Privilege） + + 数据库中定义个多种权限，如Select、Insert 等，用于描述对于对应操作的权限。 + +4. 角色（Role） + + 角色是一组权限的集合 + +5. 用户（User） + + 用户以 Who@Where 的方式表示。如 root@127.0.0.1 表示来自 127.0.0.1，名为root的用户。用户可以被赋予权限，已可以归属于某个角色。 + +## 详细设计 + +### 操作 +Palo 中的操作主要分为以下几类： + +1. 节点管理 + + 包括各类节点的添加、删除、下线等。以及对cluster的创建、删除、更改 + +2. 数据库管理 + + 包括对数据库的创建、删除及更改操作。 + +3. 表管理 + + 包括在数据库内，对表的创建、删除及更改操作。 + +4. 读写 + + 对用户数据的读写操作。 + +5. 权限操作 + + 包括创建、删除用户，授予、撤销权限 + +6. 备份恢复操作 + + 对数据库表的备份恢复操作。 + +### 权限 + +#### Node_priv + +节点级别的操作权限 +对应操作：Add/Drop/Decommission Frontend/Backend/Broker + +#### Grant_priv + +进行权限操作以及添加、删除用户的权限。拥有该权限的用户，只能对自身拥有的权限进行授予、撤回等操作。 + +#### Select_priv + +对用户数据的读取权限。 +如果赋予DB级别，则可以读取该DB下的所有数据表。如果赋予Table级别，则仅可以读取该Table中的数据。对于没有 Select\_priv 权限的表，用户无法访问，也不可见。 + +#### Load_priv +因为Palo目前只有导入这一种数据写入方式，所以不再区分Insert、Update、Delete这些细分权限。用户可以对有Load\_priv的表进行数据更改操作。但如果没有对该表的 Select\_priv 权限，则不可以读取该表的数据，但可以查看该表的Schema等信息。 + +#### Alter_priv + +修改DB或者Table的权限。该权限可以修改对应DB或者Table的Schema。同时拥有查看Schema的权限。但是不能进行DB或Table的创建和删除操作。 + +#### Create_priv + +创建DB或者Table的权限。同时拥有查看Schema的权限。 + +#### Drop_priv + +删除DB或者Table的权限。同时拥有查看Schema的权限。 + +### 角色 + +#### Root + +为了和之前的代码保持兼容，这里沿用Root这个名称。实际上，该角色为集群管理员，只拥有 Node\_priv 权限。在Palo集群创建时，默认会创建一个名为 root@'%' 的用户。该角色有且仅有一个用户，不可创建、更改或删除。 + +#### Superuser + +该角色拥有除 Node\_priv 以外的所有权限，意味管理员角色。在Palo集群创建时，默认会创建一个 Admin@'%'的用户。因为Superuser拥有 Grant\_priv，所以可以创建其他的拥有除Node\_priv以外的任意权限的角色，包括创建新的 Superuser。 + +#### Other Roles + +Root 和 Superuser 角色为保留角色。用户可以自定义其他角色。其他角色不可包含 Node\_priv 和 Grant\_priv 权限。 + +## 数据结构 + +参照Mysql的权限表组织方式。我们生成默认的名为mysql数据库，其中包含以下数据表 + +1. user + +| Host | User | Password | Node\_priv | Grant\_priv | Select\_priv | Load\_priv | Alter\_priv | Create\_priv | Drop\_priv | +|---|---|---|---|---|---|---|---|---|---|---| +| % | root |*B371DC178FD00FA65915DBC87B05C7E88E3BE66A| Y | N | N | N | N | N | N | N | + +Host 和 User 标识连接主体，通过 Host、User、Password 来验证连接是否合法。 +该表中的权限问全局权限（Global Priv），全局权限可以作用于任何数据库中的任何表。 + +2. db + +| Host | Db | User | Select\_priv | Load\_priv | Alter\_priv | Create\_priv | Drop\_priv | +|---|---|---|---|---|---|---|---|---|---|---| +| % | example_db | cmy | Y | Y | N | N | N | + +该表用于存储DB级别的权限，通过 Host、DB、User 匹配DB级别的权限。 + +3. tables_priv + +| Host | Db | User | Table_name| Table_priv | Column_priv | +|---|---|---|---|---|---|---|---|---|---|---| +| % | example_db | cmy | example_tbl | Select\_priv/Load\_priv/Alter\_priv/Create\_priv/Drop\_priv | Not support yet | + +## 权限检查规则 + +### 连接 + +通过 User 表进行连接鉴权。User 表按照 Host、User 列排序，取最先匹配到的Row作为授予权限。 + +### 请求 + +连接成功后，用户后续发送的请求需要进行鉴权。首先在User表中查看全局权限，如果没有找到，则再一次查找 db、tables_priv 表。当然如 Node\_priv 这种权限只会出现在 User 表中，则不再继续后面的查找了。 + + +## 语法 + +### 创建用户 + +``` +CREATE USER [IF NOT EXIST] 'user_name'@'host' IDENTIFIED BY 'password' DEFAULT ROLE 'role_name'; +``` + +1. user\_name: 用户名 +2. host: 可以是ip、hostname。允许通配符，如："%", "192.168.%", "%.baidu.com"（如何解析BNS？？） +3. 如果设置了 default role，则直接赋予default role 对应的权限。 + +### 删除用户 + +``` +DROP USER 'user_name'; +``` + +删除对应名称的用户，该用户的所有对应host的权限信息都会被删除。 + + +### 设置密码 +``` +SET PASSWORD FOR 'user_name'@'host' = PASSWORD('xxxx'); +``` + +### 授权 + +``` +GRANT PRIVILEGE 'priv1'[, 'priv2', ...] on `db`.`tbl` to 'user'@'host'[ROLE 'role']; +``` + +``` +GRANT ROLE 'role1' on `db`.`tbl` to 'user'@'host'; +``` + +1. db 和 tbl 可以是通配符，如：\*.\*, db.* 。 Palo不检查执行授权时，db或tbl是否存在。只是更新授权表。 +2. Node\_priv 和 Grant\_priv 不能通过该语句授权。不能授予 all privileges +3. 如果是授权给 Role，且Role不存在，会自动创建这个 Role。如果user不存在，则报错。 +4. 可以授权 ROLE 给某一个user。 + +### 撤权 + +``` +REVOKE 'priv1'[, 'priv2', ...]|[all privileges] on `db`.`tbl` from 'user'@'host'[ROLE 'role']; +``` + +1. db 和 tbl 可以是通配符，Palo会匹配权限表中的所有可匹配项，并撤销对应权限 +2. 该语句中的 host 会进行精确匹配，而不是通配符。 + +### 创建角色 + +``` +CREATE ROLE 'role'; +``` + +1. 创建角色后，可以通过授权语句或撤权语句对该 Role 的权限进行修改。 +2. 默认有 ROOT 和 SUPERUSER 两个 ROLE。这两个ROLE的权限不可修改。 + + +## 最佳实践 + +1. 创建集群后，会自动创建 ROOT@'%' 和 ADMIN@'%' 两个用户。以及 ROOT 和 SUPERUSER 两个角色。 ROOT@'%' 为 ROOT 角色。ADMIN@'%' 为 SUPERUSER 角色。 +2. ROOT 角色仅拥有 Node_priv。SUPERUSER 角色拥有其他所有权限。 +3. ROOT@'%' 主要由于运维人员搭建系统。或者用于云上的部署程序。 +4. SUPERUSER 角色的用户为数据库的实际使用者和管理员。 +5. superuser 用户可以创建各个普通用户。比如给外包人员创建 Alter_priv、Load_priv、Create_priv、Drop_priv 权限，用于数据创建和导入，但没有Select_priv。给用户开通 Select_priv，可以查看数据，但不能对数据进行修改。 +6. 初始时，只有 superuser 可以创建db。或者可以先授权某个普通用户对一个**不存在的DB**的创建权限，之后，该普通用户就可以创建数据库了。（这是一个先有鸡还是先有蛋的问题） +7. 任何对数据库内对象（DB、Table）的创建和删除都不会影响已经存在的权限。如果有必要，必须手动修改权限表对应的条目。 + + +## 排期 +预计6月底完成，本期实现表级别的权限管理。 + +## 遗留问题 +1. 白名单，如何解决bns和dns的问题 + 保留当前通过 add whitelist 的方式添加白名单的机制。通过这个机制添加的白名单，默认都是DNS或BNS。Palo会有后台线程定期解析这些域名，将解析出来的host或ip加入到权限列表中。加入权限表时，取此user在db权限表中，各db最大权限集合。 + + 这种方式，能够兼容之前创建的用户权限。但存在一些问题，假设用户之前的权限为： + + GRANT ALL on db1 to user; + GRANT ALL on db2 to user; + + 如果该user的dns解析后的host有10个，那么更新后的 db 权限表中，会产生 10 * 2 个条目。但我们认为当前数量级不会影响性能。 + + + +2. 自动生成的 ROOT@'%' 和 ADMIN@'%'，如何更改其中的 host + + +## 权限逻辑 + +### CREATE USER + +1. create user cmy@'ip' identified by '12345'; + + 检查 cmy@'ip' 是否存在，如果存在则报错。 + +2. create user cmy@'ip' identified by '12345' default role role1; + + 检查 cmy@'ip' 是否存在，如果存在则报错。 + 赋予 cmy@'ip' role1 的所有权限，即时生效。 + role1 中加入 cmy@'ip'。 + +3. create user cmy@['domian'] identified by '12345'; + + 在白名单中检查 cmy@['domian'] 是否存在，如果存在则报错。 + +4. create user cmy@['domian'] identified by '12345' default role role1; + + 在白名单中检查 cmy@['domian'] 是否存在，如果存在则报错。 + 赋予该白名单，role1的所有权限。后台线程轮询生效。 + role1 中加入 cmy@['domian'] + +### GRANT + +1. grant select on \*.* to cmy@'ip'; + + 检查 cmy@'ip' 是否存在，不存在则报错。 + 将 select 权限赋给 cmy@'ip' on \*.* + +2. grant select on \*.* to cmy@['domain']; + + 检查 cmy@['domain'] 是否存在，不存在则报错。 + 将 select 权限赋给 cmy@['domain'] on \*.* + +3. grant select on \*.* to ROLE role1; + + 检查 role1 是否存在，不存在则报错。 + 将 select 权限赋给 role1 on \*.*。 + 将 select 权限赋给所有 role1 的用户。 + + + + + + + diff --git a/docs/design/real_time_loading.md b/docs/design/real_time_loading.md new file mode 100644 index 0000000000..ef68d5a306 --- /dev/null +++ b/docs/design/real_time_loading.md @@ -0,0 +1,206 @@ +# 一些名词 + +- Transaction： 在描述中事务和导入是同一个概念，为了方便以后事务的集成，这里设计框架的时候考虑了事务的内容，而导入是事务的子内容，所以很多名词直接用事务了。 + +# 数据结构 +## FE 中的数据结构 +### 事务的状态 +TransactionState 有4个状态： + +- PREPARE 表示事务已经开始了，Coordinator已经开始导入数据了。对于我们改造的旧系统来说就是Loading阶段开始了，由FE启动一个模块当做Coordinator来发起任务。 +- COMMITTED 表示事务已经结束了，此时数据在BE上已经quorum存在了，但是FE并没有通知BE这个导入的version是多少，所以数据对查询还不可见。 +- VISIBLE 表示这次导入事务对查询可见了。 +- ABORTED 表示导入由于内部故障导致失败或者由用户显示的取消了导入任务，由reason字段来说明具体的原因 + +### 事务表相关的结构 + +table_version 表 + +table_id | visible_version | next_version +---|---|--- +1001 | 11 | 20 +1002 | 13 | 19 + +- visible_version 表示这个table可见的版本号，也就是查询时给查询计划指定的版本号 +- next_version 是version通知机制运行时给一个导入事务分配的版本号，分配完之后自动+1 + +transaction_state 表 + +transaction_id | affected_table_version | coordinator | transaction_state |start_time| commit_time | finish_time +---|---|---|---|--|--|--| + 11 | <1001, 12><1002,13> + 13 | <1002,14> + +- transactiond_id 表示一个导入事务唯一的ID +- affected_table_version 表示一个导入事务涉及到的所有table，这个列表对检测导入冲突有用。这里记录的是一个tuple list， 的list，在publishVersion的时候会用。 +- coordinator 记录当前的协调节点，为以后的实时导入服务的，目前就是FE +- start_time 是导入事务进入prepare 状态的时间 +- commit_time 是load完成进入committed状态的时间 +- finish_time 是一个事务完成的时间，可以是visible的时间，可以是abort的时间 +- 这里没有记录label，需要在现有的导入元数据中记录一个transactionid，把label映射到实时导入框架的transactionid上 + +## BE 中的数据结构 +### BE 文件目录结构 + +目前没有看过BE的目录结构，这里的只是原理上的设计，我们以后可以改，但是思路是这样的。 假定每个tablet都有一个目录，目录结构如下： + +- data 存储的就是现在的tablet的数据。 +- staging 是数据刚导入的时候把数据存放在这个目录下，等修改version的时候把数据再迁移到deltas和data目录下，在version通知机制的api里会详细写一下。 +- delta 存储的是delta文件，这个文件不会被compaction，会等待一定的时间（比如30 min后自动删除），这个文件夹主要用于副本恢复使用。 + +# API 设计 +## FE 实时导入相关API + + service GlobalTransactionManager { + // 这个API 供Coordinator发起导入任务时调用， 这个API的预期行为是： + // 1 递增的生成一个TransactionID + // 2 当前发起请求的节点作为Coordinator节点 + // 3 事务的初始状态为PREPARE + // 4 把TransactionID， Label，Coordinator, TransactionState 信息写入事务表，TableList和CommitTime为空 + // 3. 把TransactionID信息返回给调用的Coordinator节点 + int64 beginTransaction(); + + // 当coordinator导入完成后，把导入的完成情况汇报给FE时使用 + // label 是导入时用户指定的唯一标识 + // tabletsStatus 表示的是本次导入过程中涉及到的各个table的各个分片的副本的导入情况，status表示导入成功还是失败 + // fe收到这个信息后，做以下两个动作： + // 1. 判断这次导入是否成功，这里主要是一致性方面，如果根据一致性协议判定导入失败，那么就在事务表中把事务状态标记为FAILED，然后返回客户端失败； 另外也需要判断这个导入是否被CANCEL，如果已经CANCEL直接返回失败信息即可。 + // 2. 如果判定导入成功，那么 + // 2.1 将导入失败的tablet标记为CLONE状态，然后生成RepairTabletTask，如果RepairTabletTask的version比现在commit的version小的话，那么要生成一个新的task，然后下发给BE + // 2.2 计算本次导入涉及到的所有的tableId，将tableid的信息写入事务表，在事务表中把事务的状态标记为COMMITTED + Status commitTransaction(int64 transactionId, list> tabletsStatus) + + Status rollbackTransaction(int64 transactionId) + + // 获取一个table的一个transaction对应的version + map getTransactionVersion(list>) + } + + // 这个模块用来充当DPP,小批量，BROKER方式导入的Coordinator + Service TransactionCoordinator { + // BE 执行完load命令后，调用这个API给FE汇报导入完成情况，FE收到这个消息后，在内存中保存下来 + status finishLoad(list>) + } + + Service TabletReplicationManager { + status finishClone() + } + +## BE Agent的API + struct RepairTabletTask { + // 表示从哪个BE上来读取数据 + string sourceIp; + // 表示当前故障的BE的IP + string targetIp; + // 修复数据的version号 + string version; + } + + service LocalTransactonManager { + // FE 调用BE，让BE从sourceBE上clone数据， version表示clone到哪个版本的数据，这个API注意以下几点： + // 1. BE 在执行时需要跟sourceIp比对一下需要传递哪些数据，哪些数据本地有，哪些数据本地没有，是否需要全量恢复。 + // 2. sourceIP 对应的BE节点上version的数据可能还暂时不存在，那么BE需要等待一下。 + // 3. 这个API 要实现成为一个能够增量执行的API，因为FE会不断的发送不同的Repair任务给BE，比如FE发送一个version=8的，后来又发送一个version=9的，那么BE需要判断本地是否有repair任务在执行，如果在执行，那么就更新一下执行信息，没有就启动。 + // 4. repair过程中下载的文件先放到delta文件夹下，然后再放到data目录下。 + Status repairTablet(RepairTabletTask task) + + // 这个API只是这里假定的，这个应该跟BE现有的Load API 合成一个 + Status loadData(dataUrl, transactionid) + // 当BE 收到这个消息时，读取本地staging目录下的文件的后缀名.stage3的文件，如果tableid 和 transactionid匹配，那么将文件，重命名，逻辑是： + // 1. 首先在 delta 目录下建立硬链指向staing目录下的文件。 + // 2. 判断一下data目录中version-1的文件是否存在，如果存在并且当前tablet是正常状态时，那么在data 目录中也建立硬链。 + // 3. 把staging目录下的文件删除。 + // 如果BE 修改version时发生问题，那么把异常的tablet返回给FE + void publishVersion(list >) + } + +# 相关流程和机制 +## 导入流程 + +由于这一阶段只针对现有的导入流程优化，所以我们从loading阶段开始描述我们的方案， loading任务之前的阶段保持不变。 + +- 当Extract和Transform阶段执行完毕后，FE会收到通知，那么FE开始执行Loading任务，此时FE充当了Coordinator的角色，FE调用beginTransaction API在事务表中注册这个导入任务。 +- FE 调用BE的loadData API 向BE发送loading 任务，在消息中要附加一个transactionid，表明这是一个新式的导入任务。【这里会不会成为瓶颈？但是以后实时导入不会有这个瓶颈，因为实时导入中load的通知是由coordinator通知的，是分散化的】对于通知不成功的，那么就一直轮训通知即可，实在是通知不成功，那么就把副本设置为异常状态即可。 +- BE 收到loading任务后执行loading任务，不论是小批量还是DPP，都从目标源（对于DPP来说就是HDFS，对于小批量来说就是那个BE）上下载文件, 但是要把数据放到 staging 文件夹下，文件的后缀名是.stage1。 +- 当BE执行完loading任务时，把文件的后缀名改成 .stage2 +- BE 启动一个定时服务，扫描 staging 文件夹下的.stage2 的文件，如果涉及到一个transaction的所有的tablet的导入都完成了，那么BE调用 finishLoad API向FE汇报load完成信息, FE在内存中记录这个导入完成情况，注意这里没有持久化。 +- BE 收到返回后，在内存中记录这个文件已经汇报过了，下次汇报的时候不再汇报，这样就能够避免每次都发送所有的导入完成的列表给FE了，但是这引入了一个新的问题，就是FE仅仅把状态保存在内存中，当FE重启或者FE迁移了内存状态就不在了，这个在FE故障处理章节介绍一下。 +- FE 启动一个定时任务扫描所有的导入任务，如果一个transaction涉及到的所有的tablet都导入完成，或者连续的quorum完成时，那么FE 调用commitTransaction API 来完成事务，这一步FE要判断一个BE是否是导入不成功了，因为commitTransaction那个API中会根据状态设置tablet的状态。注意这里没有采用触发式的，是为了降低FE的元数据修改次数。 + +此时导入流程完毕，但是BE端并不知道transactionid和version的对应关系，数据也不可以查询。后续由version 通知机制完成BE端从transactionid到version的变更。 + +## version 通知机制 + +这个服务只运行在FE Leader上，不是Leader不运行。 + +- 遍历transaction_state 表，选择可以通知的version， 选择的方式是： + - 事务的状态为COMMITTED + - 事务的tableidlist 中每个table的version都等于 table_version 表中的visible_version + 1 + - 还要检查一下副本的状态是不是健康，如果有不健康的，那么也直接把load任务设置为失败 +- 将遍历后获取的结果组装成list >格式，调用BE的publishVersoin API通知BE。 +- BE 收到通知后， 根据tableId和transactionid从staging目录下找导入文件，完成文件名的变更，具体看API的描述，如果遇到异常那么尝试3次，如果还有错误那么跳过，依赖补救措施来搞定。 +- 如果所有的BE 都返回成功，那么修改transaction_state 中的transaction修改为VISIBLE，同时把table_version 表中的visible_version 修改为对应的tableidlist 中记录的table的version。 +- 如果只有部分BE返回成功，那么把那些没返回成功的BE的状态 + +## version通知机制的保证/补救措施 + +在某些情况下可能publishVersion不能更新tablet的transactionid到version，比如我们假设一个现象，BE在接收publishVersion时，一个磁盘掉了，然后又回来了，那么这个磁盘上的tablet的version实际是没有变的，但是FE会误认为所有的tablet的version都更新成功了。 + +所以在这里我们引入一个强制保证的机制： + +- BE 定期的跟FE比对本地所有tablet的version，如果本地的version < FE中记录的visible version，那么判定本地是有问题的。 +- BE 扫描本地tablet的staging目录，从中获得所有的transactionid，从FE中获取对应的version，然后更新本地的version。 + +另外，当BE收到publishVersion做变更时，BE要检查下tablet对应的version-1数据是否存在，如果不存在，说明上次的publish并没有成功，可能有什么意外情况没考虑到，那么BE也需要从FE中主动拉一下transactionid对应的version。 + + +## BE 启动过程 + +- BE 启动时要完成自检的过程，跟FE通信汇报本地的tablet信息，如果一个磁盘坏了，那么需要向FE汇报，FE需要将这个tablet标记为异常。 + +## Compaction 逻辑 + +现在BE上一个tablet目录下的data目录里的数据 +保持现有的compaction逻辑不变，因为version的通知是顺序的，所以最后一个version仍然是未决version，仍然不compaction。 + +## delta目录的清空逻辑 + +delta目录下的数据单纯是为了副本修复服务的，所以我们这里采取简单的定时的策略，目前考虑半小时自动删除，另外如果磁盘空间不足的话，可以考虑删除。 + +## 副本故障处理 + +副本修复的触发机制和类型： + +- 当一个tablet的BE从故障中恢复汇报本地的tablet给FE时，FE决定这个tablet仍然由这个BE来存储时，FE直接生成RepairTabletTask。 +- 当Coordinator调用commitTransaction的API时，检测到一个tablet没有完成不完整时： + - 如果这个BE在元数据中标记为正常，那么此时立即生成一个RepairTabletTask。 + - 如果这个BE在元数据中标记为异常了，那么不执行副本修复。 +- FE定时检查副本的状态，如果一个副本处于故障状态超过一定的时间（比如15min，这个时间要参考两个机器同时宕机的概率），那么立即生成RepairTabletTask。 +- relbance 过程，当一个tablet从A机器迁移到B机器时，直接把B机器当做一个故障的副本，生成一个RepairTabletTask。 +- 副本数目调大，新加的副本认为是异常副本，生成一个RepaireTabletTask。 + +注意： 生成RepairTabletTask的同时把故障的tablet标记为clone状态，对于未来的loading任务也要向这个节点发送，version通知也要发送，只是在计算quorum的时候不计算。 + +RepairReplicaTask的执行流程： + +- FE 调用BE的repairTablet API，将RepairTabletTask发送给对应的BE。 +- BE 执行RepairTabletTask。 +- 当BE下载完后，跟data目录下的内容比对一下，如果完全对了，那么把数据在data目录下建立硬链，给FE汇报成功消息。 + +# 一些异常处理的思路 +## BE 的一个磁盘掉了，然后重启了，收到publishVersion 消息后如何处理？ +此时我们不做处理，仍然认为version通知成功了。 其实现在的BE也是没处理的，这相当于BE给FE汇报导入成功后，BE自己的磁盘又掉了的情况，我们不做处理，等BE的定时汇报由FE发现异常，标记tablet为异常。 + +## FE 宕机重启 +- 内存中保存的导入完成情况丢失： 此时FE 成为Leader时，需要向所有的BE发送一个invalid消息，告诉BE本地的内存状态无效，从硬盘上汇报所有的导入进度信息给FE。 +- RepairTabletTask丢失：在创建RepairTabletTask时把tablet设置为clone状态，此时可以遍历CLONE状态的tablet生成RepairTabletTask，让BE重新做即可，这里的sourceIP可能发生变化，但是我们不管了。 + +# 遗留的导入的改造方式 +## 导入的改进方式 +- FE Leader 重启时根据元数据中已经loading finished的任务的version，将这个version的最大值根据table写入table_version 的visible_version 字段中。 +- FE Leader 根据正在loading的任务【尚未完成导入，ET阶段已经结束，Loading阶段未结束】，获取这些任务的version，获取最大的，把version写入table_version 的next_version 字段中。 +- 对于已经开始执行loading阶段的导入，FE Leader继续用轮训的方式，让他们执行loading任务，在loading完成后，要增加修改table对应的visible_version的逻辑。 这里可能不仅仅是quorum导入完成，要等尽量长的时间，让所有的副本都导入成功才可以。 如果副本导入没成功，那么标记为故障。 【或许这一步我们可以直接cancel掉之前所有的导入】 +- FE Leader上把现在追副本的任务停止。 + +# 其他一些考虑 +- 处于loading 阶段的任务不能太多，否则BE端压力太大，过去version lock能达到这个效果，现在没有version lock了，需要增加一个限制。 我们是不是有这个机制了呢？ \ No newline at end of file diff --git a/docs/help/Contents/Account Management/help.md b/docs/help/Contents/Account Management/help.md index 2b5cd75486..f1b0da9bb2 100644 --- a/docs/help/Contents/Account Management/help.md +++ b/docs/help/Contents/Account Management/help.md @@ -1,30 +1,44 @@ # CREATE USER ## description - Syntax: - CREATE USER user_specification [SUPERUSER] +Syntax: + + CREATE USER user_identity [IDENTIFIED BY 'password'] [DEFAULT ROLE 'role_name'] - user_specification: - 'user_name' [IDENTIFIED BY [PASSWORD] 'password'] - - CREATE USER 命令可用于创建一个 palo 用户，使用这个命令需要使用者必须有管理员权限 - SUPERUSER用于指定需要创建的用户是个超级用户 + user_identity: + 'user_name'@'host' + +CREATE USER 命令用于创建一个 Palo 用户。在 Palo 中，一个 user_identity 唯一标识一个用户。user_identity 由两部分组成，user_name 和 host，其中 username 为用户名。host 标识用户端连接所在的主机地址。host 部分可以使用 % 进行模糊匹配。如果不指定 host，默认为 '%'，即表示该用户可以从任意 host 连接到 Palo。 + +host 部分也可指定为 domain，语法为：'user_name'@['domain']，即使用中括号包围，则 Palo 会认为这个是一个 domain，并尝试解析其 ip 地址。目前仅支持百度内部的 BNS 解析。 + +如果指定了角色（ROLE），则会自动将该角色所拥有的权限赋予新创建的这个用户。如果不指定，则该用户默认没有任何权限。 ## example - 1. 创建一个没有密码的用户，用户名为 jack - CREATE USER 'jack' +1. 创建一个无密码用户（不指定 host，则等价于 jack@'%'） + + CREATE USER 'jack'; - 2. 创建一个带有密码的用户，用户名为 jack，并且密码被指定为 123456 - CREATE USER 'jack' IDENTIFIED BY '123456' +2. 创建一个有密码用户，允许从 '172.10.1.10' 登陆 + + CREATE USER jack@'172.10.1.10' IDENTIFIED BY '123456'; - 3. 为了避免传递明文，用例2也可以使用下面的方式来创建 - CREATE USER 'jack' IDENTIFIED BY PASSWORD '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' - 后面加密的内容可以通过PASSWORD()获得到,例如： - SELECT PASSWORD('123456') +3. 为了避免传递明文，用例2也可以使用下面的方式来创建 + + CREATE USER jack@'172.10.1.10' IDENTIFIED BY PASSWORD '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9'; + + 后面加密的内容可以通过PASSWORD()获得到,例如： + + SELECT PASSWORD('123456'); - 4. 创建一个超级用户'jack' - CREATE USER 'jack' SUPERUSER +4. 创建一个允许从 '192.168' 子网登陆的用户，同时指定其角色为 example_role + + CREATE USER 'jack'@'192.168.%' DEFAULT ROLE 'example_role'; + +5. 创建一个允许从域名 'example_domain' 登陆的用户 + + CREATE USER 'jack'@['example_domain'] IDENTIFIED BY '12345'; ## keyword CREATE, USER @@ -32,13 +46,16 @@ # DROP USER ## description - Syntax: +Syntax: + DROP USER 'user_name' - DROP USER 命令会删除一个 palo 用户，使用这个命令需要使用者必须有管理员权限 + DROP USER 命令会删除一个 palo 用户。这里 Palo 不支持删除指定的 user_identity。当删除一个指定用户后，该用户所对应的所有 user_identity 都会被删除。比如之前通过 CREATE USER 语句创建了 jack@'192.%' 以及 jack@['domain'] 两个用户，则在执行 DROP USER 'jack' 后，jack@'192.%' 以及 jack@['domain'] 都将被删除。 ## example - 1. 删除用户 jack + +1. 删除用户 jack + DROP USER 'jack' ## keyword @@ -47,76 +64,121 @@ # SET PASSWORD ## description - Syntax: - SET PASSWORD [FOR 'user_name'] = +Syntax: + + SET PASSWORD [FOR user_identity] = [PASSWORD('plain password')]|['hashed password'] - SET PASSWORD 命令可以用于修改一个用户的登录密码。如果 [FOR 'user_name'] 字段不存在，那么修改当前用户的密码。 + SET PASSWORD 命令可以用于修改一个用户的登录密码。如果 [FOR user_identity] 字段不存在，那么修改当前用户的密码。 + + 注意这里的 user_identity 必须完全匹配在使用 CREATE USER 创建用户时指定的 user_identity，否则会报错用户不存在。如果不指定 user_identity，则当前用户为 'username'@'ip'，这个当前用户，可能无法匹配任何 user_identity。可以通过 SHOW GRANTS 查看当前用户。 + PASSWORD() 方式输入的是明文密码; 而直接使用字符串，需要传递的是已加密的密码。 如果修改其他用户的密码，需要具有管理员权限。 ## example - 1. 修改当前用户的密码为 123456 - SET PASSWORD = PASSWORD('123456') - SET PASSWORD = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' +1. 修改当前用户的密码 + + SET PASSWORD = PASSWORD('123456') + SET PASSWORD = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' - 2. 修改用户 jack 的密码为 123456 - SET PASSWORD FOR 'jack' = PASSWORD('123456') - SET PASSWORD FOR 'jack' = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' +2. 修改指定用户密码 + + SET PASSWORD FOR 'jack'@'192.%' = PASSWORD('123456') + SET PASSWORD FOR 'jack'@['domain'] = '*6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9' ## keyword SET, PASSWORD # GRANT ## description - GRANT 命令是将一个数据库的具体权限授权给具体用户。调用者必须是管理员身份。 - 权限当前只包括只读 (READ_ONLY)，读写 (READ_WRITE) 两种权限，如果指定为 ALL， - 那么就是将全部权限授予该用户。 - Syntax: - GRANT privilege_list ON db_name TO 'user_name' +GRANT 命令用于赋予指定用户或角色指定的权限。 - privilege_list: - privilege [, privilege] ... +Syntax: - privilege: - READ_ONLY - | READ_WRITE - | ALL + GRANT privilege_list ON db_name[.tbl_name] TO user_identity [ROLE role_name] + + +privilege_list 是需要赋予的权限列表，以逗号分隔。当前Palo支持如下权限： + + NODE_PRIV：集群节点操作权限，包括节点上下线等操作，只有 root 用户有该权限，不可赋予其他用户。 + ADMIN_PRIV：除 NODE_PRIV 以外的所有权限。 + SELECT_PRIV：对指定的库或表的读取权限 + LOAD_PRIV：对指定的库或表的导入权限 + ALTER_PRIV：对指定的库或表的schema变更权限 + CREATE_PRIV：对指定的库或表的创建权限 + DROP_PRIV：对指定的库或表的删除权限 + + 旧版权限中的 ALL 和 READ_WRITE 会被转换成：SELECT_PRIV,LOAD_PRIV,ALTER_PRIV,CREATE_PRIV,DROP_PRIV； + READ_ONLY 会被转换为 SELECT_PRIV。 + +db_name[.tbl_name] 支持以下三种形式： + + 1. *.* 权限可以应用于所有库及其中所有表 + 2. db.* 权限可以应用于指定库下的所有表 + 3. db.tbl 权限可以应用于指定库下的指定表 + + 这里指定的库或表可以是不存在的库和表。 + +user_identity： + + 这里的 user_identity 语法同 CREATE USER。且必须为使用 CREATE USER 创建过的 user_identity。user_identity 中的host可以是域名，如果是域名的话，权限的生效时间可能会有1分钟左右的延迟。 + + 也可以将权限赋予指定的 ROLE，如果指定的 ROLE 不存在，则会自动创建。 ## example - 1. 授予用户 jack 数据库 testDb 的写权限 - GRANT READ_ONLY ON testDb to 'jack'; + 1. 授予所有库和表的权限给用户 + + GRANT SELECT_PRIV ON *.* TO 'jack'@'%'; - 2. 授予用户 jack 数据库 testDb 全部权限 - GRANT ALL ON testDb to 'jack'; + 2. 授予指定库表的权限给用户 + + GRANT SELECT_PRIV,ALTER_PRIVS,LOAD_PRIV ON db1.tbl1 TO 'jack'@'192.8.%'; + + 3. 授予指定库表的权限给角色 + + GRANT ADMIN_PRIV ON db1.* TO ROLE admin_role ## keyword GRANT # REVOKE ## description - REVOKE 命令用于撤销用户对于某一个数据库的权限（当前只支持撤销所有权限） - 语法： - REVOKE ALL ON db_name FROM 'user_name' + + REVOKE 命令用于撤销指定用户或角色指定的权限。 + Syntax： + REVOKE privilege_list ON db_name[.tbl_name] FROM user_identity [ROLE role_name] + + user_identity： + + 这里的 user_identity 语法同 CREATE USER。且必须为使用 CREATE USER 创建过的 user_identity。user_identity 中的host可以是域名，如果是域名的话，权限的撤销时间可能会有1分钟左右的延迟。 + + 也可以撤销指定的 ROLE 的权限，执行的 ROLE 必须存在。 ## example - 1. 撤销用户 jack 数据库 testDb 的权限 - REVOKE ALL ON testDb FROM 'jack'; + + 1. 撤销用户 jack 数据库 testDb 的权限 + + REVOKE SELECT_PRIV ON db1.* FROM 'jack'@'192.%'; ## keyword - REVOKE + + REVOKE # SET PROPERTY ## description + Syntax: + SET PROPERTY [FOR 'user'] 'key' = 'value' [, 'key' = 'value'] - 设置用户的属性，包括分配给用户的资源、导入cluster等。 + 设置用户的属性，包括分配给用户的资源、导入cluster等。这里设置的用户属性，是针对 user 的，而不是 user_identity。即假设通过 CREATE USER 语句创建了两个用户 'jack'@'%' 和 'jack'@'192.%'，则使用 SET PROPERTY 语句，只能针对 jack 这个用户，而不是 'jack'@'%' 或 'jack'@'192.%' key: + 超级用户权限: max_user_connections: 最大连接数。 resource.cpu_share: cpu资源分配。 @@ -132,6 +194,7 @@ default_load_cluster: 默认的导入cluster。 ## example + 1. 修改用户 jack 最大连接数为1000 SET PROPERTY FOR 'jack' 'max_user_connections' = '1000'; @@ -157,19 +220,108 @@ ## keyword SET, PROPERTY + +# CREATE ROLE -# SHOW USER ## description - 用于显示当前用户有权限查看的所有用户的权限信息。 - 如果是普通用户，仅可以查看自己的权限信息。 - 如果是 superuser 用户，可以看到自己所属 cluster 的所有用户的权限信息。 - 如果是 admin（root）用户，可以看到所有用户的权限信息。 + 该语句用户创建一个角色 + 语法： - SHOW USER; + CREATE ROLE role1; + 该语句创建一个无权限的角色，可以后续通过 GRANT 命令赋予该角色权限。 + ## example - 1. 查看用户权限信息 - SHOW USER; + + 1. 创建一个角色 + + CREATE ROLE role1; + +## keyword + CREATE, ROLE + + +# DROP ROLE + +## description + 该语句用户删除一个角色 + + 语法： + DROP ROLE role1; + + 删除一个角色，不会影响之前属于该角色的用户的权限。仅相当于将该角色与用户解耦。用户已经从该角色中获取到的权限，不会改变。 + +## example + + 1. 删除一个角色 + + DROP ROLE role1; + +## keyword + DROP, ROLE + +# SHOW ROLES + +## description + 该语句用于展示所有已创建的角色信息，包括角色名称，包含的用户以及权限。 + + 语法： + SHOW ROLES; + +## example + + 1. 查看已创建的角色： + + SHOW ROELS; ## keyword - SHOW,USER + SHOW,ROLES + +## description + 该语句用户删除一个角色 + + 语法： + DROP ROLE role1; + + 删除一个角色，不会影响之前属于该角色的用户的权限。仅相当于将该角色与用户解耦。用户已经从该角色中获取到的权限，不会改变。 + +## example + + 1. 删除一个角色 + + DROP ROLE role1; + +## keyword + DROP, ROLE + +# SHOW GRANTS + +## description + + 该语句用于查看用户权限。 + + 语法： + SHOW [ALL] GRANTS [FOR user_identity]; + + 说明： + 1. SHOW ALL GRANTS 可以查看所有用户的权限。 + 2. 如果指定 user_identity，则查看该指定用户的权限。且该 user_identity 必须为通过 CREATE USER 命令创建的。 + 3. 如果不指定 user_identity，则查看当前用户的权限。 + + +## example + + 1. 查看所有用户权限信息 + + SHOW ALL GRANTS; + + 2. 查看指定 user 的权限 + + SHOW GRANTS FOR jack@'%'; + + 3. 查看当前用户的权限 + + SHOW GRANTS; + +## keyword + SHOW, GRANTS diff --git a/docs/help/Contents/Data Definition/ddl_stmt.md b/docs/help/Contents/Data Definition/ddl_stmt.md index ceab2b99e1..ae6f78251c 100644 --- a/docs/help/Contents/Data Definition/ddl_stmt.md +++ b/docs/help/Contents/Data Definition/ddl_stmt.md @@ -513,19 +513,19 @@ TO example_rollup_index; 2. 向example_rollup_index的col1后添加一个value列new_col(非聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT DEFAULT "0" AFTER col1 - TO example_rollup_index; + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT DEFAULT "0" AFTER col1 + TO example_rollup_index; 3. 向example_rollup_index的col1后添加一个key列new_col(聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT DEFAULT "0" AFTER col1 - TO example_rollup_index; + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT DEFAULT "0" AFTER col1 + TO example_rollup_index; 4. 向example_rollup_index的col1后添加一个value列new_col SUM聚合类型(聚合模型) - ALTER TABLE example_db.my_table - ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 - TO example_rollup_index; + ALTER TABLE example_db.my_table + ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 + TO example_rollup_index; 5. 向 example_rollup_index 添加多列(聚合模型) ALTER TABLE example_db.my_table @@ -687,8 +687,8 @@ 重命名数据库后，如需要，请使用 REVOKE 和 GRANT 命令修改相应的用户权限。 ## example - 1. 设置指定数据库数据量配额为 1GB - ALTER DATABASE example_db SET DATA QUOTA 1073741824; + 1. 设置指定数据库数据量配额为 10 TB + ALTER DATABASE example_db SET DATA QUOTA 10995116277760; 2. 将数据库额 example_db 重命名为 example_db2 ALTER DATABASE example_db RENAME example_db2; @@ -696,48 +696,106 @@ ## keyword ALTER,DATABASE,RENAME -# BACKUP +# CREATE REPOSITORY ## description - 该语句用于备份指定数据库下的数据。该命令为异步操作。提交成功后，需通过 SHOW BACKUP 命令查看进度。 + 该语句用于创建仓库。仓库用于属于备份或恢复。仅 root 或 superuser 用户可以创建仓库。 语法： - BACKUP LABEL [db_name.label] [backup_objs] INTO "remote_path" - PROPERTIES ("key"="value", ...) - - backup_objs: 需要备份的表名或分区名 - 语法： - (table_name[.partition_name], ...) + CREATE [READ ONLY] REPOSITORY `repo_name` + WITH BROKER `broker_name` + ON LOCATION `repo_location` + PROPERTIES ("key"="value", ...); 说明： - 1. 同一数据库下只能有一个正在执行的 BACKUP 任务。 - 2. 如果 backup_objs 中不写 partition_name，则默认备份该 table 下所有分区。 - 如果完全不写 backup_objs，则默认备份整个 database。 - 3. PROPERTIES 需要填写访问远端备份系统所需信息。 - 4. 统一数据库下，BACKUP 任务的 label 不能重复。 - + 1. 仓库的创建，依赖于已存在的 broker + 2. 如果是只读仓库，则只能在仓库上进行恢复。如果不是，则可以进行备份和恢复操作。 + 3. 根据 broker 的不同类型，PROPERTIES 有所不同，具体见示例。 + ## example - 1. 备份 example_db 下的所有数据，备份到 Hdfs 路径：/user/cmy/backup/ 下 - BACKUP LABEL example_db.backup_label1 - INTO "/dir/backup/" - PROPERTIES( - "server_type" = "hadoop", - "host" = "hdfs://host", - "port" = "port", - "user" = "user", - "password" = "passwd", - "opt_properties" = "" + 1. 创建名为 bos_repo 的仓库，依赖 BOS broker "bos_broker"，数据根目录为：bos://palo_backup + CREATE REPOSITORY `bos_repo` + WITH BROKER `bos_broker ` + ON LOCATION "bos://palo_backup" + PROPERTIES + ( + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", + "bos_secret_accesskey"="70999999999999de274d59eaa980a" + ); + + 2. 创建和示例 1 相同的仓库，但属性为只读： + CREATE READ ONLY REPOSITORY `bos_repo` + WITH BROKER `bos_broker ` + ON LOCATION "bos://palo_backup" + PROPERTIES + ( + "bos_endpoint" = "http://gz.bcebos.com", + "bos_accesskey" = "069fc2786e664e63a5f111111114ddbs22", + "bos_secret_accesskey"="70999999999999de274d59eaa980a" + ); + + 3. 创建名为 hdfs_repo 的仓库，依赖 Baidu hdfs broker "hdfs_broker"，数据根目录为：hdfs://hadoop-name-node:54310/path/to/repo/ + CREATE REPOSITORY `hdfs_repo` + WITH BROKER `hdfs_broker ` + ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/" + PROPERTIES + ( + "username" = "user", + "password" = "password" ); - 2. 备份 example_db 下的表 example_tbl，备份到 Hdfs 路径：/user/cmy/backup/ 下 - BACKUP LABEL example_db.backup_label1 - (example_tbl) - INTO "/dir/backup/" - PROPERTIES ( - "server_type" = "hadoop", - "host" = "hdfs://host", - "port" = "port", - "user" = "user", - "password" = "passwd", - "opt_properties" = "" +## keyword + CREATE REPOSITORY + +# DROP REPOSITORY +## description + 该语句用于删除一个已创建的仓库。仅 root 或 superuser 用户可以删除仓库。 + 语法： + DROP REPOSITORY `repo_name`; + + 说明： + 1. 删除仓库，仅仅是删除该仓库在 Palo 中的映射，不会删除实际的仓库数据。删除后，可以再次通过指定相同的 broker 和 LOCATION 映射到该仓库。 + +## example + 1. 删除名为 bos_repo 的仓库： + DROP REPOSITORY `bos_repo`; + +## keyword + DROP REPOSITORY + +# BACKUP +## description + 该语句用于备份指定数据库下的数据。该命令为异步操作。提交成功后，需通过 SHOW BACKUP 命令查看进度。仅支持备份 OLAP 类型的表。 + 语法： + BACKUP SNAPSHOT [db_name].{snapshot_name} + TO `repository_name` + ON ( + `table_name` [PARTITION (`p1`, ...)], + ... + ) + PROPERTIES ("key"="value", ...); + + 说明： + 1. 同一数据库下只能有一个正在执行的 BACKUP 或 RESTORE 任务。 + 2. ON 子句中标识需要备份的表和分区。如果不指定分区，则默认备份该表的所有分区。 + 3. PROPERTIES 目前支持以下属性： + "type" = "full"：表示这是一次全量更新（默认）。 + "timeout" = "3600"：任务超时时间，默认为一天。单位秒。 + +## example + + 1. 全量备份 example_db 下的表 example_tbl 到仓库 example_repo 中： + BACKUP SNAPSHOT example_db.snapshot_label1 + TO example_repo + ON (example_tbl) + PROPERTIES ("type" = "full"); + + 2. 全量备份 example_db 下，表 example_tbl 的 p1, p2 分区，以及表 example_tbl2 到仓库 example_repo 中： + BACKUP SNAPSHOT example_db.snapshot_label2 + TO example_repo + ON + ( + example_tbl PARTITION (p1,p2), + example_tbl2 ); ## keyword @@ -746,48 +804,49 @@ # RESTORE ## description 1. RESTORE - 该语句用于将之前通过 BACKUP 命令备份的数据，恢复到指定数据库下。该命令为异步操作。提交成功后，需通过 SHOW RESTORE 命令查看进度。 + 该语句用于将之前通过 BACKUP 命令备份的数据，恢复到指定数据库下。该命令为异步操作。提交成功后，需通过 SHOW RESTORE 命令查看进度。仅支持恢复 OLAP 类型的表。 语法： - RESTORE LABEL [db_name.label] [restore_objs] FROM "remote_path" - PROPERTIES ("key"="value", ...) - - restore_objs: 需要恢复的表名或分区名 - 语法： - (table_name[.partition_name] [AS new_table_name[.partition_name]], ...) + RESTORE SNAPSHOT [db_name].{snapshot_name} + FROM `repository_name` + ON ( + `table_name` [PARTITION (`p1`, ...)] [AS `tbl_alias`], + ... + ) + PROPERTIES ("key"="value", ...); 说明： - 1. 同一数据库下只能有一个正在执行的 RESTORE 任务。 - 2. 需恢复的 database 已经创建，并且在备份路径中存在 - 3. 如果 restore_objs 中不写 partition_name，则默认恢复该 table 下所有分区。 - 如果完全不写 restore_objs，则默认恢复所有备份过的数据。支持重命名需要恢复的表名 - 4. remote_path 需指定到之前 BACKUP 任务中，remote_path/backup_label/ 下。 - 5. PROPERTIES 需要填写访问远端备份系统所需信息。 - 6. 同一数据库下，RESTORE 任务的 label 不能重复。 + 1. 同一数据库下只能有一个正在执行的 BACKUP 或 RESTORE 任务。 + 2. ON 子句中标识需要恢复的表和分区。如果不指定分区，则默认恢复该表的所有分区。所指定的表和分区必须已存在于仓库备份中。 + 3. 可以通过 AS 语句将仓库中备份的表名恢复为新的表。但新表名不能已存在于数据库中。分区名称不能修改。 + 4. 可以将仓库中备份的表恢复替换数据库中已有的同名表，但须保证两张表的表结构完全一致。表结构包括：表名、列、分区、Rollup等等。 + 5. 可以指定恢复表的部分分区，系统会检查分区 Range 是否能够匹配。 + 6. PROPERTIES 目前支持以下属性： + "backup_timestamp" = "2018-05-04-16-45-08"：指定了恢复对应备份的哪个时间版本，必填。该信息可以通过 `SHOW SNAPSHOT ON repo;` 语句获得。 + "replication_num" = "3"：指定恢复的表或分区的副本数。默认为3。若恢复已存在的表或分区，则副本数必须和已存在表或分区的副本数相同。同时，必须有足够的 host 容纳多个副本。 + "timeout" = "3600"：任务超时时间，默认为一天。单位秒。 ## example - 1. 恢复 Hdfs 路径：/user/cmy/backup/backup_label1 下所有备份数据，恢复到 example_db 中 - RESTORE LABEL example_db.restore_label1 - FROM "/dir/backup/backup_label1" - PROPERTIES( - "server_type" = "hadoop", - "host" = "hdfs://host", - "port" = "port", - "user" = "user", - "password" = "passwd", - "opt_properties" = "" + 1. 从 example_repo 中恢复备份 snapshot_1 中的表 backup_tbl 到数据库 example_db1，时间版本为 "2018-05-04-16-45-08"。恢复为 1 个副本： + RESTORE SNAPSHOT example_db1.`snapshot_1 ` + FROM `example_repo` + ON ( `backup_tbl` ) + PROPERTIES + ( + "backup_timestamp"="2018-05-04-16-45-08", + "replication_num" = "1" ); - 2. 恢复 Hdfs 路径：/user/cmy/backup/backup_label1 下表 example_tbl 的数据。 - RESTORE LABEL example_db.restore_label1 - (example_tbl) - FROM "/dir/backup/backup_label1" - PROPERTIES ( - "server_type" = "hadoop", - "host" = "hdfs://host", - "port" = "port", - "user" = "user", - "password" = "passwd", - "opt_properties" = "" + 2. 从 example_repo 中恢复备份 snapshot_2 中的表 backup_tbl 的分区 p1,p2，以及表 backup_tbl2 到数据库 example_db1，并重命名为 new_tbl，时间版本为 "2018-05-04-17-11-01"。默认恢复为 3 个副本： + RESTORE SNAPSHOT example_db1.`snapshot_2 ` + FROM `example_repo ` + ON + ( + `backup_tbl` PARTITION (`p1`, `p2`) AS `backup_tbl2`, + `backup_tbl2` + ) + PROPERTIES + ( + "backup_timestamp"="2018-05-04-17-11-01" ); ## keyword @@ -811,6 +870,9 @@ 该语句用于取消一个正在进行的 RESTORE 任务。 语法： CANCEL RESTORE FROM db_name; + + 注意： + 当取消处于 COMMIT 或之后阶段的恢复左右时，可能导致被恢复的表无法访问。此时只能通过再次执行恢复作业进行数据恢复。 ## example 1. 取消 example_db 下的 RESTORE 任务。 @@ -825,18 +887,18 @@ 通过聚合来不断的减少数据量，以此来实现加快查询的目的，基于它到的是一个估算结果，误差大概在1%左右 hll列是通过其它列或者导入数据里面的数据生成的，导入的时候通过hll_hash函数来指定数据中哪一列用于生成hll列 它常用于替代count distinct，通过结合rollup在业务上用于快速计算uv等 - - 相关函数: - - HLL_UNION_AGG(hll) - 此函数为聚合函数，用于计算满足条件的所有数据的基数估算。 - - HLL_CARDINALITY(hll) - 此函数用于计算单条hll列的基数估算 - - HLL_HASH(column_name) - 生成HLL列类型，用于insert或导入的时候，导入的使用见相关说明 - + + 相关函数: + + HLL_UNION_AGG(hll) + 此函数为聚合函数，用于计算满足条件的所有数据的基数估算。 + + HLL_CARDINALITY(hll) + 此函数用于计算单条hll列的基数估算 + + HLL_HASH(column_name) + 生成HLL列类型，用于insert或导入的时候，导入的使用见相关说明 + ## example 1. 首先创建一张含有hll列的表 create table test( @@ -848,7 +910,7 @@ set1 hll hll_union, set2 hll hll_union) distributed by hash(id) buckets 32; - + 2. 导入数据，导入的方式见相关help curl a. 使用表中的列生成hll列 @@ -862,32 +924,33 @@ a. 创建一个rollup，让hll列产生聚合， alter table test add rollup test_rollup(date, set1); - + b. 创建另外一张专门计算uv的表，然后insert数据） - + create table test_uv( time date, uv_set hll hll_union) distributed by hash(id) buckets 32; insert into test_uv select date, set1 from test; - + c. 创建另外一张专门计算uv的表，然后insert并通过hll_hash根据test其它非hll列生成hll列 create table test_uv( time date, id_set hll hll_union) distributed by hash(id) buckets 32; - + insert into test_uv select date, hll_hash(id) from test; - + 4. 查询，hll列不允许直接查询它的原始值，可以通过配套的函数进行查询 - + a. 求总uv select HLL_UNION_AGG(uv_set) from test_uv; - + b. 求每一天的uv select HLL_CARDINALITY(uv_set) from test_uv; ## keyword - HLL + HLL + diff --git a/docs/help/Contents/Data Manipulation/manipulation_stmt.md b/docs/help/Contents/Data Manipulation/manipulation_stmt.md index 3fa0148e26..0bbb667c14 100644 --- a/docs/help/Contents/Data Manipulation/manipulation_stmt.md +++ b/docs/help/Contents/Data Manipulation/manipulation_stmt.md @@ -98,7 +98,10 @@ - dfs.ha.namenodes.xxx：ha模式中指定namenode的名字,多个名字以逗号分隔,ha模式中必须配置。其中xxx表示dfs.nameservices配置的value.例子: "dfs.ha.namenodes.palo" = "nn1,nn2" - dfs.namenode.rpc-address.xxx.nn: ha模式中指定namenode的rpc地址信息,ha模式中必须配置。其中nn表示dfs.ha.namenodes.xxx中配置的一个namenode的名字。例子: "dfs.namenode.rpc-address.palo.nn1" = "host:port" - dfs.client.failover.proxy.provider: ha模式中指定client连接namenode的provider,默认为:org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - + - hadoop.security.authentication: 鉴权方式，包含simple和kerberos两种值。默认是SIMPLE模式。如果要使用kerberos，那必须配置以下配置。 + - kerberos_principal: 指定kerberos的principal + - kerberos_keytab: 指定kerberos的keytab文件路径 + - kerberos_keytab_content: 指定kerberos中keytab文件内容经过base64编码之后的内容。这个跟kerberos_keytab配置二选一就可以. 4. opt_properties 用于指定一些特殊参数。 @@ -130,7 +133,7 @@ DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file") INTO TABLE `my_table` ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password") + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password") PROPERTIES ( "timeout"="3600", @@ -220,7 +223,7 @@ v2 = hll_hash(k2) ) ) - WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); + WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password"); LOAD LABEL example_db.label8 ( @@ -254,6 +257,48 @@ "max_filter_ratio"="0.1" ); + 9. 基于keytab的kerberos鉴权访问hdfs + LOAD LABEL table1_20170707 ( + DATA INFILE("hdfs://bdos/palo/table1_data") + INTO TABLE table1 + ) + WITH BROKER hdfs ( + "fs.defaultFS"="hdfs://bdos", + "username"="hdfs_user", + "password"="hdfs_password", + "dfs.nameservices"="bdos", + "dfs.ha.namenodes.bdos"="nn1,nn2", + "dfs.namenode.rpc-address.bdos.nn1"="host1:port1", + "dfs.namenode.rpc-address.bdos.nn2"="host2:port2", + "hadoop.security.authentication"="kerberos", + "kerberos_principal"="palo@BAIDU.COM", + "kerberos_keytab"="/home/palo/palo.keytab")) + PROPERTIES ( + "timeout"="3600", + "max_filter_ratio"="0.1" + ); + + 10. 使用keytab content的kerberos鉴权访问hdfs + LOAD LABEL table1_20170707 ( + DATA INFILE("hdfs://bdos/palo/table1_data") + INTO TABLE table1 + ) + WITH BROKER hdfs ( + "fs.defaultFS"="hdfs://bdos", + "username"="hdfs_user", + "password"="hdfs_password", + "dfs.nameservices"="bdos", + "dfs.ha.namenodes.bdos"="nn1,nn2", + "dfs.namenode.rpc-address.bdos.nn1"="host1:port1", + "dfs.namenode.rpc-address.bdos.nn2"="host2:port2", + "hadoop.security.authentication"="kerberos", + "kerberos_principal"="palo@BAIDU.COM", + "kerberos_keytab_content"="BQIAAABEAAEACUJBSURVLkNPTQAEcGFsbw")) + PROPERTIES ( + "timeout"="3600", + "max_filter_ratio"="0.1" + ); + ## keyword LOAD,TABLE @@ -369,7 +414,7 @@ timeout: 指定 load 作业的超时时间，单位是秒。当load执行时间超过该阈值时，会自动取消。默认超时时间是 86400 秒。 建议指定 timeout 时间小于 86400 秒。 - hll: 用于指定数据里面和表里面的HLL列的对应关系，表中的列和数据里面指定的列 + hll: 用于指定数据里面和表里面的HLL列的对应关系，表中的列和数据里面指定的列 （如果不指定columns，则数据列面的列也可以是表里面的其它非HLL列）通过","分割 指定多个hll列使用“:”分割，例如: 'hll1,cuid:hll2,device' @@ -535,7 +580,7 @@ EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"=",") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy"); ## keyword - EXPORT + EXPORT # SHOW DATABASES ## description @@ -625,9 +670,9 @@ 3. 展示指定 db 的导出任务，state 为 "exporting", 并按 StartTime 降序排序 SHOW EXPORT FROM example_db WHERE STATE = "exporting" ORDER BY StartTime DESC; - + 4. 展示指定db，指定job_id的导出任务 - SHOW EXPORT FROM example_db WHERE EXPORT_JOB_ID = job_id; + SHOW EXPORT FROM example_db WHERE EXPORT_JOB_ID = job_id; ## keyword SHOW,EXPORT @@ -744,30 +789,35 @@ 该语句用于查看 BACKUP 任务 语法： SHOW BACKUP [FROM db_name] - [WHERE LABEL = "backup_label" | LABEL LIKE "pattern"]; 说明： - BACKUP 任务的状态（State）有以下几种： - PENDING：刚提交的 - SNAPSHOT：正在执行快照 - UPLOAD：准备上传数据 - CHECK_UPLOAD：正在上传数据 - FINISHING：即将结束 - FINISHED：任务完成 - CANCELLED：任务失败 - - 失败的任务可以通过 ErrMsg 列失败查看原因。 + 1. Palo 中仅保存最近一次 BACKUP 任务。 + 2. 各列含义如下： + JobId： 唯一作业id + SnapshotName： 备份的名称 + DbName： 所属数据库 + State： 当前阶段 + PENDING： 提交作业后的初始状态 + SNAPSHOTING： 执行快照中 + UPLOAD_SNAPSHOT：快照完成，准备上传 + UPLOADING： 快照上传中 + SAVE_META： 将作业元信息保存为本地文件 + UPLOAD_INFO： 上传作业元信息 + FINISHED： 作业成功 + CANCELLED： 作业失败 + BackupObjs： 备份的表和分区 + CreateTime： 任务提交时间 + SnapshotFinishedTime： 快照完成时间 + UploadFinishedTime： 快照上传完成时间 + FinishedTime： 作业结束时间 + UnfinishedTasks： 在 SNAPSHOTING 和 UPLOADING 阶段会显示还未完成的子任务id + Status： 如果作业失败，显示失败信息 + Timeout： 作业超时时间，单位秒 ## example - 1. 查看 example_db 下的所有 BACKUP 任务。 + 1. 查看 example_db 下最后一次 BACKUP 任务。 SHOW BACKUP FROM example_db; - 2. 查看 example_db 下 LABEL 为 "backup_label" 的任务。 - SHOW BACKUP FROM example_db WHERE LABEL = "backup_label"; - - 3. 查看 example_db 下 LABEL 前缀为 "backup" 的任务。 - SHOW BACKUP FROM example_db WHERE LABEL LIKE "backup%"; - ## keyword SHOW, BACKUP @@ -776,34 +826,96 @@ 该语句用于查看 RESTORE 任务 语法： SHOW RESTORE [FROM db_name] - [WHERE LABEL = "restore_label" | LABEL LIKE "pattern"]; 说明： - BACKUP 任务的状态（State）有以下几种： - PENDING：刚提交的 - RESTORE_META：正在恢复元数据 - DOWNLOAD_OBJS：准备恢复数据 - DOWNLOADING：正在恢复数据 - FINISHING：恢复完成，等待确认 - FINISHED：任务完成 - CANCELLED：任务失败 - - FINISHING 状态的任务需要通过 RESTORE COMMIT 语句进行确认生效，详见 RESTORE 语句 - 失败的任务可以通过 ErrMsg 列失败查看原因。 + 1. Palo 中仅保存最近一次 RESTORE 任务。 + 2. 各列含义如下： + JobId： 唯一作业id + Label： 要恢复的备份的名称 + Timestamp： 要恢复的备份的时间版本 + DbName： 所属数据库 + State： 当前阶段 + PENDING： 提交作业后的初始状态 + SNAPSHOTING： 执行快照中 + DOWNLOAD： 快照完成，准备下载仓库中的快照 + DOWNLOADING： 快照下载中 + COMMIT： 快照下载完成，准备生效 + COMMITING： 生效中 + FINISHED： 作业成功 + CANCELLED： 作业失败 + AllowLoad： 恢复时是否允许导入（当前不支持） + ReplicationNum： 指定恢复的副本数 + RestoreJobs： 要恢复的表和分区 + CreateTime： 任务提交时间 + MetaPreparedTime： 元数据准备完成时间 + SnapshotFinishedTime： 快照完成时间 + DownloadFinishedTime： 快照下载完成时间 + FinishedTime： 作业结束时间 + UnfinishedTasks： 在 SNAPSHOTING、DOWNLOADING 和 COMMITING 阶段会显示还未完成的子任务id + Status： 如果作业失败，显示失败信息 + Timeout： 作业超时时间，单位秒 ## example - 1. 查看 example_db 下的所有 RESTORE 任务。 + 1. 查看 example_db 下最近一次 RESTORE 任务。 SHOW RESTORE FROM example_db; - 2. 查看 example_db 下 LABEL 为 "restore_label" 的任务。 - SHOW RESTORE FROM example_db WHERE LABEL = "restore_label"; - - 3. 查看 example_db 下 LABEL 前缀为 "restore" 的任务。 - SHOW RESTORE FROM example_db WHERE LABEL LIKE "restore%"; - ## keyword SHOW, RESTORE - + +# SHOW REPOSITORIES +## description + 该语句用于查看当前已创建的仓库。 + 语法： + SHOW REPOSITORIES; + + 说明： + 1. 各列含义如下： + RepoId： 唯一的仓库ID + RepoName： 仓库名称 + CreateTime： 第一次创建该仓库的时间 + IsReadOnly： 是否为只读仓库 + Location： 仓库中用于备份数据的根目录 + Broker： 依赖的 Broker + ErrMsg： Palo 会定期检查仓库的连通性，如果出现问题，这里会显示错误信息 + +## example + 1. 查看已创建的仓库： + SHOW REPOSITORIES; + +## keyword + SHOW, REPOSITORY, REPOSITORIES + +# SHOW SNAPSHOT +## description + 该语句用于查看仓库中已存在的备份。 + 语法： + SHOW SNAPSHOT ON `repo_name` + [WHERE SNAPSHOT = "snapshot" [AND TIMESTAMP = "backup_timestamp"]]; + + 说明： + 1. 各列含义如下： + Snapshot： 备份的名称 + Timestamp： 对应备份的时间版本 + Status： 如果备份正常，则显示 OK，否则显示错误信息 + + 2. 如果指定了 TIMESTAMP，则会额外显示如下信息： + Database： 备份数据原属的数据库名称 + Details： 以 Json 的形式，展示整个备份的数据目录及文件结构 + +## example + 1. 查看仓库 example_repo 中已有的备份： + SHOW SNAPSHOT ON example_repo; + + 2. 仅查看仓库 example_repo 中名称为 backup1 的备份： + SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "backup1"; + + 2. 查看仓库 example_repo 中名称为 backup1 的备份，时间版本为 "2018-05-05-15-34-26" 的详细信息： + SHOW SNAPSHOT ON example_repo + WHERE SNAPSHOT = "backup1" AND TIMESTAMP = "2018-05-05-15-34-26"; + +## keyword + SHOW, SNAPSHOT + # SHOW BACKENDS ## description 该语句用于查看cluster内的节点 @@ -812,3 +924,4 @@ ## keyword SHOW, BACKENDS + diff --git a/fe/build.xml b/fe/build.xml index 57f2c26308..c156590768 100644 --- a/fe/build.xml +++ b/fe/build.xml @@ -136,4 +136,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fe/src/com/baidu/palo/alter/RollupHandler.java b/fe/src/com/baidu/palo/alter/RollupHandler.java index 0da1f16ac2..177dbdbed5 100644 --- a/fe/src/com/baidu/palo/alter/RollupHandler.java +++ b/fe/src/com/baidu/palo/alter/RollupHandler.java @@ -47,6 +47,7 @@ import com.baidu.palo.common.util.ListComparator; import com.baidu.palo.common.util.PropertyAnalyzer; import com.baidu.palo.common.util.TimeUtils; import com.baidu.palo.common.util.Util; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.persist.DropInfo; import com.baidu.palo.persist.EditLog; import com.baidu.palo.qe.ConnectContext; @@ -546,14 +547,34 @@ public class RollupHandler extends AlterHandler { } } - private List getJobInfo(RollupJob rollupJob, String tableName) { + private void getJobInfo(List> rollupJobInfos, + RollupJob rollupJob, Database db) { + if (rollupJob.getDbId() != db.getId()) { + return; + } + + OlapTable olapTable = (OlapTable) db.getTable(rollupJob.getTableId()); + if (olapTable == null) { + return; + } + + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), db.getFullName(), + olapTable.getName(), + PrivPredicate.ALTER)) { + // no priv, return + LOG.debug("No priv for user {} to table {}.{}", ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), db.getFullName(), olapTable.getName()); + return; + } + List jobInfo = new ArrayList(); // job id jobInfo.add(rollupJob.getTableId()); // table name - jobInfo.add(tableName); + jobInfo.add(olapTable.getName()); // create time long createTime = rollupJob.getCreateTimeMs(); @@ -584,7 +605,8 @@ public class RollupHandler extends AlterHandler { jobInfo.add("N/A"); } - return jobInfo; + rollupJobInfos.add(jobInfo); + return; } @Override @@ -668,31 +690,12 @@ public class RollupHandler extends AlterHandler { this.jobsLock.readLock().lock(); try { for (AlterJob alterJob : this.alterJobs.values()) { - if (alterJob.getDbId() != dbId) { - continue; - } - - OlapTable olapTable = (OlapTable) db.getTable(alterJob.getTableId()); - if (olapTable == null) { - continue; - } - - rollupJobInfos.add(getJobInfo((RollupJob) alterJob, olapTable.getName())); - } // end for rollupJobs + getJobInfo(rollupJobInfos, (RollupJob) alterJob, db); + } for (AlterJob alterJob : this.finishedOrCancelledAlterJobs) { - if (alterJob.getDbId() != dbId) { - continue; - } - - String tableName = ""; - OlapTable olapTable = (OlapTable) db.getTable(alterJob.getTableId()); - if (olapTable != null) { - tableName = olapTable.getName(); - } - - rollupJobInfos.add(getJobInfo((RollupJob) alterJob, tableName)); - } // end for rollupJobs + getJobInfo(rollupJobInfos, (RollupJob) alterJob, db); + } // sort by // "JobId", "TableName", "CreateTime", "FinishedTime", "BaseIndexName", "RollupIndexName" diff --git a/fe/src/com/baidu/palo/alter/SchemaChangeHandler.java b/fe/src/com/baidu/palo/alter/SchemaChangeHandler.java index 6229fce888..2d31e743ea 100644 --- a/fe/src/com/baidu/palo/alter/SchemaChangeHandler.java +++ b/fe/src/com/baidu/palo/alter/SchemaChangeHandler.java @@ -54,6 +54,7 @@ import com.baidu.palo.common.util.ListComparator; import com.baidu.palo.common.util.PropertyAnalyzer; import com.baidu.palo.common.util.TimeUtils; import com.baidu.palo.common.util.Util; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.thrift.TResourceInfo; import com.baidu.palo.thrift.TStorageType; @@ -1369,6 +1370,16 @@ public class SchemaChangeHandler extends AlterHandler { if (olapTable == null) { return; } + + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), db.getFullName(), + olapTable.getName(), + PrivPredicate.ALTER)) { + // no priv, return + LOG.debug("No priv for user {} to table {}.{}", ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), db.getFullName(), olapTable.getName()); + return; + } // create time long createTime = schemaChangeJob.getCreateTimeMs(); diff --git a/fe/src/com/baidu/palo/analysis/AbstractBackupStmt.java b/fe/src/com/baidu/palo/analysis/AbstractBackupStmt.java index b7f01a782e..49b16294cb 100644 --- a/fe/src/com/baidu/palo/analysis/AbstractBackupStmt.java +++ b/fe/src/com/baidu/palo/analysis/AbstractBackupStmt.java @@ -20,90 +20,116 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.Config; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.List; import java.util.Map; public class AbstractBackupStmt extends DdlStmt { - private static final String SERVER_TYPE = "server_type"; - private static final String HOST = "host"; - private static final String PORT = "port"; - private static final String USER = "user"; - private static final String PASSWORD = "password"; - private static final String OPT_PROPERTIES = "opt_properties"; + private static final Logger LOG = LogManager.getLogger(AbstractBackupStmt.class); + + private final static String PROP_TIMEOUT = "timeout"; + private final static long MIN_TIMEOUT_MS = 600 * 1000L; // 10 min protected LabelName labelName; - protected List objNames; - protected String remotePath; + protected String repoName; + protected List tblRefs; protected Map properties; - public AbstractBackupStmt(LabelName labelName, List objNames, - String remotePath, Map properties) { + protected long timeoutMs; + + public AbstractBackupStmt(LabelName labelName, String repoName, List tableRefs, + Map properties) { this.labelName = labelName; - this.objNames = objNames; - if (this.objNames == null) { - this.objNames = Lists.newArrayList(); + this.repoName = repoName; + this.tblRefs = tableRefs; + if (this.tblRefs == null) { + this.tblRefs = Lists.newArrayList(); } - this.remotePath = remotePath; - this.properties = properties; + this.properties = properties == null ? Maps.newHashMap() : properties; } @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { labelName.analyze(analyzer); - - // check authenticate - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), labelName.getDbName(), AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), - labelName.getDbName()); + + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - for (PartitionName partitionName : objNames) { - partitionName.analyze(analyzer); - } - - if (Strings.isNullOrEmpty(remotePath)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_MISSING_PARAM, "restore path is not specified"); - } + checkAndNormalizeBackupObjs(); analyzeProperties(); - - // check if partition names has intersection - PartitionName.checkIntersect(objNames); } - private void analyzeProperties() throws AnalysisException { - if (properties == null || properties.isEmpty()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_MISSING_PARAM, "romote source info is not specified"); + private void checkAndNormalizeBackupObjs() throws AnalysisException { + for (TableRef tblRef : tblRefs) { + if (!Strings.isNullOrEmpty(tblRef.getName().getDb())) { + throw new AnalysisException("Cannot specify database name on backup objects: " + + tblRef.getName().getTbl() + ". Sepcify database name before label"); + } + // set db name because we can not persist empty string when writing bdbje log + tblRef.getName().setDb(labelName.getDbName()); + } + + // normalize + // table name => table ref + Map tblPartsMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + for (TableRef tblRef : tblRefs) { + String tblName = tblRef.getName().getTbl(); + + if (!tblPartsMap.containsKey(tblName)) { + tblPartsMap.put(tblName, tblRef); + } else { + throw new AnalysisException("Duplicated restore table: " + tblName); + } + } + + // update table ref + tblRefs.clear(); + for (TableRef tableRef : tblPartsMap.values()) { + tblRefs.add(tableRef); } - Map tmpProp = Maps.newHashMap(); - for (Map.Entry entry : properties.entrySet()) { - tmpProp.put(entry.getKey().toLowerCase(), entry.getValue()); - } - properties = tmpProp; + LOG.debug("table refs after normalization: \n{}", Joiner.on("\n").join(tblRefs)); + } - if (!properties.containsKey(SERVER_TYPE) - || !properties.containsKey(HOST) - || !properties.containsKey(PORT) - || !properties.containsKey(USER) - || !properties.containsKey(PASSWORD)) { - throw new AnalysisException("Properties should contains required params."); - } + protected void analyzeProperties() throws AnalysisException { + // timeout + if (properties.containsKey("timeout")) { + try { + timeoutMs = Long.valueOf(properties.get(PROP_TIMEOUT)); + } catch (NumberFormatException e) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Invalid timeout format: " + + properties.get(PROP_TIMEOUT)); + } - if (!properties.containsKey(OPT_PROPERTIES)) { - properties.put(OPT_PROPERTIES, ""); + if (timeoutMs * 1000 < MIN_TIMEOUT_MS) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "timeout must be at least 10 min"); + } + + timeoutMs = timeoutMs * 1000; + properties.remove(PROP_TIMEOUT); + } else { + timeoutMs = Config.backup_job_default_timeout_ms; } } @@ -119,15 +145,20 @@ public class AbstractBackupStmt extends DdlStmt { return labelName; } - public List getObjNames() { - return objNames; + public String getRepoName() { + return repoName; } - public String getRemotePath() { - return remotePath; + public List getTableRefs() { + return tblRefs; } public Map getProperties() { return properties; } + + public long getTimeoutMs() { + return timeoutMs; + } } + diff --git a/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java b/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java index 3822c7e965..8a88903238 100644 --- a/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java @@ -20,12 +20,15 @@ package com.baidu.palo.analysis; -import java.util.Map; - +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + +import java.util.Map; public class AlterClusterStmt extends DdlStmt { @@ -41,9 +44,8 @@ public class AlterClusterStmt extends DdlStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_AUTHORITY, analyzer.getUser()); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_AUTHORITY, "NODE"); } if (properties == null || properties.size() == 0 diff --git a/fe/src/com/baidu/palo/analysis/AlterDatabaseQuotaStmt.java b/fe/src/com/baidu/palo/analysis/AlterDatabaseQuotaStmt.java index 080ad2b061..64405651e0 100644 --- a/fe/src/com/baidu/palo/analysis/AlterDatabaseQuotaStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterDatabaseQuotaStmt.java @@ -20,11 +20,15 @@ package com.baidu.palo.analysis; -import com.baidu.palo.cluster.ClusterNamespace; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.ErrorCode; -import com.baidu.palo.common.ErrorReport; -import com.baidu.palo.common.InternalException; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Strings; public class AlterDatabaseQuotaStmt extends DdlStmt { @@ -46,11 +50,12 @@ public class AlterDatabaseQuotaStmt extends DdlStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - super.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); - } - + super.analyze(analyzer); + + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), dbName); + } + if (Strings.isNullOrEmpty(dbName)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); } diff --git a/fe/src/com/baidu/palo/analysis/AlterDatabaseRename.java b/fe/src/com/baidu/palo/analysis/AlterDatabaseRename.java index 951ef09ec9..a7227908aa 100644 --- a/fe/src/com/baidu/palo/analysis/AlterDatabaseRename.java +++ b/fe/src/com/baidu/palo/analysis/AlterDatabaseRename.java @@ -20,12 +20,19 @@ package com.baidu.palo.analysis; -import com.baidu.palo.cluster.ClusterNamespace; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.ErrorCode; -import com.baidu.palo.common.ErrorReport; -import com.baidu.palo.common.FeNameFormat; -import com.baidu.palo.common.InternalException; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Strings; public class AlterDatabaseRename extends DdlStmt { @@ -50,10 +57,13 @@ public class AlterDatabaseRename extends DdlStmt { super.analyze(analyzer); if (Strings.isNullOrEmpty(dbName)) { throw new AnalysisException("Database name is not set"); - } - - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + } + + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), dbName, + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.ALTER_PRIV), + Operator.OR))) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), dbName); } if (Strings.isNullOrEmpty(newDbName)) { diff --git a/fe/src/com/baidu/palo/analysis/AlterLoadErrorUrlClause.java b/fe/src/com/baidu/palo/analysis/AlterLoadErrorUrlClause.java index 4c05863b97..b7b68fa8f9 100644 --- a/fe/src/com/baidu/palo/analysis/AlterLoadErrorUrlClause.java +++ b/fe/src/com/baidu/palo/analysis/AlterLoadErrorUrlClause.java @@ -21,8 +21,6 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.ErrorCode; -import com.baidu.palo.common.ErrorReport; import com.baidu.palo.load.LoadErrorHub; import org.apache.logging.log4j.LogManager; @@ -48,12 +46,7 @@ public class AlterLoadErrorUrlClause extends AlterClause { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - // only root can do it - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, - "SET LOAD_ERROR_URL"); - } - + // auth is checked in Alter System Stmt this.param = LoadErrorHub.analyzeUrl(url); } diff --git a/fe/src/com/baidu/palo/analysis/AlterSystemStmt.java b/fe/src/com/baidu/palo/analysis/AlterSystemStmt.java index c09e21a999..de5d3ff019 100644 --- a/fe/src/com/baidu/palo/analysis/AlterSystemStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterSystemStmt.java @@ -15,10 +15,13 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -36,8 +39,10 @@ public class AlterSystemStmt extends DdlStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ALTER SYSTEM"); + + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "NODE"); } Preconditions.checkState((alterClause instanceof AddBackendClause) diff --git a/fe/src/com/baidu/palo/analysis/AlterTableStmt.java b/fe/src/com/baidu/palo/analysis/AlterTableStmt.java index 35fb0dc5e9..2485e31b1a 100644 --- a/fe/src/com/baidu/palo/analysis/AlterTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterTableStmt.java @@ -20,12 +20,14 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.io.Writable; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.collect.Lists; @@ -76,8 +78,13 @@ public class AlterTableStmt extends DdlStmt implements Writable { op.analyze(analyzer); } - // check access - analyzer.checkPrivilege(tbl.getDb(), AccessPrivilege.READ_WRITE); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tbl.getDb(), tbl.getTbl(), + PrivPredicate.ALTER)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "ALTER TABLE", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tbl.getTbl()); + } } @Override diff --git a/fe/src/com/baidu/palo/analysis/AlterUserStmt.java b/fe/src/com/baidu/palo/analysis/AlterUserStmt.java index 6fdf9ae54c..b1b234b8f2 100644 --- a/fe/src/com/baidu/palo/analysis/AlterUserStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterUserStmt.java @@ -20,98 +20,44 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.Config; -import com.baidu.palo.common.DdlException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; - -import com.google.common.base.Strings; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import org.apache.commons.lang.NotImplementedException; import java.util.List; +@Deprecated public class AlterUserStmt extends DdlStmt { - private String userName; + private UserIdentity userIdent; private AlterUserClause clause; - public AlterUserStmt(String userName, AlterClause clause) { - this.userName = userName; + public AlterUserStmt(UserIdentity userIdent, AlterClause clause) { + this.userIdent = userIdent; this.clause = (AlterUserClause) clause; } - private boolean hasRightToModify(Analyzer analyzer) { - String user = analyzer.getUser(); - String toUser = userName; - - // own can modify own - if (user.equals(toUser)) { - return true; - } - - // admin can modify all - if (analyzer.getCatalog().getUserMgr().isAdmin(user)) { - return true; - } - - // superuse can modify Ordinary user - if (analyzer.getCatalog().getUserMgr().isSuperuser(user) - && !analyzer.getCatalog().getUserMgr().isSuperuser(toUser)) { - return true; - } - return false; - } - - private void checkWhiteListSize(Analyzer analyzer) throws AnalysisException { - if (clause.getAlterUserType() == AlterUserType.ADD_USER_WHITELIST) { - try { - if (analyzer.getCatalog().getUserMgr().getWhiteListSize(userName) - > Config.per_user_white_list_limit) { - throw new AnalysisException("whitelist size excced the max (" - + Config.per_user_white_list_limit + ")"); - } - } catch (DdlException e) { - throw new AnalysisException(e.getMessage()); - } - } - } - @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - // check toUser - if (Strings.isNullOrEmpty(userName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "empty user"); - } - userName = ClusterNamespace.getFullName(getClusterName(), userName); - // check destination user if exists - try { - analyzer.getCatalog().getUserMgr().checkUserIfExist(userName); - } catch (DdlException e) { - throw new AnalysisException(e.getMessage()); - } - // check destination user's whitelist if ecceed max value - checkWhiteListSize(analyzer); + userIdent.analyze(analyzer.getClusterName()); - // only write user can modify - analyzer.checkPrivilege(analyzer.getDefaultDb(), AccessPrivilege.READ_WRITE); - - // check if has the right - if (!hasRightToModify(analyzer)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ALTER CLUSTER"); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ALTER USER"); } - + // alter clause analysis clause.analyze(analyzer); } - public String getUser() { - return userName; + public UserIdentity getUserIdent() { + return userIdent; } public List getHosts() { diff --git a/fe/src/com/baidu/palo/analysis/Analyzer.java b/fe/src/com/baidu/palo/analysis/Analyzer.java index 197aaaac92..48bb6f8c62 100644 --- a/fe/src/com/baidu/palo/analysis/Analyzer.java +++ b/fe/src/com/baidu/palo/analysis/Analyzer.java @@ -20,12 +20,14 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.InfoSchemaDb; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.OlapTable.OlapTableState; import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Table.TableType; import com.baidu.palo.catalog.Type; import com.baidu.palo.catalog.View; import com.baidu.palo.cluster.ClusterNamespace; @@ -219,7 +221,7 @@ public class Analyzer { public final Map ijClauseByConjunct = Maps.newHashMap(); // TODO chenhao16, to save conjuncts, which children are constant - public final Map constantConjunct = Maps.newHashMap(); + public final Map> constantConjunct = Maps.newHashMap(); // map from slot id to the analyzer/block in which it was registered public final Map blockBySlot = Maps.newHashMap(); @@ -471,6 +473,11 @@ public class Analyzer { ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName.getTbl()); } + if (table.getType() == TableType.OLAP && (((OlapTable) table).getState() == OlapTableState.RESTORE + || ((OlapTable) table).getState() == OlapTableState.RESTORE_WITH_LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_STATE, "RESTORING"); + } + TableName tblName = new TableName(database.getFullName(), table.getName()); if (table instanceof View) { return new InlineViewRef((View) table, tableRef); @@ -739,7 +746,12 @@ public class Analyzer { private void registerConstantConjunct(TupleId id, Expr e) { if (id != null && e.isConstant()) { - globalState.constantConjunct.put(id, e); + Set set = globalState.constantConjunct.get(id); + if (set == null) { + set = Sets.newHashSet(); + globalState.constantConjunct.put(id, set); + } + set.add(e); } } @@ -866,7 +878,8 @@ public class Analyzer { if (e.isConstant()) { boolean isBoundByTuple = false; for (TupleId id : tupleIds) { - if (globalState.constantConjunct.containsKey(id)) { + final Set exprSet = globalState.constantConjunct.get(id); + if (exprSet != null && exprSet.contains(e)) { isBoundByTuple = true; break; } @@ -875,6 +888,7 @@ public class Analyzer { continue; } } + if (e.isBoundByTupleIds(tupleIds) && !e.isAuxExpr() && !globalState.assignedConjuncts.contains(e.getId()) @@ -1378,8 +1392,8 @@ public class Analyzer { return globalState.context.getClusterName(); } - public String getUser() { - return globalState.context.getUser(); + public String getQualifiedUser() { + return globalState.context.getQualifiedUser(); } public String getSchemaDb() { @@ -1588,12 +1602,6 @@ public class Analyzer { public Map getLocalViews() { return localViews_; } - public void checkPrivilege(String db, AccessPrivilege priv) throws AnalysisException { - if (!globalState.catalog.getUserMgr().checkAccess(getUser(), db, priv)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, priv.toString()); - } - } - public boolean isOuterJoined(TupleId tid) { return globalState.outerJoinedTupleIds.containsKey(tid); } diff --git a/fe/src/com/baidu/palo/analysis/BackupStmt.java b/fe/src/com/baidu/palo/analysis/BackupStmt.java index 5d0da6db7e..feda2627d9 100644 --- a/fe/src/com/baidu/palo/analysis/BackupStmt.java +++ b/fe/src/com/baidu/palo/analysis/BackupStmt.java @@ -20,18 +20,72 @@ package com.baidu.palo.analysis; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; import com.baidu.palo.common.util.PrintableMap; import com.google.common.base.Joiner; +import com.google.common.collect.Maps; import java.util.List; import java.util.Map; public class BackupStmt extends AbstractBackupStmt { + private final static String PROP_TYPE = "type"; - public BackupStmt(LabelName labelName, List backupObjNames, String backupPath, - Map properties) { - super(labelName, backupObjNames, backupPath, properties); + public enum BackupType { + INCREMENTAL, FULL + } + + private BackupType type = BackupType.FULL; + + public BackupStmt(LabelName labelName, String repoName, List tblRefs, Map properties) { + super(labelName, repoName, tblRefs, properties); + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public BackupType getType() { + return type; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + + // tbl refs can not set alias in backup + for (TableRef tblRef : tblRefs) { + if (tblRef.hasExplicitAlias()) { + throw new AnalysisException("Can not set alias for table in Backup Stmt: " + tblRef); + } + } + } + + @Override + protected void analyzeProperties() throws AnalysisException { + super.analyzeProperties(); + + Map copiedProperties = Maps.newHashMap(properties); + // type + if (copiedProperties.containsKey(PROP_TYPE)) { + try { + type = BackupType.valueOf(copiedProperties.get(PROP_TYPE).toUpperCase()); + } catch (Exception e) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Invalid backup job type: " + + copiedProperties.get(PROP_TYPE)); + } + copiedProperties.remove(PROP_TYPE); + } + + if (!copiedProperties.isEmpty()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Unknown backup job properties: " + copiedProperties.keySet()); + } } @Override @@ -42,15 +96,14 @@ public class BackupStmt extends AbstractBackupStmt { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("BACKUP LABEL ").append(labelName.toSql()); - if (!objNames.isEmpty()) { - sb.append(" ("); - sb.append(Joiner.on(", ").join(objNames)); - sb.append(")"); - } - sb.append(" INTO \"").append(remotePath).append("\" PROPERTIES("); - sb.append(new PrintableMap(properties, "=", true, false)); - sb.append(")"); + sb.append("BACKUP SNAPSHOT ").append(labelName.toSql()); + sb.append("\n").append("TO ").append(repoName).append("\nON\n("); + + sb.append(Joiner.on(",\n").join(tblRefs)); + + sb.append("\n)\nPROPERTIES\n("); + sb.append(new PrintableMap(properties, " = ", true, true)); + sb.append("\n)"); return sb.toString(); } } \ No newline at end of file diff --git a/fe/src/com/baidu/palo/analysis/CancelAlterTableStmt.java b/fe/src/com/baidu/palo/analysis/CancelAlterTableStmt.java index 20911f9aa1..78050fe933 100644 --- a/fe/src/com/baidu/palo/analysis/CancelAlterTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/CancelAlterTableStmt.java @@ -15,9 +15,13 @@ package com.baidu.palo.analysis; -import com.baidu.palo.analysis.ShowAlterStmt.AlterType; -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.analysis.ShowAlterStmt.AlterType; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; /* * CANCEL ALTER COLUMN|ROLLUP FROM db_name.table_name @@ -50,9 +54,13 @@ public class CancelAlterTableStmt extends CancelStmt { dbTableName.analyze(analyzer); // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbTableName.getDb(), AccessPrivilege.READ_WRITE)) { - throw new AnalysisException("No privilege to access database[" + dbTableName.getDb() + "]"); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbTableName.getDb(), + dbTableName.getTbl(), + PrivPredicate.ALTER)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "CANCEL ALTER TABLE", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbTableName.getTbl()); } } diff --git a/fe/src/com/baidu/palo/analysis/CancelBackupStmt.java b/fe/src/com/baidu/palo/analysis/CancelBackupStmt.java index f86161bba8..c45398edc7 100644 --- a/fe/src/com/baidu/palo/analysis/CancelBackupStmt.java +++ b/fe/src/com/baidu/palo/analysis/CancelBackupStmt.java @@ -15,11 +15,14 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -43,16 +46,19 @@ public class CancelBackupStmt extends CancelStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); if (Strings.isNullOrEmpty(dbName)) { dbName = analyzer.getDefaultDb(); if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); + throw new AnalysisException("No database selected"); } + } else { + dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } } diff --git a/fe/src/com/baidu/palo/analysis/CancelLoadStmt.java b/fe/src/com/baidu/palo/analysis/CancelLoadStmt.java index 3a99f8b087..9483335d55 100644 --- a/fe/src/com/baidu/palo/analysis/CancelLoadStmt.java +++ b/fe/src/com/baidu/palo/analysis/CancelLoadStmt.java @@ -16,7 +16,6 @@ package com.baidu.palo.analysis; import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; @@ -46,7 +45,6 @@ public class CancelLoadStmt extends DdlStmt { this.whereClause = whereClause; } - @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); @@ -59,11 +57,7 @@ public class CancelLoadStmt extends DdlStmt { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - throw new AnalysisException("No privilege of db(" + dbName + ")."); - } + // check auth after we get real load job // analyze expr if not null boolean valid = true; diff --git a/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java b/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java index fc0d7acfac..40cdc18f7b 100644 --- a/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java @@ -15,16 +15,20 @@ package com.baidu.palo.analysis; -import java.util.Map; - +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.FeNameFormat; import com.baidu.palo.common.InternalException; import com.baidu.palo.mysql.MysqlPassword; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Strings; +import java.util.Map; + public class CreateClusterStmt extends DdlStmt { public static String CLUSTER_INSTANCE_NUM = "instance_num"; public static String CLUSTER_SUPERMAN_PASSWORD = "password"; @@ -62,8 +66,8 @@ public class CreateClusterStmt extends DdlStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { FeNameFormat.checkDbName(clusterName); - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PERMISSIONS); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_AUTHORITY, analyzer.getQualifiedUser()); } if (properties == null || properties.size() == 0 || !properties.containsKey(CLUSTER_INSTANCE_NUM)) { diff --git a/fe/src/com/baidu/palo/analysis/CreateDbStmt.java b/fe/src/com/baidu/palo/analysis/CreateDbStmt.java index d8692267f6..c3efafcbe7 100644 --- a/fe/src/com/baidu/palo/analysis/CreateDbStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateDbStmt.java @@ -20,12 +20,16 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.FeNameFormat; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Strings; // 用于描述CREATE DATABASE的内部结构 @@ -54,8 +58,9 @@ public class CreateDbStmt extends DdlStmt { } FeNameFormat.checkDbName(dbName); dbName = ClusterNamespace.getFullName(getClusterName(), dbName); - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), dbName, PrivPredicate.CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), dbName); } } diff --git a/fe/src/com/baidu/palo/analysis/CreateRepositoryStmt.java b/fe/src/com/baidu/palo/analysis/CreateRepositoryStmt.java new file mode 100644 index 0000000000..952d4cc275 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/CreateRepositoryStmt.java @@ -0,0 +1,103 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations + +package com.baidu.palo.analysis; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.common.util.PrintableMap; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + +import com.google.common.base.Strings; + +import java.util.Map; + +public class CreateRepositoryStmt extends DdlStmt { + private boolean isReadOnly; + private String name; + private String brokerName; + private String location; + private Map properties; + + public CreateRepositoryStmt(boolean isReadOnly, String name, String brokerName, String location, + Map properties) { + this.isReadOnly = isReadOnly; + this.name = name; + this.brokerName = brokerName; + this.location = location; + this.properties = properties; + } + + public boolean isReadOnly() { + return isReadOnly; + } + + public String getName() { + return name; + } + + public String getBrokerName() { + return brokerName; + } + + public String getLocation() { + return location; + } + + public Map getProperties() { + return properties; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + FeNameFormat.checkCommonName("repository", name); + + if (Strings.isNullOrEmpty(brokerName)) { + throw new AnalysisException("You must specify the broker of the repository"); + } + + if (Strings.isNullOrEmpty(location)) { + throw new AnalysisException("You must specify a location on the repository"); + } + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (isReadOnly) { + sb.append("READ_ONLY "); + } + sb.append("REPOSITORY `").append(name).append("` ").append("WITH BROKER `").append(brokerName).append("` "); + sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")"); + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/analysis/CreateRoleStmt.java b/fe/src/com/baidu/palo/analysis/CreateRoleStmt.java new file mode 100644 index 0000000000..84b11d59f8 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/CreateRoleStmt.java @@ -0,0 +1,46 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.InternalException; + +public class CreateRoleStmt extends DdlStmt { + + private String role; + + public CreateRoleStmt(String role) { + this.role = role; + } + + public String getQualifiedRole() { + return role; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + FeNameFormat.checkRoleName(role, false /* can not be admin */); + role = ClusterNamespace.getFullName(analyzer.getClusterName(), role); + } + + @Override + public String toSql() { + return "CREATE ROLE " + role; + } +} diff --git a/fe/src/com/baidu/palo/analysis/CreateTableStmt.java b/fe/src/com/baidu/palo/analysis/CreateTableStmt.java index 820ef7a707..f2d45530f0 100644 --- a/fe/src/com/baidu/palo/analysis/CreateTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateTableStmt.java @@ -20,7 +20,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.AggregateType; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; @@ -37,6 +36,8 @@ import com.baidu.palo.common.io.Text; import com.baidu.palo.common.io.Writable; import com.baidu.palo.common.util.KuduUtil; import com.baidu.palo.common.util.PrintableMap; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -191,10 +192,9 @@ public class CreateTableStmt extends DdlStmt implements Writable { tableName.analyze(analyzer); FeNameFormat.checkTableName(tableName.getTbl()); - // check authenticate - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), tableName.getDb(), AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), tableName.getDb()); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tableName.getDb(), + tableName.getTbl(), PrivPredicate.CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); } analyzeEngineName(); @@ -358,7 +358,7 @@ public class CreateTableStmt extends DdlStmt implements Writable { if (idx != 0) { sb.append(",\n"); } - sb.append(column.toSql()); + sb.append(" ").append(column.toSql()); idx++; } sb.append("\n)"); diff --git a/fe/src/com/baidu/palo/analysis/CreateUserStmt.java b/fe/src/com/baidu/palo/analysis/CreateUserStmt.java index 7d675fc370..459509adfe 100644 --- a/fe/src/com/baidu/palo/analysis/CreateUserStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateUserStmt.java @@ -20,6 +20,7 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; @@ -27,78 +28,86 @@ import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.FeNameFormat; import com.baidu.palo.common.InternalException; import com.baidu.palo.mysql.MysqlPassword; +import com.baidu.palo.mysql.privilege.PaloRole; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -// this is memory struct from CREATE USER statement -// CREATE USER user_name [IDENTIFIED BY [PASSWORD] 'password'] -// -// CREATE USER user_name -// this clause create user without password -// eg. CREATE USER 'jeffrey' -// -// CREATE USER user_name IDENTIFIED BY 'password' -// this clause create user with password in plaintext mode -// eg. CREATE USER 'jeffrey' IDENTIFIED BY 'mypass' -// -// CREATE USER user_name IDENTIFIED BY PASSWORD 'password' -// this clause create user with password in hashed mode. -// eg. CREATE USER 'jeffrey' IDENTIFIED BY PASSWORD '*90E462C37378CED12064BB3388827D2BA3A9B689' +/* + * We support the following create user stmt + * 1. create user user@ip [identified by 'password'] + * specify the user name at a certain ip(wildcard is accepted), with optional password. + * the user@ip must not exist in system + * + * 2. create user user@[domain] [identified by 'password'] + * specify the user name at a certain domain, with optional password. + * the user@[domain] must not exist in system + * the daemon thread will resolve this domain to user@ip format + * + * 3. create user user@xx [identified by 'password'] role role_name + * not only create the specified user, but also grant all privs of the specified role to the user. + */ public class CreateUserStmt extends DdlStmt { private static final Logger LOG = LogManager.getLogger(CreateUserStmt.class); - private String user; + private boolean ifNotExist; + private UserIdentity userIdent; private String password; private byte[] scramblePassword; private boolean isPlain; - private boolean isSuperuser; + private String role; public CreateUserStmt() { } public CreateUserStmt(UserDesc userDesc) { - user = userDesc.getUser(); + userIdent = userDesc.getUserIdent(); password = userDesc.getPassword(); isPlain = userDesc.isPlain(); } - public CreateUserStmt(UserDesc userDesc, boolean isSuperuser) { - user = userDesc.getUser(); + public CreateUserStmt(boolean ifNotExist, UserDesc userDesc, String role) { + this.ifNotExist = ifNotExist; + userIdent = userDesc.getUserIdent(); password = userDesc.getPassword(); isPlain = userDesc.isPlain(); - this.isSuperuser = isSuperuser; + this.role = role; + } + + public boolean isIfNotExist() { + return ifNotExist; } public boolean isSuperuser() { - return isSuperuser; + return role.equalsIgnoreCase(PaloRole.ADMIN_ROLE); + } + + public boolean hasRole() { + return role != null; + } + + public String getQualifiedRole() { + return role; } public byte[] getPassword() { return scramblePassword; } - public String getUser() { - return user; - } - - private void checkUser() throws AnalysisException { - if (Strings.isNullOrEmpty(user)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CANNOT_USER, "CREATE USER", user); - } - - FeNameFormat.checkUserName(user); + public UserIdentity getUserIdent() { + return userIdent; } @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - checkUser(); + userIdent.analyze(analyzer.getClusterName()); // convert plain password to hashed password if (!Strings.isNullOrEmpty(password)) { - // TODO(zhaochun): convert password if (isPlain) { // convert plain password to scramble scramblePassword = MysqlPassword.makeScrambledPassword(password); @@ -108,32 +117,39 @@ public class CreateUserStmt extends DdlStmt { } else { scramblePassword = new byte[0]; } - user = ClusterNamespace.getFullName(getClusterName(), user); - // check authenticate - if (isSuperuser) { - // Only root can create superuser - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_PERMISSION_TO_CREATE_USER, analyzer.getUser()); - } - } else { - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_PERMISSION_TO_CREATE_USER, analyzer.getUser()); + + if (role != null) { + if (role.equalsIgnoreCase("SUPERUSER")) { + // for forward compatibility + role = PaloRole.ADMIN_ROLE; } + FeNameFormat.checkRoleName(role, true /* can be admin */); + role = ClusterNamespace.getFullName(analyzer.getClusterName(), role); + } + + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE USER"); } } @Override public String toSql() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("CREATE USER '").append(user).append("'"); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE USER ").append(userIdent); if (!Strings.isNullOrEmpty(password)) { if (isPlain) { - stringBuilder.append(" IDENTIFIED BY '").append(password).append("'"); + sb.append(" IDENTIFIED BY '").append(password).append("'"); } else { - stringBuilder.append(" IDENTIFIED BY PASSWORD '").append(password).append("'"); + sb.append(" IDENTIFIED BY PASSWORD '").append(password).append("'"); } } - return stringBuilder.toString(); + + if (!Strings.isNullOrEmpty(role)) { + sb.append(" DEFAULT ROLE '").append(role).append("'"); + + } + + return sb.toString(); } @Override diff --git a/fe/src/com/baidu/palo/analysis/CreateViewStmt.java b/fe/src/com/baidu/palo/analysis/CreateViewStmt.java index 8b29a58002..43afeb5256 100644 --- a/fe/src/com/baidu/palo/analysis/CreateViewStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateViewStmt.java @@ -20,7 +20,7 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.catalog.PrimitiveType; @@ -28,12 +28,14 @@ import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.List; import java.util.Set; @@ -157,7 +159,11 @@ public class CreateViewStmt extends DdlStmt { viewDefStmt.analyze(viewAnalyzer); // check privilege - analyzer.checkPrivilege(tableName.getDb(), AccessPrivilege.READ_WRITE); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tableName.getDb(), + tableName.getTbl(), PrivPredicate.CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); + } + createColumnAndViewDefs(analyzer); } } diff --git a/fe/src/com/baidu/palo/analysis/DataDescription.java b/fe/src/com/baidu/palo/analysis/DataDescription.java index 728e576632..e0269fb262 100644 --- a/fe/src/com/baidu/palo/analysis/DataDescription.java +++ b/fe/src/com/baidu/palo/analysis/DataDescription.java @@ -20,17 +20,17 @@ package com.baidu.palo.analysis; -import java.util.List; -import java.util.Map; -import java.util.Set; - import com.baidu.palo.analysis.BinaryPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.Pair; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.thrift.TNetworkAddress; + import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Strings; @@ -38,9 +38,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Set; // used to describe data info which is needed to import. // // data_desc: @@ -391,10 +394,19 @@ public class DataDescription { } } - public void analyze() throws AnalysisException { + public void analyze(String fullDbName) throws AnalysisException { if (Strings.isNullOrEmpty(tableName)) { throw new AnalysisException("No table name in load statement."); } + + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), fullDbName, tableName, + PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), tableName); + } + if (filePathes == null || filePathes.isEmpty()) { throw new AnalysisException("No file path in load statement."); } diff --git a/fe/src/com/baidu/palo/analysis/DeleteStmt.java b/fe/src/com/baidu/palo/analysis/DeleteStmt.java index 5fb55d61f5..b0aa53ad04 100644 --- a/fe/src/com/baidu/palo/analysis/DeleteStmt.java +++ b/fe/src/com/baidu/palo/analysis/DeleteStmt.java @@ -21,12 +21,14 @@ package com.baidu.palo.analysis; import com.baidu.palo.analysis.CompoundPredicate.Operator; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.util.PrintableMap; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -93,9 +95,11 @@ public class DeleteStmt extends DdlStmt { analyzePredicate(wherePredicate); // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), tbl.getDb(), AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), tbl.getDb()); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tbl.getDb(), tbl.getTbl(), + PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), tbl.getTbl()); } } diff --git a/fe/src/com/baidu/palo/analysis/DescribeStmt.java b/fe/src/com/baidu/palo/analysis/DescribeStmt.java index a47b6f70c8..498be872a0 100644 --- a/fe/src/com/baidu/palo/analysis/DescribeStmt.java +++ b/fe/src/com/baidu/palo/analysis/DescribeStmt.java @@ -20,7 +20,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; @@ -37,10 +36,13 @@ import com.baidu.palo.common.proc.ProcNodeInterface; import com.baidu.palo.common.proc.ProcResult; import com.baidu.palo.common.proc.ProcService; import com.baidu.palo.common.proc.TableProcDir; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; + import org.apache.commons.lang.StringUtils; import java.util.Arrays; @@ -95,10 +97,13 @@ public class DescribeStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { dbTableName.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbTableName.getDb(), AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, - analyzer.getUser(), dbTableName.getDb()); + + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbTableName.getDb(), + dbTableName.getTbl(), PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbTableName.getTbl()); } Database db = Catalog.getInstance().getDb(dbTableName.getDb()); diff --git a/fe/src/com/baidu/palo/analysis/DropClusterStmt.java b/fe/src/com/baidu/palo/analysis/DropClusterStmt.java index ace081ffa8..f6ed0bcdf0 100644 --- a/fe/src/com/baidu/palo/analysis/DropClusterStmt.java +++ b/fe/src/com/baidu/palo/analysis/DropClusterStmt.java @@ -15,10 +15,13 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.system.SystemInfoService; import com.google.common.base.Strings; @@ -42,7 +45,7 @@ public class DropClusterStmt extends DdlStmt { throw new AnalysisException("Can not drop " + SystemInfoService.DEFAULT_CLUSTER); } - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PERMISSIONS); } } diff --git a/fe/src/com/baidu/palo/analysis/DropDbStmt.java b/fe/src/com/baidu/palo/analysis/DropDbStmt.java index f211f605e1..2921243a76 100644 --- a/fe/src/com/baidu/palo/analysis/DropDbStmt.java +++ b/fe/src/com/baidu/palo/analysis/DropDbStmt.java @@ -15,12 +15,15 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.InfoSchemaDb; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -51,10 +54,12 @@ public class DropDbStmt extends DdlStmt { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); // Don't allowed to drop 'information_schema' if (dbName.equalsIgnoreCase(ClusterNamespace.getFullName(getClusterName(), InfoSchemaDb.DATABASE_NAME))) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), dbName); } - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), dbName, PrivPredicate.DROP)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, + ConnectContext.get().getQualifiedUser(), dbName); } } diff --git a/fe/src/com/baidu/palo/analysis/DropRepositoryStmt.java b/fe/src/com/baidu/palo/analysis/DropRepositoryStmt.java new file mode 100644 index 0000000000..0658ecb9d1 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/DropRepositoryStmt.java @@ -0,0 +1,63 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + +public class DropRepositoryStmt extends DdlStmt { + + private String repoName; + + public DropRepositoryStmt(String repoName) { + this.repoName = repoName; + } + + public String getRepoName() { + return repoName; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + FeNameFormat.checkCommonName("repository", repoName); + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("DROP "); + sb.append("REPOSITORY `").append(repoName).append("`"); + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/analysis/DropRoleStmt.java b/fe/src/com/baidu/palo/analysis/DropRoleStmt.java new file mode 100644 index 0000000000..3ec5efca1f --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/DropRoleStmt.java @@ -0,0 +1,46 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.InternalException; + +public class DropRoleStmt extends DdlStmt { + + private String role; + + public DropRoleStmt(String role) { + this.role = role; + } + + public String getQualifiedRole() { + return role; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + FeNameFormat.checkRoleName(role, false /* can not be superuser */); + role = ClusterNamespace.getFullName(analyzer.getClusterName(), role); + } + + @Override + public String toSql() { + return "DROP ROLE " + role; + } +} diff --git a/fe/src/com/baidu/palo/analysis/DropTableStmt.java b/fe/src/com/baidu/palo/analysis/DropTableStmt.java index bbf03f2ec2..cfc96dee7f 100644 --- a/fe/src/com/baidu/palo/analysis/DropTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/DropTableStmt.java @@ -15,11 +15,13 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -64,10 +66,10 @@ public class DropTableStmt extends DdlStmt { } tableName.analyze(analyzer); // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), tableName.getDb(), AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, - analyzer.getUser(), tableName.getDb()); + + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tableName.getDb(), + tableName.getTbl(), PrivPredicate.DROP)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "DROP"); } } diff --git a/fe/src/com/baidu/palo/analysis/DropUserStmt.java b/fe/src/com/baidu/palo/analysis/DropUserStmt.java index 51dfb4516b..80ec770091 100644 --- a/fe/src/com/baidu/palo/analysis/DropUserStmt.java +++ b/fe/src/com/baidu/palo/analysis/DropUserStmt.java @@ -15,48 +15,46 @@ package com.baidu.palo.analysis; -import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; -import com.google.common.base.Strings; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; -// DROP USER statement +// drop user cmy; public class DropUserStmt extends DdlStmt { - private String user; + private UserIdentity userIdent; - public DropUserStmt(String user) { - this.user = user; + public DropUserStmt(UserIdentity userIdent) { + this.userIdent = userIdent; } - public String getUser() { - return user; + public UserIdentity getUserIdentity() { + return userIdent; } @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - if (Strings.isNullOrEmpty(user)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CANNOT_USER, "DROP USER", user); + userIdent.analyze(analyzer.getClusterName()); + + if (!userIdent.getHost().equals("%")) { + throw new AnalysisException("Can not drop user with specified host: " + userIdent.getHost()); } - user = ClusterNamespace.getFullName(getClusterName(), user); - // check access - if (analyzer.getCatalog().getUserMgr().isSuperuser(user)) { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "DROP USER"); - } - } else { - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "DROP USER"); - } + + // check authenticate + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "DROP USER"); } } @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("DROP USER '").append(user).append("'"); + sb.append("DROP USER ").append(userIdent); return sb.toString(); } diff --git a/fe/src/com/baidu/palo/analysis/ExportStmt.java b/fe/src/com/baidu/palo/analysis/ExportStmt.java index f48343a9df..08646a94ae 100644 --- a/fe/src/com/baidu/palo/analysis/ExportStmt.java +++ b/fe/src/com/baidu/palo/analysis/ExportStmt.java @@ -15,7 +15,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.BrokerMgr; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Database; @@ -27,6 +26,8 @@ import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.util.PrintableMap; import com.baidu.palo.common.util.PropertyAnalyzer; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -61,8 +62,6 @@ public class ExportStmt extends StatementBase { private TableRef tableRef; - private String user; - public ExportStmt(TableRef tableRef, String path, Map properties, BrokerDesc brokerDesc) { this.tableRef = tableRef; @@ -71,8 +70,6 @@ public class ExportStmt extends StatementBase { this.brokerDesc = brokerDesc; this.columnSeparator = DEFAULT_COLUMN_SEPARATOR; this.lineDelimiter = DEFAULT_LINE_DELIMITER; - - this.user = null; } public TableRef getTableRef() { @@ -99,10 +96,6 @@ public class ExportStmt extends StatementBase { return properties; } - public String getUser() { - return user; - } - public String getColumnSeparator() { return this.columnSeparator; } @@ -123,9 +116,13 @@ public class ExportStmt extends StatementBase { this.partitions = tableRef.getPartitions(); // check auth - user = analyzer.getUser(); - if (!analyzer.getCatalog().getUserMgr().checkAccess(user, tblName.getDb(), AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, user, tblName.getDb()); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), + tblName.getDb(), tblName.getTbl(), + PrivPredicate.SELECT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "EXPORT", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tblName.getTbl()); } // check table && partitions whether exist diff --git a/fe/src/com/baidu/palo/analysis/Expr.java b/fe/src/com/baidu/palo/analysis/Expr.java index a1ec5695e3..90d90b2f5f 100644 --- a/fe/src/com/baidu/palo/analysis/Expr.java +++ b/fe/src/com/baidu/palo/analysis/Expr.java @@ -1141,23 +1141,17 @@ abstract public class Expr extends TreeNode implements ParseNode, Cloneabl * failure to convert a string literal to a date literal */ public final Expr castTo(Type targetType) throws AnalysisException { - final Type type = Type.getAssignmentCompatibleType(this.type, targetType, false); - if (!type.isValid()) { - throw new AnalysisException("can't cast " + this.type + " to " + targetType); - } // If the targetType is NULL_TYPE then ignore the cast because NULL_TYPE // is compatible with all types and no cast is necessary. if (targetType.isNull()) { return this; } - if (!targetType.isDecimal()) { - // requested cast must be to assignment-compatible type - // (which implies no loss of precision) - if (!targetType.equals(type)) { - throw new AnalysisException("can't cast " + this.type + " to " + targetType); - } - } + + if ((targetType.isStringType() || targetType.isHllType()) + && (this.type.isStringType() || this.type.isHllType())) { + return this; + } // Preconditions.checkState(PrimitiveType.isImplicitCast(type, targetType), "cast %s to %s", this.type, targetType); // TODO(zc): use implicit cast Preconditions.checkState(Type.canCastTo(this.type, targetType), "cast %s to %s", this.type, targetType); diff --git a/fe/src/com/baidu/palo/analysis/FrontendClause.java b/fe/src/com/baidu/palo/analysis/FrontendClause.java index 9ee4a1e4fe..eebe3579bc 100644 --- a/fe/src/com/baidu/palo/analysis/FrontendClause.java +++ b/fe/src/com/baidu/palo/analysis/FrontendClause.java @@ -20,17 +20,21 @@ package com.baidu.palo.analysis; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.ErrorCode; -import com.baidu.palo.common.ErrorReport; -import com.baidu.palo.common.Pair; -import com.baidu.palo.ha.FrontendNodeType; -import com.baidu.palo.system.SystemInfoService; - -import org.apache.commons.lang.NotImplementedException; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; - +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.Pair; +import com.baidu.palo.ha.FrontendNodeType; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.system.SystemInfoService; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import org.apache.commons.lang.NotImplementedException; + import java.util.Map; public class FrontendClause extends AlterClause { @@ -54,9 +58,9 @@ public class FrontendClause extends AlterClause { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, - "ADD/DROP OBSERVER/REPLICA"); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + analyzer.getQualifiedUser()); } Pair pair = SystemInfoService.validateHostAndPort(hostPort); diff --git a/fe/src/com/baidu/palo/analysis/GrantStmt.java b/fe/src/com/baidu/palo/analysis/GrantStmt.java index a11be2b012..d160f4a5cb 100644 --- a/fe/src/com/baidu/palo/analysis/GrantStmt.java +++ b/fe/src/com/baidu/palo/analysis/GrantStmt.java @@ -21,11 +21,19 @@ package com.baidu.palo.analysis; import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeNameFormat; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + +import com.google.common.base.Joiner; import com.google.common.base.Strings; import java.util.List; @@ -33,65 +41,80 @@ import java.util.List; // GRANT STMT // grant privilege to some user, this is an administrator operation. // -// GRANT privilege [, privilege] ON db_name TO user +// GRANT privilege [, privilege] ON db.tbl TO user [ROLE 'role']; public class GrantStmt extends DdlStmt { - private String user; - private String db; - private List privileges; + private UserIdentity userIdent; + private String role; + private TablePattern tblPattern; + private List privileges; - public GrantStmt(String user, String db, List privileges) { - this.user = user; - this.db = db; - this.privileges = privileges; + public GrantStmt(UserIdentity userIdent, String role, TablePattern tblPattern, List privileges) { + this.userIdent = userIdent; + this.role = role; + this.tblPattern = tblPattern; + PrivBitSet privs = PrivBitSet.of(); + for (AccessPrivilege accessPrivilege : privileges) { + privs.or(accessPrivilege.toPaloPrivilege()); + } + this.privileges = privs.toPrivilegeList(); } - public String getUser() { - return user; + public UserIdentity getUserIdent() { + return userIdent; } - public String getDb() { - return db; + public TablePattern getTblPattern() { + return tblPattern; } - public AccessPrivilege getPrivilege() { - return AccessPrivilege.merge(privileges); + public boolean hasRole() { + return !Strings.isNullOrEmpty(role); + } + + public String getQualifiedRole() { + return role; + } + + public List getPrivileges() { + return privileges; } @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - if (Strings.isNullOrEmpty(user)) { - throw new AnalysisException("No user in grant statement."); + if (userIdent != null) { + userIdent.analyze(analyzer.getClusterName()); + } else { + FeNameFormat.checkUserName(role); } - if (Strings.isNullOrEmpty(db)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - db = ClusterNamespace.getFullName(getClusterName(), db); - user = ClusterNamespace.getFullName(getClusterName(), user); - + + tblPattern.analyze(analyzer.getClusterName()); + if (privileges == null || privileges.isEmpty()) { throw new AnalysisException("No privileges in grant statement."); } - if (!analyzer.getCatalog().getUserMgr().checkUserAccess(analyzer.getUser(), user)) { - throw new AnalysisException("No privilege to grant."); + if (role != null) { + FeNameFormat.checkRoleName(role, false /* can not be superuser */); + role = ClusterNamespace.getFullName(analyzer.getClusterName(), role); + } + + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "GRANT"); } } @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("GRANT "); - int idx = 0; - for (AccessPrivilege privilege : privileges) { - if (idx != 0) { - sb.append(", "); - } - sb.append(privilege); - idx++; + sb.append("GRANT ").append(Joiner.on(", ").join(privileges)); + sb.append(" ON ").append(tblPattern).append(" TO "); + if (!Strings.isNullOrEmpty(role)) { + sb.append(" ROLE '").append(role).append("'"); + } else { + sb.append(userIdent); } - sb.append(" ON ").append(db).append(" TO '").append(user).append("'"); - return sb.toString(); } diff --git a/fe/src/com/baidu/palo/analysis/InformationFunction.java b/fe/src/com/baidu/palo/analysis/InformationFunction.java index b139feb8c0..daa999e90b 100644 --- a/fe/src/com/baidu/palo/analysis/InformationFunction.java +++ b/fe/src/com/baidu/palo/analysis/InformationFunction.java @@ -58,10 +58,10 @@ public class InformationFunction extends Expr { strValue = analyzer.getDefaultDb(); } else if (funcType.equalsIgnoreCase("USER")) { type = Type.VARCHAR; - strValue = analyzer.getUser(); + strValue = analyzer.getQualifiedUser(); } else if (funcType.equalsIgnoreCase("CURRENT_USER")) { type = Type.VARCHAR; - strValue = analyzer.getUser(); + strValue = analyzer.getQualifiedUser(); } else if (funcType.equalsIgnoreCase("CONNECTION_ID")) { type = Type.BIGINT; intValue = analyzer.getConnectId(); diff --git a/fe/src/com/baidu/palo/analysis/InsertStmt.java b/fe/src/com/baidu/palo/analysis/InsertStmt.java index 986bd2cd21..c276dbcff6 100644 --- a/fe/src/com/baidu/palo/analysis/InsertStmt.java +++ b/fe/src/com/baidu/palo/analysis/InsertStmt.java @@ -20,8 +20,8 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.BrokerTable; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.MysqlTable; @@ -29,14 +29,17 @@ import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.PartitionType; import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Type; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.planner.DataPartition; import com.baidu.palo.planner.DataSink; import com.baidu.palo.planner.DataSplitSink; import com.baidu.palo.planner.ExportSink; +import com.baidu.palo.qe.ConnectContext; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -136,9 +139,11 @@ public class InsertStmt extends DdlStmt { } // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tblName.getDb(), tblName.getTbl(), + PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), tblName.getTbl()); } dbs.put(dbName, db); @@ -152,11 +157,18 @@ public class InsertStmt extends DdlStmt { public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - // Check privilege if (targetTable == null) { tblName.analyze(analyzer); - analyzer.checkPrivilege(tblName.getDb(), AccessPrivilege.READ_WRITE); } + + // Check privilege + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tblName.getDb(), + tblName.getTbl(), PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), tblName.getTbl()); + } + // check partition if (targetPartitions != null && targetPartitions.isEmpty()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_PARTITION_CLAUSE_ON_NONPARTITIONED); @@ -315,6 +327,20 @@ public class InsertStmt extends DdlStmt { } private Expr checkTypeCompatibility(Column col, Expr expr) throws AnalysisException { + // TargeTable's hll column must be hll_hash's result + if (col.getType().equals(Type.HLL)) { + final String hllAnalysisErrorLog = "Column's type is HLL," + + " it must be hll_hash function's result, column=" + col.getName(); + if (!(expr instanceof FunctionCallExpr)) { + throw new AnalysisException(hllAnalysisErrorLog); + } + + final FunctionCallExpr functionExpr = (FunctionCallExpr) expr; + if (!functionExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) { + throw new AnalysisException(hllAnalysisErrorLog); + } + } + if (col.getDataType().equals(expr.getType())) { return expr; } @@ -350,6 +376,9 @@ public class InsertStmt extends DdlStmt { dataPartition = dataSink.getOutputPartition(); } else if (targetTable instanceof BrokerTable) { BrokerTable table = (BrokerTable) targetTable; + // TODO(lingbin): think use which one if have more than one path + // Map brokerProperties = Maps.newHashMap(); + // BrokerDesc brokerDesc = new BrokerDesc("test_broker", brokerProperties); BrokerDesc brokerDesc = new BrokerDesc(table.getBrokerName(), table.getBrokerProperties()); dataSink = new ExportSink( table.getWritablePath(), diff --git a/fe/src/com/baidu/palo/analysis/LabelName.java b/fe/src/com/baidu/palo/analysis/LabelName.java index eb53d3a4ad..d02b1bae09 100644 --- a/fe/src/com/baidu/palo/analysis/LabelName.java +++ b/fe/src/com/baidu/palo/analysis/LabelName.java @@ -28,6 +28,7 @@ import com.baidu.palo.common.FeNameFormat; import com.baidu.palo.common.io.Text; import com.baidu.palo.common.io.Writable; import com.baidu.palo.system.SystemInfoService; + import com.google.common.base.Strings; import org.apache.commons.lang.builder.HashCodeBuilder; diff --git a/fe/src/com/baidu/palo/analysis/LinkDbStmt.java b/fe/src/com/baidu/palo/analysis/LinkDbStmt.java index a1b69fb1fd..be504df7ea 100644 --- a/fe/src/com/baidu/palo/analysis/LinkDbStmt.java +++ b/fe/src/com/baidu/palo/analysis/LinkDbStmt.java @@ -20,11 +20,15 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Strings; public class LinkDbStmt extends DdlStmt { @@ -62,8 +66,9 @@ public class LinkDbStmt extends DdlStmt { src.analyze(analyzer); dest.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PERMISSIONS); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ADMIN"); } if (Strings.isNullOrEmpty(src.getCluster()) || Strings.isNullOrEmpty(dest.getCluster()) diff --git a/fe/src/com/baidu/palo/analysis/LoadStmt.java b/fe/src/com/baidu/palo/analysis/LoadStmt.java index 47e1c030d9..cc71e3e0ae 100644 --- a/fe/src/com/baidu/palo/analysis/LoadStmt.java +++ b/fe/src/com/baidu/palo/analysis/LoadStmt.java @@ -20,13 +20,12 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.ErrorCode; -import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.util.PrintableMap; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; @@ -190,20 +189,16 @@ public class LoadStmt extends DdlStmt { if (brokerDesc != null) { dataDescription.setIsPullLoad(true); } - dataDescription.analyze(); + dataDescription.analyze(label.getDbName()); } - // check auth - user = analyzer.getUser(); - if (!analyzer.getCatalog().getUserMgr().checkAccess(user, label.getDbName(), AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, user, label.getDbName()); - } - try { checkProperties(properties); } catch (DdlException e) { throw new AnalysisException(e.getMessage()); } + + user = ConnectContext.get().getQualifiedUser(); } @Override diff --git a/fe/src/com/baidu/palo/analysis/MigrateDbStmt.java b/fe/src/com/baidu/palo/analysis/MigrateDbStmt.java index ef170752b5..09c0d16b89 100644 --- a/fe/src/com/baidu/palo/analysis/MigrateDbStmt.java +++ b/fe/src/com/baidu/palo/analysis/MigrateDbStmt.java @@ -20,11 +20,14 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; public class MigrateDbStmt extends DdlStmt { @@ -61,8 +64,9 @@ public class MigrateDbStmt extends DdlStmt { src.analyze(analyzer); dest.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PERMISSIONS); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ADMIN"); } srcCluster = src.getCluster(); diff --git a/fe/src/com/baidu/palo/analysis/ModifyTablePropertiesClause.java b/fe/src/com/baidu/palo/analysis/ModifyTablePropertiesClause.java index 7159a91b37..2f232dc3e1 100644 --- a/fe/src/com/baidu/palo/analysis/ModifyTablePropertiesClause.java +++ b/fe/src/com/baidu/palo/analysis/ModifyTablePropertiesClause.java @@ -20,10 +20,13 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.util.PrintableMap; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import java.util.Map; @@ -40,16 +43,22 @@ public class ModifyTablePropertiesClause extends AlterClause { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, - "Modify table property"); - } - if (properties == null || properties.isEmpty()) { throw new AnalysisException("Properties is not set"); } + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ALTER)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ALTER"); + } + if (properties.containsKey(KEY_STORAGE_TYPE)) { + // if set storage type, we need ADMIN privs. + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ADMIN"); + } + if (!properties.get(KEY_STORAGE_TYPE).equals("column")) { throw new AnalysisException("Can only change storage type to COLUMN"); } diff --git a/fe/src/com/baidu/palo/analysis/RecoverDbStmt.java b/fe/src/com/baidu/palo/analysis/RecoverDbStmt.java index fd5979640c..dddbc32377 100644 --- a/fe/src/com/baidu/palo/analysis/RecoverDbStmt.java +++ b/fe/src/com/baidu/palo/analysis/RecoverDbStmt.java @@ -20,12 +20,17 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -47,9 +52,13 @@ public class RecoverDbStmt extends DdlStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName); } dbName = ClusterNamespace.getFullName(getClusterName(), dbName); - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); + + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), dbName, + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.CREATE_PRIV, + PaloPrivilege.ADMIN_PRIV), + Operator.OR))) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), dbName); } } diff --git a/fe/src/com/baidu/palo/analysis/RecoverPartitionStmt.java b/fe/src/com/baidu/palo/analysis/RecoverPartitionStmt.java index aac5d9a5b1..b4d753be8d 100644 --- a/fe/src/com/baidu/palo/analysis/RecoverPartitionStmt.java +++ b/fe/src/com/baidu/palo/analysis/RecoverPartitionStmt.java @@ -20,11 +20,16 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -52,10 +57,16 @@ public class RecoverPartitionStmt extends DdlStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { dbTblName.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), - dbTblName.getDb(), AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), - dbTblName.getDb()); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbTblName.getDb(), + dbTblName.getTbl(), + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.CREATE_PRIV, + PaloPrivilege.ADMIN_PRIV), + Operator.OR))) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "RECOVERY", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbTblName.getTbl()); } } diff --git a/fe/src/com/baidu/palo/analysis/RecoverTableStmt.java b/fe/src/com/baidu/palo/analysis/RecoverTableStmt.java index 578a7357e9..00e1bcc660 100644 --- a/fe/src/com/baidu/palo/analysis/RecoverTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/RecoverTableStmt.java @@ -20,11 +20,16 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -47,9 +52,16 @@ public class RecoverTableStmt extends DdlStmt { public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { dbTblName.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbTblName.getDb(), - AccessPrivilege.READ_WRITE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbTblName.getDb()); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbTblName.getDb(), + dbTblName.getTbl(), + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.CREATE_PRIV, + PaloPrivilege.ADMIN_PRIV), + Operator.OR))) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "RECOVERY", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbTblName.getTbl()); } } diff --git a/fe/src/com/baidu/palo/analysis/RestoreStmt.java b/fe/src/com/baidu/palo/analysis/RestoreStmt.java index 0810229b16..0cc4d495ad 100644 --- a/fe/src/com/baidu/palo/analysis/RestoreStmt.java +++ b/fe/src/com/baidu/palo/analysis/RestoreStmt.java @@ -20,18 +20,107 @@ package com.baidu.palo.analysis; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.common.InternalException; import com.baidu.palo.common.util.PrintableMap; import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import java.util.List; import java.util.Map; +import java.util.Set; public class RestoreStmt extends AbstractBackupStmt { + private final static String PROP_ALLOW_LOAD = "allow_load"; + private final static String PROP_REPLICATION_NUM = "replication_num"; + private final static String PROP_BACKUP_TIMESTAMP = "backup_timestamp"; - public RestoreStmt(LabelName labelName, List restoreObjNames, - String restorePath, Map properties) { - super(labelName, restoreObjNames, restorePath, properties); + private boolean allowLoad = false; + private int replicationNum = FeConstants.default_replication_num; + private String backupTimestamp = null; + + public RestoreStmt(LabelName labelName, String repoName, List tblRefs, Map properties) { + super(labelName, repoName, tblRefs, properties); + } + + public boolean allowLoad() { + return allowLoad; + } + + public int getReplicationNum() { + return replicationNum; + } + + public String getBackupTimestamp() { + return backupTimestamp; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + + // check if alias is duplicated + Set aliasSet = Sets.newHashSet(); + for (TableRef tblRef : tblRefs) { + aliasSet.add(tblRef.getName().getTbl()); + } + + for (TableRef tblRef : tblRefs) { + if (tblRef.hasExplicitAlias() && !aliasSet.add(tblRef.getExplicitAlias())) { + throw new AnalysisException("Duplicated alias name: " + tblRef.getExplicitAlias()); + } + } + } + + @Override + public void analyzeProperties() throws AnalysisException { + super.analyzeProperties(); + + Map copiedProperties = Maps.newHashMap(properties); + // allow load + if (copiedProperties.containsKey(PROP_ALLOW_LOAD)) { + if (copiedProperties.get(PROP_ALLOW_LOAD).equalsIgnoreCase("true")) { + allowLoad = true; + } else if (copiedProperties.get(PROP_ALLOW_LOAD).equalsIgnoreCase("false")) { + allowLoad = false; + } else { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Invalid allow load value: " + + copiedProperties.get(PROP_ALLOW_LOAD)); + } + copiedProperties.remove(PROP_ALLOW_LOAD); + } + + // replication num + if (copiedProperties.containsKey(PROP_REPLICATION_NUM)) { + try { + replicationNum = Integer.valueOf(copiedProperties.get(PROP_REPLICATION_NUM)); + } catch (NumberFormatException e) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Invalid replication num format: " + + copiedProperties.get(PROP_REPLICATION_NUM)); + } + copiedProperties.remove(PROP_REPLICATION_NUM); + } + + // backup timestamp + if (copiedProperties.containsKey(PROP_BACKUP_TIMESTAMP)) { + backupTimestamp = copiedProperties.get(PROP_BACKUP_TIMESTAMP); + copiedProperties.remove(PROP_BACKUP_TIMESTAMP); + } else { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Missing " + PROP_BACKUP_TIMESTAMP + " property"); + } + + if (!copiedProperties.isEmpty()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Unknown restore job properties: " + copiedProperties.keySet()); + } } @Override @@ -42,15 +131,14 @@ public class RestoreStmt extends AbstractBackupStmt { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("RESTORE LABEL ").append(labelName.toSql()); - if (!objNames.isEmpty()) { - sb.append(" ("); - sb.append(Joiner.on(", ").join(objNames)); - sb.append(")"); - } - sb.append(" FROM \"").append(remotePath).append("\" PROPERTIES("); - sb.append(new PrintableMap(properties, "=", true, false)); - sb.append(")"); + sb.append("RESTORE SNAPSHOT ").append(labelName.toSql()); + sb.append("\n").append("FROM ").append(repoName).append("\nON\n("); + + sb.append(Joiner.on(",\n").join(tblRefs)); + + sb.append("\n)\nPROPERTIES\n("); + sb.append(new PrintableMap(properties, " = ", true, true)); + sb.append("\n)"); return sb.toString(); } } diff --git a/fe/src/com/baidu/palo/analysis/RevokeStmt.java b/fe/src/com/baidu/palo/analysis/RevokeStmt.java index d698a2e7e8..ed83d9be4b 100644 --- a/fe/src/com/baidu/palo/analysis/RevokeStmt.java +++ b/fe/src/com/baidu/palo/analysis/RevokeStmt.java @@ -20,52 +20,91 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.google.common.base.Joiner; import com.google.common.base.Strings; +import java.util.List; + // REVOKE STMT // revoke privilege from some user, this is an administrator operation. // -// REVOKE ALL ON db_name FROM user +// REVOKE privilege [, privilege] ON db.tbl FROM user [ROLE 'role']; public class RevokeStmt extends DdlStmt { - private String user; - private String db; + private UserIdentity userIdent; + private String role; + private TablePattern tblPattern; + private List privileges; - public RevokeStmt(String user, String db) { - this.user = user; - this.db = db; + public RevokeStmt(UserIdentity userIdent, String role, TablePattern tblPattern, List privileges) { + this.userIdent = userIdent; + this.role = role; + this.tblPattern = tblPattern; + PrivBitSet privs = PrivBitSet.of(); + for (AccessPrivilege accessPrivilege : privileges) { + privs.or(accessPrivilege.toPaloPrivilege()); + } + this.privileges = privs.toPrivilegeList(); } - public String getUser() { - return user; + public UserIdentity getUserIdent() { + return userIdent; } - public String getDb() { - return db; + public TablePattern getTblPattern() { + return tblPattern; + } + + public String getQualifiedRole() { + return role; + } + + public List getPrivileges() { + return privileges; } @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (Strings.isNullOrEmpty(user)) { - throw new AnalysisException("No user in grant statement."); + if (userIdent != null) { + userIdent.analyze(analyzer.getClusterName()); + } else { + FeNameFormat.checkRoleName(role, false /* can not be superuser */); + role = ClusterNamespace.getFullName(analyzer.getClusterName(), role); } - user = ClusterNamespace.getFullName(analyzer.getClusterName(), user); - if (Strings.isNullOrEmpty(db)) { - throw new AnalysisException("No database in grant statement."); + + tblPattern.analyze(analyzer.getClusterName()); + + if (privileges == null || privileges.isEmpty()) { + throw new AnalysisException("No privileges in revoke statement."); } - db = ClusterNamespace.getFullName(analyzer.getClusterName(), db); - if (!analyzer.getCatalog().getUserMgr().checkUserAccess(analyzer.getUser(), user)) { - throw new AnalysisException("No privilege to grant."); + + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "REVOKE"); } } @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("REVOKE ALL ON ").append(db).append(" FROM '").append(user).append("'"); - + sb.append("REVOKE ").append(Joiner.on(", ").join(privileges)); + sb.append(" ON ").append(tblPattern).append(" FROM "); + if (!Strings.isNullOrEmpty(role)) { + sb.append(" ROLE '").append(role).append("'"); + } else { + sb.append(userIdent); + } return sb.toString(); } diff --git a/fe/src/com/baidu/palo/analysis/SelectStmt.java b/fe/src/com/baidu/palo/analysis/SelectStmt.java index 927dc997cd..7292fcdeef 100644 --- a/fe/src/com/baidu/palo/analysis/SelectStmt.java +++ b/fe/src/com/baidu/palo/analysis/SelectStmt.java @@ -20,7 +20,7 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.OlapTable; @@ -37,26 +37,26 @@ import com.baidu.palo.common.Pair; import com.baidu.palo.common.TableAliasGenerator; import com.baidu.palo.common.TreeNode; import com.baidu.palo.common.util.SqlUtils; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.rewrite.ExprRewriter; -import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; - import com.google.common.collect.Sets; -import org.apache.logging.log4j.Logger; + import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.Set; @@ -231,11 +231,16 @@ public class SelectStmt extends QueryStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, dbName); } - // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), db); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, + tblRef.getName().getTbl(), + PrivPredicate.SELECT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tblRef.getName().getTbl()); } + dbs.put(dbName, db); } } diff --git a/fe/src/com/baidu/palo/analysis/SetPassVar.java b/fe/src/com/baidu/palo/analysis/SetPassVar.java index c30d7a99d3..a10b235c6d 100644 --- a/fe/src/com/baidu/palo/analysis/SetPassVar.java +++ b/fe/src/com/baidu/palo/analysis/SetPassVar.java @@ -20,27 +20,30 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.mysql.MysqlPassword; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; public class SetPassVar extends SetVar { - private String user; + private UserIdentity userIdent; private String passwdParam; private byte[] passwdBytes; // The password in parameter is a hashed password. - public SetPassVar(String user, String passwd) { - this.user = user; + public SetPassVar(UserIdentity userIdent, String passwd) { + this.userIdent = userIdent; this.passwdParam = passwd; } - public String getUser() { - return user; + public UserIdentity getUserIdent() { + return userIdent; } public byte[] getPassword() { @@ -52,16 +55,29 @@ public class SetPassVar extends SetVar { if (Strings.isNullOrEmpty(analyzer.getClusterName())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_SELECT_CLUSTER); } - if (Strings.isNullOrEmpty(user)) { - user = analyzer.getUser(); - } else { - user = ClusterNamespace.getFullName(analyzer.getClusterName(), user); + + boolean isSelf = false; + ConnectContext ctx = ConnectContext.get(); + if (userIdent == null) { + // set userIdent as itself + userIdent = new UserIdentity(ClusterNamespace.getNameFromFullName(analyzer.getQualifiedUser()), + ctx.getRemoteIP()); + isSelf = true; } + userIdent.analyze(analyzer.getClusterName()); + // Check password passwdBytes = MysqlPassword.checkPassword(passwdParam); - // Check user - if (!analyzer.getCatalog().getUserMgr().checkUserAccess(analyzer.getUser(), user)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_PASSWORD_NOT_ALLOWED); + + // check privs. + // 1. this is user itself + if (isSelf) { + return; + } + + // 2. user has grant privs + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "GRANT"); } } @@ -72,6 +88,6 @@ public class SetPassVar extends SetVar { @Override public String toSql() { - return "SET PASSWORD FOR '" + user + "' = '" + new String(passwdBytes) + "'"; + return "SET PASSWORD FOR " + userIdent + " = '" + new String(passwdBytes) + "'"; } } diff --git a/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java b/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java index 5c4f15687b..54b5cf8f7b 100644 --- a/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java +++ b/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java @@ -23,6 +23,8 @@ package com.baidu.palo.analysis; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -51,10 +53,10 @@ public class SetUserPropertyStmt extends DdlStmt { if (Strings.isNullOrEmpty(user)) { // If param 'user' is not set, use the login user name. // The login user name is full-qualified with cluster name. - user = analyzer.getUser(); + user = ConnectContext.get().getQualifiedUser(); } else { // If param 'user' is set, check if it need to be full-qualified - if (!analyzer.getCatalog().getUserMgr().isAdmin(user)) { + if (!user.equals(PaloAuth.ROOT_USER) && !user.equals(PaloAuth.ADMIN_USER)) { user = ClusterNamespace.getFullName(getClusterName(), user); } } @@ -62,8 +64,10 @@ public class SetUserPropertyStmt extends DdlStmt { if (propertyList == null || propertyList.isEmpty()) { throw new AnalysisException("Empty properties"); } + + boolean isSelf = user.equals(ConnectContext.get().getQualifiedUser()); for (SetVar var : propertyList) { - ((SetUserPropertyVar) var).analyze(analyzer, user); + ((SetUserPropertyVar) var).analyze(analyzer, isSelf); } } diff --git a/fe/src/com/baidu/palo/analysis/SetUserPropertyVar.java b/fe/src/com/baidu/palo/analysis/SetUserPropertyVar.java index 5a114ad6fe..871a89d4af 100644 --- a/fe/src/com/baidu/palo/analysis/SetUserPropertyVar.java +++ b/fe/src/com/baidu/palo/analysis/SetUserPropertyVar.java @@ -20,10 +20,13 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.UserProperty; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.mysql.privilege.UserProperty; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; @@ -49,23 +52,22 @@ public class SetUserPropertyVar extends SetVar { return value; } - public void analyze(Analyzer analyzer, String user) throws AnalysisException { + public void analyze(Analyzer analyzer, boolean isSelf) throws AnalysisException { if (Strings.isNullOrEmpty(key)) { throw new AnalysisException("User property key is null"); } - checkAcess(analyzer, user); + checkAccess(analyzer, isSelf); } - private void checkAcess(Analyzer analyzer, String user) throws AnalysisException { + private void checkAccess(Analyzer analyzer, boolean isSelf) throws AnalysisException { for (Pattern advPattern : UserProperty.ADVANCED_PROPERTIES) { Matcher matcher = advPattern.matcher(key); if (matcher.find()) { - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, - "SET PROPERTY FOR " + user + " " + key); + "ADMIN"); } - return; } } @@ -73,11 +75,11 @@ public class SetUserPropertyVar extends SetVar { for (Pattern commPattern : UserProperty.COMMON_PROPERTIES) { Matcher matcher = commPattern.matcher(key); if (matcher.find()) { - if (!analyzer.getCatalog().getUserMgr().checkUserAccess(analyzer.getUser(), user)) { + if (!isSelf && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.ADMIN)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, - "SET PROPERTY FOR " + user + " " + key); + "GRANT"); } - return; } } diff --git a/fe/src/com/baidu/palo/analysis/SetVar.java b/fe/src/com/baidu/palo/analysis/SetVar.java index 9ecc4c8c2f..a7084fa5cd 100644 --- a/fe/src/com/baidu/palo/analysis/SetVar.java +++ b/fe/src/com/baidu/palo/analysis/SetVar.java @@ -20,11 +20,14 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.UserResource; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.mysql.privilege.UserResource; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.SessionVariable; import com.google.common.base.Strings; @@ -75,8 +78,9 @@ public class SetVar { throw new AnalysisException("No variable name in set statement."); } if (type == SetType.GLOBAL) { - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SET GLOBAL"); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ADMIN"); } } if (value == null) { diff --git a/fe/src/com/baidu/palo/analysis/ShowAlterStmt.java b/fe/src/com/baidu/palo/analysis/ShowAlterStmt.java index bec0cf183a..0597aa558b 100644 --- a/fe/src/com/baidu/palo/analysis/ShowAlterStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowAlterStmt.java @@ -15,7 +15,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.catalog.Database; @@ -34,8 +33,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /* * ShowAlterStmt: used to show process state of alter statement. @@ -74,7 +73,6 @@ public class ShowAlterStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - final String dbNameWithoutPrefix = dbName; if (Strings.isNullOrEmpty(dbName)) { dbName = analyzer.getDefaultDb(); if (Strings.isNullOrEmpty(dbName)) { @@ -85,11 +83,9 @@ public class ShowAlterStmt extends ShowStmt { } Preconditions.checkNotNull(type); - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), - dbNameWithoutPrefix); - } + + // check auth when get job info + handleShowAlterTable(analyzer); } @@ -101,21 +97,21 @@ public class ShowAlterStmt extends ShowStmt { } // build proc path - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("/jobs/"); - stringBuilder.append(db.getId()); + StringBuilder sb = new StringBuilder(); + sb.append("/jobs/"); + sb.append(db.getId()); if (type == AlterType.COLUMN) { - stringBuilder.append("/schema_change"); + sb.append("/schema_change"); } else if (type == AlterType.ROLLUP) { - stringBuilder.append("/rollup"); + sb.append("/rollup"); } else { throw new InternalException("SHOW " + type.name() + " does not implement yet"); } - LOG.debug("process SHOW PROC '{}';", stringBuilder.toString()); + LOG.debug("process SHOW PROC '{}';", sb.toString()); // create show proc stmt // '/jobs/db_name/rollup|schema_change/ - node = ProcService.getInstance().open(stringBuilder.toString()); + node = ProcService.getInstance().open(sb.toString()); if (node == null) { throw new AnalysisException("Failed to show alter table"); } diff --git a/fe/src/com/baidu/palo/analysis/ShowBackendsStmt.java b/fe/src/com/baidu/palo/analysis/ShowBackendsStmt.java index 5564ad5a13..0099772e10 100644 --- a/fe/src/com/baidu/palo/analysis/ShowBackendsStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowBackendsStmt.java @@ -15,9 +15,16 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; import com.baidu.palo.common.proc.BackendsProcDir; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; public class ShowBackendsStmt extends ShowStmt { @@ -25,11 +32,21 @@ public class ShowBackendsStmt extends ShowStmt { public ShowBackendsStmt() { } + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + } + @Override public ShowResultSetMetaData getMetaData() { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); for (String title : BackendsProcDir.TITLE_NAMES) { - if (title.equals("HostName") || title.contains("Port")) { + // hide hostname for SHOW BACKENDS stmt + if (title.equals("HostName")) { continue; } builder.addColumn(new Column(title, ColumnType.createVarchar(30))); @@ -37,3 +54,4 @@ public class ShowBackendsStmt extends ShowStmt { return builder.build(); } } + diff --git a/fe/src/com/baidu/palo/analysis/ShowBackupStmt.java b/fe/src/com/baidu/palo/analysis/ShowBackupStmt.java index ea282f1a9e..1684b54fcc 100644 --- a/fe/src/com/baidu/palo/analysis/ShowBackupStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowBackupStmt.java @@ -15,146 +15,60 @@ package com.baidu.palo.analysis; -import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.backup.BackupJob; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; -import com.baidu.palo.catalog.Database; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; -import com.baidu.palo.common.PatternMatcher; -import com.baidu.palo.common.proc.BackupProcNode; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import com.google.common.collect.ImmutableList; public class ShowBackupStmt extends ShowStmt { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("JobId").add("SnapshotName").add("DbName").add("State").add("BackupObjs").add("CreateTime") + .add("SnapshotFinishedTime").add("UploadFinishedTime").add("FinishedTime").add("UnfinishedTasks") + .add("TaskErrMsg").add("Status").add("Timeout") + .build(); private String dbName; - private Expr where; - private String label; - public ShowBackupStmt(String dbName, Expr where) { + public ShowBackupStmt(String dbName) { this.dbName = dbName; - this.where = where; } public String getDbName() { return dbName; } - public String getLabel() { - return label; - } - @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); if (Strings.isNullOrEmpty(dbName)) { dbName = analyzer.getDefaultDb(); if (Strings.isNullOrEmpty(dbName)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); } + } else { + dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); - } - - analyzeWhere(); - } - - private void analyzeWhere() throws AnalysisException { - boolean valid = true; - if (where == null) { - return; - } - - CHECK: { - if (where instanceof BinaryPredicate) { - BinaryPredicate binaryPredicate = (BinaryPredicate) where; - if (binaryPredicate.getOp() != Operator.EQ) { - valid = false; - break CHECK; - } - } else if (where instanceof LikePredicate) { - LikePredicate likePredicate = (LikePredicate) where; - if (likePredicate.getOp() != LikePredicate.Operator.LIKE) { - valid = false; - break CHECK; - } - } else { - valid = false; - break CHECK; - } - - // left child - if (!(where.getChild(0) instanceof SlotRef)) { - valid = false; - break CHECK; - } - String leftKey = ((SlotRef) where.getChild(0)).getColumnName(); - if (!leftKey.equalsIgnoreCase("label")) { - valid = false; - break CHECK; - } - - // right child - if (!(where.getChild(1) instanceof StringLiteral)) { - valid = false; - break CHECK; - } - - label = ((StringLiteral) where.getChild(1)).getStringValue(); - if (Strings.isNullOrEmpty(label)) { - valid = false; - break CHECK; - } - } - - if (!valid) { - throw new AnalysisException("Where clause should looks like: LABEL = \"your_backup_label\"," - + " or LABEL LIKE \"matcher\""); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } } - public List> getResultRows() throws AnalysisException { - List> result = new LinkedList>(); - Database db = Catalog.getInstance().getDb(dbName); - if (db == null) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, dbName); - } - - BackupHandler backupHandler = Catalog.getInstance().getBackupHandler(); - PatternMatcher matcher = null; - if (!Strings.isNullOrEmpty(label)) { - matcher = PatternMatcher.createMysqlPattern(label); - } - List> backupJobInfos = backupHandler.getJobInfosByDb(db.getId(), BackupJob.class, matcher); - for (List infoStr : backupJobInfos) { - List oneInfo = new ArrayList(BackupProcNode.TITLE_NAMES.size()); - for (Comparable element : infoStr) { - oneInfo.add(element.toString()); - } - result.add(oneInfo); - } - return result; - } - - @Override public ShowResultSetMetaData getMetaData() { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - for (String title : BackupProcNode.TITLE_NAMES) { + for (String title : TITLE_NAMES) { builder.addColumn(new Column(title, ColumnType.createVarchar(30))); } return builder.build(); @@ -168,7 +82,6 @@ public class ShowBackupStmt extends ShowStmt { builder.append(" FROM `").append(dbName).append("` "); } - builder.append(where.toSql()); return builder.toString(); } diff --git a/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java b/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java index f5d10d6a62..0277d588d7 100644 --- a/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java @@ -15,8 +15,15 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; // Show @@ -31,7 +38,12 @@ public class ShowBrokerStmt extends ShowStmt { } @Override - public void analyze(Analyzer analyzer) { + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } } @Override diff --git a/fe/src/com/baidu/palo/analysis/ShowClusterStmt.java b/fe/src/com/baidu/palo/analysis/ShowClusterStmt.java index 2924bf47b3..3b0a9bec22 100644 --- a/fe/src/com/baidu/palo/analysis/ShowClusterStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowClusterStmt.java @@ -15,13 +15,20 @@ package com.baidu.palo.analysis; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; + import com.google.common.collect.ImmutableList; public class ShowClusterStmt extends ShowStmt { @@ -50,8 +57,11 @@ public class ShowClusterStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PERMISSIONS); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.NODE_PRIV), + Operator.OR))) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowCreateDbStmt.java b/fe/src/com/baidu/palo/analysis/ShowCreateDbStmt.java index c6ca987798..ba87d6280e 100644 --- a/fe/src/com/baidu/palo/analysis/ShowCreateDbStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowCreateDbStmt.java @@ -15,7 +15,8 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; @@ -23,6 +24,10 @@ import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -54,9 +59,15 @@ public class ShowCreateDbStmt extends ShowStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, db); } db = ClusterNamespace.getFullName(getClusterName(), db); - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), db, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), db); + + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), db, + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.ALTER_PRIV, + PaloPrivilege.CREATE_PRIV, + PaloPrivilege.DROP_PRIV), + Operator.OR))) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, + ConnectContext.get().getQualifiedUser(), db); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowCreateTableStmt.java b/fe/src/com/baidu/palo/analysis/ShowCreateTableStmt.java index 790214761e..7e3a41f72a 100644 --- a/fe/src/com/baidu/palo/analysis/ShowCreateTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowCreateTableStmt.java @@ -15,12 +15,14 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; // SHOW CREATE TABLE statement. @@ -73,9 +75,13 @@ public class ShowCreateTableStmt extends ShowStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_TABLES_USED); } tbl.analyze(analyzer); - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), tbl.getDb(), AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), tbl.getDb()); + + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), tbl.getDb(), tbl.getTbl(), + PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW CREATE TABLE", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tbl.getTbl()); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowDataStmt.java b/fe/src/com/baidu/palo/analysis/ShowDataStmt.java index 46b95f221a..bccb892ffe 100644 --- a/fe/src/com/baidu/palo/analysis/ShowDataStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowDataStmt.java @@ -15,7 +15,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; @@ -35,6 +34,8 @@ import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.Pair; import com.baidu.palo.common.util.DebugUtil; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -85,12 +86,6 @@ public class ShowDataStmt extends ShowStmt { } else { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - - // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); - } Database db = Catalog.getInstance().getDb(dbName); if (db == null) { @@ -111,6 +106,11 @@ public class ShowDataStmt extends ShowStmt { }); for (Table table : tables) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, + table.getName(), + PrivPredicate.SHOW)) { + continue; + } sortedTables.add(table); } @@ -166,6 +166,15 @@ public class ShowDataStmt extends ShowStmt { List leftRow = Arrays.asList("Left", readableLeft); totalRows.add(leftRow); } else { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, + tableName, + PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW DATA", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tableName); + } + Table table = db.getTable(tableName); if (table == null) { ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); diff --git a/fe/src/com/baidu/palo/analysis/ShowDeleteStmt.java b/fe/src/com/baidu/palo/analysis/ShowDeleteStmt.java index 4bfb00072c..1fb0af34ff 100644 --- a/fe/src/com/baidu/palo/analysis/ShowDeleteStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowDeleteStmt.java @@ -15,7 +15,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; @@ -43,8 +42,7 @@ public class ShowDeleteStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); - final String dbNameWithoutPrefix = dbName; - final String userNameWithoutPrefix = ClusterNamespace.getNameFromFullName(analyzer.getUser()); + if (Strings.isNullOrEmpty(dbName)) { dbName = analyzer.getDefaultDb(); if (Strings.isNullOrEmpty(dbName)) { @@ -53,12 +51,6 @@ public class ShowDeleteStmt extends ShowStmt { } else { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, userNameWithoutPrefix, - dbNameWithoutPrefix); - } } @Override diff --git a/fe/src/com/baidu/palo/analysis/ShowExportStmt.java b/fe/src/com/baidu/palo/analysis/ShowExportStmt.java index bafc84b8c1..758683ffec 100644 --- a/fe/src/com/baidu/palo/analysis/ShowExportStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowExportStmt.java @@ -16,7 +16,6 @@ package com.baidu.palo.analysis; import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; @@ -101,13 +100,6 @@ public class ShowExportStmt extends ShowStmt { } else { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - final String userNameWithoutPrefix = ClusterNamespace.getNameFromFullName(dbName); - final String dbNameWithoutPrefix = ClusterNamespace.getNameFromFullName(dbName); - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, userNameWithoutPrefix, - dbNameWithoutPrefix); - } // analyze where clause if not null if (whereClause != null) { diff --git a/fe/src/com/baidu/palo/analysis/ShowFrontendsStmt.java b/fe/src/com/baidu/palo/analysis/ShowFrontendsStmt.java new file mode 100644 index 0000000000..2c99f0ca98 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/ShowFrontendsStmt.java @@ -0,0 +1,53 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.common.proc.FrontendsProcNode; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.ShowResultSetMetaData; + +public class ShowFrontendsStmt extends ShowStmt { + + public ShowFrontendsStmt() { + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : FrontendsProcNode.TITLE_NAMES) { + builder.addColumn(new Column(title, ColumnType.createVarchar(30))); + } + return builder.build(); + } +} + diff --git a/fe/src/com/baidu/palo/analysis/ShowGrantsStmt.java b/fe/src/com/baidu/palo/analysis/ShowGrantsStmt.java new file mode 100644 index 0000000000..331328e3cc --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/ShowGrantsStmt.java @@ -0,0 +1,101 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.common.proc.AuthProcDir; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.ShowResultSetMetaData; + +/* + * SHOW ALL GRANTS; + * show all grants. + * + * SHOW GRANTS: + * show grants of current user + * + * SHOW GRANTS FOR user@'xxx'; + * show grants for specified user identity + */ +// +// SHOW GRANTS; +// SHOW GRANTS FOR user@'xxx' +public class ShowGrantsStmt extends ShowStmt { + + private static final ShowResultSetMetaData META_DATA; + static { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String col : AuthProcDir.TITLE_NAMES) { + builder.addColumn(new Column(col, ColumnType.createVarchar(100))); + } + META_DATA = builder.build(); + } + + private boolean isAll; + private UserIdentity userIdent; + + public ShowGrantsStmt(UserIdentity userIdent, boolean isAll) { + this.userIdent = userIdent; + this.isAll = isAll; + } + + public UserIdentity getUserIdent() { + return userIdent; + } + + public boolean isAll() { + return isAll; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + if (userIdent != null) { + if (isAll) { + throw new AnalysisException("Can not specified keyword ALL when specified user"); + } + userIdent.analyze(analyzer.getClusterName()); + } else { + if (!isAll) { + // self + userIdent = new UserIdentity(ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP()); + userIdent.setIsAnalyzed(); + } + } + + UserIdentity self = new UserIdentity(ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP()); + + if (isAll || !self.equals(userIdent)) { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "GRANT"); + } + } + } + + @Override + public ShowResultSetMetaData getMetaData() { + return META_DATA; + } + +} diff --git a/fe/src/com/baidu/palo/analysis/ShowLoadStmt.java b/fe/src/com/baidu/palo/analysis/ShowLoadStmt.java index 5a5373aa58..26012ed987 100644 --- a/fe/src/com/baidu/palo/analysis/ShowLoadStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowLoadStmt.java @@ -16,7 +16,6 @@ package com.baidu.palo.analysis; import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; @@ -117,13 +116,6 @@ public class ShowLoadStmt extends ShowStmt { } else { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - final String userNameWithoutPrefix = ClusterNamespace.getNameFromFullName(dbName); - final String dbNameWithoutPrefix = ClusterNamespace.getNameFromFullName(dbName); - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, userNameWithoutPrefix, - dbNameWithoutPrefix); - } // analyze where clause if not null if (whereClause != null) { diff --git a/fe/src/com/baidu/palo/analysis/ShowLoadWarningsStmt.java b/fe/src/com/baidu/palo/analysis/ShowLoadWarningsStmt.java index b97978ad8f..8fee298109 100644 --- a/fe/src/com/baidu/palo/analysis/ShowLoadWarningsStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowLoadWarningsStmt.java @@ -15,7 +15,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; @@ -26,6 +25,7 @@ import com.baidu.palo.common.InternalException; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -95,13 +95,6 @@ public class ShowLoadWarningsStmt extends ShowStmt { } else { dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - final String userNameWithoutPrefix = ClusterNamespace.getNameFromFullName(dbName); - final String dbNameWithoutPrefix = ClusterNamespace.getNameFromFullName(dbName); - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, userNameWithoutPrefix, - dbNameWithoutPrefix); - } // analyze where clause if not null if (whereClause == null) { diff --git a/fe/src/com/baidu/palo/analysis/ShowMigrationsStmt.java b/fe/src/com/baidu/palo/analysis/ShowMigrationsStmt.java index 38422d5b00..44537c8822 100644 --- a/fe/src/com/baidu/palo/analysis/ShowMigrationsStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowMigrationsStmt.java @@ -15,11 +15,17 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; + import com.google.common.collect.ImmutableList; public class ShowMigrationsStmt extends ShowStmt { @@ -51,8 +57,9 @@ public class ShowMigrationsStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - throw new AnalysisException("No privilege to grant."); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ADMIN"); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowPartitionsStmt.java b/fe/src/com/baidu/palo/analysis/ShowPartitionsStmt.java index 59329ab5ad..5ee6c7e864 100644 --- a/fe/src/com/baidu/palo/analysis/ShowPartitionsStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowPartitionsStmt.java @@ -15,24 +15,27 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.catalog.ColumnType; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.OlapTable; -import com.baidu.palo.catalog.Table; -import com.baidu.palo.cluster.ClusterNamespace; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; -import com.baidu.palo.common.proc.ProcNodeInterface; -import com.baidu.palo.common.proc.ProcResult; -import com.baidu.palo.common.proc.ProcService; -import com.baidu.palo.qe.ShowResultSetMetaData; - -import com.google.common.base.Strings; - -import org.apache.logging.log4j.LogManager; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.common.proc.ProcNodeInterface; +import com.baidu.palo.common.proc.ProcResult; +import com.baidu.palo.common.proc.ProcService; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.ShowResultSetMetaData; + +import com.google.common.base.Strings; + +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; public class ShowPartitionsStmt extends ShowStmt { @@ -80,9 +83,12 @@ public class ShowPartitionsStmt extends ShowStmt { } // check access - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - throw new AnalysisException("No privilege of db(" + dbName + ")."); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, tableName, + PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW PARTITIONS", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tableName); } Database db = Catalog.getInstance().getDb(dbName); diff --git a/fe/src/com/baidu/palo/analysis/ShowProcStmt.java b/fe/src/com/baidu/palo/analysis/ShowProcStmt.java index dbbce5465d..fc37c34726 100644 --- a/fe/src/com/baidu/palo/analysis/ShowProcStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowProcStmt.java @@ -15,6 +15,7 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.common.AnalysisException; @@ -24,6 +25,8 @@ import com.baidu.palo.common.InternalException; import com.baidu.palo.common.proc.ProcNodeInterface; import com.baidu.palo.common.proc.ProcResult; import com.baidu.palo.common.proc.ProcService; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; // SHOW PROC statement. Used to show proc information, only admin can use. @@ -41,9 +44,11 @@ public class ShowProcStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SHOW PROC"); + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "ADMIN"); } + node = ProcService.getInstance().open(path); if (node == null) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_PROC_PATH, path); diff --git a/fe/src/com/baidu/palo/analysis/ShowRepositoriesStmt.java b/fe/src/com/baidu/palo/analysis/ShowRepositoriesStmt.java new file mode 100644 index 0000000000..ba5b4a2407 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/ShowRepositoriesStmt.java @@ -0,0 +1,48 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.qe.ShowResultSetMetaData; + +import com.google.common.collect.ImmutableList; + +public class ShowRepositoriesStmt extends ShowStmt { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("RepoId").add("RepoName").add("CreateTime").add("IsReadOnly").add("Location") + .add("Broker").add("ErrMsg") + .build(); + + public ShowRepositoriesStmt() { + + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TITLE_NAMES) { + builder.addColumn(new Column(title, ColumnType.createVarchar(30))); + } + return builder.build(); + } + +} diff --git a/fe/src/com/baidu/palo/analysis/ShowRestoreStmt.java b/fe/src/com/baidu/palo/analysis/ShowRestoreStmt.java index c807376508..d8bb7f1fe8 100644 --- a/fe/src/com/baidu/palo/analysis/ShowRestoreStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowRestoreStmt.java @@ -15,29 +15,29 @@ package com.baidu.palo.analysis; -import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.backup.RestoreJob; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; -import com.baidu.palo.catalog.Database; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; -import com.baidu.palo.common.PatternMatcher; -import com.baidu.palo.common.proc.RestoreProcNode; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import com.google.common.collect.ImmutableList; public class ShowRestoreStmt extends ShowStmt { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("JobId").add("Label").add("Timestamp").add("DbName").add("State") + .add("AllowLoad").add("ReplicationNum") + .add("RestoreObjs").add("CreateTime").add("MetaPreparedTime").add("SnapshotFinishedTime") + .add("DownloadFinishedTime").add("FinishedTime").add("UnfinishedTasks").add("TaskErrMsg") + .add("Status").add("Timeout") + .build(); private String dbName; private Expr where; @@ -58,103 +58,26 @@ public class ShowRestoreStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); if (Strings.isNullOrEmpty(dbName)) { dbName = analyzer.getDefaultDb(); if (Strings.isNullOrEmpty(dbName)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); } + } else { + dbName = ClusterNamespace.getFullName(getClusterName(), dbName); } - // check access - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), dbName); - } - - analyzeWhere(); - } - - private void analyzeWhere() throws AnalysisException { - boolean valid = true; - if (where == null) { - return; - } - - CHECK: { - if (where instanceof BinaryPredicate) { - BinaryPredicate binaryPredicate = (BinaryPredicate) where; - if (binaryPredicate.getOp() != Operator.EQ) { - valid = false; - break CHECK; - } - } else if (where instanceof LikePredicate) { - LikePredicate likePredicate = (LikePredicate) where; - if (likePredicate.getOp() != LikePredicate.Operator.LIKE) { - valid = false; - break CHECK; - } - } else { - valid = false; - break CHECK; - } - - // left child - if (!(where.getChild(0) instanceof SlotRef)) { - valid = false; - break CHECK; - } - String leftKey = ((SlotRef) where.getChild(0)).getColumnName(); - if (!leftKey.equalsIgnoreCase("label")) { - valid = false; - break CHECK; - } - - // right child - if (!(where.getChild(1) instanceof StringLiteral)) { - valid = false; - break CHECK; - } - - label = ((StringLiteral) where.getChild(1)).getStringValue(); - if (Strings.isNullOrEmpty(label)) { - valid = false; - break CHECK; - } - } - - if (!valid) { - throw new AnalysisException("Where clause should looks like: LABEL = \"your_restore_label\"," - + " or LABEL LIKE \"matcher\""); + // check auth + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } } - public List> getResultRows() throws AnalysisException { - List> result = new LinkedList>(); - Database db = Catalog.getInstance().getDb(dbName); - if (db == null) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, dbName); - } - - BackupHandler backupHandler = Catalog.getInstance().getBackupHandler(); - PatternMatcher matcher = null; - if (!Strings.isNullOrEmpty(label)) { - matcher = PatternMatcher.createMysqlPattern(label); - } - List> backupJobInfos = backupHandler.getJobInfosByDb(db.getId(), RestoreJob.class, matcher); - for (List infoStr : backupJobInfos) { - List oneInfo = new ArrayList(RestoreProcNode.TITLE_NAMES.size()); - for (Comparable element : infoStr) { - oneInfo.add(element.toString()); - } - result.add(oneInfo); - } - return result; - } - - @Override public ShowResultSetMetaData getMetaData() { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - for (String title : RestoreProcNode.TITLE_NAMES) { + for (String title : TITLE_NAMES) { builder.addColumn(new Column(title, ColumnType.createVarchar(30))); } return builder.build(); @@ -177,3 +100,4 @@ public class ShowRestoreStmt extends ShowStmt { return toSql(); } } + diff --git a/fe/src/com/baidu/palo/analysis/ShowRolesStmt.java b/fe/src/com/baidu/palo/analysis/ShowRolesStmt.java new file mode 100644 index 0000000000..1e86f04aa6 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/ShowRolesStmt.java @@ -0,0 +1,59 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.ShowResultSetMetaData; + +public class ShowRolesStmt extends ShowStmt { + private static final ShowResultSetMetaData META_DATA; + static { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + + builder.addColumn(new Column("Name", ColumnType.createVarchar(100))); + builder.addColumn(new Column("Users", ColumnType.createVarchar(100))); + builder.addColumn(new Column("GlobalPrivs", ColumnType.createVarchar(300))); + builder.addColumn(new Column("DatabasePrivs", ColumnType.createVarchar(300))); + builder.addColumn(new Column("TablePrivs", ColumnType.createVarchar(300))); + + META_DATA = builder.build(); + } + + public ShowRolesStmt() { + + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "GRANT"); + } + } + + @Override + public ShowResultSetMetaData getMetaData() { + return META_DATA; + } + +} diff --git a/fe/src/com/baidu/palo/analysis/ShowSnapshotStmt.java b/fe/src/com/baidu/palo/analysis/ShowSnapshotStmt.java new file mode 100644 index 0000000000..531a628164 --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/ShowSnapshotStmt.java @@ -0,0 +1,156 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.qe.ShowResultSetMetaData; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; + +public class ShowSnapshotStmt extends ShowStmt { + public static final ImmutableList SNAPSHOT_ALL = new ImmutableList.Builder() + .add("Snapshot").add("Timestamp").add("Status") + .build(); + public static final ImmutableList SNAPSHOT_DETAIL = new ImmutableList.Builder() + .add("Snapshot").add("Timestamp").add("Database").add("Details").add("Status") + .build(); + + private String repoName; + private Expr where; + private String snapshotName; + private String timestamp; + + public ShowSnapshotStmt(String repoName, Expr where) { + this.repoName = repoName; + this.where = where; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + + // analyze where clause if not null + if (where != null) { + // eg: WHERE snapshot="snapshot_label" [and timestamp="2018-04-19-11-11:11"]; + boolean ok = true; + CHECK: { + if (where instanceof BinaryPredicate) { + if (!analyzeSubExpr((BinaryPredicate) where)) { + ok = false; + break CHECK; + } + } else if (where instanceof CompoundPredicate) { + CompoundPredicate cp = (CompoundPredicate) where; + if (cp.getOp() != Operator.AND) { + ok = false; + break CHECK; + } + + if (!(cp.getChild(0) instanceof BinaryPredicate) + || !(cp.getChild(1) instanceof BinaryPredicate)) { + ok = false; + break CHECK; + } + + if (!analyzeSubExpr((BinaryPredicate) cp.getChild(0)) + || !analyzeSubExpr((BinaryPredicate) cp.getChild(1))) { + ok = false; + break CHECK; + } + } + } + + if (ok && (Strings.isNullOrEmpty(snapshotName) && !Strings.isNullOrEmpty(timestamp))) { + // can not only set timestamp + ok = false; + } + + if (!ok) { + throw new AnalysisException("Where clause should looks like: SNAPSHOT = 'your_snapshot_name'" + + " [AND TIMESTAMP = '2018-04-18-19-19-10']"); + } + } + } + + private boolean analyzeSubExpr(BinaryPredicate expr) { + Expr key = expr.getChild(0); + Expr val = expr.getChild(1); + + if (!(key instanceof SlotRef)) { + return false; + } + if (!(val instanceof StringLiteral)) { + return false; + } + + String name = ((SlotRef) key).getColumnName(); + if (name.equalsIgnoreCase("snapshot")) { + snapshotName = ((StringLiteral) val).getStringValue(); + if (Strings.isNullOrEmpty(snapshotName)) { + return false; + } + return true; + } else if (name.equalsIgnoreCase("timestamp")) { + timestamp = ((StringLiteral) val).getStringValue(); + if (Strings.isNullOrEmpty(timestamp)) { + return false; + } + return true; + } + + return false; + + } + + public String getRepoName() { + return repoName; + } + + public String getSnapshotName() { + return snapshotName; + } + + public String getTimestamp() { + return timestamp; + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + if (!Strings.isNullOrEmpty(snapshotName) && !Strings.isNullOrEmpty(timestamp)) { + for (String title : SNAPSHOT_DETAIL) { + builder.addColumn(new Column(title, ColumnType.createVarchar(30))); + } + } else { + for (String title : SNAPSHOT_ALL) { + builder.addColumn(new Column(title, ColumnType.createVarchar(30))); + } + } + return builder.build(); + } + +} + diff --git a/fe/src/com/baidu/palo/analysis/ShowTableStatusStmt.java b/fe/src/com/baidu/palo/analysis/ShowTableStatusStmt.java index 6d3c5b7f88..ad12d0b1bf 100644 --- a/fe/src/com/baidu/palo/analysis/ShowTableStatusStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowTableStatusStmt.java @@ -15,7 +15,7 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.catalog.InfoSchemaDb; @@ -24,6 +24,8 @@ import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -83,9 +85,8 @@ public class ShowTableStatusStmt extends ShowStmt { } else { db = ClusterNamespace.getFullName(analyzer.getClusterName(), db); } - if (!analyzer.getCatalog().getUserMgr() - .checkAccess(analyzer.getUser(), db, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), db); + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), db, PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), db); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowTableStmt.java b/fe/src/com/baidu/palo/analysis/ShowTableStmt.java index 47754c5e59..18d2fac4fc 100644 --- a/fe/src/com/baidu/palo/analysis/ShowTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowTableStmt.java @@ -15,7 +15,6 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.catalog.InfoSchemaDb; @@ -79,9 +78,9 @@ public class ShowTableStmt extends ShowStmt { } else { db = ClusterNamespace.getFullName(analyzer.getClusterName(), db); } - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), db, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), db); - } + + // we do not check db privs here. because user may not have any db privs, + // but if it has privs of tbls inside this db,it should be allowed to see this db. } @Override diff --git a/fe/src/com/baidu/palo/analysis/ShowTabletStmt.java b/fe/src/com/baidu/palo/analysis/ShowTabletStmt.java index e2c0693969..c646e14651 100644 --- a/fe/src/com/baidu/palo/analysis/ShowTabletStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowTabletStmt.java @@ -15,6 +15,7 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; @@ -23,6 +24,8 @@ import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.proc.TabletsProcDir; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -77,7 +80,7 @@ public class ShowTabletStmt extends ShowStmt { } // check access - if (!analyzer.getCatalog().getUserMgr().isSuperuser(analyzer.getUser())) { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SHOW TABLET"); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowUserPropertyStmt.java b/fe/src/com/baidu/palo/analysis/ShowUserPropertyStmt.java index e8244feda5..ce588b78e7 100644 --- a/fe/src/com/baidu/palo/analysis/ShowUserPropertyStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowUserPropertyStmt.java @@ -15,17 +15,19 @@ package com.baidu.palo.analysis; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.PatternMatcher; -import com.baidu.palo.common.proc.ProcNodeInterface; -import com.baidu.palo.common.proc.ProcService; import com.baidu.palo.common.proc.UserPropertyProcNode; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -45,8 +47,6 @@ public class ShowUserPropertyStmt extends ShowStmt { private String user; private String pattern; - private ProcNodeInterface node; - public ShowUserPropertyStmt(String user, String pattern) { this.user = user; this.pattern = pattern; @@ -56,39 +56,29 @@ public class ShowUserPropertyStmt extends ShowStmt { public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); if (Strings.isNullOrEmpty(user)) { - user = analyzer.getUser(); + user = analyzer.getQualifiedUser(); + // user can see itself's property, no need to check privs } else { user = ClusterNamespace.getFullName(getClusterName(), user); - if (!analyzer.getCatalog().getUserMgr().checkUserAccess(analyzer.getUser(), user)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SHOW PROPERTY"); + + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "GRANT"); } } pattern = Strings.emptyToNull(pattern); } - public void handleShow() throws AnalysisException { - // build proc path - // /access_resource/user - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("/access_resource/"); - stringBuilder.append(user); - LOG.debug("process SHOW PROC '{}';", stringBuilder.toString()); - - node = ProcService.getInstance().open(stringBuilder.toString()); - if (node == null) { - throw new AnalysisException("Failed to show user property"); - } - } - public List> getRows() throws AnalysisException { - List> rows = node.fetchResult().getRows(); + List> rows = Catalog.getCurrentCatalog().getAuth().getUserProperties(user); + if (pattern == null) { return rows; } List> result = Lists.newArrayList(); - PatternMatcher matcher = PatternMatcher.createMysqlPattern(pattern); + PatternMatcher matcher = PatternMatcher.createMysqlPattern(pattern, + CaseSensibility.USER.getCaseSensibility()); for (List row : rows) { String key = row.get(0).split("\\" + SetUserPropertyVar.DOT_SEPARATOR)[0]; if (matcher.match(key)) { diff --git a/fe/src/com/baidu/palo/analysis/ShowUserStmt.java b/fe/src/com/baidu/palo/analysis/ShowUserStmt.java index 8946fb6b2d..c855d0d5aa 100644 --- a/fe/src/com/baidu/palo/analysis/ShowUserStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowUserStmt.java @@ -4,7 +4,7 @@ import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; -import com.baidu.palo.common.proc.AccessResourceProcDir; +import com.baidu.palo.common.proc.AuthProcDir; import com.baidu.palo.qe.ShowResultSetMetaData; public class ShowUserStmt extends ShowStmt { @@ -12,7 +12,7 @@ public class ShowUserStmt extends ShowStmt { static { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - for (String title : AccessResourceProcDir.TITLE_NAMES) { + for (String title : AuthProcDir.TITLE_NAMES) { builder.addColumn(new Column(title, ColumnType.createVarchar(30))); } META_DATA = builder.build(); @@ -30,7 +30,7 @@ public class ShowUserStmt extends ShowStmt { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - user = analyzer.getUser(); + user = analyzer.getQualifiedUser(); } @Override diff --git a/fe/src/com/baidu/palo/analysis/TablePattern.java b/fe/src/com/baidu/palo/analysis/TablePattern.java new file mode 100644 index 0000000000..e5192f23cc --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/TablePattern.java @@ -0,0 +1,141 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.mysql.privilege.PaloAuth.PrivLevel; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +// only the following 3 formats are allowed +// db.tbl +// *.* +// db.* +public class TablePattern implements Writable { + private String db; + private String tbl; + boolean isAnalyzed = false; + + public static TablePattern ALL; + static { + ALL = new TablePattern("*", "*"); + try { + ALL.analyze(""); + } catch (AnalysisException e) { + // will not happen + } + } + + private TablePattern() { + } + + public TablePattern(String db, String tbl) { + this.db = Strings.isNullOrEmpty(db) ? "*" : db; + this.tbl = Strings.isNullOrEmpty(tbl) ? "*" : tbl; + } + + public String getQuolifiedDb() { + Preconditions.checkState(isAnalyzed); + return db; + } + + public String getTbl() { + return tbl; + } + + public PrivLevel getPrivLevel() { + Preconditions.checkState(isAnalyzed); + if (db.equals("*")) { + return PrivLevel.GLOBAL; + } else if (!tbl.equals("*")) { + return PrivLevel.TABLE; + } else { + return PrivLevel.DATABASE; + } + } + + public void analyze(String clusterName) throws AnalysisException { + if (isAnalyzed) { + return; + } + if (db.equals("*") && !tbl.equals("*")) { + throw new AnalysisException("Do not support format: " + toString()); + } + + if (!db.equals("*")) { + FeNameFormat.checkDbName(db); + db = ClusterNamespace.getFullName(clusterName, db); + } + + if (!tbl.equals("*")) { + FeNameFormat.checkTableName(tbl); + } + isAnalyzed = true; + } + + public static TablePattern read(DataInput in) throws IOException { + TablePattern tablePattern = new TablePattern(); + tablePattern.readFields(in); + return tablePattern; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TablePattern)) { + return false; + } + TablePattern other = (TablePattern) obj; + return db.equals(other.getQuolifiedDb()) && tbl.equals(other.getTbl()); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + db.hashCode(); + result = 31 * result + tbl.hashCode(); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(db).append(".").append(tbl); + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + Preconditions.checkState(isAnalyzed); + Text.writeString(out, db); + Text.writeString(out, tbl); + } + + @Override + public void readFields(DataInput in) throws IOException { + db = Text.readString(in); + tbl = Text.readString(in); + isAnalyzed = true; + } +} diff --git a/fe/src/com/baidu/palo/analysis/TableRef.java b/fe/src/com/baidu/palo/analysis/TableRef.java index 95f0e8c50a..6fa466db9d 100644 --- a/fe/src/com/baidu/palo/analysis/TableRef.java +++ b/fe/src/com/baidu/palo/analysis/TableRef.java @@ -25,6 +25,8 @@ import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; import com.baidu.palo.rewrite.ExprRewriter; import com.google.common.base.Joiner; @@ -35,6 +37,9 @@ import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -68,7 +73,7 @@ import java.util.Set; * TODO for 2.3: Rename this class to CollectionRef and re-consider the naming and * structure of all subclasses. */ -public class TableRef implements ParseNode { +public class TableRef implements ParseNode, Writable { private static final Logger LOG = LogManager.getLogger(TableRef.class); protected TableName name; private List partitions = null; @@ -126,6 +131,10 @@ public class TableRef implements ParseNode { // END: Members that need to be reset() // /////////////////////////////////////// + public TableRef() { + // for persist + } + public TableRef(TableName name, String alias) { this(name, alias, null); } @@ -641,4 +650,58 @@ public class TableRef implements ParseNode { public TableRef clone() { return new TableRef(this); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name); + if (partitions != null && !partitions.isEmpty()) { + sb.append(" PARTITIONS("); + sb.append(Joiner.on(", ").join(partitions)).append(")"); + } + if (aliases_ != null && aliases_.length > 0) { + sb.append(" AS ").append(aliases_[0]); + } + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + name.write(out); + if (partitions == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeInt(partitions.size()); + for (String partName : partitions) { + Text.writeString(out, partName); + } + } + + if (hasExplicitAlias()) { + out.writeBoolean(true); + Text.writeString(out, getExplicitAlias()); + } else { + out.writeBoolean(false); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + name = new TableName(); + name.readFields(in); + if (in.readBoolean()) { + partitions = Lists.newArrayList(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String partName = Text.readString(in); + partitions.add(partName); + } + } + + if (in.readBoolean()) { + String alias = Text.readString(in); + aliases_ = new String[] { alias }; + } + } } diff --git a/fe/src/com/baidu/palo/analysis/TupleIsNullPredicate.java b/fe/src/com/baidu/palo/analysis/TupleIsNullPredicate.java index 84fc5196b4..68221f973a 100644 --- a/fe/src/com/baidu/palo/analysis/TupleIsNullPredicate.java +++ b/fe/src/com/baidu/palo/analysis/TupleIsNullPredicate.java @@ -142,6 +142,9 @@ public class TupleIsNullPredicate extends Predicate { */ private static boolean requiresNullWrapping(Expr expr, Analyzer analyzer) throws InternalException { + if (expr.isConstant()) { + return false; + } return true; } diff --git a/fe/src/com/baidu/palo/analysis/UseStmt.java b/fe/src/com/baidu/palo/analysis/UseStmt.java index 11ebc32bd2..c4513ee38a 100644 --- a/fe/src/com/baidu/palo/analysis/UseStmt.java +++ b/fe/src/com/baidu/palo/analysis/UseStmt.java @@ -20,17 +20,20 @@ package com.baidu.palo.analysis; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + /** * Representation of a USE db statement. */ @@ -63,8 +66,8 @@ public class UseStmt extends StatementBase { } database = ClusterNamespace.getFullName(getClusterName(), database); - if (!analyzer.getCatalog().getUserMgr().checkAccess(analyzer.getUser(), database, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getUser(), database); + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), database, PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, analyzer.getQualifiedUser(), database); } } diff --git a/fe/src/com/baidu/palo/analysis/UserDesc.java b/fe/src/com/baidu/palo/analysis/UserDesc.java index 081e53b26c..ac57a47d73 100644 --- a/fe/src/com/baidu/palo/analysis/UserDesc.java +++ b/fe/src/com/baidu/palo/analysis/UserDesc.java @@ -22,26 +22,22 @@ package com.baidu.palo.analysis; // Description of user in SQL statement public class UserDesc { - private String user; + private UserIdentity userIdent; private String password; private boolean isPlain; - public UserDesc(String user) { - this(user, "", false); + public UserDesc(UserIdentity userIdent) { + this(userIdent, "", false); } - public UserDesc(String user, String password, boolean isPlain) { - this.user = user; + public UserDesc(UserIdentity userIdent, String password, boolean isPlain) { + this.userIdent = userIdent; this.password = password; this.isPlain = isPlain; } - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; + public UserIdentity getUserIdent() { + return userIdent; } public String getPassword() { @@ -51,8 +47,4 @@ public class UserDesc { public boolean isPlain() { return isPlain; } - - public void setPlain(boolean isPlain) { - this.isPlain = isPlain; - } } diff --git a/fe/src/com/baidu/palo/analysis/UserIdentity.java b/fe/src/com/baidu/palo/analysis/UserIdentity.java new file mode 100644 index 0000000000..257db4c84c --- /dev/null +++ b/fe/src/com/baidu/palo/analysis/UserIdentity.java @@ -0,0 +1,205 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.analysis; + +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.PatternMatcher; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.mysql.privilege.PaloAuth; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +// https://dev.mysql.com/doc/refman/8.0/en/account-names.html +// user name must be literally matched. +// host name can take many forms, and wildcards are permitted. +// cmy@% +// cmy@192.168.% +// cmy@[domain.name] +public class UserIdentity implements Writable { + private String user; + private String host; + private boolean isDomain; + private boolean isAnalyzed = false; + + private UserIdentity() { + } + + public UserIdentity(String user, String host) { + this.user = Strings.emptyToNull(user); + this.host = Strings.emptyToNull(host); + this.isDomain = false; + } + + public UserIdentity(String user, String host, boolean isDomain) { + this.user = Strings.emptyToNull(user); + this.host = Strings.emptyToNull(host); + this.isDomain = isDomain; + } + + public String getQualifiedUser() { + Preconditions.checkState(isAnalyzed); + return user; + } + + public String getHost() { + return host; + } + + public boolean isDomain() { + return isDomain; + } + + public void setIsAnalyzed() { + this.isAnalyzed = true; + } + + public void analyze(String clusterName) throws AnalysisException { + if (isAnalyzed) { + return; + } + if (Strings.isNullOrEmpty(user)) { + throw new AnalysisException("Does not support anonymous user"); + } + + FeNameFormat.checkUserName(user); + if (!user.equals(PaloAuth.ROOT_USER) && !user.equals(PaloAuth.ADMIN_USER)) { + user = ClusterNamespace.getFullName(clusterName, user); + } + + // reuse createMysqlPattern to validate host pattern + PatternMatcher.createMysqlPattern(host, CaseSensibility.HOST.getCaseSensibility()); + isAnalyzed = true; + } + + public boolean include(UserIdentity self) { + Preconditions.checkState(isAnalyzed && self.isAnalyzed); + if (!user.equals(self.user)) { + return false; + } + + if (host.equals(self.host)) { + return true; + } + + // same user with different host + try { + PatternMatcher patternMatcher = PatternMatcher.createMysqlPattern(host, + CaseSensibility.HOST.getCaseSensibility()); + + return patternMatcher.match(self.host); + } catch (AnalysisException e) { + Preconditions.checkNotNull(null, e.getMessage()); + } + return false; + } + + public static UserIdentity fromString(String userIdentStr) { + if (Strings.isNullOrEmpty(userIdentStr)) { + return null; + } + + String[] parts = userIdentStr.split("@"); + if (parts.length != 2) { + return null; + } + + String user = parts[0]; + if (!user.startsWith("'") || !user.endsWith("'")) { + return null; + } + + String host = parts[1]; + if (host.startsWith("['") && host.endsWith("']")) { + UserIdentity userIdent = new UserIdentity(user.substring(1, user.length() - 1), + host.substring(2, host.length() - 2), true); + userIdent.setIsAnalyzed(); + return userIdent; + } else if (host.startsWith("'") && host.endsWith("'")) { + UserIdentity userIdent = new UserIdentity(user.substring(1, user.length() - 1), + host.substring(1, host.length() - 1)); + userIdent.setIsAnalyzed(); + return userIdent; + } + + return null; + } + + public static UserIdentity read(DataInput in) throws IOException { + UserIdentity userIdentity = new UserIdentity(); + userIdentity.readFields(in); + return userIdentity; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof UserIdentity)) { + return false; + } + UserIdentity other = (UserIdentity) obj; + return user.equals(other.getQualifiedUser()) && host.equals(other.getHost()); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + user.hashCode(); + result = 31 * result + host.hashCode(); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("'"); + if (!Strings.isNullOrEmpty(user)) { + sb.append(user); + } + sb.append("'@"); + if (!Strings.isNullOrEmpty(host)) { + if (isDomain) { + sb.append("['").append(host).append("']"); + } else { + sb.append("'").append(host).append("'"); + } + } + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + Preconditions.checkState(isAnalyzed); + Text.writeString(out, user); + Text.writeString(out, host); + out.writeBoolean(isDomain); + } + + @Override + public void readFields(DataInput in) throws IOException { + user = Text.readString(in); + host = Text.readString(in); + isDomain = in.readBoolean(); + isAnalyzed = true; + } +} diff --git a/fe/src/com/baidu/palo/backup/AbstractBackupJob.java b/fe/src/com/baidu/palo/backup/AbstractBackupJob_D.java similarity index 96% rename from fe/src/com/baidu/palo/backup/AbstractBackupJob.java rename to fe/src/com/baidu/palo/backup/AbstractBackupJob_D.java index 8e756a956b..8199307d95 100644 --- a/fe/src/com/baidu/palo/backup/AbstractBackupJob.java +++ b/fe/src/com/baidu/palo/backup/AbstractBackupJob_D.java @@ -39,8 +39,9 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -public class AbstractBackupJob implements Writable { - private static final Logger LOG = LogManager.getLogger(AbstractBackupJob.class); +@Deprecated +public class AbstractBackupJob_D implements Writable { + private static final Logger LOG = LogManager.getLogger(AbstractBackupJob_D.class); protected long jobId; protected long dbId; @@ -62,11 +63,11 @@ public class AbstractBackupJob implements Writable { protected Future future; - public AbstractBackupJob() { + public AbstractBackupJob_D() { unfinishedTabletIds = HashMultimap.create(); } - public AbstractBackupJob(long jobId, long dbId, LabelName labelName, String remotePath, + public AbstractBackupJob_D(long jobId, long dbId, LabelName labelName, String remotePath, Map remoteProperties) { this.jobId = jobId; this.dbId = dbId; diff --git a/fe/src/com/baidu/palo/backup/AbstractJob.java b/fe/src/com/baidu/palo/backup/AbstractJob.java new file mode 100644 index 0000000000..e033b9af8f --- /dev/null +++ b/fe/src/com/baidu/palo/backup/AbstractJob.java @@ -0,0 +1,244 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; + +/* + * The design of JobI is as follows + * 1. Here are only two methods: run() and cancel() that can modify the internal state of a Job. + * And each method is implemented as synchronized to avoid handling concurrent modify things. + * + * 2. isDone() method is used to check whether we can submit the next job. + */ +public abstract class AbstractJob implements Writable { + + public enum JobType { + BACKUP, RESTORE + } + + protected JobType type; + + // must be set right before job's running + protected Catalog catalog; + // repo will be set at first run() + protected Repository repo; + protected long repoId; + + /* + * In BackupJob, jobId will be generated every time before we call prepareAndSendSnapshotTask(); + * Because prepareAndSendSnapshotTask() may be called several times due to FE restart. + * And each time this method is called, the snapshot tasks will be sent with (maybe) different + * version and version hash. So we have to use different job id to identify the tasks in different batches. + */ + protected long jobId = -1; + + protected String label; + protected long dbId; + protected String dbName; + + protected Status status = Status.OK; + + protected long createTime = -1; + protected long finishedTime = -1; + protected long timeoutMs; + + protected boolean isTypeRead = false; + + // save err msg of tasks + protected Map taskErrMsg = Maps.newHashMap(); + + protected AbstractJob(JobType type) { + this.type = type; + } + + protected AbstractJob(JobType type, String label, long dbId, String dbName, + long timeoutMs, Catalog catalog, long repoId) { + this.type = type; + this.label = label; + this.dbId = dbId; + this.dbName = dbName; + this.createTime = System.currentTimeMillis(); + this.timeoutMs = timeoutMs; + this.catalog = catalog; + this.repoId = repoId; + } + + public JobType getType() { + return type; + } + + public long getJobId() { + return jobId; + } + + public String getLabel() { + return label; + } + + public long getDbId() { + return dbId; + } + + public String getDbName() { + return dbName; + } + + public Status getStatus() { + return status; + } + + public long getCreateTime() { + return createTime; + } + + public long getFinishedTime() { + return finishedTime; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public void setCatalog(Catalog catalog) { + this.catalog = catalog; + } + + public long getRepoId() { + return repoId; + } + + public void setTypeRead(boolean isTypeRead) { + this.isTypeRead = isTypeRead; + } + + public abstract void run(); + + public abstract Status cancel(); + + public abstract void replayRun(); + + public abstract void replayCancel(); + + public abstract boolean isDone(); + + public abstract boolean isPending(); + + public abstract boolean isCancelled(); + + public static AbstractJob read(DataInput in) throws IOException { + AbstractJob job = null; + JobType type = JobType.valueOf(Text.readString(in)); + if (type == JobType.BACKUP) { + job = new BackupJob(); + } else if (type == JobType.RESTORE) { + job = new RestoreJob(); + } else { + throw new IOException("Unknown job type: " + type.name()); + } + + job.setTypeRead(true); + job.readFields(in); + return job; + } + + @Override + public void write(DataOutput out) throws IOException { + // ATTN: must write type first + Text.writeString(out, type.name()); + + out.writeLong(repoId); + Text.writeString(out, label); + out.writeLong(jobId); + out.writeLong(dbId); + Text.writeString(out, dbName); + + out.writeLong(createTime); + out.writeLong(finishedTime); + out.writeLong(timeoutMs); + + if (!taskErrMsg.isEmpty()) { + out.writeBoolean(true); + // we only save at most 3 err msgs + int savedNum = Math.min(3, taskErrMsg.size()); + out.writeInt(savedNum); + for (Map.Entry entry : taskErrMsg.entrySet()) { + if (savedNum == 0) { + break; + } + out.writeLong(entry.getKey()); + Text.writeString(out, entry.getValue()); + savedNum--; + } + Preconditions.checkState(savedNum == 0, savedNum); + } else { + out.writeBoolean(false); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + if (!isTypeRead) { + type = JobType.valueOf(Text.readString(in)); + isTypeRead = true; + } + + repoId = in.readLong(); + label = Text.readString(in); + jobId = in.readLong(); + dbId = in.readLong(); + dbName = Text.readString(in); + + createTime = in.readLong(); + finishedTime = in.readLong(); + timeoutMs = in.readLong(); + + if (in.readBoolean()) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long taskId = in.readLong(); + String msg = Text.readString(in); + taskErrMsg.put(taskId, msg); + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(type.name()); + sb.append(" repo id: ").append(repoId).append(", label: ").append(label); + sb.append(", job id: ").append(jobId).append(", db id: ").append(dbId).append(", db name: ").append(dbName); + sb.append(", status: ").append(status); + sb.append(", timeout: ").append(timeoutMs); + return sb.toString(); + } +} + diff --git a/fe/src/com/baidu/palo/backup/BackupHandler.java b/fe/src/com/baidu/palo/backup/BackupHandler.java index b6d4aa9361..7f16ee5b51 100644 --- a/fe/src/com/baidu/palo/backup/BackupHandler.java +++ b/fe/src/com/baidu/palo/backup/BackupHandler.java @@ -17,767 +17,538 @@ package com.baidu.palo.backup; import com.baidu.palo.analysis.AbstractBackupStmt; import com.baidu.palo.analysis.BackupStmt; +import com.baidu.palo.analysis.BackupStmt.BackupType; import com.baidu.palo.analysis.CancelBackupStmt; -import com.baidu.palo.analysis.LabelName; -import com.baidu.palo.analysis.PartitionName; +import com.baidu.palo.analysis.CreateRepositoryStmt; +import com.baidu.palo.analysis.DropRepositoryStmt; import com.baidu.palo.analysis.RestoreStmt; +import com.baidu.palo.analysis.TableRef; import com.baidu.palo.backup.BackupJob.BackupJobState; -import com.baidu.palo.backup.RestoreJob.RestoreJobState; +import com.baidu.palo.backup.BackupJobInfo.BackupTableInfo; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.OlapTable; -import com.baidu.palo.catalog.OlapTable.OlapTableState; import com.baidu.palo.catalog.Partition; -import com.baidu.palo.catalog.PartitionType; import com.baidu.palo.catalog.Table; import com.baidu.palo.catalog.Table.TableType; -import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; -import com.baidu.palo.common.FeNameFormat; -import com.baidu.palo.common.PatternMatcher; +import com.baidu.palo.common.io.Writable; import com.baidu.palo.common.util.Daemon; -import com.baidu.palo.common.util.TimeUtils; -import com.baidu.palo.task.RestoreTask; +import com.baidu.palo.task.DirMoveTask; +import com.baidu.palo.task.DownloadTask; import com.baidu.palo.task.SnapshotTask; import com.baidu.palo.task.UploadTask; +import com.baidu.palo.thrift.TFinishTaskRequest; import com.google.common.base.Preconditions; -import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Iterator; -import java.util.LinkedList; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; -public class BackupHandler extends Daemon { +public class BackupHandler extends Daemon implements Writable { private static final Logger LOG = LogManager.getLogger(BackupHandler.class); + + public static final int SIGNATURE_VERSION = 1; + public static final Path BACKUP_ROOT_DIR = Paths.get(Config.tmp_dir, "backup").normalize(); + public static final Path RESTORE_ROOT_DIR = Paths.get(Config.tmp_dir, "restore").normalize(); - private Map dbIdToBackupJob; - private Map dbIdToRestoreJob; - private List finishedOrCancelledBackupJobs; - private List finishedOrCancelledRestoreJobs; + private RepositoryMgr repoMgr = new RepositoryMgr(); - private Multimap dbIdToLabels; + // db id -> last running or finished backup/restore jobs + // We only save the last backup/restore job of a database. + // Newly submitted job will replace the current job, only if current job is finished or cancelled. + // If the last job is finished, user can get the job info from repository. If the last job is cancelled, + // user can get the error message before submitting the next one. + // Use ConcurrentMap to get rid of locks. + private Map dbIdToBackupOrRestoreJob = Maps.newConcurrentMap(); - // lock before db.lock - private ReentrantReadWriteLock lock; + // this lock is used for handling one backup or restore request at a time. + private ReentrantLock seqlock = new ReentrantLock(); - private final AsynchronousCmdExecutor cmdExecutor; + private boolean isInit = false; - public BackupHandler() { - super("backupHandler", 5000L); - dbIdToBackupJob = Maps.newHashMap(); - dbIdToRestoreJob = Maps.newHashMap(); - finishedOrCancelledBackupJobs = Lists.newArrayList(); - finishedOrCancelledRestoreJobs = Lists.newArrayList(); + private Catalog catalog; - dbIdToLabels = HashMultimap.create(); - - lock = new ReentrantReadWriteLock(); - - cmdExecutor = new AsynchronousCmdExecutor(); + private BackupHandler() { + // for persist } - public void readLock() { - lock.readLock().lock(); + public BackupHandler(Catalog catalog) { + super("backupHandler", 3000L); + this.catalog = catalog; } - public void readUnlock() { - lock.readLock().unlock(); + public void setCatalog(Catalog catalog) { + this.catalog = catalog; } - private void writeLock() { - lock.writeLock().lock(); + @Override + public synchronized void start() { + Preconditions.checkNotNull(catalog); + super.start(); + repoMgr.start(); } - private void writeUnlock() { - lock.writeLock().unlock(); + public RepositoryMgr getRepoMgr() { + return repoMgr; } - public AsynchronousCmdExecutor getAsynchronousCmdExecutor() { - return cmdExecutor; - } - - public void process(AbstractBackupStmt stmt) throws DdlException { - String dbName = stmt.getDbName(); - Database db = Catalog.getInstance().getDb(dbName); - if (db == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); - } - - long dbId = db.getId(); - String label = stmt.getLabel(); - writeLock(); - try { - // 1. check if db has job underway - checkJobExist(dbId); - - // 2. check if label is used - checkAndAddLabel(dbId, label); - - // create job - if (stmt instanceof BackupStmt) { - createAndAddBackupJob(db, stmt.getLabelName(), stmt.getObjNames(), stmt.getRemotePath(), - stmt.getProperties()); - } else if (stmt instanceof RestoreStmt) { - createAndAddRestoreJob(db, stmt.getLabelName(), stmt.getObjNames(), stmt.getRemotePath(), - stmt.getProperties()); + private boolean init() { + // Check and create backup dir if necessarily + File backupDir = new File(BACKUP_ROOT_DIR.toString()); + if (!backupDir.exists()) { + if (!backupDir.mkdirs()) { + LOG.warn("failed to create backup dir: " + BACKUP_ROOT_DIR); + return false; } - } catch (DdlException e) { - // remove label - removeLabel(dbId, label); - throw e; - } finally { - writeUnlock(); - } - } - - private void checkJobExist(long dbId) throws DdlException { - if (dbIdToBackupJob.containsKey(dbId)) { - throw new DdlException("Database[" + dbId + "] has backup job underway"); - } - - if (dbIdToRestoreJob.containsKey(dbId)) { - throw new DdlException("Database[" + dbId + "] has restore job underway"); - } - } - - private void checkAndAddLabel(long dbId, String label) throws DdlException { - try { - FeNameFormat.checkLabel(label); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - - if (dbIdToLabels.containsKey(dbId)) { - if (dbIdToLabels.get(dbId).contains(label)) { - throw new DdlException("label " + label + " is already used"); - } - } - - dbIdToLabels.put(dbId, label); - } - - private void removeLabel(long dbId, String label) { - writeLock(); - try { - if (dbIdToLabels.containsKey(dbId)) { - dbIdToLabels.get(dbId).remove(label); - } - } finally { - writeUnlock(); - } - } - - private BackupJob createAndAddBackupJob(Database db, LabelName labelName, List backupObjNames, - String backupPath, Map properties) throws DdlException { - long jobId = Catalog.getInstance().getNextId(); - BackupJob job = new BackupJob(jobId, db.getId(), labelName, backupPath, properties); - db.writeLock(); - try { - if (backupObjNames.isEmpty()) { - // backup all tables - for (String tableName : db.getTableNamesWithLock()) { - backupObjNames.add(new PartitionName(tableName, null, null, null)); - } - } - - if (backupObjNames.isEmpty()) { - throw new DdlException("Database[" + db.getFullName() + "] is empty. no need to backup"); - } - - List backupTables = Lists.newArrayList(); - for (PartitionName backupObj : backupObjNames) { - String tableName = backupObj.getTableName(); - Table table = db.getTable(tableName); - if (table == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); - } - - long tableId = table.getId(); - if (table.getType() == TableType.OLAP) { - OlapTable olapTable = (OlapTable) table; - - // check state - if (olapTable.getState() != OlapTableState.NORMAL) { - throw new DdlException("Table[" + table.getName() + "]' state is not NORMAL"); - } - - // add partition - String partitionName = backupObj.getPartitionName(); - if (partitionName.isEmpty()) { - // add all partitions - for (Partition partition : olapTable.getPartitions()) { - job.addPartitionId(tableId, partition.getId()); - } - } else { - if (olapTable.getPartitionInfo().getType() != PartitionType.RANGE) { - throw new DdlException("Table[" + table.getName() + "] is not range partitioned"); - } - // find and add specified partition - Partition partition = olapTable.getPartition(partitionName); - if (partition == null) { - throw new DdlException("Partition[" + partitionName + "] does not exist in table[" - + tableName + "]"); - } - - job.addPartitionId(tableId, partition.getId()); - } - - // add all indices - for (long indexId : olapTable.getIndexIdToSchema().keySet()) { - job.addIndexId(tableId, indexId); - } - - backupTables.add(olapTable); - } else { - // non-olap table; - job.addPartitionId(tableId, -1L); - } - } // end for backup objs - - // set table state - for (OlapTable olapTable : backupTables) { - Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL); - olapTable.setState(OlapTableState.BACKUP); - } - } finally { - db.writeUnlock(); - } - - // add - Preconditions.checkState(!dbIdToBackupJob.containsKey(db.getId())); - dbIdToBackupJob.put(db.getId(), job); - - // log - Catalog.getInstance().getEditLog().logBackupStart(job); - - LOG.info("finished create backup job[{}]", job.getJobId()); - return job; - } - - private RestoreJob createAndAddRestoreJob(Database db, LabelName labelName, List restoreObjNames, - String restorePath, Map properties) throws DdlException { - Map> tableToPartitionNames = Maps.newHashMap(); - Map tableRenameMap = Maps.newHashMap(); - List existTables = Lists.newArrayList(); - db.writeLock(); - try { - for (PartitionName partitionName : restoreObjNames) { - String newTableName = partitionName.getNewTableName(); - String newPartitionName = partitionName.getNewPartitionName(); - - Table table = db.getTable(newTableName); - if (table != null) { - if (newPartitionName.isEmpty()) { - // do not allow overwrite entire table - throw new DdlException("Table[" + table.getName() + "]' already exist. " - + "Drop table first or restore to another table"); - } - - existTables.add(table); - } - - Set partitionNames = tableToPartitionNames.get(partitionName.getTableName()); - if (partitionNames == null) { - partitionNames = Sets.newHashSet(); - tableToPartitionNames.put(newTableName, partitionNames); - } - - if (!newPartitionName.isEmpty()) { - partitionNames.add(newPartitionName); - } - - tableRenameMap.put(newTableName, partitionName.getTableName()); - } - - // set exist table's state - for (Table table : existTables) { - if (table.getType() == TableType.OLAP) { - ((OlapTable) table).setState(OlapTableState.RESTORE); - } - } - } finally { - db.writeUnlock(); - } - - long jobId = Catalog.getInstance().getNextId(); - RestoreJob job = new RestoreJob(jobId, db.getId(), labelName, restorePath, properties, - tableToPartitionNames, tableRenameMap); - - // add - Preconditions.checkState(!dbIdToRestoreJob.containsKey(db.getId())); - dbIdToRestoreJob.put(db.getId(), job); - - // log - Catalog.getInstance().getEditLog().logRestoreJobStart(job); - - LOG.info("finished create restore job[{}]", job.getJobId()); - return job; - } - - public void handleFinishedSnapshot(SnapshotTask snapshotTask, String snapshotPath) { - long dbId = snapshotTask.getDbId(); - long tabletId = snapshotTask.getTabletId(); - readLock(); - try { - BackupJob job = (BackupJob) dbIdToBackupJob.get(dbId); - if (job == null) { - LOG.warn("db[{}] does not have backup job. tablet: {}", dbId, tabletId); - return; - } - - if (job.getJobId() != snapshotTask.getJobId()) { - LOG.warn("tablet[{}] does not belong to backup job[{}]. tablet job[{}], tablet db[{}]", - tabletId, job.getJobId(), snapshotTask.getJobId(), dbId); - return; - } - - job.handleFinishedSnapshot(tabletId, snapshotTask.getBackendId(), snapshotPath); - - } finally { - readUnlock(); - } - } - - public void handleFinishedUpload(UploadTask uploadTask) { - long dbId = uploadTask.getDbId(); - long tabletId = uploadTask.getTabletId(); - readLock(); - try { - BackupJob job = (BackupJob) dbIdToBackupJob.get(dbId); - if (job == null) { - LOG.warn("db[{}] does not have backup job. tablet: {}", dbId, tabletId); - return; - } - - if (job.getJobId() != uploadTask.getJobId()) { - LOG.warn("tablet[{}] does not belong to backup job[{}]. tablet job[{}], tablet db[{}]", - tabletId, job.getJobId(), uploadTask.getJobId(), dbId); - return; - } - - job.handleFinishedUpload(tabletId, uploadTask.getBackendId()); - } finally { - readUnlock(); - } - } - - public void handleFinishedRestore(RestoreTask restoreTask) { - long dbId = restoreTask.getDbId(); - long tabletId = restoreTask.getTabletId(); - readLock(); - try { - RestoreJob job = (RestoreJob) dbIdToRestoreJob.get(dbId); - if (job == null) { - LOG.warn("db[{}] does not have restore job. tablet: {}", dbId, tabletId); - return; - } - - if (job.getJobId() != restoreTask.getJobId()) { - LOG.warn("tablet[{}] does not belong to restore job[{}]. tablet job[{}], tablet db[{}]", - tabletId, job.getJobId(), restoreTask.getJobId(), dbId); - return; - } - - job.handleFinishedRestore(tabletId, restoreTask.getBackendId()); - } finally { - readUnlock(); - } - } - - public void cancel(CancelBackupStmt stmt) throws DdlException { - String dbName = stmt.getDbName(); - Database db = Catalog.getInstance().getDb(stmt.getDbName()); - if (db == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); - } - - Map dbIdToJob = null; - List finishedOrCancelledJobs = null; - if (stmt.isRestore()) { - dbIdToJob = dbIdToRestoreJob; - finishedOrCancelledJobs = finishedOrCancelledRestoreJobs; } else { - dbIdToJob = dbIdToBackupJob; - finishedOrCancelledJobs = finishedOrCancelledBackupJobs; + if (!backupDir.isDirectory()) { + LOG.warn("backup dir is not a directory: " + BACKUP_ROOT_DIR); + return false; + } } - cancelInternal(db, dbIdToJob, finishedOrCancelledJobs); + // Check and create restore dir if necessarily + File restoreDir = new File(RESTORE_ROOT_DIR.toString()); + if (!restoreDir.exists()) { + if (!restoreDir.mkdirs()) { + LOG.warn("failed to create restore dir: " + RESTORE_ROOT_DIR); + return false; + } + } else { + if (!restoreDir.isDirectory()) { + LOG.warn("restore dir is not a directory: " + RESTORE_ROOT_DIR); + return false; + } + } + isInit = true; + return true; } - private void cancelInternal(Database db, Map dbIdToJob, - List finishedOrCancelledJobs) throws DdlException { - writeLock(); - try { - long dbId = db.getId(); - AbstractBackupJob job = dbIdToJob.get(dbId); - if (job == null) { - throw new DdlException("There is no job in database[" + db.getFullName() + "]"); - } - - job.setErrMsg("user cancelled"); - if (job instanceof BackupJob) { - ((BackupJob) job).setState(BackupJobState.CANCELLED); - } else { - ((RestoreJob) job).setState(RestoreJobState.CANCELLED); - } - - job.end(Catalog.getInstance(), false); - - dbIdToJob.remove(dbId); - finishedOrCancelledJobs.add(job); - removeLabel(dbId, job.getLabel()); - - LOG.info("cancel job[{}] from db[{}]", job.getJobId(), dbId); - } finally { - writeUnlock(); - } + public AbstractJob getJob(long dbId) { + return dbIdToBackupOrRestoreJob.get(dbId); } @Override protected void runOneCycle() { - // backup - LOG.debug("run backup jobs once"); - runOnce(dbIdToBackupJob, finishedOrCancelledBackupJobs); + if (!isInit) { + if (!init()) { + return; + } + } - // restore - LOG.debug("run restore jobs once"); - runOnce(dbIdToRestoreJob, finishedOrCancelledRestoreJobs); + for (AbstractJob job : dbIdToBackupOrRestoreJob.values()) { + job.setCatalog(catalog); + job.run(); + } } - private void runOnce(Map dbIdToJobs, List finishedOrCancelledJobs) { - // backup jobs - writeLock(); + // handle create repository stmt + public void createRepository(CreateRepositoryStmt stmt) throws DdlException { + if (!catalog.getBrokerMgr().contaisnBroker(stmt.getBrokerName())) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "broker does not exist: " + stmt.getBrokerName()); + } + + BlobStorage storage = new BlobStorage(stmt.getBrokerName(), stmt.getProperties()); + long repoId = catalog.getNextId(); + Repository repo = new Repository(repoId, stmt.getName(), stmt.isReadOnly(), stmt.getLocation(), storage); + + Status st = repoMgr.addAndInitRepoIfNotExist(repo, false); + if (!st.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to create repository: " + st.getErrMsg()); + } + } + + // handle drop repository stmt + public void dropRepository(DropRepositoryStmt stmt) throws DdlException { + tryLock(); try { - Iterator> iterator = dbIdToJobs.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - AbstractBackupJob job = entry.getValue(); - job.runOnce(); + Repository repo = repoMgr.getRepo(stmt.getRepoName()); + if (repo == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository does not exist"); + } + + for (AbstractJob job : dbIdToBackupOrRestoreJob.values()) { + if (!job.isDone() && job.getRepoId() == repo.getId()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Backup or restore job is running on this repository." + + " Can not drop it"); + } + } + + Status st = repoMgr.removeRepo(repo.getName(), false /* not replay */); + if (!st.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to drop repository: " + st.getErrMsg()); + } + } finally { + seqlock.unlock(); + } + } - // handle finished or cancelled jobs - if (job instanceof BackupJob) { - BackupJob backupJob = (BackupJob) job; - if (backupJob.getState() == BackupJobState.FINISHED - || backupJob.getState() == BackupJobState.CANCELLED) { - finishedOrCancelledJobs.add(backupJob); + // the entry method of submitting a backup or restore job + public void process(AbstractBackupStmt stmt) throws DdlException { + // check if repo exist + String repoName = stmt.getRepoName(); + Repository repository = repoMgr.getRepo(repoName); + if (repository == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository " + repoName + " does not exist"); + } - if (backupJob.getState() == BackupJobState.CANCELLED) { - // remove label - removeLabel(backupJob.getDbId(), backupJob.getLabel()); + // check if db exist + String dbName = stmt.getDbName(); + Database db = catalog.getDb(dbName); + if (db == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } + + // Try to get sequence lock. + // We expect at most one operation on a repo at same time. + // But this operation may take a few seconds with lock held. + // So we use tryLock() to give up this operation if we can not get lock. + tryLock(); + try { + // Check if there is backup or restore job running on this database + AbstractJob currentJob = dbIdToBackupOrRestoreJob.get(db.getId()); + if (currentJob != null && !currentJob.isDone()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Can only run one backup or restore job of a database at same time"); + } + + if (stmt instanceof BackupStmt) { + backup(repository, db, (BackupStmt) stmt); + } else if (stmt instanceof RestoreStmt) { + restore(repository, db, (RestoreStmt) stmt); + } + } finally { + seqlock.unlock(); + } + } + + private void tryLock() throws DdlException { + try { + if (!seqlock.tryLock(10, TimeUnit.SECONDS)) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Another backup or restore job" + + " is being submitted. Please wait and try again"); + } + } catch (InterruptedException e) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Got interrupted exception when " + + "try locking. Try again"); + } + } + + private void backup(Repository repository, Database db, BackupStmt stmt) throws DdlException { + if (repository.isReadOnly()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository " + repository.getName() + + " is read only"); + } + + // Check if backup objects are valid + // This is just a pre-check to avoid most of invalid backup requests. + // Also calculate the signature for incremental backup check. + List tblRefs = stmt.getTableRefs(); + BackupMeta curBackupMeta = null; + db.readLock(); + try { + List
backupTbls = Lists.newArrayList(); + for (TableRef tblRef : tblRefs) { + String tblName = tblRef.getName().getTbl(); + Table tbl = db.getTable(tblName); + if (tbl == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tblName); + } + if (tbl.getType() != TableType.OLAP) { + ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, tblName); + } + + OlapTable olapTbl = (OlapTable) tbl; + if (tblRef.getPartitions() != null && !tblRef.getPartitions().isEmpty()) { + for (String partName : tblRef.getPartitions()) { + Partition partition = olapTbl.getPartition(partName); + if (partition == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Unknown partition " + partName + " in table" + tblName); } - iterator.remove(); - } - } else if (job instanceof RestoreJob) { - RestoreJob restoreJob = (RestoreJob) job; - if (restoreJob.getState() == RestoreJobState.FINISHED - || restoreJob.getState() == RestoreJobState.CANCELLED) { - finishedOrCancelledJobs.add(restoreJob); - - if (restoreJob.getState() == RestoreJobState.CANCELLED) { - // remove label - removeLabel(restoreJob.getDbId(), restoreJob.getLabel()); - } - iterator.remove(); } } - } - // clear historical jobs - Iterator iter = finishedOrCancelledJobs.iterator(); - while (iter.hasNext()) { - AbstractBackupJob job = iter.next(); - if ((System.currentTimeMillis() - job.getCreateTime()) / 1000 > Config.label_keep_max_second) { - iter.remove(); - LOG.info("remove history job[{}]. created at {}", job.getJobId(), - TimeUtils.longToTimeString(job.getCreateTime())); + // copy a table with selected partitions for calculating the signature + OlapTable copiedTbl = olapTbl.selectiveCopy(tblRef.getPartitions()); + if (copiedTbl == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to copy table " + tblName + " with selected partitions"); } + backupTbls.add(copiedTbl); } - + curBackupMeta = new BackupMeta(backupTbls); } finally { - writeUnlock(); - } - } - - public int getBackupJobNum(BackupJobState state, long dbId) { - int jobNum = 0; - readLock(); - try { - if (dbIdToBackupJob.containsKey(dbId)) { - BackupJob job = (BackupJob) dbIdToBackupJob.get(dbId); - if (job.getState() == state) { - ++jobNum; - } - } - - if (state == BackupJobState.FINISHED || state == BackupJobState.CANCELLED) { - for (AbstractBackupJob job : finishedOrCancelledBackupJobs) { - if (job.getDbId() != dbId) { - continue; - } - - if (((BackupJob) job).getState() == state) { - ++jobNum; - } - } - } - } finally { - readUnlock(); + db.readUnlock(); } - return jobNum; - } + // Check if label already be used + List existSnapshotNames = Lists.newArrayList(); + Status st = repository.listSnapshots(existSnapshotNames); + if (!st.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, st.getErrMsg()); + } + if (existSnapshotNames.contains(stmt.getLabel())) { + if (stmt.getType() == BackupType.FULL) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Snapshot with name '" + + stmt.getLabel() + "' already exist in repository"); + } else { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Currently does not support " + + "incremental backup"); - public int getRestoreJobNum(RestoreJobState state, long dbId) { - int jobNum = 0; - readLock(); - try { - if (dbIdToRestoreJob.containsKey(dbId)) { - RestoreJob job = (RestoreJob) dbIdToRestoreJob.get(dbId); - if (job.getState() == state) { - ++jobNum; + // TODO: + // This is a incremental backup, the existing snapshot in repository will be treated + // as base snapshot. + // But first we need to check if the existing snapshot has same meta. + List backupMetas = Lists.newArrayList(); + st = repository.getSnapshotMetaFile(stmt.getLabel(), backupMetas); + if (!st.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to get existing meta info for repository: " + + st.getErrMsg()); + } + Preconditions.checkState(backupMetas.size() == 1); + + if (!curBackupMeta.compatibleWith(backupMetas.get(0))) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Can not make incremental backup. Meta does not compatible"); } } - - if (state == RestoreJobState.FINISHED || state == RestoreJobState.CANCELLED) { - for (AbstractBackupJob job : finishedOrCancelledRestoreJobs) { - if (job.getDbId() != dbId) { - continue; - } - - if (((RestoreJob) job).getState() == state) { - ++jobNum; - } - } - } - } finally { - readUnlock(); } - return jobNum; + // Create a backup job + BackupJob backupJob = new BackupJob(stmt.getLabel(), db.getId(), + ClusterNamespace.getNameFromFullName(db.getFullName()), + tblRefs, stmt.getTimeoutMs(), + catalog, repository.getId()); + dbIdToBackupOrRestoreJob.put(db.getId(), backupJob); + + // write log + catalog.getEditLog().logBackupJob(backupJob); + + LOG.info("finished to submit backup job: {}", backupJob); } - public List> getJobInfosByDb(long dbId, Class jobClass, - PatternMatcher matcher) { - Map dbIdToJob = null; - List finishedOrCancelledJobs = null; - if (jobClass.equals(BackupJob.class)) { - dbIdToJob = dbIdToBackupJob; - finishedOrCancelledJobs = finishedOrCancelledBackupJobs; + private void restore(Repository repository, Database db, RestoreStmt stmt) throws DdlException { + // Check if snapshot exist in repository + List infos = Lists.newArrayList(); + Status status = repository.getSnapshotInfoFile(stmt.getLabel(), stmt.getBackupTimestamp(), infos); + if (!status.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to get info of snapshot '" + stmt.getLabel() + "' because: " + + status.getErrMsg() + ". Maybe specified wrong backup timestamp"); + } + + // Check if all restore objects are exist in this snapshot. + // Also remove all unrelated objs + Preconditions.checkState(infos.size() == 1); + BackupJobInfo jobInfo = infos.get(0); + checkAndFilterRestoreObjsExistInSnapshot(jobInfo, stmt.getTableRefs()); + + // Create a restore job + RestoreJob restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(), + db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicationNum(), + stmt.getTimeoutMs(), catalog, repository.getId()); + dbIdToBackupOrRestoreJob.put(db.getId(), restoreJob); + + catalog.getEditLog().logRestoreJob(restoreJob); + LOG.info("finished to submit restore job: {}", restoreJob); + } + + private void checkAndFilterRestoreObjsExistInSnapshot(BackupJobInfo jobInfo, List tblRefs) + throws DdlException { + Set allTbls = Sets.newHashSet(); + for (TableRef tblRef : tblRefs) { + String tblName = tblRef.getName().getTbl(); + if (!jobInfo.containsTbl(tblName)) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Table " + tblName + " does not exist in snapshot " + jobInfo.name); + } + BackupTableInfo tblInfo = jobInfo.getTableInfo(tblName); + if (tblRef.getPartitions() != null && !tblRef.getPartitions().isEmpty()) { + // check the selected partitions + for (String partName : tblRef.getPartitions()) { + if (!tblInfo.containsPart(partName)) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Partition " + partName + " of table " + tblName + + " does not exist in snapshot " + jobInfo.name); + } + } + } + + // set alias + if (tblRef.hasExplicitAlias()) { + jobInfo.setAlias(tblName, tblRef.getExplicitAlias()); + } + + // only retain restore partitions + tblInfo.retainPartitions(tblRef.getPartitions()); + allTbls.add(tblName); + } + + // only retain restore tables + jobInfo.retainTables(allTbls); + } + + public void cancel(CancelBackupStmt stmt) throws DdlException { + String dbName = stmt.getDbName(); + Database db = catalog.getDb(dbName); + if (db == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } + + AbstractJob job = dbIdToBackupOrRestoreJob.get(db.getId()); + if (job == null || (job instanceof BackupJob && stmt.isRestore()) + || (job instanceof RestoreJob && !stmt.isRestore())) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "No " + + (stmt.isRestore() ? "restore" : "backup" + " job") + + " is currently running"); + } + + Status status = job.cancel(); + if (!status.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Failed to cancel job: " + status.getErrMsg()); + } + + LOG.info("finished to cancel {} job: {}", (stmt.isRestore() ? "restore" : "backup"), job); + } + + public boolean handleFinishedSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { + AbstractJob job = dbIdToBackupOrRestoreJob.get(task.getDbId()); + if (job == null) { + LOG.warn("failed to find backup or restore job for task: {}", task); + // return true to remove this task from AgentTaskQueue + return true; + } + if (job instanceof BackupJob) { + if (task.isRestoreTask()) { + LOG.warn("expect finding restore job, but get backup job {} for task: {}", job, task); + // return true to remove this task from AgentTaskQueue + return true; + } + + return ((BackupJob) job).finishTabletSnapshotTask(task, request); } else { - dbIdToJob = dbIdToRestoreJob; - finishedOrCancelledJobs = finishedOrCancelledRestoreJobs; - } - - List> jobInfos = new LinkedList>(); - readLock(); - try { - AbstractBackupJob abstractJob = dbIdToJob.get(dbId); - if (abstractJob != null) { - List jobInfo = abstractJob.getJobInfo(); - if (matcher != null) { - String label = jobInfo.get(1).toString(); - if (matcher.match(label)) { - jobInfos.add(jobInfo); - } - } else { - jobInfos.add(jobInfo); - } + if (!task.isRestoreTask()) { + LOG.warn("expect finding backup job, but get restore job {} for task: {}", job, task); + // return true to remove this task from AgentTaskQueue + return true; } + return ((RestoreJob) job).finishTabletSnapshotTask(task, request); + } + } - for (AbstractBackupJob job : finishedOrCancelledJobs) { - if (job.getDbId() != dbId) { - continue; - } - List jobInfo = job.getJobInfo(); - if (matcher != null) { - String label = jobInfo.get(1).toString(); - if (matcher.match(label)) { - jobInfos.add(jobInfo); - } - } else { - jobInfos.add(jobInfo); - } + public boolean handleFinishedSnapshotUploadTask(UploadTask task, TFinishTaskRequest request) { + AbstractJob job = dbIdToBackupOrRestoreJob.get(task.getDbId()); + if (job == null || (job instanceof RestoreJob)) { + LOG.info("invalid upload task: {}, no backup job is found. db id: {}", task, task.getDbId()); + return false; + } + BackupJob restoreJob = (BackupJob) job; + if (restoreJob.getJobId() != task.getJobId() || restoreJob.getState() != BackupJobState.UPLOADING) { + LOG.info("invalid upload task: {}, job id: {}, job state: {}", + task, restoreJob.getJobId(), restoreJob.getState().name()); + return false; + } + return restoreJob.finishSnapshotUploadTask(task, request); + } + + public boolean handleDownloadSnapshotTask(DownloadTask task, TFinishTaskRequest request) { + AbstractJob job = dbIdToBackupOrRestoreJob.get(task.getDbId()); + if (job == null || !(job instanceof RestoreJob)) { + LOG.warn("failed to find restore job for task: {}", task); + // return true to remove this task from AgentTaskQueue + return true; + } + + return ((RestoreJob) job).finishTabletDownloadTask(task, request); + } + + public boolean handleDirMoveTask(DirMoveTask task, TFinishTaskRequest request) { + AbstractJob job = dbIdToBackupOrRestoreJob.get(task.getDbId()); + if (job == null || !(job instanceof RestoreJob)) { + LOG.warn("failed to find restore job for task: {}", task); + // return true to remove this task from AgentTaskQueue + return true; + } + + return ((RestoreJob) job).finishDirMoveTask(task, request); + } + + public void replayAddJob(AbstractJob job) { + if (job.isCancelled()) { + AbstractJob existingJob = dbIdToBackupOrRestoreJob.get(job.getDbId()); + if (existingJob == null || existingJob.isDone()) { + LOG.error("invalid existing job: {}. current replay job is: {}", + existingJob, job); + return; } - } finally { - readUnlock(); - } - return jobInfos; - } - - public List> getJobUnfinishedTablet(long dbId, Class jobClass) { - Map dbIdToJob = null; - if (jobClass.equals(BackupJob.class)) { - dbIdToJob = dbIdToBackupJob; - } else { - dbIdToJob = dbIdToRestoreJob; - } - - List> jobInfos = Lists.newArrayList(); - readLock(); - try { - AbstractBackupJob abstractJob = dbIdToJob.get(dbId); - if (abstractJob != null) { - jobInfos = abstractJob.getUnfinishedInfos(); + existingJob.setCatalog(catalog); + existingJob.replayCancel(); + } else if (!job.isPending()) { + AbstractJob existingJob = dbIdToBackupOrRestoreJob.get(job.getDbId()); + if (existingJob == null || existingJob.isDone()) { + LOG.error("invalid existing job: {}. current replay job is: {}", + existingJob, job); + return; } - } finally { - readUnlock(); + // We use replayed job, not the existing job, to do the replayRun(). + // Because if we use the existing job to run again, + // for example: In restore job, PENDING will transfer to SNAPSHOTING, not DOWNLOAD. + job.replayRun(); } - return jobInfos; + dbIdToBackupOrRestoreJob.put(job.getDbId(), job); } - private void setTableState(Catalog catalog, BackupJob job) { - Database db = catalog.getDb(job.getDbId()); - db.writeLock(); - try { - for (long tableId : job.getTableIdToPartitionIds().keySet()) { - Table table = db.getTable(tableId); - if (table.getType() == TableType.OLAP) { - ((OlapTable) table).setState(OlapTableState.BACKUP); - } - } - } finally { - db.writeUnlock(); + public static BackupHandler read(DataInput in) throws IOException { + BackupHandler backupHandler = new BackupHandler(); + backupHandler.readFields(in); + return backupHandler; + } + + @Override + public void write(DataOutput out) throws IOException { + repoMgr.write(out); + + out.writeInt(dbIdToBackupOrRestoreJob.size()); + for (AbstractJob job : dbIdToBackupOrRestoreJob.values()) { + job.write(out); } - LOG.info("finished set backup tables' state to BACKUP. job: {}", job.getJobId()); } - private void setTableState(Catalog catalog, RestoreJob job) { - Database db = catalog.getDb(job.getDbId()); - db.writeLock(); - try { - for (String tableName : job.getTableToPartitionNames().keySet()) { - Table table = db.getTable(tableName); - if (table != null && table.getType() == TableType.OLAP) { - ((OlapTable) table).setState(OlapTableState.RESTORE); - } - } - } finally { - db.writeUnlock(); + @Override + public void readFields(DataInput in) throws IOException { + repoMgr = RepositoryMgr.read(in); + + int size = in.readInt(); + for (int i = 0; i < size; i++) { + AbstractJob job = AbstractJob.read(in); + dbIdToBackupOrRestoreJob.put(job.getDbId(), job); } - LOG.info("finished set restore tables' state to RESTORE. job: {}", job.getJobId()); - } - - public void replayBackupStart(Catalog catalog, BackupJob job) { - setTableState(catalog, job); - - writeLock(); - try { - dbIdToLabels.put(job.getDbId(), job.getLabel()); - Preconditions.checkState(!dbIdToBackupJob.containsKey(job.getDbId())); - dbIdToBackupJob.put(job.getDbId(), job); - LOG.debug("replay start backup job, put {} to map", job.getDbId()); - } finally { - writeUnlock(); - } - - LOG.info("finished replay start backup job[{}]", job.getJobId()); - } - - public void replayBackupFinishSnapshot(BackupJob job) { - Preconditions.checkState(job.getState() == BackupJobState.UPLOAD); - writeLock(); - try { - long dbId = job.getDbId(); - BackupJob currentJob = (BackupJob) dbIdToBackupJob.get(dbId); - Preconditions.checkState(currentJob.getJobId() == job.getJobId()); - dbIdToBackupJob.remove(dbId); - dbIdToBackupJob.put(dbId, job); - } finally { - writeUnlock(); - } - - LOG.info("finished replay backup finish snapshot. job[{}]", job.getJobId()); - } - - public void replayBackupFinish(Catalog catalog, BackupJob job) { - job.end(catalog, true); - - writeLock(); - try { - BackupJob currentJob = (BackupJob) dbIdToBackupJob.remove(job.getDbId()); - Preconditions.checkNotNull(currentJob, job.getDbId()); - - finishedOrCancelledBackupJobs.add(job); - if (job.getState() == BackupJobState.CANCELLED) { - removeLabel(job.getDbId(), job.getLabel()); - } - } finally { - writeUnlock(); - } - - LOG.info("finished replay backup job finish. job: {}", job.getJobId()); - } - - public void replayRestoreStart(Catalog catalog, RestoreJob job) { - setTableState(catalog, job); - writeLock(); - try { - dbIdToLabels.put(job.getDbId(), job.getLabel()); - Preconditions.checkState(!dbIdToRestoreJob.containsKey(job.getDbId())); - dbIdToRestoreJob.put(job.getDbId(), job); - LOG.debug("replay start restore job, put {} to map", job.getDbId()); - } finally { - writeUnlock(); - } - - LOG.info("finished replay start restore job[{}]", job.getJobId()); - } - - public void replayRestoreFinish(Catalog catalog, RestoreJob job) { - try { - job.finishing(catalog, true); - job.end(catalog, true); - } catch (DdlException e) { - LOG.error("should not happend", e); - } - - writeLock(); - try { - RestoreJob currentJob = (RestoreJob) dbIdToRestoreJob.remove(job.getDbId()); - Preconditions.checkNotNull(currentJob, job.getDbId()); - - finishedOrCancelledRestoreJobs.add(job); - if (job.getState() == RestoreJobState.CANCELLED) { - removeLabel(job.getDbId(), job.getLabel()); - } - } finally { - writeUnlock(); - } - - LOG.info("finished replay restore job finish. job: {}", job.getJobId()); - } - - public Map unprotectedGetBackupJobs() { - return dbIdToBackupJob; - } - - public List unprotectedGetFinishedOrCancelledBackupJobs() { - return finishedOrCancelledBackupJobs; - } - - public Map unprotectedGetRestoreJobs() { - return dbIdToRestoreJob; - } - - public List unprotectedGetFinishedOrCancelledRestoreJobs() { - return finishedOrCancelledRestoreJobs; - } - - public Multimap unprotectedGetDbIdToLabels() { - return dbIdToLabels; } } + + diff --git a/fe/src/com/baidu/palo/backup/BackupJob.java b/fe/src/com/baidu/palo/backup/BackupJob.java index 14c8377c57..6c8a5030d5 100644 --- a/fe/src/com/baidu/palo/backup/BackupJob.java +++ b/fe/src/com/baidu/palo/backup/BackupJob.java @@ -1,62 +1,37 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - package com.baidu.palo.backup; -import com.baidu.palo.analysis.AlterTableStmt; -import com.baidu.palo.analysis.CreateTableStmt; -import com.baidu.palo.analysis.LabelName; +import com.baidu.palo.analysis.TableRef; +import com.baidu.palo.backup.Status.ErrCode; +import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.MaterializedIndex; import com.baidu.palo.catalog.OlapTable; -import com.baidu.palo.catalog.OlapTable.OlapTableState; import com.baidu.palo.catalog.Partition; -import com.baidu.palo.catalog.PartitionInfo; -import com.baidu.palo.catalog.PartitionKey; -import com.baidu.palo.catalog.PartitionType; -import com.baidu.palo.catalog.RangePartitionInfo; import com.baidu.palo.catalog.Replica; import com.baidu.palo.catalog.Table; import com.baidu.palo.catalog.Table.TableType; import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.Pair; import com.baidu.palo.common.io.Text; -import com.baidu.palo.common.io.Writable; -import com.baidu.palo.common.util.LoadBalancer; import com.baidu.palo.common.util.TimeUtils; -import com.baidu.palo.common.util.Util; -import com.baidu.palo.load.DeleteInfo; -import com.baidu.palo.load.Load; -import com.baidu.palo.load.LoadJob; import com.baidu.palo.task.AgentBatchTask; import com.baidu.palo.task.AgentTask; import com.baidu.palo.task.AgentTaskExecutor; import com.baidu.palo.task.AgentTaskQueue; -import com.baidu.palo.task.ReleaseSnapshotTask; import com.baidu.palo.task.SnapshotTask; import com.baidu.palo.task.UploadTask; +import com.baidu.palo.thrift.TFinishTaskRequest; +import com.baidu.palo.thrift.TStatusCode; import com.baidu.palo.thrift.TTaskType; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import com.google.common.collect.HashMultimap; +import com.google.common.base.Predicates; +import com.google.common.base.Strings; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; @@ -66,810 +41,711 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.File; import java.io.IOException; -import java.util.Collection; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; -public class BackupJob extends AbstractBackupJob { + +public class BackupJob extends AbstractJob { private static final Logger LOG = LogManager.getLogger(BackupJob.class); - private static final long SNAPSHOT_TIMEOUT_MS = 2000; // 1s for one tablet - public enum BackupJobState { - PENDING, - SNAPSHOT, - UPLOAD, - UPLOADING, - FINISHING, - FINISHED, - CANCELLED + PENDING, // Job is newly created. Send snapshot tasks and save copied meta info, then transfer to SNAPSHOTING + SNAPSHOTING, // Wait for finishing snapshot tasks. When finished, transfer to UPLOAD_SNAPSHOT + UPLOAD_SNAPSHOT, // Begin to send upload task to BE, then transfer to UPLOADING + UPLOADING, // Wait for finishing upload tasks. When finished, transfer to SAVE_META + SAVE_META, // Save copied meta info to local file. When finished, transfer to UPLOAD_INFO + UPLOAD_INFO, // Upload meta and job info file to repository. When finished, transfer to FINISHED + FINISHED, // Job is finished. + CANCELLED // Job is cancelled. } + // all objects which need backup + private List tableRefs = Lists.newArrayList(); + private BackupJobState state; - private String lastestLoadLabel; - private DeleteInfo lastestDeleteInfo; + private long snapshotFinishedTime = -1; + private long snapshopUploadFinishedTime = -1; - // all partitions need to be backuped - private Map> tableIdToPartitionIds; - private Multimap tableIdToIndexIds; - // partition id -> (version, version hash) - private Map> partitionIdToVersionInfo; - - private Map> tabletIdToSnapshotPath; + // save all tablets which tasks are not finished. + private Set unfinishedTaskIds = Sets.newConcurrentHashSet(); + // tablet id -> snapshot info + private Map snapshotInfos = Maps.newConcurrentMap(); + // save all related table[partition] info + private BackupMeta backupMeta; + // job info file content + private BackupJobInfo jobInfo; - private long metaSavedTime; - private long snapshotFinishedTime; - private long uploadFinishedTime; - - private long phasedTimeoutMs; - - private String readableManifestPath; + // save the local dir of this backup job + // after job is done, this dir should be deleted + private Path localJobDirPath = null; + // save the local file path of meta info and job info file + private String localMetaInfoFilePath = null; + private String localJobInfoFilePath = null; public BackupJob() { - super(); - tableIdToPartitionIds = Maps.newHashMap(); - tableIdToIndexIds = HashMultimap.create(); - partitionIdToVersionInfo = Maps.newHashMap(); - tabletIdToSnapshotPath = Maps.newHashMap(); + super(JobType.BACKUP); } - public BackupJob(long jobId, long dbId, LabelName labelName, String backupPath, - Map remoteProperties) { - super(jobId, dbId, labelName, backupPath, remoteProperties); + public BackupJob(String label, long dbId, String dbName, List tableRefs, long timeoutMs, + Catalog catalog, long repoId) { + super(JobType.BACKUP, label, dbId, dbName, timeoutMs, catalog, repoId); + this.tableRefs = tableRefs; this.state = BackupJobState.PENDING; - - tableIdToPartitionIds = Maps.newHashMap(); - tableIdToIndexIds = HashMultimap.create(); - partitionIdToVersionInfo = Maps.newHashMap(); - - tabletIdToSnapshotPath = Maps.newHashMap(); - - metaSavedTime = -1; - snapshotFinishedTime = -1; - uploadFinishedTime = -1; - phasedTimeoutMs = -1; - - lastestLoadLabel = "N/A"; - readableManifestPath = ""; - } - - public void setState(BackupJobState state) { - this.state = state; } public BackupJobState getState() { return state; } - public String getLatestLoadLabel() { - return lastestLoadLabel; + public BackupMeta getBackupMeta() { + return backupMeta; } - public DeleteInfo getLastestDeleteInfo() { - return lastestDeleteInfo; - } - - public PathBuilder getPathBuilder() { - return pathBuilder; - } - - public long getMetaSavedTimeMs() { - return metaSavedTime; - } - - public long getSnapshotFinishedTimeMs() { - return snapshotFinishedTime; - } - - public long getUploadFinishedTimeMs() { - return uploadFinishedTime; - } - - public String getReadableManifestPath() { - return readableManifestPath; - } - - public Map> getTableIdToPartitionIds() { - return tableIdToPartitionIds; - } - - public void addPartitionId(long tableId, long partitionId) { - Set partitionIds = tableIdToPartitionIds.get(tableId); - if (partitionIds == null) { - partitionIds = Sets.newHashSet(); - tableIdToPartitionIds.put(tableId, partitionIds); - } - if (partitionId != -1L) { - partitionIds.add(partitionId); - } - - LOG.debug("add partition[{}] from table[{}], job[{}]", partitionId, tableId, jobId); - } - - public void addIndexId(long tableId, long indexId) { - tableIdToIndexIds.put(tableId, indexId); - LOG.debug("add index[{}] from table[{}], job[{}]", indexId, tableId, jobId); - } - - public void handleFinishedSnapshot(long tabletId, long backendId, String snapshotPath) { - synchronized (unfinishedTabletIds) { - if (!unfinishedTabletIds.containsKey(tabletId)) { - LOG.warn("backup job[{}] does not contains tablet[{}]", jobId, tabletId); - return; - } - - if (unfinishedTabletIds.get(tabletId) == null - || !unfinishedTabletIds.get(tabletId).contains(backendId)) { - LOG.warn("backup job[{}] does not contains tablet[{}]'s snapshot from backend[{}]. " - + "it should from backend[{}]", - jobId, tabletId, backendId, unfinishedTabletIds.get(tabletId)); - return; - } - unfinishedTabletIds.remove(tabletId, backendId); - } - - synchronized (tabletIdToSnapshotPath) { - tabletIdToSnapshotPath.put(tabletId, new Pair(backendId, snapshotPath)); - } - LOG.debug("finished add tablet[{}] from backend[{}]. snapshot path: {}", tabletId, backendId, snapshotPath); - } - - public void handleFinishedUpload(long tabletId, long backendId) { - synchronized (unfinishedTabletIds) { - if (unfinishedTabletIds.remove(tabletId, backendId)) { - LOG.debug("finished upload tablet[{}] snapshot, backend[{}]", tabletId, backendId); - } - } - } - - @Override - public List getJobInfo() { - List jobInfo = Lists.newArrayList(); - jobInfo.add(jobId); - jobInfo.add(getLabel()); - jobInfo.add(state.name()); - jobInfo.add(TimeUtils.longToTimeString(createTime)); - jobInfo.add(TimeUtils.longToTimeString(metaSavedTime)); - jobInfo.add(TimeUtils.longToTimeString(snapshotFinishedTime)); - jobInfo.add(TimeUtils.longToTimeString(uploadFinishedTime)); - jobInfo.add(TimeUtils.longToTimeString(finishedTime)); - jobInfo.add(errMsg); - jobInfo.add(PathBuilder.createPath(remotePath, getLabel())); - jobInfo.add(getReadableManifestPath()); - jobInfo.add(getLeftTasksNum()); - jobInfo.add(getLatestLoadLabel()); + public BackupJobInfo getJobInfo() { return jobInfo; } - @Override - public void runOnce() { - LOG.debug("begin to run backup job: {}, state: {}", jobId, state.name()); - try { - switch (state) { - case PENDING: - saveMetaAndMakeSnapshot(); - break; - case SNAPSHOT: - waitSnapshot(); - break; - case UPLOAD: - upload(); - break; - case UPLOADING: - waitUpload(); - break; - case FINISHING: - finishing(); - break; - default: - break; + public String getLocalJobInfoFilePath() { + return localJobInfoFilePath; + } + + public String getLocalMetaInfoFilePath() { + return localMetaInfoFilePath; + } + + public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { + Preconditions.checkState(task.getJobId() == jobId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + return false; + } + + Preconditions.checkState(request.isSetSnapshot_path()); + Preconditions.checkState(request.isSetSnapshot_files()); + // snapshot path does not contains last 'tablet_id' and 'schema_hash' dir + // eg: + // /path/to/your/be/data/snapshot/20180410102311.0/ + // Full path will look like: + // /path/to/your/be/data/snapshot/20180410102311.0/10006/352781111/ + SnapshotInfo info = new SnapshotInfo(task.getDbId(), task.getTableId(), task.getPartitionId(), + task.getIndexId(), task.getTabletId(), task.getBackendId(), + task.getSchemaHash(), request.getSnapshot_path(), + request.getSnapshot_files()); + + snapshotInfos.put(task.getTabletId(), info); + boolean res = unfinishedTaskIds.remove(task.getTabletId()); + taskErrMsg.remove(task.getTabletId()); + LOG.debug("get finished snapshot info: {}, unfinished tasks num: {}, remove result: {}. {}", + info, unfinishedTaskIds.size(), res, this); + + return res; + } + + public synchronized boolean finishSnapshotUploadTask(UploadTask task, TFinishTaskRequest request) { + Preconditions.checkState(task.getJobId() == jobId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + return false; + } + + Preconditions.checkState(request.isSetTablet_files()); + Map> tabletFileMap = request.getTablet_files(); + if (tabletFileMap.isEmpty()) { + LOG.warn("upload snapshot files failed because nothing is uploaded. be: {}. {}", + task.getBackendId(), this); + return false; + } + + // remove checksum suffix in reported file name before checking files + Map> newTabletFileMap = Maps.newHashMap(); + for (Map.Entry> entry : tabletFileMap.entrySet()) { + List files = entry.getValue().stream() + .map(name -> Repository.decodeFileNameWithChecksum(name).first).collect(Collectors.toList()); + newTabletFileMap.put(entry.getKey(), files); + } + + // check if uploaded files are correct + for (long tabletId : newTabletFileMap.keySet()) { + SnapshotInfo info = snapshotInfos.get(tabletId); + List tabletFiles = info.getFiles(); + List uploadedFiles = newTabletFileMap.get(tabletId); + + if (tabletFiles.size() != uploadedFiles.size()) { + LOG.warn("upload snapshot files failed because file num is wrong. " + + "expect: {}, actual:{}, tablet: {}, be: {}. {}", + tabletFiles.size(), uploadedFiles.size(), tabletId, task.getBackendId(), this); + return false; + } + + if (!Collections2.filter(tabletFiles, Predicates.not(Predicates.in(uploadedFiles))).isEmpty()) { + LOG.warn("upload snapshot files failed because file is different. " + + "expect: [{}], actual: [{}], tablet: {}, be: {}. {}", + tabletFiles, uploadedFiles, tabletId, task.getBackendId(), this); + return false; + } + + // reset files in snapshot info with checksum filename + info.setFiles(tabletFileMap.get(tabletId)); + } + + boolean res = unfinishedTaskIds.remove(task.getSignature()); + taskErrMsg.remove(task.getTabletId()); + LOG.debug("get finished upload snapshot task, unfinished tasks num: {}, remove result: {}. {}", + unfinishedTaskIds.size(), res, this); + return res; + } + + @Override + public synchronized void replayRun() { + // Backup process does not change any current catalog state, + // So nothing need to be done when replaying log + } + + @Override + public synchronized void replayCancel() { + // nothing to do + } + + @Override + public boolean isPending() { + return state == BackupJobState.PENDING; + } + + @Override + public boolean isCancelled() { + return state == BackupJobState.CANCELLED; + } + + // Polling the job state and do the right things. + @Override + public synchronized void run() { + if (state == BackupJobState.FINISHED || state == BackupJobState.CANCELLED) { + return; + } + + // check timeout + if (System.currentTimeMillis() - createTime > timeoutMs) { + status = new Status(ErrCode.TIMEOUT, ""); + cancelInternal(); + return; + } + + // get repo if not set + if (repo == null) { + repo = catalog.getBackupHandler().getRepoMgr().getRepo(repoId); + if (repo == null) { + status = new Status(ErrCode.COMMON_ERROR, "failed to get repository: " + repoId); + cancelInternal(); + return; } - } catch (Exception e) { - errMsg = e.getMessage() == null ? "Unknown Exception" : e.getMessage(); - LOG.warn("failed to backup: " + errMsg + ", job[" + jobId + "]", e); - state = BackupJobState.CANCELLED; } + LOG.debug("run backup job: {}", this); + + // run job base on current state + switch (state) { + case PENDING: + prepareAndSendSnapshotTask(); + break; + case SNAPSHOTING: + waitingAllSnapshotsFinished(); + break; + case UPLOAD_SNAPSHOT: + uploadSnapshot(); + break; + case UPLOADING: + waitingAllUploadingFinished(); + break; + case SAVE_META: + saveMetaInfo(); + break; + case UPLOAD_INFO: + uploadMetaAndJobInfoFile(); + break; + default: + break; + } + + if (!status.ok()) { + cancelInternal(); + } + } + + // cancel by user + @Override + public synchronized Status cancel() { + if (isDone()) { + return new Status(ErrCode.COMMON_ERROR, + "Job with label " + label + " can not be cancelled. state: " + state); + } + + status = new Status(ErrCode.COMMON_ERROR, "user cancelled"); + cancelInternal(); + return Status.OK; + } + + @Override + public synchronized boolean isDone() { if (state == BackupJobState.FINISHED || state == BackupJobState.CANCELLED) { - end(Catalog.getInstance(), false); + return true; } + return false; } - private void saveMetaAndMakeSnapshot() throws DdlException, IOException { - Database db = Catalog.getInstance().getDb(dbId); + private void prepareAndSendSnapshotTask() { + Database db = catalog.getDb(dbId); if (db == null) { - throw new DdlException("[" + getDbName() + "] does not exist"); - } - - try { - pathBuilder = PathBuilder.createPathBuilder(getLocalDirName()); - } catch (IOException e) { - pathBuilder = null; - throw e; - } - - // file path -> writable objs - Map> pathToWritables = Maps.newHashMap(); - // 1. get meta - getMeta(db, pathToWritables); - - // 2. write meta - // IO ops should be done outside db.lock - try { - writeMeta(pathToWritables); - } catch (IOException e) { - errMsg = e.getMessage(); - state = BackupJobState.CANCELLED; + status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); return; } - metaSavedTime = System.currentTimeMillis(); - LOG.info("save meta finished. path: {}, job: {}", pathBuilder.getRoot().getFullPath(), jobId); - - // 3. send snapshot tasks - snapshot(db); - } - - private void getMeta(Database db, Map> pathToWritables) throws DdlException { - db.readLock(); - try { - for (long tableId : tableIdToPartitionIds.keySet()) { - Table table = db.getTable(tableId); - if (table == null) { - throw new DdlException("table[" + tableId + "] does not exist"); - } - - // 1. get table meta - getTableMeta(db.getFullName(), table, pathToWritables); - - if (table.getType() != TableType.OLAP) { - // this is not a OLAP table. just save table meta - continue; - } - - OlapTable olapTable = (OlapTable) table; - - // 2. get rollup meta - // 2.1 check all indices exist - for (Long indexId : tableIdToIndexIds.get(tableId)) { - if (olapTable.getIndexNameById(indexId) == null) { - errMsg = "Index[" + indexId + "] does not exist"; - state = BackupJobState.CANCELLED; - return; - } - } - getRollupMeta(db.getFullName(), olapTable, pathToWritables); - - // 3. save partition meta - Collection partitionIds = tableIdToPartitionIds.get(tableId); - PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - if (partitionInfo.getType() == PartitionType.RANGE) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - List>> rangeMap = rangePartitionInfo.getSortedRangeMap(); - for (Map.Entry> entry : rangeMap) { - long partitionId = entry.getKey(); - if (!partitionIds.contains(partitionId)) { - continue; - } - - Partition partition = olapTable.getPartition(partitionId); - if (partition == null) { - throw new DdlException("partition[" + partitionId + "] does not exist"); - } - getPartitionMeta(db.getFullName(), olapTable, partition.getName(), pathToWritables); - - // save version info - partitionIdToVersionInfo.put(partitionId, - new Pair(partition.getCommittedVersion(), - partition.getCommittedVersionHash())); - } - } else { - Preconditions.checkState(partitionIds.size() == 1); - for (Long partitionId : partitionIds) { - Partition partition = olapTable.getPartition(partitionId); - // save version info - partitionIdToVersionInfo.put(partitionId, - new Pair(partition.getCommittedVersion(), - partition.getCommittedVersionHash())); - } - } - } // end for tables - - // get last finished load job and delele job label - Load load = Catalog.getInstance().getLoadInstance(); - LoadJob lastestLoadJob = load.getLastestFinishedLoadJob(dbId); - if (lastestLoadJob == null) { - // there is no load job, or job info has been removed - lastestLoadLabel = "N/A"; - } else { - lastestLoadLabel = lastestLoadJob.getLabel(); - } - LOG.info("get lastest load job label: {}, job: {}", lastestLoadJob, jobId); - - lastestDeleteInfo = load.getLastestFinishedDeleteInfo(dbId); - LOG.info("get lastest delete info: {}, job: {}", lastestDeleteInfo, jobId); - - LOG.info("get meta finished. job[{}]", jobId); - } finally { - db.readUnlock(); - } - } - - private void getTableMeta(String dbName, Table table, Map> pathToWritables) { - CreateTableStmt stmt = table.toCreateTableStmt(dbName); - int tableSignature = table.getSignature(BackupVersion.VERSION_1); - stmt.setTableSignature(tableSignature); - List stmts = Lists.newArrayList(stmt); - String filePath = pathBuilder.createTableStmt(dbName, table.getName()); - - Preconditions.checkState(!pathToWritables.containsKey(filePath)); - pathToWritables.put(filePath, stmts); - } - - private void getRollupMeta(String dbName, OlapTable olapTable, - Map> pathToWritables) { - Set indexIds = Sets.newHashSet(tableIdToIndexIds.get(olapTable.getId())); - if (indexIds.size() == 1) { - // only contains base index. do nothing - return; - } else { - // remove base index id - Preconditions.checkState(indexIds.size() > 1); - indexIds.remove(olapTable.getId()); - } - AlterTableStmt stmt = olapTable.toAddRollupStmt(dbName, indexIds); - String filePath = pathBuilder.addRollupStmt(dbName, olapTable.getName()); - List stmts = Lists.newArrayList(stmt); - - Preconditions.checkState(!pathToWritables.containsKey(filePath)); - pathToWritables.put(filePath, stmts); - } - - private void getPartitionMeta(String dbName, OlapTable olapTable, String partitionName, - Map> pathToWritables) { - AlterTableStmt stmt = olapTable.toAddPartitionStmt(dbName, partitionName); - String filePath = pathBuilder.addPartitionStmt(dbName, olapTable.getName(), partitionName); - List stmts = Lists.newArrayList(stmt); - - Preconditions.checkState(!pathToWritables.containsKey(filePath)); - pathToWritables.put(filePath, stmts); - } - - private void writeMeta(Map> pathToWritables) throws IOException { - // 1. write meta - for (Map.Entry> entry : pathToWritables.entrySet()) { - String filePath = entry.getKey(); - List writables = entry.getValue(); - ObjectWriter.write(filePath, writables); - } - } - - private void snapshot(Database db) throws DdlException { + // generate job id + jobId = catalog.getNextId(); AgentBatchTask batchTask = new AgentBatchTask(); - LoadBalancer loadBalancer = new LoadBalancer(1L); - long dbId = db.getId(); db.readLock(); try { - for (Map.Entry> entry : tableIdToPartitionIds.entrySet()) { - long tableId = entry.getKey(); - Set partitionIds = entry.getValue(); - - Table table = db.getTable(tableId); - if (table == null) { - throw new DdlException("table[" + tableId + "] does not exist"); + // check all backup tables again + for (TableRef tableRef : tableRefs) { + String tblName = tableRef.getName().getTbl(); + Table tbl = db.getTable(tblName); + if (tbl == null) { + status = new Status(ErrCode.NOT_FOUND, "table " + tblName + " does not exist"); + return; + } + if (tbl.getType() != TableType.OLAP) { + status = new Status(ErrCode.COMMON_ERROR, "table " + tblName + " is not OLAP table"); + return; } - if (table.getType() != TableType.OLAP) { - continue; + OlapTable olapTbl = (OlapTable) tbl; + if (tableRef.getPartitions() != null && !tableRef.getPartitions().isEmpty()) { + for (String partName : tableRef.getPartitions()) { + Partition partition = olapTbl.getPartition(partName); + if (partition == null) { + status = new Status(ErrCode.NOT_FOUND, "partition " + partName + + " does not exist in table" + tblName); + return; + } + } + } + } + + unfinishedTaskIds.clear(); + taskErrMsg.clear(); + // create snapshot tasks + for (TableRef tblRef : tableRefs) { + String tblName = tblRef.getName().getTbl(); + OlapTable tbl = (OlapTable) db.getTable(tblName); + List partitions = Lists.newArrayList(); + if (tblRef.getPartitions() == null || tblRef.getPartitions().isEmpty()) { + partitions.addAll(tbl.getPartitions()); + } else { + for (String partName : tblRef.getPartitions()) { + Partition partition = tbl.getPartition(partName); + partitions.add(partition); + } } - OlapTable olapTable = (OlapTable) table; - - for (Long partitionId : partitionIds) { - Partition partition = olapTable.getPartition(partitionId); - if (partition == null) { - throw new DdlException("partition[" + partitionId + "] does not exist"); + // snapshot partitions + for (Partition partition : partitions) { + long committedVersion = partition.getCommittedVersion(); + long committedVersionHash = partition.getCommittedVersionHash(); + List indexes = partition.getMaterializedIndices(); + for (MaterializedIndex index : indexes) { + int schemaHash = tbl.getSchemaHashByIndexId(index.getId()); + List tablets = index.getTablets(); + for (Tablet tablet : tablets) { + Replica replica = chooseReplica(tablet, committedVersion, committedVersionHash); + if (replica == null) { + status = new Status(ErrCode.COMMON_ERROR, + "faild to choose replica to make snapshot for tablet " + tablet.getId() + + ". committed version: " + committedVersion + + ", committed version hash: " + committedVersionHash); + return; + } + SnapshotTask task = new SnapshotTask(null, replica.getBackendId(), tablet.getId(), + jobId, dbId, tbl.getId(), partition.getId(), + index.getId(), tablet.getId(), + committedVersion, committedVersionHash, + schemaHash, timeoutMs, false /* not restore task */); + batchTask.addTask(task); + unfinishedTaskIds.add(tablet.getId()); + } } - Pair versionInfo = partitionIdToVersionInfo.get(partitionId); - for (Long indexId : tableIdToIndexIds.get(tableId)) { - int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - MaterializedIndex index = partition.getIndex(indexId); - if (index == null) { - throw new DdlException("index[" + indexId + "] does not exist"); - } - - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - List backendIds = Lists.newArrayList(); - for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(versionInfo.first, versionInfo.second)) { - backendIds.add(replica.getBackendId()); - } - } - - if (backendIds.isEmpty()) { - String msg = "tablet[" + tabletId + "] does not check up with version: " - + versionInfo.first + "-" + versionInfo.second; - // this should not happen - LOG.error(msg); - throw new DdlException(msg); - } - - long chosenBackendId = loadBalancer.chooseKey(backendIds); - SnapshotTask task = new SnapshotTask(null, chosenBackendId, jobId, dbId, tableId, - partitionId, indexId, tabletId, - versionInfo.first, versionInfo.second, - schemaHash, -1L); - LOG.debug("choose backend[{}] to make snapshot for tablet[{}]", chosenBackendId, tabletId); - batchTask.addTask(task); - unfinishedTabletIds.put(tabletId, chosenBackendId); - } // end for tablet - } // end for indices - } // end for partitions - } // end for tables + LOG.info("snapshot for partition {}, version: {}, version hash: {}", + partition.getId(), committedVersion, committedVersionHash); + } + } + // copy all related schema at this moment + List
copiedTables = Lists.newArrayList(); + for (TableRef tableRef : tableRefs) { + String tblName = tableRef.getName().getTbl(); + OlapTable tbl = (OlapTable) db.getTable(tblName); + OlapTable copiedTbl = tbl.selectiveCopy(tableRef.getPartitions()); + if (copiedTbl == null) { + status = new Status(ErrCode.COMMON_ERROR, "faild to copy table: " + tblName); + return; + } + copiedTables.add(copiedTbl); + } + backupMeta = new BackupMeta(copiedTables); } finally { db.readUnlock(); } - phasedTimeoutMs = unfinishedTabletIds.size() * SNAPSHOT_TIMEOUT_MS; - LOG.debug("estimate snapshot timeout: {}, tablet size: {}", phasedTimeoutMs, unfinishedTabletIds.size()); - - // send task + // send tasks for (AgentTask task : batchTask.getAllTasks()) { AgentTaskQueue.addTask(task); } AgentTaskExecutor.submit(batchTask); - state = BackupJobState.SNAPSHOT; - LOG.info("finish send snapshot task. job: {}", jobId); + state = BackupJobState.SNAPSHOTING; + + // DO NOT write log here, state will be reset to PENDING after FE restart. Then all snapshot tasks + // will be re-generated and be sent again + LOG.info("finished to send snapshot tasks to backend. {}", this); } - private synchronized void waitSnapshot() throws DdlException { - if (unfinishedTabletIds.isEmpty()) { + private void waitingAllSnapshotsFinished() { + if (unfinishedTaskIds.isEmpty()) { snapshotFinishedTime = System.currentTimeMillis(); - state = BackupJobState.UPLOAD; + state = BackupJobState.UPLOAD_SNAPSHOT; - Catalog.getInstance().getEditLog().logBackupFinishSnapshot(this); - LOG.info("backup job[{}] is finished making snapshot", jobId); - - return; - } else if (System.currentTimeMillis() - metaSavedTime > phasedTimeoutMs) { - // remove task in AgentTaskQueue - for (Map.Entry entry : unfinishedTabletIds.entries()) { - AgentTaskQueue.removeTask(entry.getValue(), TTaskType.MAKE_SNAPSHOT, entry.getKey()); - } - - // check timeout - String msg = "snapshot timeout. " + phasedTimeoutMs + "s."; - LOG.warn("{}. job[{}]", msg, jobId); - throw new DdlException(msg); - } else { - LOG.debug("waiting {} tablets to make snapshot", unfinishedTabletIds.size()); - } - } - - private void upload() throws IOException, DdlException, InterruptedException, ExecutionException { - LOG.debug("start upload. job[{}]", jobId); - - if (commandBuilder == null) { - String remotePropFilePath = pathBuilder.remoteProperties(); - commandBuilder = CommandBuilder.create(remotePropFilePath, remoteProperties); - } - Preconditions.checkNotNull(commandBuilder); - - // 1. send meta to remote source - if (!uploadMetaObjs()) { + // log + catalog.getEditLog().logBackupJob(this); + LOG.info("finished to make snapshots. {}", this); return; } - // 2. send upload task to be - sendUploadTasks(); + LOG.info("waiting {} tablets to make snapshot. {}", unfinishedTaskIds.size(), this); } - private boolean uploadMetaObjs() throws IOException, InterruptedException, ExecutionException { - if (future == null) { - LOG.info("begin to submit upload meta objs. job: {}", jobId); - String dest = PathBuilder.createPath(remotePath, getLabel()); - String uploadCmd = commandBuilder.uploadCmd(getLabel(), pathBuilder.getRoot().getFullPath(), dest); + private void uploadSnapshot() { + // reuse this set to save all unfinished tablets + unfinishedTaskIds.clear(); + taskErrMsg.clear(); - MetaUploadTask uploadTask = new MetaUploadTask(uploadCmd); - future = Catalog.getInstance().getBackupHandler().getAsynchronousCmdExecutor().submit(uploadTask); - return false; - } else { - return checkFuture("upload meta objs"); + // We classify the snapshot info by backend + ArrayListMultimap beToSnapshots = ArrayListMultimap.create(); + for (SnapshotInfo info : snapshotInfos.values()) { + beToSnapshots.put(info.getBeId(), info); } - } - - private synchronized void sendUploadTasks() throws DdlException { - Preconditions.checkState(unfinishedTabletIds.isEmpty()); AgentBatchTask batchTask = new AgentBatchTask(); - Database db = Catalog.getInstance().getDb(dbId); - if (db == null) { - throw new DdlException("database[" + getDbName() + "] does not exist"); - } - db.readLock(); - try { - String dbName = db.getFullName(); - for (Map.Entry> entry : tableIdToPartitionIds.entrySet()) { - long tableId = entry.getKey(); - Set partitionIds = entry.getValue(); + for (Long beId : beToSnapshots.keySet()) { + List infos = beToSnapshots.get(beId); + int totalNum = infos.size(); + // each backend allot at most 3 tasks + int batchNum = Math.min(totalNum, 3); + // each task contains several upload sub tasks + int taskNumPerBatch = Math.max(totalNum / batchNum, 1); + LOG.debug("backend {} has {} batch, total {} tasks, {}", beId, batchNum, totalNum, this); - Table table = db.getTable(tableId); - if (table == null) { - throw new DdlException("table[" + tableId + "] does not exist"); + List brokerAddrs = Lists.newArrayList(); + Status st = repo.getBrokerAddress(beId, catalog, brokerAddrs); + if (!st.ok()) { + status = st; + return; + } + Preconditions.checkState(brokerAddrs.size() == 1); + + // allot tasks + int index = 0; + for (int batch = 0; batch < batchNum; batch++) { + Map srcToDest = Maps.newHashMap(); + int currentBatchTaskNum = (batch == batchNum - 1) ? totalNum - index : taskNumPerBatch; + for (int j = 0; j < currentBatchTaskNum; j++) { + SnapshotInfo info = infos.get(index++); + String src = info.getTabletPath(); + String dest = repo.getRepoTabletPathBySnapshotInfo(label, info); + srcToDest.put(src, dest); } - - if (table.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTable = (OlapTable) table; - String tableName = olapTable.getName(); - for (Long partitionId : partitionIds) { - Partition partition = olapTable.getPartition(partitionId); - if (partition == null) { - throw new DdlException("partition[" + partitionId + "] does not exist"); - } - - String partitionName = partition.getName(); - for (Long indexId : tableIdToIndexIds.get(tableId)) { - MaterializedIndex index = partition.getIndex(indexId); - if (index == null) { - throw new DdlException("index[" + index + "] does not exist"); - } - - String indexName = olapTable.getIndexNameById(indexId); - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - if (!tabletIdToSnapshotPath.containsKey(tabletId)) { - // this should not happend - String msg = "tablet[" + tabletId + "]'s snapshot is missing"; - LOG.error(msg); - throw new DdlException(msg); - } - - Pair snapshotInfo = tabletIdToSnapshotPath.get(tabletId); - String dest = pathBuilder.tabletRemotePath(dbName, tableName, partitionName, - indexName, tabletId, remotePath, getLabel()); - UploadTask task = new UploadTask(null, snapshotInfo.first, jobId, dbId, tableId, - partitionId, indexId, tabletId, - snapshotInfo.second, dest, - remoteProperties); - - batchTask.addTask(task); - unfinishedTabletIds.put(tabletId, snapshotInfo.first); - } // end for tablet - } // end for indices - } // end for partitions - } // end for tables - - } finally { - db.readUnlock(); + long signature = catalog.getNextId(); + UploadTask task = new UploadTask(null, beId, signature, jobId, dbId, srcToDest, + brokerAddrs.get(0), repo.getStorage().getProperties()); + batchTask.addTask(task); + unfinishedTaskIds.add(signature); + } } - // send task + // send tasks for (AgentTask task : batchTask.getAllTasks()) { AgentTaskQueue.addTask(task); } AgentTaskExecutor.submit(batchTask); state = BackupJobState.UPLOADING; - LOG.info("finish send upload task. job: {}", jobId); + + // DO NOT write log here, upload tasks will be resend after FE crashed. + LOG.info("finished to send update tasks. {}", this); } - private synchronized void waitUpload() throws DdlException { - if (unfinishedTabletIds.isEmpty()) { - LOG.info("backup job[{}] is finished upload snapshot", jobId); - uploadFinishedTime = System.currentTimeMillis(); - state = BackupJobState.FINISHING; + private void waitingAllUploadingFinished() { + if (unfinishedTaskIds.isEmpty()) { + snapshopUploadFinishedTime = System.currentTimeMillis(); + state = BackupJobState.SAVE_META; + + // log + catalog.getEditLog().logBackupJob(this); + LOG.info("finished uploading snapshots. {}", this); return; - } else { - LOG.debug("waiting {} tablets to upload snapshot", unfinishedTabletIds.size()); } + + LOG.debug("waiting {} tablets to upload snapshot. {}", unfinishedTaskIds.size(), this); } - private void finishing() throws DdlException, InterruptedException, ExecutionException, IOException { - // save manifest and upload - // manifest contain all file under {label}/ + private void saveMetaInfo() { + String createTimeStr = TimeUtils.longToTimeString(createTime, new SimpleDateFormat( + "yyyy-MM-dd-HH-mm-ss")); + // local job dir: backup/label__createtime/ + localJobDirPath = Paths.get(BackupHandler.BACKUP_ROOT_DIR.toString(), + label + "__" + createTimeStr).normalize(); - if (future == null) { - LOG.info("begin to submit save and upload manifest. job: {}", jobId); - String deleteInfo = lastestDeleteInfo == null ? "" : lastestDeleteInfo.toString(); - SaveManifestTask task = new SaveManifestTask(jobId, getLabel(), remotePath, getLocalDirName(), - lastestLoadLabel, deleteInfo, pathBuilder, commandBuilder); - future = Catalog.getInstance().getBackupHandler().getAsynchronousCmdExecutor().submit(task); - } else { - boolean finished = checkFuture("save and upload manifest"); - if (finished) { - // reset future - readableManifestPath = - PathBuilder.createPath(remotePath, getLabel(), PathBuilder.READABLE_MANIFEST_NAME); - future = null; - state = BackupJobState.FINISHED; + try { + // 1. create local job dir of this backup job + File jobDir = new File(localJobDirPath.toString()); + if (jobDir.exists()) { + // if dir exists, delete it first + Files.walk(localJobDirPath, + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); } - } - } - - public void restoreTableState(Catalog catalog) { - Database db = catalog.getDb(dbId); - if (db != null) { - db.writeLock(); - try { - for (long tableId : tableIdToPartitionIds.keySet()) { - Table table = db.getTable(tableId); - if (table != null && table.getType() == TableType.OLAP) { - if (((OlapTable) table).getState() == OlapTableState.BACKUP) { - ((OlapTable) table).setState(OlapTableState.NORMAL); - LOG.debug("set table[{}] state to NORMAL", table.getName()); - } - } - } - } finally { - db.writeUnlock(); - } - } - } - - private void removeLeftTasks() { - for (Map.Entry entry : unfinishedTabletIds.entries()) { - AgentTaskQueue.removeTask(entry.getValue(), TTaskType.MAKE_SNAPSHOT, entry.getKey()); - AgentTaskQueue.removeTask(entry.getValue(), TTaskType.UPLOAD, entry.getKey()); - } - } - - @Override - public void end(Catalog catalog, boolean isReplay) { - // 1. set table state - restoreTableState(catalog); - - if (!isReplay) { - // 2. remove agent tasks if left - removeLeftTasks(); - - if (pathBuilder == null) { - finishedTime = System.currentTimeMillis(); - Catalog.getInstance().getEditLog().logBackupFinish(this); - LOG.info("finished end job[{}]. state: {}", jobId, state.name()); + if (!jobDir.mkdir()) { + status = new Status(ErrCode.COMMON_ERROR, "Failed to create tmp dir: " + localJobDirPath); return; } - // 3. remove local file - String labelDir = pathBuilder.getRoot().getFullPath(); - Util.deleteDirectory(new File(labelDir)); - LOG.debug("delete local dir: {}", labelDir); - - // 4. release snapshot - synchronized (tabletIdToSnapshotPath) { - AgentBatchTask batchTask = new AgentBatchTask(); - for (Long tabletId : tabletIdToSnapshotPath.keySet()) { - long backendId = tabletIdToSnapshotPath.get(tabletId).first; - String snapshotPath = tabletIdToSnapshotPath.get(tabletId).second; - ReleaseSnapshotTask task = new ReleaseSnapshotTask(null, backendId, dbId, tabletId, snapshotPath); - batchTask.addTask(task); - } - // no need to add to AgentTaskQueue - AgentTaskExecutor.submit(batchTask); + // 2. save meta info file + File metaInfoFile = new File(jobDir, Repository.FILE_META_INFO); + if (!metaInfoFile.createNewFile()) { + status = new Status(ErrCode.COMMON_ERROR, + "Failed to create meta info file: " + metaInfoFile.toString()); + return; } + backupMeta.writeToFile(metaInfoFile); + localMetaInfoFilePath = metaInfoFile.getAbsolutePath(); - finishedTime = System.currentTimeMillis(); - - Catalog.getInstance().getEditLog().logBackupFinish(this); + // 3. save job info file + jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, backupMeta.getTables().values(), + snapshotInfos); + LOG.debug("job info: {}. {}", jobInfo, this); + File jobInfoFile = new File(jobDir, Repository.PREFIX_JOB_INFO + createTimeStr); + if (!jobInfoFile.createNewFile()) { + status = new Status(ErrCode.COMMON_ERROR, "Failed to create job info file: " + jobInfoFile.toString()); + return; + } + jobInfo.writeToFile(jobInfoFile); + localJobInfoFilePath = jobInfoFile.getAbsolutePath(); + } catch (Exception e) { + status = new Status(ErrCode.COMMON_ERROR, "failed to save meta info and job info file: " + e.getMessage()); + return; } - clearJob(); + state = BackupJobState.UPLOAD_INFO; - LOG.info("finished end job[{}]. state: {}, replay: {}", jobId, state.name(), isReplay); + // meta info and job info has been saved to local file, this can be cleaned to reduce log size + backupMeta = null; + jobInfo = null; + snapshotInfos.clear(); + + // log + catalog.getEditLog().logBackupJob(this); + LOG.info("finished to save meta the backup job info file to local.[{}], [{}] {}", + localMetaInfoFilePath, localJobInfoFilePath, this); } - @Override - protected void clearJob() { - tableIdToPartitionIds = null; - tableIdToIndexIds = null; - partitionIdToVersionInfo = null; - tabletIdToSnapshotPath = null; + private void uploadMetaAndJobInfoFile() { + String remoteMetaInfoFile = repo.assembleMetaInfoFilePath(label); + if (!uploadFile(localMetaInfoFilePath, remoteMetaInfoFile)) { + return; + } - unfinishedTabletIds = null; - remoteProperties = null; - pathBuilder = null; - commandBuilder = null; + String remoteJobInfoFile = repo.assembleJobInfoFilePath(label, createTime); + if (!uploadFile(localJobInfoFilePath, remoteJobInfoFile)) { + return; + } + + finishedTime = System.currentTimeMillis(); + state = BackupJobState.FINISHED; + + // log + catalog.getEditLog().logBackupJob(this); + LOG.info("job is finished. {}", this); + } + + private boolean uploadFile(String localFilePath, String remoteFilePath) { + if (!validateLocalFile(localFilePath)) { + return false; + } + + status = repo.upload(localFilePath, remoteFilePath); + if (!status.ok()) { + return false; + } + return true; + } + + private boolean validateLocalFile(String filePath) { + File file = new File(filePath); + if (!file.exists() || !file.canRead()) { + status = new Status(ErrCode.COMMON_ERROR, "file is invalid: " + filePath); + return false; + } + return true; + } + + /* + * Choose a replica order by replica id. + * This is to expect to choose the same replica at each backup job. + */ + private Replica chooseReplica(Tablet tablet, long committedVersion, long committedVersionHash) { + List replicaIds = Lists.newArrayList(); + for (Replica replica : tablet.getReplicas()) { + replicaIds.add(replica.getId()); + } + + Collections.sort(replicaIds); + for (Long replicaId : replicaIds) { + Replica replica = tablet.getReplicaById(replicaId); + if (replica.getVersion() > committedVersion + || (replica.getVersion() == committedVersion && replica.getVersionHash()==committedVersionHash)) { + return replica; + } + } + return null; + } + + private void cancelInternal() { + // We need to clean the residual due to current state + switch (state) { + case SNAPSHOTING: + // remove all snapshot tasks in AgentTaskQueue + for (Long taskId : unfinishedTaskIds) { + AgentTaskQueue.removeTaskOfType(TTaskType.MAKE_SNAPSHOT, taskId); + } + break; + case UPLOADING: + // remove all upload tasks in AgentTaskQueue + for (Long taskId : unfinishedTaskIds) { + AgentTaskQueue.removeTaskOfType(TTaskType.UPLOAD, taskId); + } + break; + default: + break; + } + + // clean the backup job dir + if (localJobDirPath != null) { + try { + File jobDir = new File(localJobDirPath.toString()); + if (jobDir.exists()) { + Files.walk(localJobDirPath, + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } catch (Exception e) { + LOG.warn("failed to clean the backup job dir: " + localJobDirPath.toString()); + } + } + + BackupJobState curState = state; + finishedTime = System.currentTimeMillis(); + state = BackupJobState.CANCELLED; + + // log + catalog.getEditLog().logBackupJob(this); + LOG.info("finished to cancel backup job. current state: {}. {}", curState.name(), this); + } + + public List getInfo() { + List info = Lists.newArrayList(); + info.add(String.valueOf(jobId)); + info.add(label); + info.add(dbName); + info.add(state.name()); + info.add(getBackupObjs()); + info.add(TimeUtils.longToTimeString(createTime)); + info.add(TimeUtils.longToTimeString(snapshotFinishedTime)); + info.add(TimeUtils.longToTimeString(snapshopUploadFinishedTime)); + info.add(TimeUtils.longToTimeString(finishedTime)); + info.add(Joiner.on(", ").join(unfinishedTaskIds)); + List msgs = taskErrMsg.entrySet().stream().map(n -> "[" + n.getKey() + ": " + n.getValue() + + "]").collect(Collectors.toList()); + info.add(Joiner.on(", ").join(msgs)); + info.add(status.toString()); + info.add(String.valueOf(timeoutMs / 1000)); + return info; + } + + private String getBackupObjs() { + List list = tableRefs.stream().map(n -> "[" + n.toString() + "]").collect(Collectors.toList()); + return Joiner.on(", ").join(list); + } + + public static BackupJob read(DataInput in) throws IOException { + BackupJob job = new BackupJob(); + job.readFields(in); + return job; } @Override public void write(DataOutput out) throws IOException { super.write(out); + // table refs + out.writeInt(tableRefs.size()); + for (TableRef tblRef : tableRefs) { + tblRef.write(out); + } + + // state Text.writeString(out, state.name()); - Text.writeString(out, lastestLoadLabel); - if (lastestDeleteInfo == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - lastestDeleteInfo.write(out); - } - - if (tableIdToPartitionIds == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tableIdToPartitionIds.size(); - out.writeInt(size); - for (Map.Entry> entry : tableIdToPartitionIds.entrySet()) { - out.writeLong(entry.getKey()); - size = entry.getValue().size(); - out.writeInt(size); - for (Long partitionId : entry.getValue()) { - out.writeLong(partitionId); - } - } - } - - if (tableIdToIndexIds == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Collection> entries = tableIdToIndexIds.entries(); - int size = entries.size(); - out.writeInt(size); - for (Map.Entry entry : entries) { - out.writeLong(entry.getKey()); - out.writeLong(entry.getValue()); - } - } - - if (partitionIdToVersionInfo == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = partitionIdToVersionInfo.size(); - out.writeInt(size); - for (Map.Entry> entry : partitionIdToVersionInfo.entrySet()) { - out.writeLong(entry.getKey()); - Pair pair = entry.getValue(); - out.writeLong(pair.first); - out.writeLong(pair.second); - } - } - - if (tabletIdToSnapshotPath == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tabletIdToSnapshotPath.size(); - out.writeInt(size); - for (Map.Entry> entry : tabletIdToSnapshotPath.entrySet()) { - out.writeLong(entry.getKey()); - Pair pair = entry.getValue(); - out.writeLong(pair.first); - Text.writeString(out, pair.second); - } - } - - out.writeLong(metaSavedTime); + // times out.writeLong(snapshotFinishedTime); - out.writeLong(uploadFinishedTime); - out.writeLong(phasedTimeoutMs); + out.writeLong(snapshopUploadFinishedTime); - Text.writeString(out, readableManifestPath); - - if (pathBuilder == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - pathBuilder.write(out); + // snapshot info + out.writeInt(snapshotInfos.size()); + for (SnapshotInfo info : snapshotInfos.values()) { + info.write(out); } - if (commandBuilder == null) { + // backup meta + if (backupMeta == null) { out.writeBoolean(false); } else { out.writeBoolean(true); - commandBuilder.write(out); + backupMeta.write(out); + } + + // No need to persist job info. It is generated then write to file + + // metaInfoFilePath and jobInfoFilePath + if (Strings.isNullOrEmpty(localMetaInfoFilePath)) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Text.writeString(out, localMetaInfoFilePath); + } + + if (Strings.isNullOrEmpty(localJobInfoFilePath)) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Text.writeString(out, localJobInfoFilePath); } } @@ -877,73 +753,51 @@ public class BackupJob extends AbstractBackupJob { public void readFields(DataInput in) throws IOException { super.readFields(in); + // table refs + int size = in.readInt(); + tableRefs = Lists.newArrayList(); + for (int i = 0; i < size; i++) { + TableRef tblRef = new TableRef(); + tblRef.readFields(in); + tableRefs.add(tblRef); + } + state = BackupJobState.valueOf(Text.readString(in)); - lastestLoadLabel = Text.readString(in); - if (in.readBoolean()) { - lastestDeleteInfo = new DeleteInfo(); - lastestDeleteInfo.readFields(in); - } - - if (in.readBoolean()) { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - long tableId = in.readLong(); - Set partitionIds = Sets.newHashSet(); - tableIdToPartitionIds.put(tableId, partitionIds); - int count = in.readInt(); - for (int j = 0; j < count; j++) { - long partitionId = in.readLong(); - partitionIds.add(partitionId); - } - } - } - - if (in.readBoolean()) { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - long tableId = in.readLong(); - long indexId = in.readLong(); - tableIdToIndexIds.put(tableId, indexId); - } - } - - if (in.readBoolean()) { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - long partitionId = in.readLong(); - long version = in.readLong(); - long versionHash = in.readLong(); - partitionIdToVersionInfo.put(partitionId, new Pair(version, versionHash)); - } - } - - if (in.readBoolean()) { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - long tabletId = in.readLong(); - long backendId = in.readLong(); - String path = Text.readString(in); - tabletIdToSnapshotPath.put(tabletId, new Pair(backendId, path)); - } - } - - metaSavedTime = in.readLong(); + // times snapshotFinishedTime = in.readLong(); - uploadFinishedTime = in.readLong(); - phasedTimeoutMs = in.readLong(); + snapshopUploadFinishedTime = in.readLong(); - readableManifestPath = Text.readString(in); + // snapshot info + size = in.readInt(); + for (int i = 0; i < size; i++) { + SnapshotInfo snapshotInfo = new SnapshotInfo(); + snapshotInfo.readFields(in); + snapshotInfos.put(snapshotInfo.getTabletId(), snapshotInfo); + } + // backup meta if (in.readBoolean()) { - pathBuilder = new PathBuilder(); - pathBuilder.readFields(in); + backupMeta = BackupMeta.read(in); + } + + // No need to persist job info. It is generated then write to file + + // metaInfoFilePath and jobInfoFilePath + if (in.readBoolean()) { + localMetaInfoFilePath = Text.readString(in); } if (in.readBoolean()) { - commandBuilder = new CommandBuilder(); - commandBuilder.readFields(in); + localJobInfoFilePath = Text.readString(in); } + } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.append(", state: ").append(state.name()); + return sb.toString(); } } + diff --git a/fe/src/com/baidu/palo/backup/BackupJobInfo.java b/fe/src/com/baidu/palo/backup/BackupJobInfo.java new file mode 100644 index 0000000000..87af8da69b --- /dev/null +++ b/fe/src/com/baidu/palo/backup/BackupJobInfo.java @@ -0,0 +1,481 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.backup.RestoreFileMapping.IdChain; +import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.Partition; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/* + * This is a memory structure mapping the job info file in repository. + * It contains all content of a job info file. + * It also be used to save the info of a restore job, such as alias of table and meta info file path + */ +public class BackupJobInfo implements Writable { + private static final Logger LOG = LogManager.getLogger(BackupJobInfo.class); + + public String name; + public String dbName; + public long dbId; + public long backupTime; + public Map tables = Maps.newHashMap(); + public boolean success; + + // This map is used to save the table alias mapping info when processing a restore job. + // origin -> alias + public Map tblAlias = Maps.newHashMap(); + + public boolean containsTbl(String tblName) { + return tables.containsKey(tblName); + } + + public BackupTableInfo getTableInfo(String tblName) { + return tables.get(tblName); + } + + public void retainTables(Set tblNames) { + Iterator> iter = tables.entrySet().iterator(); + while (iter.hasNext()) { + if (!tblNames.contains(iter.next().getKey())) { + iter.remove(); + } + } + } + + public void setAlias(String orig, String alias) { + tblAlias.put(orig, alias); + } + + public String getAliasByOriginNameIfSet(String orig) { + return tblAlias.containsKey(orig) ? tblAlias.get(orig) : orig; + } + + public String getOrginNameByAlias(String alias) { + for (Map.Entry entry : tblAlias.entrySet()) { + if (entry.getValue().equals(alias)) { + return entry.getKey(); + } + } + return alias; + } + + public static class BackupTableInfo { + public String name; + public long id; + public Map partitions = Maps.newHashMap(); + + public boolean containsPart(String partName) { + return partitions.containsKey(partName); + } + + public BackupPartitionInfo getPartInfo(String partName) { + return partitions.get(partName); + } + + public void retainPartitions(Collection partNames) { + if (partNames == null || partNames.isEmpty()) { + // retain all + return; + } + Iterator> iter = partitions.entrySet().iterator(); + while (iter.hasNext()) { + if (!partNames.contains(iter.next().getKey())) { + iter.remove(); + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("name: ").append(name).append(", id: ").append(id); + sb.append(", partitions: [").append(Joiner.on(", ").join(partitions.keySet())).append("]"); + return sb.toString(); + } + } + + public static class BackupPartitionInfo { + public String name; + public long id; + public long version; + public long versionHash; + public Map indexes = Maps.newHashMap(); + + public BackupIndexInfo getIdx(String idxName) { + return indexes.get(idxName); + } + } + + public static class BackupIndexInfo { + public String name; + public long id; + public int schemaHash; + public List tablets = Lists.newArrayList(); + + public BackupTabletInfo getTablet(long tabletId) { + for (BackupTabletInfo backupTabletInfo : tablets) { + if (backupTabletInfo.id == tabletId) { + return backupTabletInfo; + } + } + return null; + } + } + + public static class BackupTabletInfo { + public long id; + public List files = Lists.newArrayList(); + } + + // eg: __db_10001/__tbl_10002/__part_10003/__idx_10002/__10004 + public String getFilePath(String db, String tbl, String part, String idx, long tabletId) { + if (!db.equalsIgnoreCase(dbName)) { + LOG.debug("db name does not equal: {}-{}", dbName, db); + return null; + } + + BackupTableInfo tblInfo = tables.get(tbl); + if (tblInfo == null) { + LOG.debug("tbl {} does not exist", tbl); + return null; + } + + BackupPartitionInfo partInfo = tblInfo.getPartInfo(part); + if (partInfo == null) { + LOG.debug("part {} does not exist", part); + return null; + } + + BackupIndexInfo idxInfo = partInfo.getIdx(idx); + if (idxInfo == null) { + LOG.debug("idx {} does not exist", idx); + return null; + } + + List pathSeg = Lists.newArrayList(); + pathSeg.add(Repository.PREFIX_DB + dbId); + pathSeg.add(Repository.PREFIX_TBL + tblInfo.id); + pathSeg.add(Repository.PREFIX_PART + partInfo.id); + pathSeg.add(Repository.PREFIX_IDX + idxInfo.id); + pathSeg.add(Repository.PREFIX_COMMON + tabletId); + + return Joiner.on("/").join(pathSeg); + } + + // eg: __db_10001/__tbl_10002/__part_10003/__idx_10002/__10004 + public String getFilePath(IdChain ids) { + List pathSeg = Lists.newArrayList(); + pathSeg.add(Repository.PREFIX_DB + dbId); + pathSeg.add(Repository.PREFIX_TBL + ids.getTblId()); + pathSeg.add(Repository.PREFIX_PART + ids.getPartId()); + pathSeg.add(Repository.PREFIX_IDX + ids.getIdxId()); + pathSeg.add(Repository.PREFIX_COMMON + ids.getTabletId()); + + return Joiner.on("/").join(pathSeg); + } + + public static BackupJobInfo fromCatalog(long backupTime, String label, String dbName, long dbId, + Collection
tbls, Map snapshotInfos) { + + BackupJobInfo jobInfo = new BackupJobInfo(); + jobInfo.backupTime = backupTime; + jobInfo.name = label; + jobInfo.dbName = dbName; + jobInfo.dbId = dbId; + jobInfo.success = true; + + // tbls + for (Table tbl : tbls) { + OlapTable olapTbl = (OlapTable) tbl; + BackupTableInfo tableInfo = new BackupTableInfo(); + tableInfo.id = tbl.getId(); + tableInfo.name = tbl.getName(); + jobInfo.tables.put(tableInfo.name, tableInfo); + // partitions + for (Partition partition : olapTbl.getPartitions()) { + BackupPartitionInfo partitionInfo = new BackupPartitionInfo(); + partitionInfo.id = partition.getId(); + partitionInfo.name = partition.getName(); + partitionInfo.version = partition.getCommittedVersion(); + partitionInfo.versionHash = partition.getCommittedVersionHash(); + tableInfo.partitions.put(partitionInfo.name, partitionInfo); + // indexes + for (MaterializedIndex index : partition.getMaterializedIndices()) { + BackupIndexInfo idxInfo = new BackupIndexInfo(); + idxInfo.id = index.getId(); + idxInfo.name = olapTbl.getIndexNameById(index.getId()); + idxInfo.schemaHash = olapTbl.getSchemaHashByIndexId(index.getId()); + partitionInfo.indexes.put(idxInfo.name, idxInfo); + // tablets + for (Tablet tablet : index.getTablets()) { + BackupTabletInfo tabletInfo = new BackupTabletInfo(); + tabletInfo.id = tablet.getId(); + tabletInfo.files.addAll(snapshotInfos.get(tablet.getId()).getFiles()); + idxInfo.tablets.add(tabletInfo); + } + } + } + } + + return jobInfo; + } + + public static BackupJobInfo fromFile(String path) throws IOException { + byte[] bytes = Files.readAllBytes(Paths.get(path)); + String json = new String(bytes, StandardCharsets.UTF_8); + BackupJobInfo jobInfo = new BackupJobInfo(); + genFromJson(json, jobInfo); + return jobInfo; + } + + private static void genFromJson(String json, BackupJobInfo jobInfo) { + /* parse the json string: + * { + * "backup_time": 1522231864000, + * "name": "snapshot1", + * "database": "db1" + * "id": 10000 + * "backup_result": "succeed", + * "backup_objects": { + * "table1": { + * "partitions": { + * "partition2": { + * "indexes": { + * "rollup1": { + * "id": 10009, + * "schema_hash": 3473401 + * "tablets": { + * "10008": ["__10029_seg1.dat", "__10029_seg2.dat"], + * "10007": ["__10029_seg1.dat", "__10029_seg2.dat"] + * } + * }, + * "table1": { + * "id": 10008, + * "schema_hash": 9845021 + * "tablets": { + * "10004": ["__10027_seg1.dat", "__10027_seg2.dat"], + * "10005": ["__10028_seg1.dat", "__10028_seg2.dat"] + * } + * } + * }, + * "id": 10007 + * "version": 10 + * "version_hash": 1273047329538 + * }, + * }, + * "id": 10001 + * } + * } + * } + */ + JSONObject root = new JSONObject(json); + jobInfo.name = (String) root.get("name"); + jobInfo.dbName = (String) root.get("database"); + jobInfo.dbId = root.getLong("id"); + jobInfo.backupTime = root.getLong("backup_time"); + JSONObject backupObjs = root.getJSONObject("backup_objects"); + String[] tblNames = JSONObject.getNames(backupObjs); + for (String tblName : tblNames) { + BackupTableInfo tblInfo = new BackupTableInfo(); + tblInfo.name = tblName; + JSONObject tbl = backupObjs.getJSONObject(tblName); + tblInfo.id = tbl.getLong("id"); + JSONObject parts = tbl.getJSONObject("partitions"); + String[] partsNames = JSONObject.getNames(parts); + for (String partName : partsNames) { + BackupPartitionInfo partInfo = new BackupPartitionInfo(); + partInfo.name = partName; + JSONObject part = parts.getJSONObject(partName); + partInfo.id = part.getLong("id"); + partInfo.version = part.getLong("version"); + partInfo.versionHash = part.getLong("version_hash"); + JSONObject indexes = part.getJSONObject("indexes"); + String[] indexNames = JSONObject.getNames(indexes); + for (String idxName : indexNames) { + BackupIndexInfo indexInfo = new BackupIndexInfo(); + indexInfo.name = idxName; + JSONObject idx = indexes.getJSONObject(idxName); + indexInfo.id = idx.getLong("id"); + indexInfo.schemaHash = idx.getInt("schema_hash"); + JSONObject tablets = idx.getJSONObject("tablets"); + String[] tabletIds = JSONObject.getNames(tablets); + for (String tabletId : tabletIds) { + BackupTabletInfo tabletInfo = new BackupTabletInfo(); + tabletInfo.id = Long.valueOf(tabletId); + JSONArray files = tablets.getJSONArray(tabletId); + for (Object object : files) { + tabletInfo.files.add((String) object); + } + indexInfo.tablets.add(tabletInfo); + } + partInfo.indexes.put(indexInfo.name, indexInfo); + } + tblInfo.partitions.put(partName, partInfo); + } + jobInfo.tables.put(tblName, tblInfo); + } + + String result = root.getString("backup_result"); + if (result.equals("succeed")) { + jobInfo.success = true; + } else { + jobInfo.success = false; + } + } + + public void writeToFile(File jobInfoFile) throws FileNotFoundException { + PrintWriter printWriter = new PrintWriter(jobInfoFile); + try { + printWriter.print(toJson().toString()); + printWriter.flush(); + } finally { + printWriter.close(); + } + } + + public JSONObject toJson() { + JSONObject root = new JSONObject(); + root.put("name", name); + root.put("database", dbName); + root.put("id", dbId); + root.put("backup_time", backupTime); + JSONObject backupObj = new JSONObject(); + root.put("backup_objects", backupObj); + + for (BackupTableInfo tblInfo : tables.values()) { + JSONObject tbl = new JSONObject(); + tbl.put("id", tblInfo.id); + JSONObject parts = new JSONObject(); + tbl.put("partitions", parts); + for (BackupPartitionInfo partInfo : tblInfo.partitions.values()) { + JSONObject part = new JSONObject(); + part.put("id", partInfo.id); + part.put("version", partInfo.version); + part.put("version_hash", partInfo.versionHash); + JSONObject indexes = new JSONObject(); + part.put("indexes", indexes); + for (BackupIndexInfo idxInfo : partInfo.indexes.values()) { + JSONObject idx = new JSONObject(); + idx.put("id", idxInfo.id); + idx.put("schema_hash", idxInfo.schemaHash); + JSONObject tablets = new JSONObject(); + idx.put("tablets", tablets); + for (BackupTabletInfo tabletInfo : idxInfo.tablets) { + JSONArray files = new JSONArray(); + tablets.put(String.valueOf(tabletInfo.id), files); + for (String fileName : tabletInfo.files) { + files.put(fileName); + } + } + indexes.put(idxInfo.name, idx); + } + parts.put(partInfo.name, part); + } + backupObj.put(tblInfo.name, tbl); + } + + root.put("backup_result", "succeed"); + return root; + } + + public String toString(int indentFactor) { + return toJson().toString(indentFactor); + } + + public String getInfo() { + List objs = Lists.newArrayList(); + for (BackupTableInfo tblInfo : tables.values()) { + StringBuilder sb = new StringBuilder(); + sb.append(tblInfo.name); + List partNames = tblInfo.partitions.values().stream() + .filter(n -> !n.name.equals(tblInfo.name)).map(n -> n.name).collect(Collectors.toList()); + if (!partNames.isEmpty()) { + sb.append(" PARTITIONS [").append(Joiner.on(", ").join(partNames)).append("]"); + } + objs.add(sb.toString()); + } + return Joiner.on(", ").join(objs); + } + + public static BackupJobInfo read(DataInput in) throws IOException { + BackupJobInfo jobInfo = new BackupJobInfo(); + jobInfo.readFields(in); + return jobInfo; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, toJson().toString()); + out.writeInt(tblAlias.size()); + for (Map.Entry entry : tblAlias.entrySet()) { + Text.writeString(out, entry.getKey()); + Text.writeString(out, entry.getValue()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + String json = Text.readString(in); + genFromJson(json, this); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tbl = Text.readString(in); + String alias = Text.readString(in); + tblAlias.put(tbl, alias); + } + } + + @Override + public String toString() { + return toJson().toString(); + } +} + diff --git a/fe/src/com/baidu/palo/backup/BackupJob_D.java b/fe/src/com/baidu/palo/backup/BackupJob_D.java new file mode 100644 index 0000000000..c1b5b4a7f6 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/BackupJob_D.java @@ -0,0 +1,944 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.analysis.AlterTableStmt; +import com.baidu.palo.analysis.CreateTableStmt; +import com.baidu.palo.analysis.LabelName; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.OlapTable.OlapTableState; +import com.baidu.palo.catalog.Partition; +import com.baidu.palo.catalog.PartitionInfo; +import com.baidu.palo.catalog.PartitionKey; +import com.baidu.palo.catalog.PartitionType; +import com.baidu.palo.catalog.RangePartitionInfo; +import com.baidu.palo.catalog.Replica; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Table.TableType; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.Pair; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.common.util.LoadBalancer; +import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.common.util.Util; +import com.baidu.palo.load.DeleteInfo; +import com.baidu.palo.load.Load; +import com.baidu.palo.load.LoadJob; +import com.baidu.palo.task.AgentBatchTask; +import com.baidu.palo.task.AgentTask; +import com.baidu.palo.task.AgentTaskExecutor; +import com.baidu.palo.task.AgentTaskQueue; +import com.baidu.palo.task.ReleaseSnapshotTask; +import com.baidu.palo.task.SnapshotTask; +import com.baidu.palo.thrift.TTaskType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Range; +import com.google.common.collect.Sets; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; + +@Deprecated +public class BackupJob_D extends AbstractBackupJob_D { + private static final Logger LOG = LogManager.getLogger(BackupJob_D.class); + + private static final long SNAPSHOT_TIMEOUT_MS = 2000; // 1s for one tablet + + public enum BackupJobState { + PENDING, + SNAPSHOT, + UPLOAD, + UPLOADING, + FINISHING, + FINISHED, + CANCELLED + } + + private BackupJobState state; + + private String lastestLoadLabel; + private DeleteInfo lastestDeleteInfo; + + // all partitions need to be backuped + private Map> tableIdToPartitionIds; + private Multimap tableIdToIndexIds; + // partition id -> (version, version hash) + private Map> partitionIdToVersionInfo; + + private Map> tabletIdToSnapshotPath; + + private long metaSavedTime; + private long snapshotFinishedTime; + private long uploadFinishedTime; + + private long phasedTimeoutMs; + + private String readableManifestPath; + + public BackupJob_D() { + super(); + tableIdToPartitionIds = Maps.newHashMap(); + tableIdToIndexIds = HashMultimap.create(); + partitionIdToVersionInfo = Maps.newHashMap(); + tabletIdToSnapshotPath = Maps.newHashMap(); + } + + public BackupJob_D(long jobId, long dbId, LabelName labelName, String backupPath, + Map remoteProperties) { + super(jobId, dbId, labelName, backupPath, remoteProperties); + this.state = BackupJobState.PENDING; + + tableIdToPartitionIds = Maps.newHashMap(); + tableIdToIndexIds = HashMultimap.create(); + partitionIdToVersionInfo = Maps.newHashMap(); + + tabletIdToSnapshotPath = Maps.newHashMap(); + + metaSavedTime = -1; + snapshotFinishedTime = -1; + uploadFinishedTime = -1; + phasedTimeoutMs = -1; + + lastestLoadLabel = "N/A"; + readableManifestPath = ""; + } + + public void setState(BackupJobState state) { + this.state = state; + } + + public BackupJobState getState() { + return state; + } + + public String getLatestLoadLabel() { + return lastestLoadLabel; + } + + public DeleteInfo getLastestDeleteInfo() { + return lastestDeleteInfo; + } + + public PathBuilder getPathBuilder() { + return pathBuilder; + } + + public long getMetaSavedTimeMs() { + return metaSavedTime; + } + + public long getSnapshotFinishedTimeMs() { + return snapshotFinishedTime; + } + + public long getUploadFinishedTimeMs() { + return uploadFinishedTime; + } + + public String getReadableManifestPath() { + return readableManifestPath; + } + + public Map> getTableIdToPartitionIds() { + return tableIdToPartitionIds; + } + + public void addPartitionId(long tableId, long partitionId) { + Set partitionIds = tableIdToPartitionIds.get(tableId); + if (partitionIds == null) { + partitionIds = Sets.newHashSet(); + tableIdToPartitionIds.put(tableId, partitionIds); + } + if (partitionId != -1L) { + partitionIds.add(partitionId); + } + + LOG.debug("add partition[{}] from table[{}], job[{}]", partitionId, tableId, jobId); + } + + public void addIndexId(long tableId, long indexId) { + tableIdToIndexIds.put(tableId, indexId); + LOG.debug("add index[{}] from table[{}], job[{}]", indexId, tableId, jobId); + } + + public void handleFinishedSnapshot(long tabletId, long backendId, String snapshotPath) { + synchronized (unfinishedTabletIds) { + if (!unfinishedTabletIds.containsKey(tabletId)) { + LOG.warn("backup job[{}] does not contains tablet[{}]", jobId, tabletId); + return; + } + + if (unfinishedTabletIds.get(tabletId) == null + || !unfinishedTabletIds.get(tabletId).contains(backendId)) { + LOG.warn("backup job[{}] does not contains tablet[{}]'s snapshot from backend[{}]. " + + "it should from backend[{}]", + jobId, tabletId, backendId, unfinishedTabletIds.get(tabletId)); + return; + } + unfinishedTabletIds.remove(tabletId, backendId); + } + + synchronized (tabletIdToSnapshotPath) { + tabletIdToSnapshotPath.put(tabletId, new Pair(backendId, snapshotPath)); + } + LOG.debug("finished add tablet[{}] from backend[{}]. snapshot path: {}", tabletId, backendId, snapshotPath); + } + + public void handleFinishedUpload(long tabletId, long backendId) { + synchronized (unfinishedTabletIds) { + if (unfinishedTabletIds.remove(tabletId, backendId)) { + LOG.debug("finished upload tablet[{}] snapshot, backend[{}]", tabletId, backendId); + } + } + } + + @Override + public List getJobInfo() { + List jobInfo = Lists.newArrayList(); + jobInfo.add(jobId); + jobInfo.add(getLabel()); + jobInfo.add(state.name()); + jobInfo.add(TimeUtils.longToTimeString(createTime)); + jobInfo.add(TimeUtils.longToTimeString(metaSavedTime)); + jobInfo.add(TimeUtils.longToTimeString(snapshotFinishedTime)); + jobInfo.add(TimeUtils.longToTimeString(uploadFinishedTime)); + jobInfo.add(TimeUtils.longToTimeString(finishedTime)); + jobInfo.add(errMsg); + jobInfo.add(PathBuilder.createPath(remotePath, getLabel())); + jobInfo.add(getReadableManifestPath()); + jobInfo.add(getLeftTasksNum()); + jobInfo.add(getLatestLoadLabel()); + return jobInfo; + } + + @Override + public void runOnce() { + LOG.debug("begin to run backup job: {}, state: {}", jobId, state.name()); + try { + switch (state) { + case PENDING: + saveMetaAndMakeSnapshot(); + break; + case SNAPSHOT: + waitSnapshot(); + break; + case UPLOAD: + upload(); + break; + case UPLOADING: + waitUpload(); + break; + case FINISHING: + finishing(); + break; + default: + break; + } + } catch (Exception e) { + errMsg = e.getMessage() == null ? "Unknown Exception" : e.getMessage(); + LOG.warn("failed to backup: " + errMsg + ", job[" + jobId + "]", e); + state = BackupJobState.CANCELLED; + } + + if (state == BackupJobState.FINISHED || state == BackupJobState.CANCELLED) { + end(Catalog.getInstance(), false); + } + } + + private void saveMetaAndMakeSnapshot() throws DdlException, IOException { + Database db = Catalog.getInstance().getDb(dbId); + if (db == null) { + throw new DdlException("[" + getDbName() + "] does not exist"); + } + + try { + pathBuilder = PathBuilder.createPathBuilder(getLocalDirName()); + } catch (IOException e) { + pathBuilder = null; + throw e; + } + + // file path -> writable objs + Map> pathToWritables = Maps.newHashMap(); + // 1. get meta + getMeta(db, pathToWritables); + + // 2. write meta + // IO ops should be done outside db.lock + try { + writeMeta(pathToWritables); + } catch (IOException e) { + errMsg = e.getMessage(); + state = BackupJobState.CANCELLED; + return; + } + + metaSavedTime = System.currentTimeMillis(); + LOG.info("save meta finished. path: {}, job: {}", pathBuilder.getRoot().getFullPath(), jobId); + + // 3. send snapshot tasks + snapshot(db); + } + + private void getMeta(Database db, Map> pathToWritables) throws DdlException { + db.readLock(); + try { + for (long tableId : tableIdToPartitionIds.keySet()) { + Table table = db.getTable(tableId); + if (table == null) { + throw new DdlException("table[" + tableId + "] does not exist"); + } + + // 1. get table meta + getTableMeta(db.getFullName(), table, pathToWritables); + + if (table.getType() != TableType.OLAP) { + // this is not a OLAP table. just save table meta + continue; + } + + OlapTable olapTable = (OlapTable) table; + + // 2. get rollup meta + // 2.1 check all indices exist + for (Long indexId : tableIdToIndexIds.get(tableId)) { + if (olapTable.getIndexNameById(indexId) == null) { + errMsg = "Index[" + indexId + "] does not exist"; + state = BackupJobState.CANCELLED; + return; + } + } + getRollupMeta(db.getFullName(), olapTable, pathToWritables); + + // 3. save partition meta + Collection partitionIds = tableIdToPartitionIds.get(tableId); + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + if (partitionInfo.getType() == PartitionType.RANGE) { + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + List>> rangeMap = rangePartitionInfo.getSortedRangeMap(); + for (Map.Entry> entry : rangeMap) { + long partitionId = entry.getKey(); + if (!partitionIds.contains(partitionId)) { + continue; + } + + Partition partition = olapTable.getPartition(partitionId); + if (partition == null) { + throw new DdlException("partition[" + partitionId + "] does not exist"); + } + getPartitionMeta(db.getFullName(), olapTable, partition.getName(), pathToWritables); + + // save version info + partitionIdToVersionInfo.put(partitionId, + new Pair(partition.getCommittedVersion(), + partition.getCommittedVersionHash())); + } + } else { + Preconditions.checkState(partitionIds.size() == 1); + for (Long partitionId : partitionIds) { + Partition partition = olapTable.getPartition(partitionId); + // save version info + partitionIdToVersionInfo.put(partitionId, + new Pair(partition.getCommittedVersion(), + partition.getCommittedVersionHash())); + } + } + } // end for tables + + // get last finished load job and delele job label + Load load = Catalog.getInstance().getLoadInstance(); + LoadJob lastestLoadJob = load.getLastestFinishedLoadJob(dbId); + if (lastestLoadJob == null) { + // there is no load job, or job info has been removed + lastestLoadLabel = "N/A"; + } else { + lastestLoadLabel = lastestLoadJob.getLabel(); + } + LOG.info("get lastest load job label: {}, job: {}", lastestLoadJob, jobId); + + lastestDeleteInfo = load.getLastestFinishedDeleteInfo(dbId); + LOG.info("get lastest delete info: {}, job: {}", lastestDeleteInfo, jobId); + + LOG.info("get meta finished. job[{}]", jobId); + } finally { + db.readUnlock(); + } + } + + private void getTableMeta(String dbName, Table table, Map> pathToWritables) { + CreateTableStmt stmt = table.toCreateTableStmt(dbName); + int tableSignature = table.getSignature(BackupVersion.VERSION_1); + stmt.setTableSignature(tableSignature); + List stmts = Lists.newArrayList(stmt); + String filePath = pathBuilder.createTableStmt(dbName, table.getName()); + + Preconditions.checkState(!pathToWritables.containsKey(filePath)); + pathToWritables.put(filePath, stmts); + } + + private void getRollupMeta(String dbName, OlapTable olapTable, + Map> pathToWritables) { + Set indexIds = Sets.newHashSet(tableIdToIndexIds.get(olapTable.getId())); + if (indexIds.size() == 1) { + // only contains base index. do nothing + return; + } else { + // remove base index id + Preconditions.checkState(indexIds.size() > 1); + indexIds.remove(olapTable.getId()); + } + AlterTableStmt stmt = olapTable.toAddRollupStmt(dbName, indexIds); + String filePath = pathBuilder.addRollupStmt(dbName, olapTable.getName()); + List stmts = Lists.newArrayList(stmt); + + Preconditions.checkState(!pathToWritables.containsKey(filePath)); + pathToWritables.put(filePath, stmts); + } + + private void getPartitionMeta(String dbName, OlapTable olapTable, String partitionName, + Map> pathToWritables) { + AlterTableStmt stmt = olapTable.toAddPartitionStmt(dbName, partitionName); + String filePath = pathBuilder.addPartitionStmt(dbName, olapTable.getName(), partitionName); + List stmts = Lists.newArrayList(stmt); + + Preconditions.checkState(!pathToWritables.containsKey(filePath)); + pathToWritables.put(filePath, stmts); + } + + private void writeMeta(Map> pathToWritables) throws IOException { + // 1. write meta + for (Map.Entry> entry : pathToWritables.entrySet()) { + String filePath = entry.getKey(); + List writables = entry.getValue(); + ObjectWriter.write(filePath, writables); + } + } + + private void snapshot(Database db) throws DdlException { + AgentBatchTask batchTask = new AgentBatchTask(); + LoadBalancer loadBalancer = new LoadBalancer(1L); + long dbId = db.getId(); + db.readLock(); + try { + for (Map.Entry> entry : tableIdToPartitionIds.entrySet()) { + long tableId = entry.getKey(); + Set partitionIds = entry.getValue(); + + Table table = db.getTable(tableId); + if (table == null) { + throw new DdlException("table[" + tableId + "] does not exist"); + } + + if (table.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTable = (OlapTable) table; + + for (Long partitionId : partitionIds) { + Partition partition = olapTable.getPartition(partitionId); + if (partition == null) { + throw new DdlException("partition[" + partitionId + "] does not exist"); + } + + Pair versionInfo = partitionIdToVersionInfo.get(partitionId); + for (Long indexId : tableIdToIndexIds.get(tableId)) { + int schemaHash = olapTable.getSchemaHashByIndexId(indexId); + MaterializedIndex index = partition.getIndex(indexId); + if (index == null) { + throw new DdlException("index[" + indexId + "] does not exist"); + } + + for (Tablet tablet : index.getTablets()) { + long tabletId = tablet.getId(); + List backendIds = Lists.newArrayList(); + for (Replica replica : tablet.getReplicas()) { + if (replica.checkVersionCatchUp(versionInfo.first, versionInfo.second)) { + backendIds.add(replica.getBackendId()); + } + } + + if (backendIds.isEmpty()) { + String msg = "tablet[" + tabletId + "] does not check up with version: " + + versionInfo.first + "-" + versionInfo.second; + // this should not happen + LOG.error(msg); + throw new DdlException(msg); + } + + long chosenBackendId = loadBalancer.chooseKey(backendIds); + SnapshotTask task = new SnapshotTask(null, chosenBackendId, tabletId, jobId, dbId, tableId, + partitionId, indexId, tabletId, + versionInfo.first, versionInfo.second, + schemaHash, -1L, false); + LOG.debug("choose backend[{}] to make snapshot for tablet[{}]", chosenBackendId, tabletId); + batchTask.addTask(task); + unfinishedTabletIds.put(tabletId, chosenBackendId); + } // end for tablet + } // end for indices + } // end for partitions + } // end for tables + + } finally { + db.readUnlock(); + } + + phasedTimeoutMs = unfinishedTabletIds.size() * SNAPSHOT_TIMEOUT_MS; + LOG.debug("estimate snapshot timeout: {}, tablet size: {}", phasedTimeoutMs, unfinishedTabletIds.size()); + + // send task + for (AgentTask task : batchTask.getAllTasks()) { + AgentTaskQueue.addTask(task); + } + AgentTaskExecutor.submit(batchTask); + + state = BackupJobState.SNAPSHOT; + LOG.info("finish send snapshot task. job: {}", jobId); + } + + private synchronized void waitSnapshot() throws DdlException { + if (unfinishedTabletIds.isEmpty()) { + snapshotFinishedTime = System.currentTimeMillis(); + state = BackupJobState.UPLOAD; + + Catalog.getInstance().getEditLog().logBackupFinishSnapshot(this); + LOG.info("backup job[{}] is finished making snapshot", jobId); + + return; + } else if (System.currentTimeMillis() - metaSavedTime > phasedTimeoutMs) { + // remove task in AgentTaskQueue + for (Map.Entry entry : unfinishedTabletIds.entries()) { + AgentTaskQueue.removeTask(entry.getValue(), TTaskType.MAKE_SNAPSHOT, entry.getKey()); + } + + // check timeout + String msg = "snapshot timeout. " + phasedTimeoutMs + "s."; + LOG.warn("{}. job[{}]", msg, jobId); + throw new DdlException(msg); + } else { + LOG.debug("waiting {} tablets to make snapshot", unfinishedTabletIds.size()); + } + } + + private void upload() throws IOException, DdlException, InterruptedException, ExecutionException { + LOG.debug("start upload. job[{}]", jobId); + + if (commandBuilder == null) { + String remotePropFilePath = pathBuilder.remoteProperties(); + commandBuilder = CommandBuilder.create(remotePropFilePath, remoteProperties); + } + Preconditions.checkNotNull(commandBuilder); + + // 1. send meta to remote source + if (!uploadMetaObjs()) { + return; + } + + // 2. send upload task to be + sendUploadTasks(); + } + + private boolean uploadMetaObjs() throws IOException, InterruptedException, ExecutionException { + if (future == null) { + LOG.info("begin to submit upload meta objs. job: {}", jobId); + String dest = PathBuilder.createPath(remotePath, getLabel()); + String uploadCmd = commandBuilder.uploadCmd(getLabel(), pathBuilder.getRoot().getFullPath(), dest); + + MetaUploadTask uploadTask = new MetaUploadTask(uploadCmd); + // future = Catalog.getInstance().getBackupHandler().getAsynchronousCmdExecutor().submit(uploadTask); + return false; + } else { + return checkFuture("upload meta objs"); + } + } + + private synchronized void sendUploadTasks() throws DdlException { + Preconditions.checkState(unfinishedTabletIds.isEmpty()); + + AgentBatchTask batchTask = new AgentBatchTask(); + Database db = Catalog.getInstance().getDb(dbId); + if (db == null) { + throw new DdlException("database[" + getDbName() + "] does not exist"); + } + db.readLock(); + try { + String dbName = db.getFullName(); + for (Map.Entry> entry : tableIdToPartitionIds.entrySet()) { + long tableId = entry.getKey(); + Set partitionIds = entry.getValue(); + + Table table = db.getTable(tableId); + if (table == null) { + throw new DdlException("table[" + tableId + "] does not exist"); + } + + if (table.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTable = (OlapTable) table; + String tableName = olapTable.getName(); + for (Long partitionId : partitionIds) { + Partition partition = olapTable.getPartition(partitionId); + if (partition == null) { + throw new DdlException("partition[" + partitionId + "] does not exist"); + } + + String partitionName = partition.getName(); + for (Long indexId : tableIdToIndexIds.get(tableId)) { + MaterializedIndex index = partition.getIndex(indexId); + if (index == null) { + throw new DdlException("index[" + index + "] does not exist"); + } + + String indexName = olapTable.getIndexNameById(indexId); + for (Tablet tablet : index.getTablets()) { + long tabletId = tablet.getId(); + if (!tabletIdToSnapshotPath.containsKey(tabletId)) { + // this should not happend + String msg = "tablet[" + tabletId + "]'s snapshot is missing"; + LOG.error(msg); + throw new DdlException(msg); + } + + Pair snapshotInfo = tabletIdToSnapshotPath.get(tabletId); + String dest = pathBuilder.tabletRemotePath(dbName, tableName, partitionName, + indexName, tabletId, remotePath, getLabel()); + + unfinishedTabletIds.put(tabletId, snapshotInfo.first); + } // end for tablet + } // end for indices + } // end for partitions + } // end for tables + + } finally { + db.readUnlock(); + } + + // send task + for (AgentTask task : batchTask.getAllTasks()) { + AgentTaskQueue.addTask(task); + } + AgentTaskExecutor.submit(batchTask); + + state = BackupJobState.UPLOADING; + LOG.info("finish send upload task. job: {}", jobId); + } + + private synchronized void waitUpload() throws DdlException { + if (unfinishedTabletIds.isEmpty()) { + LOG.info("backup job[{}] is finished upload snapshot", jobId); + uploadFinishedTime = System.currentTimeMillis(); + state = BackupJobState.FINISHING; + return; + } else { + LOG.debug("waiting {} tablets to upload snapshot", unfinishedTabletIds.size()); + } + } + + private void finishing() throws DdlException, InterruptedException, ExecutionException, IOException { + // save manifest and upload + // manifest contain all file under {label}/ + + if (future == null) { + LOG.info("begin to submit save and upload manifest. job: {}", jobId); + String deleteInfo = lastestDeleteInfo == null ? "" : lastestDeleteInfo.toString(); + SaveManifestTask task = new SaveManifestTask(jobId, getLabel(), remotePath, getLocalDirName(), + lastestLoadLabel, deleteInfo, pathBuilder, commandBuilder); + // future = Catalog.getInstance().getBackupHandler().getAsynchronousCmdExecutor().submit(task); + } else { + boolean finished = checkFuture("save and upload manifest"); + if (finished) { + // reset future + readableManifestPath = + PathBuilder.createPath(remotePath, getLabel(), PathBuilder.READABLE_MANIFEST_NAME); + future = null; + state = BackupJobState.FINISHED; + } + } + } + + public void restoreTableState(Catalog catalog) { + Database db = catalog.getDb(dbId); + if (db != null) { + db.writeLock(); + try { + for (long tableId : tableIdToPartitionIds.keySet()) { + Table table = db.getTable(tableId); + if (table != null && table.getType() == TableType.OLAP) { + if (((OlapTable) table).getState() == OlapTableState.BACKUP) { + ((OlapTable) table).setState(OlapTableState.NORMAL); + LOG.debug("set table[{}] state to NORMAL", table.getName()); + } + } + } + } finally { + db.writeUnlock(); + } + } + } + + private void removeLeftTasks() { + for (Map.Entry entry : unfinishedTabletIds.entries()) { + AgentTaskQueue.removeTask(entry.getValue(), TTaskType.MAKE_SNAPSHOT, entry.getKey()); + AgentTaskQueue.removeTask(entry.getValue(), TTaskType.UPLOAD, entry.getKey()); + } + } + + @Override + public void end(Catalog catalog, boolean isReplay) { + // 1. set table state + restoreTableState(catalog); + + if (!isReplay) { + // 2. remove agent tasks if left + removeLeftTasks(); + + if (pathBuilder == null) { + finishedTime = System.currentTimeMillis(); + Catalog.getInstance().getEditLog().logBackupFinish(this); + LOG.info("finished end job[{}]. state: {}", jobId, state.name()); + return; + } + + // 3. remove local file + String labelDir = pathBuilder.getRoot().getFullPath(); + Util.deleteDirectory(new File(labelDir)); + LOG.debug("delete local dir: {}", labelDir); + + // 4. release snapshot + synchronized (tabletIdToSnapshotPath) { + AgentBatchTask batchTask = new AgentBatchTask(); + for (Long tabletId : tabletIdToSnapshotPath.keySet()) { + long backendId = tabletIdToSnapshotPath.get(tabletId).first; + String snapshotPath = tabletIdToSnapshotPath.get(tabletId).second; + ReleaseSnapshotTask task = new ReleaseSnapshotTask(null, backendId, dbId, tabletId, snapshotPath); + batchTask.addTask(task); + } + // no need to add to AgentTaskQueue + AgentTaskExecutor.submit(batchTask); + } + + finishedTime = System.currentTimeMillis(); + + Catalog.getInstance().getEditLog().logBackupFinish(this); + } + + clearJob(); + + LOG.info("finished end job[{}]. state: {}, replay: {}", jobId, state.name(), isReplay); + } + + @Override + protected void clearJob() { + tableIdToPartitionIds = null; + tableIdToIndexIds = null; + partitionIdToVersionInfo = null; + tabletIdToSnapshotPath = null; + + unfinishedTabletIds = null; + remoteProperties = null; + pathBuilder = null; + commandBuilder = null; + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + + Text.writeString(out, state.name()); + Text.writeString(out, lastestLoadLabel); + + if (lastestDeleteInfo == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + lastestDeleteInfo.write(out); + } + + if (tableIdToPartitionIds == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableIdToPartitionIds.size(); + out.writeInt(size); + for (Map.Entry> entry : tableIdToPartitionIds.entrySet()) { + out.writeLong(entry.getKey()); + size = entry.getValue().size(); + out.writeInt(size); + for (Long partitionId : entry.getValue()) { + out.writeLong(partitionId); + } + } + } + + if (tableIdToIndexIds == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Collection> entries = tableIdToIndexIds.entries(); + int size = entries.size(); + out.writeInt(size); + for (Map.Entry entry : entries) { + out.writeLong(entry.getKey()); + out.writeLong(entry.getValue()); + } + } + + if (partitionIdToVersionInfo == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = partitionIdToVersionInfo.size(); + out.writeInt(size); + for (Map.Entry> entry : partitionIdToVersionInfo.entrySet()) { + out.writeLong(entry.getKey()); + Pair pair = entry.getValue(); + out.writeLong(pair.first); + out.writeLong(pair.second); + } + } + + if (tabletIdToSnapshotPath == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tabletIdToSnapshotPath.size(); + out.writeInt(size); + for (Map.Entry> entry : tabletIdToSnapshotPath.entrySet()) { + out.writeLong(entry.getKey()); + Pair pair = entry.getValue(); + out.writeLong(pair.first); + Text.writeString(out, pair.second); + } + } + + out.writeLong(metaSavedTime); + out.writeLong(snapshotFinishedTime); + out.writeLong(uploadFinishedTime); + out.writeLong(phasedTimeoutMs); + + Text.writeString(out, readableManifestPath); + + if (pathBuilder == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + pathBuilder.write(out); + } + + if (commandBuilder == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + commandBuilder.write(out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + state = BackupJobState.valueOf(Text.readString(in)); + lastestLoadLabel = Text.readString(in); + + if (in.readBoolean()) { + lastestDeleteInfo = new DeleteInfo(); + lastestDeleteInfo.readFields(in); + } + + if (in.readBoolean()) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long tableId = in.readLong(); + Set partitionIds = Sets.newHashSet(); + tableIdToPartitionIds.put(tableId, partitionIds); + int count = in.readInt(); + for (int j = 0; j < count; j++) { + long partitionId = in.readLong(); + partitionIds.add(partitionId); + } + } + } + + if (in.readBoolean()) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long tableId = in.readLong(); + long indexId = in.readLong(); + tableIdToIndexIds.put(tableId, indexId); + } + } + + if (in.readBoolean()) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long partitionId = in.readLong(); + long version = in.readLong(); + long versionHash = in.readLong(); + partitionIdToVersionInfo.put(partitionId, new Pair(version, versionHash)); + } + } + + if (in.readBoolean()) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long tabletId = in.readLong(); + long backendId = in.readLong(); + String path = Text.readString(in); + tabletIdToSnapshotPath.put(tabletId, new Pair(backendId, path)); + } + } + + metaSavedTime = in.readLong(); + snapshotFinishedTime = in.readLong(); + uploadFinishedTime = in.readLong(); + phasedTimeoutMs = in.readLong(); + + readableManifestPath = Text.readString(in); + + if (in.readBoolean()) { + pathBuilder = new PathBuilder(); + pathBuilder.readFields(in); + } + + if (in.readBoolean()) { + commandBuilder = new CommandBuilder(); + commandBuilder.readFields(in); + } + + } +} diff --git a/fe/src/com/baidu/palo/backup/BackupMeta.java b/fe/src/com/baidu/palo/backup/BackupMeta.java new file mode 100644 index 0000000000..edb1b6c3f9 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/BackupMeta.java @@ -0,0 +1,119 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.catalog.Table; +import com.baidu.palo.common.io.Writable; + +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class BackupMeta implements Writable { + private static final Logger LOG = LogManager.getLogger(BackupMeta.class); + + // tbl name -> tbl + private Map tblNameMap = Maps.newHashMap(); + // tbl id -> tbl + private Map tblIdMap = Maps.newHashMap(); + + private BackupMeta() { + + } + + public BackupMeta(List
tables) { + for (Table table : tables) { + tblNameMap.put(table.getName(), table); + tblIdMap.put(table.getId(), table); + } + } + + public Map getTables() { + return tblNameMap; + } + + public Table getTable(String tblName) { + return tblNameMap.get(tblName); + } + + public Table getTable(Long tblId) { + return tblIdMap.get(tblId); + } + + public static BackupMeta fromFile(String filePath) throws IOException { + File file = new File(filePath); + try (DataInputStream dis = new DataInputStream(new FileInputStream(file))) { + BackupMeta backupMeta = BackupMeta.read(dis); + return backupMeta; + } + } + + public void writeToFile(File metaInfoFile) throws IOException { + DataOutputStream dos = new DataOutputStream(new FileOutputStream(metaInfoFile)); + try { + write(dos); + dos.flush(); + } finally { + dos.close(); + } + } + + public boolean compatibleWith(BackupMeta other) { + // TODO + return false; + } + + public static BackupMeta read(DataInput in) throws IOException { + BackupMeta backupMeta = new BackupMeta(); + backupMeta.readFields(in); + return backupMeta; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(tblNameMap.size()); + for (Table table : tblNameMap.values()) { + table.write(out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + Table tbl = Table.read(in); + tblNameMap.put(tbl.getName(), tbl); + tblIdMap.put(tbl.getId(), tbl); + } + } +} diff --git a/fe/src/com/baidu/palo/backup/BlobStorage.java b/fe/src/com/baidu/palo/backup/BlobStorage.java new file mode 100644 index 0000000000..97df20066a --- /dev/null +++ b/fe/src/com/baidu/palo/backup/BlobStorage.java @@ -0,0 +1,700 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.backup.Status.ErrCode; +import com.baidu.palo.catalog.BrokerMgr; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.ClientPool; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.Pair; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.service.FrontendOptions; +import com.baidu.palo.thrift.TBrokerCheckPathExistRequest; +import com.baidu.palo.thrift.TBrokerCheckPathExistResponse; +import com.baidu.palo.thrift.TBrokerCloseReaderRequest; +import com.baidu.palo.thrift.TBrokerCloseWriterRequest; +import com.baidu.palo.thrift.TBrokerFD; +import com.baidu.palo.thrift.TBrokerFileStatus; +import com.baidu.palo.thrift.TBrokerListPathRequest; +import com.baidu.palo.thrift.TBrokerListResponse; +import com.baidu.palo.thrift.TBrokerOpenMode; +import com.baidu.palo.thrift.TBrokerOpenReaderRequest; +import com.baidu.palo.thrift.TBrokerOpenReaderResponse; +import com.baidu.palo.thrift.TBrokerOpenWriterRequest; +import com.baidu.palo.thrift.TBrokerOpenWriterResponse; +import com.baidu.palo.thrift.TBrokerOperationStatus; +import com.baidu.palo.thrift.TBrokerOperationStatusCode; +import com.baidu.palo.thrift.TBrokerPReadRequest; +import com.baidu.palo.thrift.TBrokerPWriteRequest; +import com.baidu.palo.thrift.TBrokerReadResponse; +import com.baidu.palo.thrift.TBrokerRenamePathRequest; +import com.baidu.palo.thrift.TBrokerVersion; +import com.baidu.palo.thrift.TNetworkAddress; +import com.baidu.palo.thrift.TPaloBrokerService; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +public class BlobStorage implements Writable { + private static final Logger LOG = LogManager.getLogger(BlobStorage.class); + + private String brokerName; + private Map properties = Maps.newHashMap(); + + private BlobStorage() { + // for persist + } + + public BlobStorage(String brokerName, Map properties) { + this.brokerName = brokerName; + this.properties = properties; + } + + public String getBrokerName() { + return brokerName; + } + + public Map getProperties() { + return properties; + } + + public Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize) { + LOG.debug("download from {} to {}, file size: {}.", + remoteFilePath, localFilePath, fileSize); + + long start = System.currentTimeMillis(); + + // 1. get a proper broker + Pair pair = new Pair( + null, null); + Status st = getBroker(pair); + if (!st.ok()) { + return st; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + // 2. open file reader with broker + TBrokerFD fd = null; + try { + TBrokerOpenReaderRequest req = new TBrokerOpenReaderRequest(TBrokerVersion.VERSION_ONE, remoteFilePath, + 0, clientId(), properties); + TBrokerOpenReaderResponse rep = client.openReader(req); + TBrokerOperationStatus opst = rep.getOpStatus(); + if (opst.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to open reader on broker " + brokerName + " for file: " + + remoteFilePath + ". msg: " + opst.getMessage()); + } + + fd = rep.getFd(); + LOG.info("finished to open reader. fd: {}. download {} to {}.", + fd, remoteFilePath, localFilePath); + } catch (TException e) { + return new Status(ErrCode.COMMON_ERROR, + "failed to open reader on broker " + brokerName + " for file: " + + remoteFilePath + ". msg: " + e.getMessage()); + } + Preconditions.checkNotNull(fd); + + // 3. delete local file if exist + File localFile = new File(localFilePath); + if (localFile.exists()) { + try { + Files.walk(Paths.get(localFilePath), + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to delete exist local file: " + localFilePath); + } + } + + // 4. create local file + Status status = Status.OK; + try { + if (!localFile.createNewFile()) { + return new Status(ErrCode.COMMON_ERROR, "failed to create local file: " + localFilePath); + } + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to create local file: " + + localFilePath + ", msg: " + e.getMessage()); + } + + // 5. read remote file with broker and write to local + String lastErrMsg = null; + try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(localFile))) { + final long bufSize = 1024 * 1024; // 1MB + long leftSize = fileSize; + long readOffset = 0; + while (leftSize > 0) { + long readLen = leftSize > bufSize ? bufSize : leftSize; + TBrokerReadResponse rep = null; + // We only retry if we encounter a timeout thrift exception. + int tryTimes = 0; + while (tryTimes < 3) { + try { + TBrokerPReadRequest req = new TBrokerPReadRequest(TBrokerVersion.VERSION_ONE, + fd, readOffset, readLen); + rep = client.pread(req); + if (rep.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { + // pread return failure. + lastErrMsg = String.format("failed to read via broker %s. " + + "current read offset: %d, read length: %d," + + " file size: %d, file: %s, err code: %d, msg: %s", + brokerName, readOffset, readLen, fileSize, + remoteFilePath, rep.getOpStatus().getStatusCode(), + rep.getOpStatus().getMessage()); + LOG.warn(lastErrMsg); + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + } + LOG.debug("download. readLen: {}, read data len: {}, left size:{}. total size: {}", + readLen, rep.getData().length, leftSize, fileSize); + break; + } catch (TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT) { + // we only retry when we encounter timeout exception. + lastErrMsg = String.format("failed to read via broker %s. " + + "current read offset: %d, read length: %d," + + " file size: %d, file: %s, timeout.", + brokerName, readOffset, readLen, fileSize, + remoteFilePath); + tryTimes++; + continue; + } + + lastErrMsg = String.format("failed to read via broker %s. " + + "current read offset: %d, read length: %d," + + " file size: %d, file: %s. msg: %s", + brokerName, readOffset, readLen, fileSize, + remoteFilePath, e.getMessage()); + LOG.warn(lastErrMsg); + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + break; + } catch (TException e) { + lastErrMsg = String.format("failed to read via broker %s. " + + "current read offset: %d, read length: %d," + + " file size: %d, file: %s. msg: %s", + brokerName, readOffset, readLen, fileSize, + remoteFilePath, e.getMessage()); + LOG.warn(lastErrMsg); + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + break; + } + } // end of retry loop + + if (status.ok() && tryTimes < 3) { + // read succeed, write to local file + Preconditions.checkNotNull(rep); + // NOTICE(cmy): Sometimes the actual read length does not equal to the expected read length, + // even if the broker's read buffer size is large enough. + // I don't know why, but have to adapt to it. + if (rep.getData().length != readLen) { + LOG.warn("the actual read length does not equal to " + + "the expected read length: {} vs. {}, file: {}", + rep.getData().length, readLen, remoteFilePath); + } + + out.write(rep.getData()); + readOffset += rep.getData().length; + leftSize -= rep.getData().length; + } else { + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + break; + } + } // end of reading remote file + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "Got exception: " + e.getMessage()); + } finally { + // close broker reader + Status closeStatus = closeReader(client, fd); + if (!closeStatus.ok()) { + LOG.warn(closeStatus.getErrMsg()); + if (status.ok()) { + // we return close write error only if no other error has been encountered. + status = closeStatus; + } + ClientPool.brokerPool.invalidateObject(address, client); + } else { + ClientPool.brokerPool.returnObject(address, client); + } + } + + LOG.info("finished to download from {} to {} with size: {}. cost {} ms", remoteFilePath, localFilePath, + fileSize, (System.currentTimeMillis() - start)); + return status; + } + + // directly upload the content to remote file + public Status directUpload(String content, String remoteFile) { + Status status = Status.OK; + + // 1. get a proper broker + Pair pair = new Pair( + null, null); + status = getBroker(pair); + if (!status.ok()) { + return status; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + TBrokerFD fd = new TBrokerFD(); + try { + // 2. open file write with broker + status = openWriter(client, address, remoteFile, fd); + if (!status.ok()) { + return status; + } + + // 3. write content + try { + ByteBuffer bb = ByteBuffer.wrap(content.getBytes("UTF-8")); + TBrokerPWriteRequest req = new TBrokerPWriteRequest(TBrokerVersion.VERSION_ONE, fd, 0, bb); + TBrokerOperationStatus opst = client.pwrite(req); + if (opst.getStatusCode() != TBrokerOperationStatusCode.OK) { + // pwrite return failure. + status = new Status(ErrCode.COMMON_ERROR, "write failed: " + opst.getMessage()); + } + } catch (TException e) { + status = new Status(ErrCode.BAD_CONNECTION, "write exception: " + e.getMessage()); + } catch (UnsupportedEncodingException e) { + status = new Status(ErrCode.COMMON_ERROR, "unsupported encoding: " + e.getMessage()); + } + } finally { + Status closeStatus = closeWriter(client, fd); + if (closeStatus.getErrCode() == ErrCode.BAD_CONNECTION || status.getErrCode() == ErrCode.BAD_CONNECTION) { + ClientPool.brokerPool.invalidateObject(address, client); + } else { + ClientPool.brokerPool.returnObject(address, client); + } + } + + return status; + } + + public Status upload(String localPath, String remotePath) { + long start = System.currentTimeMillis(); + + Status status = Status.OK; + + // 1. get a proper broker + Pair pair = new Pair(null, null); + status = getBroker(pair); + if (!status.ok()) { + return status; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + // 2. open file write with broker + TBrokerFD fd = new TBrokerFD(); + status = openWriter(client, address, remotePath, fd); + if (!status.ok()) { + return status; + } + + // 3. read local file and write to remote with broker + File localFile = new File(localPath); + long fileLength = localFile.length(); + byte[] readBuf = new byte[1024]; + try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(localFile))) { + // save the last err msg + String lastErrMsg = null; + // save the current write offset of remote file + long writeOffset = 0; + // read local file, 1MB at a time + int bytesRead = 0; + while ((bytesRead = in.read(readBuf)) != -1) { + ByteBuffer bb = ByteBuffer.wrap(readBuf, 0, bytesRead); + + // We only retry if we encounter a timeout thrift exception. + int tryTimes = 0; + while (tryTimes < 3) { + try { + TBrokerPWriteRequest req = new TBrokerPWriteRequest(TBrokerVersion.VERSION_ONE, fd, writeOffset, bb); + TBrokerOperationStatus opst = client.pwrite(req); + if (opst.getStatusCode() != TBrokerOperationStatusCode.OK) { + // pwrite return failure. + lastErrMsg = String.format("failed to write via broker %s. " + + "current write offset: %d, write length: %d," + + " file length: %d, file: %s, err code: %d, msg: %s", + brokerName, writeOffset, bytesRead, fileLength, + remotePath, opst.getStatusCode(), opst.getMessage()); + LOG.warn(lastErrMsg); + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + } + break; + } catch (TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT) { + // we only retry when we encounter timeout exception. + lastErrMsg = String.format("failed to write via broker %s. " + + "current write offset: %d, write length: %d," + + " file length: %d, file: %s. timeout", + brokerName, writeOffset, bytesRead, fileLength, + remotePath); + tryTimes++; + continue; + } + + lastErrMsg = String.format("failed to write via broker %s. " + + "current write offset: %d, write length: %d," + + " file length: %d, file: %s. encounter TTransportException: %s", + brokerName, writeOffset, bytesRead, fileLength, + remotePath, e.getMessage()); + LOG.warn(lastErrMsg, e); + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + break; + } catch (TException e) { + lastErrMsg = String.format("failed to write via broker %s. " + + "current write offset: %d, write length: %d," + + " file length: %d, file: %s. encounter TException: %s", + brokerName, writeOffset, bytesRead, fileLength, + remotePath, e.getMessage()); + LOG.warn(lastErrMsg, e); + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + break; + } + } + + if (status.ok() && tryTimes < 3) { + // write succeed, update current write offset + writeOffset += bytesRead; + } else { + status = new Status(ErrCode.COMMON_ERROR, lastErrMsg); + break; + } + } // end of read local file loop + } catch (FileNotFoundException e1) { + return new Status(ErrCode.COMMON_ERROR, "encounter file not found exception: " + e1.getMessage()); + } catch (IOException e1) { + return new Status(ErrCode.COMMON_ERROR, "encounter io exception: " + e1.getMessage()); + } finally { + // close write + Status closeStatus = closeWriter(client, fd); + if (!closeStatus.ok()) { + LOG.warn(closeStatus.getErrMsg()); + if (status.ok()) { + // we return close write error only if no other error has been encountered. + status = closeStatus; + } + ClientPool.brokerPool.invalidateObject(address, client); + } else { + ClientPool.brokerPool.returnObject(address, client); + } + } + + if (status.ok()) { + LOG.info("finished to upload {} to remote path {}. cost: {} ms", + localPath, remotePath, (System.currentTimeMillis() - start)); + } + return status; + } + + public Status rename(String origFilePath, String destFilePath) { + long start = System.currentTimeMillis(); + Status status = Status.OK; + + // 1. get a proper broker + Pair pair = new Pair( + null, null); + status = getBroker(pair); + if (!status.ok()) { + return status; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + // 2. rename + boolean needReturn = true; + try { + TBrokerRenamePathRequest req = new TBrokerRenamePathRequest(TBrokerVersion.VERSION_ONE, origFilePath, + destFilePath, properties); + TBrokerOperationStatus ost = client.renamePath(req); + if (ost.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to rename " + origFilePath + " to " + destFilePath + ", msg: " + ost.getMessage()); + } + } catch (TException e) { + needReturn = false; + return new Status(ErrCode.COMMON_ERROR, + "failed to rename " + origFilePath + " to " + destFilePath + ", msg: " + e.getMessage()); + } finally { + if (needReturn) { + ClientPool.brokerPool.returnObject(address, client); + } else { + ClientPool.brokerPool.invalidateObject(address, client); + } + } + + LOG.info("finished to rename {} to {}. cost: {} ms", + origFilePath, destFilePath, (System.currentTimeMillis() - start)); + return Status.OK; + } + + public Status delete(String remotePath) { + return Status.OK; + } + + // List files in remotePath + // The remote file name will only contains file name only(Not full path) + public Status list(String remotePath, List result) { + // get a proper broker + Pair pair = new Pair(null, null); + Status st = getBroker(pair); + if (!st.ok()) { + return st; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + // list + boolean needReturn = true; + try { + TBrokerListPathRequest req = new TBrokerListPathRequest(TBrokerVersion.VERSION_ONE, remotePath, + false /* not recursive */, properties); + req.setFileNameOnly(true); + TBrokerListResponse rep = client.listPath(req); + TBrokerOperationStatus opst = rep.getOpStatus(); + if (opst.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to list remote path: " + remotePath + ". msg: " + opst.getMessage()); + } + + List fileStatus = rep.getFiles(); + for (TBrokerFileStatus tFile : fileStatus) { + RemoteFile file = new RemoteFile(tFile.path, !tFile.isDir, tFile.size); + result.add(file); + } + LOG.info("finished to list remote path {}. get files: {}", remotePath, result); + } catch (TException e) { + needReturn = false; + return new Status(ErrCode.COMMON_ERROR, + "failed to list remote path: " + remotePath + ". msg: " + e.getMessage()); + } finally { + if (needReturn) { + ClientPool.brokerPool.returnObject(address, client); + } else { + ClientPool.brokerPool.invalidateObject(address, client); + } + } + + return Status.OK; + } + + public Status makeDir(String remotePath) { + // 1. get a proper broker + Pair pair = new Pair( + null, null); + Status st = getBroker(pair); + if (!st.ok()) { + return st; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + // TODO: mkdir + return Status.OK; + } + + public Status checkPathExist(String remotePath) { + // 1. get a proper broker + Pair pair = new Pair( + null, null); + Status st = getBroker(pair); + if (!st.ok()) { + return st; + } + TPaloBrokerService.Client client = pair.first; + TNetworkAddress address = pair.second; + + // check path + boolean needReturn = true; + try { + TBrokerCheckPathExistRequest req = new TBrokerCheckPathExistRequest(TBrokerVersion.VERSION_ONE, + remotePath, properties); + TBrokerCheckPathExistResponse rep = client.checkPathExist(req); + TBrokerOperationStatus opst = rep.getOpStatus(); + if (opst.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to check remote path exist: " + remotePath + ". msg: " + opst.getMessage()); + } + + if (!rep.isIsPathExist()) { + return new Status(ErrCode.NOT_FOUND, "remote path does not exist: " + remotePath); + } + + return Status.OK; + } catch (TException e) { + needReturn = false; + return new Status(ErrCode.COMMON_ERROR, + "failed to check remote path exist: " + remotePath + ". msg: " + e.getMessage()); + } finally { + if (needReturn) { + ClientPool.brokerPool.returnObject(address, client); + } else { + ClientPool.brokerPool.invalidateObject(address, client); + } + } + } + + public static String clientId() { + return FrontendOptions.getLocalHostAddress() + ":" + Config.edit_log_port; + } + + private Status getBroker(Pair result) { + BrokerMgr.BrokerAddress brokerAddress = null; + try { + String localIP = FrontendOptions.getLocalHostAddress(); + brokerAddress = Catalog.getInstance().getBrokerMgr().getBroker(brokerName, localIP); + } catch (AnalysisException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to get a broker address: " + e.getMessage()); + } + TNetworkAddress address = new TNetworkAddress(brokerAddress.ip, brokerAddress.port); + TPaloBrokerService.Client client = null; + try { + client = ClientPool.brokerPool.borrowObject(address); + } catch (Exception e) { + return new Status(ErrCode.COMMON_ERROR, "failed to get broker client: " + e.getMessage()); + } + + result.first = client; + result.second = address; + return Status.OK; + } + + private Status openWriter(TPaloBrokerService.Client client, TNetworkAddress address, String remoteFile, + TBrokerFD fd) { + try { + TBrokerOpenWriterRequest req = new TBrokerOpenWriterRequest(TBrokerVersion.VERSION_ONE, + remoteFile, TBrokerOpenMode.APPEND, clientId(), properties); + TBrokerOpenWriterResponse rep = client.openWriter(req); + TBrokerOperationStatus opst = rep.getOpStatus(); + if (opst.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to open writer on broker " + brokerName + " for file: " + + remoteFile + ". msg: " + opst.getMessage()); + } + + fd.setHigh(rep.getFd().getHigh()); + fd.setLow(rep.getFd().getLow()); + LOG.info("finished to open writer. fd: {}. directly upload to remote path {}.", + fd, remoteFile); + } catch (TException e) { + return new Status(ErrCode.BAD_CONNECTION, + "failed to open writer on broker " + brokerName + ": " + e.getMessage()); + } + + return Status.OK; + } + + private Status closeWriter(TPaloBrokerService.Client client, TBrokerFD fd) { + try { + TBrokerCloseWriterRequest req = new TBrokerCloseWriterRequest(TBrokerVersion.VERSION_ONE, fd); + TBrokerOperationStatus st = client.closeWriter(req); + if (st.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to close writer on broker " + brokerName + " for fd: " + fd); + } + + LOG.info("finished to close writer. fd: {}.", fd); + } catch (TException e) { + return new Status(ErrCode.BAD_CONNECTION, + "failed to close writer on broker " + brokerName + ", fd " + fd + ", msg: " + e.getMessage()); + } + + return Status.OK; + } + + private Status closeReader(TPaloBrokerService.Client client, TBrokerFD fd) { + try { + TBrokerCloseReaderRequest req = new TBrokerCloseReaderRequest(TBrokerVersion.VERSION_ONE, fd); + TBrokerOperationStatus st = client.closeReader(req); + if (st.getStatusCode() != TBrokerOperationStatusCode.OK) { + return new Status(ErrCode.COMMON_ERROR, + "failed to close reader on broker " + brokerName + " for fd: " + fd); + } + + LOG.info("finished to close reader. fd: {}.", fd); + } catch (TException e) { + return new Status(ErrCode.BAD_CONNECTION, + "failed to close reader on broker " + brokerName + ", fd " + fd + ", msg: " + e.getMessage()); + } + + return Status.OK; + } + + public static BlobStorage read(DataInput in) throws IOException { + BlobStorage blobStorage = new BlobStorage(); + blobStorage.readFields(in); + return blobStorage; + } + + @Override + public void write(DataOutput out) throws IOException { + // must write type first + Text.writeString(out, brokerName); + + out.writeInt(properties.size()); + for (Map.Entry entry : properties.entrySet()) { + Text.writeString(out, entry.getKey()); + Text.writeString(out, entry.getValue()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + brokerName = Text.readString(in); + + // properties + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String key = Text.readString(in); + String value = Text.readString(in); + properties.put(key, value); + } + } +} diff --git a/fe/src/com/baidu/palo/backup/RemoteFile.java b/fe/src/com/baidu/palo/backup/RemoteFile.java new file mode 100644 index 0000000000..7fb15f55c8 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/RemoteFile.java @@ -0,0 +1,56 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +// represent a file or a dir in remote storage +public class RemoteFile { + // Only file name, not full path + private String name; + private boolean isFile; + private long size; + + public RemoteFile(String name, boolean isFile, long size) { + Preconditions.checkState(!Strings.isNullOrEmpty(name)); + this.name = name; + this.isFile = isFile; + this.size = size; + } + + public String getName() { + return name; + } + + public boolean isFile() { + return isFile; + } + + public long getSize() { + return size; + } + + @Override + public String toString() { + return "[name: " + name + ", is file: " + isFile + "]"; + } +} diff --git a/fe/src/com/baidu/palo/backup/Repository.java b/fe/src/com/baidu/palo/backup/Repository.java new file mode 100644 index 0000000000..61256e5cc3 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/Repository.java @@ -0,0 +1,651 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.backup.Status.ErrCode; +import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.Pair; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.system.Backend; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.List; + +/* + * Repository represents a remote storage for backup to or restore from + * File organization in repository is: + * + * * __palo_repository_repo_name/ + * * __repo_info + * * __ss_my_ss1/ + * * __meta__DJdwnfiu92n + * * __info_2018-01-01-08-00-00.OWdn90ndwpu + * * __info_2018-01-02-08-00-00.Dnvdio298da + * * __info_2018-01-03-08-00-00.O79adbneJWk + * * __ss_content/ + * * __db_10001/ + * * __tbl_10010/ + * * __tbl_10020/ + * * __part_10021/ + * * __part_10031/ + * * __idx_10041/ + * * __idx_10020/ + * * __10022/ + * * __10023/ + * * __10023_seg1.dat.NUlniklnwDN67 + * * __10023_seg2.dat.DNW231dnklawd + * * __10023.hdr.dnmwDDWI92dDko + */ +public class Repository implements Writable { + private static final Logger LOG = LogManager.getLogger(Repository.class); + + public static final String PREFIX_REPO = "__palo_repository_"; + public static final String PREFIX_SNAPSHOT_DIR = "__ss_"; + public static final String PREFIX_DB = "__db_"; + public static final String PREFIX_TBL = "__tbl_"; + public static final String PREFIX_PART = "__part_"; + public static final String PREFIX_IDX = "__idx_"; + public static final String PREFIX_COMMON = "__"; + public static final String PREFIX_JOB_INFO = "__info_"; + + public static final String SUFFIX_TMP_FILE = "part"; + + public static final String FILE_REPO_INFO = "__repo_info"; + public static final String FILE_META_INFO = "__meta"; + + public static final String DIR_SNAPSHOT_CONTENT = "__ss_content"; + + private static final String PATH_DELIMITER = "/"; + private static final String CHECKSUM_SEPARATOR = "."; + + private long id; + private String name; + private String errMsg; + private long createTime; + + // If True, user can not backup data to this repo. + private boolean isReadOnly; + + // BOS location should start with "bos://your_bucket_name/" + // and the specified bucket should exist. + private String location; + + private BlobStorage storage; + + private Repository() { + // for persist + } + + public Repository(long id, String name, boolean isReadOnly, String location, BlobStorage storage) { + this.id = id; + this.name = name; + this.isReadOnly = isReadOnly; + this.location = location; + this.storage = storage; + this.createTime = System.currentTimeMillis(); + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public boolean isReadOnly() { + return isReadOnly; + } + + public String getLocation() { + return location; + } + + public String getErrorMsg() { + return errMsg; + } + + public BlobStorage getStorage() { + return storage; + } + + public long getCreateTime() { + return createTime; + } + + // create repository dir and repo info file + public Status initRepository() { + String repoInfoFilePath = assembleRepoInfoFilePath(); + // check if the repo is already exist in remote + List remoteFiles = Lists.newArrayList(); + Status st = storage.list(repoInfoFilePath, remoteFiles); + if (!st.ok()) { + return st; + } + if (remoteFiles.size() == 1) { + RemoteFile remoteFile = remoteFiles.get(0); + if (!remoteFile.isFile()) { + return new Status(ErrCode.COMMON_ERROR, "the existing repo info is not a file"); + } + + // exist, download and parse the repo info file + String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/tmp_info_" + System.currentTimeMillis(); + try { + st = storage.downloadWithFileSize(repoInfoFilePath, localFilePath, remoteFile.getSize()); + if (!st.ok()) { + return st; + } + + byte[] bytes = Files.readAllBytes(Paths.get(localFilePath)); + String json = new String(bytes, StandardCharsets.UTF_8); + JSONObject root = new JSONObject(json); + name = (String) root.get("name"); + createTime = TimeUtils.timeStringToLong((String) root.get("create_time")); + if (createTime == -1) { + return new Status(ErrCode.COMMON_ERROR, + "failed to parse create time of repository: " + (String) root.get("create_time")); + } + return Status.OK; + + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to read repo info file: " + e.getMessage()); + } finally { + File localFile = new File(localFilePath); + localFile.delete(); + } + + } else if (remoteFiles.size() > 1) { + return new Status(ErrCode.COMMON_ERROR, + "Invalid repository dir. expected one repo info file. get more: " + remoteFiles); + } else { + // repo is already exist, get repo info + JSONObject root = new JSONObject(); + root.put("name", name); + root.put("create_time", TimeUtils.longToTimeString(createTime)); + String repoInfoContent = root.toString(); + return storage.directUpload(repoInfoContent, repoInfoFilePath); + } + } + + // eg: location/__palo_repository_repo_name/__repo_info + public String assembleRepoInfoFilePath() { + return Joiner.on(PATH_DELIMITER).join(location, + joinPrefix(PREFIX_REPO, name), + FILE_REPO_INFO); + } + + // eg: location/__palo_repository_repo_name/__my_sp1/__meta + public String assembleMetaInfoFilePath(String label) { + return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + joinPrefix(PREFIX_SNAPSHOT_DIR, label), + FILE_META_INFO); + } + + // eg: location/__palo_repository_repo_name/__my_sp1/__info_2018-01-01-08-00-00 + public String assembleJobInfoFilePath(String label, long createTime) { + return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + joinPrefix(PREFIX_SNAPSHOT_DIR, label), + jobInfoFileNameWithTimestamp(createTime)); + } + + // eg: + // __palo_repository_repo_name/__ss_my_ss1/__ss_content/__db_10001/__tbl_10020/__part_10031/__idx_10020/__10022/ + public String getRepoTabletPathBySnapshotInfo(String label, SnapshotInfo info) { + return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + joinPrefix(PREFIX_SNAPSHOT_DIR, label), + DIR_SNAPSHOT_CONTENT, + joinPrefix(PREFIX_DB, info.getDbId()), + joinPrefix(PREFIX_TBL, info.getTblId()), + joinPrefix(PREFIX_PART, info.getPartitionId()), + joinPrefix(PREFIX_IDX, info.getIndexId()), + joinPrefix(PREFIX_COMMON, info.getTabletId())); + } + + public String getRepoPath(String label, String childPath) { + return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + joinPrefix(PREFIX_SNAPSHOT_DIR, label), + DIR_SNAPSHOT_CONTENT, + childPath); + } + + // Check if this repo is available. + // If failed to connect this repo, set errMsg and return false. + public boolean ping() { + String checkPath = Joiner.on(PATH_DELIMITER).join(location, + joinPrefix(PREFIX_REPO, name)); + Status st = storage.checkPathExist(checkPath); + if (!st.ok()) { + errMsg = TimeUtils.longToTimeString(System.currentTimeMillis()) + ": " + st.getErrMsg(); + return false; + } + + // clear err msg + errMsg = null; + + return true; + } + + // Visit the repository, and list all existing snapshot names + public Status listSnapshots(List snapshotNames) { + // list with prefix: + // eg. __palo_repository_repo_name/__ss_* + String listPath = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), PREFIX_SNAPSHOT_DIR) + + "*"; + List result = Lists.newArrayList(); + Status st = storage.list(listPath, result); + if (!st.ok()) { + return st; + } + + for (RemoteFile remoteFile : result) { + if (remoteFile.isFile()) { + LOG.debug("get snapshot path{} which is not a dir", remoteFile); + continue; + } + + snapshotNames.add(disjoinPrefix(PREFIX_SNAPSHOT_DIR, remoteFile.getName())); + } + return Status.OK; + } + + // + public boolean prepareSnapshotInfo() { + return false; + } + + // create remote tablet snapshot path + // eg: + // /location/__palo_repository_repo_name/__ss_my_ss1/__ss_content/__db_10001/__tbl_10020/__part_10031/__idx_10032/__10023/__3481721 + public String assembleRemoteSnapshotPath(String label, SnapshotInfo info) { + String path = Joiner.on(PATH_DELIMITER).join(location, + joinPrefix(PREFIX_REPO, name), + joinPrefix(PREFIX_SNAPSHOT_DIR, label), + DIR_SNAPSHOT_CONTENT, + joinPrefix(PREFIX_DB, info.getDbId()), + joinPrefix(PREFIX_TBL, info.getTblId()), + joinPrefix(PREFIX_PART, info.getPartitionId()), + joinPrefix(PREFIX_IDX, info.getIndexId()), + joinPrefix(PREFIX_COMMON, info.getTabletId()), + joinPrefix(PREFIX_COMMON, info.getSchemaHash())); + LOG.debug("get remote tablet snapshot path: {}", path); + return path; + } + + public Status getSnapshotInfoFile(String label, String backupTimestamp, List infos) { + String remoteInfoFilePath = assembleJobInfoFilePath(label, -1) + backupTimestamp; + File localInfoFile = new File(BackupHandler.BACKUP_ROOT_DIR + PATH_DELIMITER + + "info_" + System.currentTimeMillis()); + try { + Status st = download(remoteInfoFilePath, localInfoFile.getPath()); + if (!st.ok()) { + return st; + } + + BackupJobInfo jobInfo = BackupJobInfo.fromFile(localInfoFile.getAbsolutePath()); + infos.add(jobInfo); + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "Failed to create job info from file: " + + "" + localInfoFile.getName() + ". msg: " + e.getMessage()); + } finally { + localInfoFile.delete(); + } + + return Status.OK; + } + + public Status getSnapshotMetaFile(String label, List backupMetas) { + String remoteMetaFilePath = assembleMetaInfoFilePath(label); + File localMetaFile = new File(BackupHandler.BACKUP_ROOT_DIR + PATH_DELIMITER + + "meta_" + System.currentTimeMillis()); + + try { + Status st = download(remoteMetaFilePath, localMetaFile.getAbsolutePath()); + if (!st.ok()) { + return st; + } + + // read file to backupMeta + BackupMeta backupMeta = BackupMeta.fromFile(localMetaFile.getAbsolutePath()); + backupMetas.add(backupMeta); + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "Failed create backup meta from file: " + + localMetaFile.getAbsolutePath() + ", msg: " + e.getMessage()); + } finally { + localMetaFile.delete(); + } + + return Status.OK; + } + + // upload the local file to specified remote file with checksum + // remoteFilePath should be FULL path + public Status upload(String localFilePath, String remoteFilePath) { + Preconditions.checkArgument(remoteFilePath.startsWith(location), remoteFilePath); + // get md5usm of local file + File file = new File(localFilePath); + String md5sum = null; + try { + md5sum = DigestUtils.md5Hex(new FileInputStream(file)); + } catch (FileNotFoundException e) { + return new Status(ErrCode.NOT_FOUND, "file " + localFilePath + " does not exist"); + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to get md5sum of file: " + localFilePath); + } + Preconditions.checkState(!Strings.isNullOrEmpty(md5sum)); + String tmpRemotePath = assembleFileNameWithSuffix(remoteFilePath, SUFFIX_TMP_FILE); + LOG.debug("get md5sum of file: {}. tmp remote path: {}", localFilePath, tmpRemotePath); + + // upload tmp file + Status st = storage.upload(localFilePath, tmpRemotePath); + if (!st.ok()) { + return st; + } + + // rename tmp file with checksum named file + String finalRemotePath = assembleFileNameWithSuffix(remoteFilePath, md5sum); + st = storage.rename(tmpRemotePath, finalRemotePath); + LOG.info("finished to upload local file {} to remote file: {}", localFilePath, finalRemotePath); + return st; + } + + // remoteFilePath must be a file(not dir) and does not contain checksum + public Status download(String remoteFilePath, String localFilePath) { + // 0. list to get to full name(with checksum) + List remoteFiles = Lists.newArrayList(); + Status status = storage.list(remoteFilePath + "*", remoteFiles); + if (!status.ok()) { + return status; + } + if (remoteFiles.size() != 1) { + return new Status(ErrCode.COMMON_ERROR, + "Expected one file with path: " + remoteFilePath + ". get: " + remoteFiles.size()); + } + if (!remoteFiles.get(0).isFile()) { + return new Status(ErrCode.COMMON_ERROR, "Expected file with path: " + remoteFilePath + ". but get dir"); + } + + String remoteFilePathWithChecksum = replaceFileNameWithChecksumFileName(remoteFilePath, + remoteFiles.get(0).getName()); + LOG.debug("get download filename with checksum: " + remoteFilePathWithChecksum); + + // 1. get checksum from remote file name + Pair pair = decodeFileNameWithChecksum(remoteFilePathWithChecksum); + if (pair == null) { + return new Status(ErrCode.COMMON_ERROR, + "file name should contains checksum: " + remoteFilePathWithChecksum); + } + if (!remoteFilePath.endsWith(pair.first)) { + return new Status(ErrCode.COMMON_ERROR, "File does not exist: " + remoteFilePath); + } + String md5sum = pair.second; + + // 2. download + status = storage.downloadWithFileSize(remoteFilePathWithChecksum, localFilePath, remoteFiles.get(0).getSize()); + if (!status.ok()) { + return status; + } + + // 3. verify checksum + String localMd5sum = null; + try { + localMd5sum = DigestUtils.md5Hex(new FileInputStream(localFilePath)); + } catch (FileNotFoundException e) { + return new Status(ErrCode.NOT_FOUND, "file " + localFilePath + " does not exist"); + } catch (IOException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to get md5sum of file: " + localFilePath); + } + + if (!localMd5sum.equals(md5sum)) { + return new Status(ErrCode.BAD_FILE, + "md5sum does not equal. local: " + localMd5sum + ", remote: " + md5sum); + } + + return Status.OK; + } + + // join job info file name with timestamp + // eg: __info_2018-01-01-08-00-00 + private static String jobInfoFileNameWithTimestamp(long createTime) { + if (createTime == -1) { + return PREFIX_JOB_INFO; + } else { + return PREFIX_JOB_INFO + + TimeUtils.longToTimeString(createTime, new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss")); + } + } + + // join the name with specified prefix + private static String joinPrefix(String prefix, Object name) { + return prefix + name; + } + + // disjoint the name with specified prefix + private static String disjoinPrefix(String prefix, String nameWithPrefix) { + return nameWithPrefix.substring(prefix.length()); + } + + private static String assembleFileNameWithSuffix(String filePath, String md5sum) { + return filePath + CHECKSUM_SEPARATOR + md5sum; + } + + public static Pair decodeFileNameWithChecksum(String fileNameWithChecksum) { + int index = fileNameWithChecksum.lastIndexOf(CHECKSUM_SEPARATOR); + if (index == -1) { + return null; + } + String fileName = fileNameWithChecksum.substring(0, index); + String md5sum = fileNameWithChecksum.substring(index + CHECKSUM_SEPARATOR.length()); + + if (md5sum.length() != 32) { + return null; + } + + return Pair.create(fileName, md5sum); + } + + // in: /path/to/orig_file + // out: /path/to/orig_file.BUWDnl831e4nldsf + public static String replaceFileNameWithChecksumFileName(String origPath, String fileNameWithChecksum) { + return origPath.substring(0, origPath.lastIndexOf(PATH_DELIMITER) + 1) + fileNameWithChecksum; + } + + public Status getBrokerAddress(Long beId, Catalog catalog, List brokerAddrs) { + // get backend + Backend be = Catalog.getCurrentSystemInfo().getBackend(beId); + if (be == null) { + return new Status(ErrCode.COMMON_ERROR, "backend " + beId + " is missing. " + + "failed to send upload snapshot task"); + } + + // get proper broker for this backend + BrokerAddress brokerAddr = null; + try { + brokerAddr = catalog.getBrokerMgr().getBroker(storage.getBrokerName(), be.getHost()); + } catch (AnalysisException e) { + return new Status(ErrCode.COMMON_ERROR, "failed to get address of broker " + + storage.getBrokerName() + " when try to send upload snapshot task: " + + e.getMessage()); + } + if (brokerAddr == null) { + return new Status(ErrCode.COMMON_ERROR, "failed to get address of broker " + + storage.getBrokerName() + " when try to send upload snapshot task"); + } + brokerAddrs.add(brokerAddr); + return Status.OK; + } + + public List getInfo() { + List info = Lists.newArrayList(); + info.add(String.valueOf(id)); + info.add(name); + info.add(TimeUtils.longToTimeString(createTime)); + info.add(String.valueOf(isReadOnly)); + info.add(location); + info.add(storage.getBrokerName()); + info.add(errMsg == null ? "N/A" : errMsg); + return info; + } + + public List> getSnapshotInfos(String snapshotName, String timestamp) + throws AnalysisException { + List> snapshotInfos = Lists.newArrayList(); + if (Strings.isNullOrEmpty(snapshotName)) { + // get all snapshot infos + List snapshotNames = Lists.newArrayList(); + Status status = listSnapshots(snapshotNames); + if (!status.ok()) { + throw new AnalysisException( + "Failed to list snapshot in repo: " + name + ", err: " + status.getErrMsg()); + } + + for (String ssName : snapshotNames) { + List info = getSnapshotInfo(ssName, null /* get all timestamp */); + snapshotInfos.add(info); + } + } else { + // get specified snapshot info + List info = getSnapshotInfo(snapshotName, timestamp); + snapshotInfos.add(info); + } + + return snapshotInfos; + } + + private List getSnapshotInfo(String snapshotName, String timestamp) { + List info = Lists.newArrayList(); + if (Strings.isNullOrEmpty(timestamp)) { + // get all timestamp + // path eg: /location/__palo_repository_repo_name/__ss_my_snap/__info_* + String infoFilePath = assembleJobInfoFilePath(snapshotName, -1); + LOG.debug("assemble infoFilePath: {}, snapshot: {}", infoFilePath, snapshotName); + List results = Lists.newArrayList(); + Status st = storage.list(infoFilePath + "*", results); + if (!st.ok()) { + info.add(snapshotName); + info.add("N/A"); + info.add("ERROR: Failed to get info: " + st.getErrMsg()); + } else { + info.add(snapshotName); + + List tmp = Lists.newArrayList(); + for (RemoteFile file : results) { + // __info_2018-04-18-20-11-00.Jdwnd9312sfdn1294343 + Pair pureFileName = decodeFileNameWithChecksum(file.getName()); + if (pureFileName == null) { + // maybe: __info_2018-04-18-20-11-00.part + tmp.add("Invalid: " + file.getName()); + continue; + } + tmp.add(disjoinPrefix(PREFIX_JOB_INFO, pureFileName.first)); + } + info.add(Joiner.on("\n").join(tmp)); + info.add(tmp.isEmpty() ? "ERROR: no snapshot" : "OK"); + } + } else { + // get specified timestamp + // path eg: /path/to/backup/__info_2081-04-19-12-59-11 + String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/" + Repository.PREFIX_JOB_INFO + timestamp; + try { + String remoteInfoFilePath = assembleJobInfoFilePath(snapshotName, -1) + timestamp; + Status st = download(remoteInfoFilePath, localFilePath); + if (!st.ok()) { + info.add(snapshotName); + info.add(timestamp); + info.add("N/A"); + info.add("N/A"); + info.add("Failed to get info: " + st.getErrMsg()); + } else { + try { + BackupJobInfo jobInfo = BackupJobInfo.fromFile(localFilePath); + info.add(snapshotName); + info.add(timestamp); + info.add(jobInfo.dbName); + info.add(jobInfo.toString(1)); + info.add("OK"); + } catch (IOException e) { + info.add(snapshotName); + info.add(timestamp); + info.add("N/A"); + info.add("N/A"); + info.add("Failed to read info from local file: " + e.getMessage()); + } + } + } finally { + // delete tmp local file + File localFile = new File(localFilePath); + if (localFile.exists()) { + localFile.delete(); + } + } + } + + return info; + } + + public static Repository read(DataInput in) throws IOException { + Repository repo = new Repository(); + repo.readFields(in); + return repo; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeLong(id); + Text.writeString(out, name); + out.writeBoolean(isReadOnly); + Text.writeString(out, location); + storage.write(out); + out.writeLong(createTime); + } + + @Override + public void readFields(DataInput in) throws IOException { + id = in.readLong(); + name = Text.readString(in); + isReadOnly = in.readBoolean(); + location = Text.readString(in); + storage = BlobStorage.read(in); + createTime = in.readLong(); + } +} diff --git a/fe/src/com/baidu/palo/backup/RepositoryMgr.java b/fe/src/com/baidu/palo/backup/RepositoryMgr.java new file mode 100644 index 0000000000..76d50c4739 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/RepositoryMgr.java @@ -0,0 +1,154 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.backup.Status.ErrCode; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.common.util.Daemon; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +/* + * A manager to manage all backup repositories + */ +public class RepositoryMgr extends Daemon implements Writable { + private static final Logger LOG = LogManager.getLogger(RepositoryMgr.class); + + // all key should be in lower case + private Map repoNameMap = Maps.newConcurrentMap(); + private Map repoIdMap = Maps.newConcurrentMap(); + + private ReentrantLock lock = new ReentrantLock(); + + public RepositoryMgr() { + super(Repository.class.getSimpleName(), 600 * 1000 /* 10min */); + } + + @Override + protected void runOneCycle() { + for (Repository repo : repoNameMap.values()) { + if (!repo.ping()) { + LOG.warn("Failed to connect repository {}. msg: {}", repo.getName(), repo.getErrorMsg()); + } + } + } + + public Status addAndInitRepoIfNotExist(Repository repo, boolean isReplay) { + lock.lock(); + try { + if (!repoNameMap.containsKey(repo.getName())) { + if (!isReplay) { + // create repository path and repo info file in remote storage + Status st = repo.initRepository(); + if (!st.ok()) { + return st; + } + } + repoNameMap.put(repo.getName(), repo); + repoIdMap.put(repo.getId(), repo); + + if (!isReplay) { + // write log + Catalog.getCurrentCatalog().getEditLog().logCreateRepository(repo); + } + + LOG.info("successfully adding repo {} to repository mgr. is replay: {}", + repo.getName(), isReplay); + return Status.OK; + } + return new Status(ErrCode.COMMON_ERROR, "repository with same name already exist: " + repo.getName()); + } finally { + lock.unlock(); + } + } + + public Repository getRepo(String repoName) { + return repoNameMap.get(repoName.toLowerCase()); + } + + public Repository getRepo(long repoId) { + return repoIdMap.get(repoId); + } + + public Status removeRepo(String repoName, boolean isReplay) { + lock.lock(); + try { + Repository repo = repoNameMap.remove(repoName.toLowerCase()); + if (repo != null) { + repoIdMap.remove(repo.getId()); + + if (!isReplay) { + // log + Catalog.getCurrentCatalog().getEditLog().logDropRepository(repoName); + } + LOG.info("sucessfully removing repo {} from repository mgr", repoName); + return Status.OK; + } + return new Status(ErrCode.NOT_FOUND, "repository does not exist"); + } finally { + lock.unlock(); + } + } + + public List> getReposInfo() { + List> infos = Lists.newArrayList(); + for (Repository repo : repoIdMap.values()) { + infos.add(repo.getInfo()); + } + return infos; + } + + public static RepositoryMgr read(DataInput in) throws IOException { + RepositoryMgr mgr = new RepositoryMgr(); + mgr.readFields(in); + return mgr; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(repoNameMap.size()); + for (Repository repo : repoNameMap.values()) { + repo.write(out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + Repository repo = Repository.read(in); + repoNameMap.put(repo.getName(), repo); + repoIdMap.put(repo.getId(), repo); + } + } +} diff --git a/fe/src/com/baidu/palo/backup/RestoreFileMapping.java b/fe/src/com/baidu/palo/backup/RestoreFileMapping.java new file mode 100644 index 0000000000..17a6bdbffe --- /dev/null +++ b/fe/src/com/baidu/palo/backup/RestoreFileMapping.java @@ -0,0 +1,203 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.common.io.Writable; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; + +public class RestoreFileMapping implements Writable { + + public static class IdChain implements Writable { + // tblId, partId, idxId, tabletId, replicaId + private Long[] chain; + + private IdChain() { + // for persist + } + + public IdChain(Long... ids) { + Preconditions.checkState(ids.length == 5); + chain = ids; + } + + public Long getTblId() { + return chain[0]; + } + + public long getPartId() { + return chain[1]; + } + + public long getIdxId() { + return chain[2]; + } + + public long getTabletId() { + return chain[3]; + } + + public long getReplicaId() { + return chain[4]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("["); + sb.append(Joiner.on("-").join(chain)); + sb.append("]"); + return sb.toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof IdChain)) { + return false; + } + + IdChain other = (IdChain) obj; + for (int i = 0; i < 5; i++) { + // DO NOT use ==, Long_1 != Long_2 + if (!chain[i].equals(other.chain[i])) { + return false; + } + } + + return true; + } + + @Override + public int hashCode() { + int code = chain[0].hashCode(); + for (int i = 1; i < 5; i++) { + code ^= chain[i].hashCode(); + } + return code; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(chain.length); + for (Long id : chain) { + out.writeLong(id); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + chain = new Long[size]; + for (int i = 0; i < size; i++) { + chain[i] = in.readLong(); + } + } + + public static IdChain read(DataInput in) throws IOException { + IdChain chain = new IdChain(); + chain.readFields(in); + return chain; + } + } + + // catalog ids -> repository ids + private Map mapping = Maps.newHashMap(); + // tablet id -> is overwrite + private Map overwriteMap = Maps.newHashMap(); + + public RestoreFileMapping() { + + } + + public void putMapping(IdChain key, IdChain value, boolean overwrite) { + mapping.put(key, value); + overwriteMap.put(key.getTabletId(), overwrite); + } + + public IdChain get(IdChain key) { + return mapping.get(key); + } + + public Map getMapping() { + return mapping; + } + + public boolean isOverwrite(long tabletId) { + if (overwriteMap.containsKey(tabletId)) { + return overwriteMap.get(tabletId); + } + return false; + } + + public static RestoreFileMapping read(DataInput in) throws IOException { + RestoreFileMapping mapping = new RestoreFileMapping(); + mapping.readFields(in); + return mapping; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(mapping.size()); + for (Map.Entry entry : mapping.entrySet()) { + entry.getKey().write(out); + entry.getValue().write(out); + } + + out.writeInt(overwriteMap.size()); + for (Map.Entry entry : overwriteMap.entrySet()) { + out.writeLong(entry.getKey()); + out.writeBoolean(entry.getValue()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + IdChain key = IdChain.read(in); + IdChain val = IdChain.read(in); + mapping.put(key, val); + } + + size = in.readInt(); + for (int i = 0; i < size; i++) { + long tabletId = in.readLong(); + boolean overwrite = in.readBoolean(); + overwriteMap.put(tabletId, overwrite); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : mapping.entrySet()) { + sb.append(entry.getKey()).append(" : ").append(entry.getValue()).append("\n"); + } + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/backup/RestoreJob.java b/fe/src/com/baidu/palo/backup/RestoreJob.java index 844234ae98..32a8578349 100644 --- a/fe/src/com/baidu/palo/backup/RestoreJob.java +++ b/fe/src/com/baidu/palo/backup/RestoreJob.java @@ -1,8 +1,13 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // @@ -15,877 +20,1456 @@ package com.baidu.palo.backup; -import com.baidu.palo.alter.RollupHandler; -import com.baidu.palo.analysis.AddPartitionClause; -import com.baidu.palo.analysis.AddRollupClause; -import com.baidu.palo.analysis.AlterClause; -import com.baidu.palo.analysis.AlterTableStmt; -import com.baidu.palo.analysis.CreateTableStmt; -import com.baidu.palo.analysis.LabelName; +import com.baidu.palo.backup.BackupJobInfo.BackupIndexInfo; +import com.baidu.palo.backup.BackupJobInfo.BackupPartitionInfo; +import com.baidu.palo.backup.BackupJobInfo.BackupTableInfo; +import com.baidu.palo.backup.BackupJobInfo.BackupTabletInfo; +import com.baidu.palo.backup.RestoreFileMapping.IdChain; +import com.baidu.palo.backup.Status.ErrCode; +import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.DataProperty; import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.KeysType; import com.baidu.palo.catalog.MaterializedIndex; import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.OlapTable.OlapTableState; import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.PartitionInfo; +import com.baidu.palo.catalog.PartitionKey; import com.baidu.palo.catalog.PartitionType; import com.baidu.palo.catalog.RangePartitionInfo; import com.baidu.palo.catalog.Replica; +import com.baidu.palo.catalog.Replica.ReplicaState; import com.baidu.palo.catalog.Table; -import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.catalog.TabletInvertedIndex; -import com.baidu.palo.catalog.TabletMeta; import com.baidu.palo.catalog.Table.TableType; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.DdlException; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.catalog.TabletMeta; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.MarkedCountDownLatch; import com.baidu.palo.common.Pair; import com.baidu.palo.common.io.Text; import com.baidu.palo.common.util.TimeUtils; -import com.baidu.palo.common.util.Util; import com.baidu.palo.task.AgentBatchTask; import com.baidu.palo.task.AgentTask; import com.baidu.palo.task.AgentTaskExecutor; import com.baidu.palo.task.AgentTaskQueue; -import com.baidu.palo.task.RestoreTask; +import com.baidu.palo.task.CreateReplicaTask; +import com.baidu.palo.task.DirMoveTask; +import com.baidu.palo.task.DownloadTask; +import com.baidu.palo.task.SnapshotTask; +import com.baidu.palo.thrift.TFinishTaskRequest; +import com.baidu.palo.thrift.TStatusCode; +import com.baidu.palo.thrift.TStorageMedium; +import com.baidu.palo.thrift.TStorageType; import com.baidu.palo.thrift.TTaskType; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import com.google.common.collect.Range; +import com.google.common.collect.Table.Cell; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; -import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -public class RestoreJob extends AbstractBackupJob { +public class RestoreJob extends AbstractJob { private static final Logger LOG = LogManager.getLogger(RestoreJob.class); public enum RestoreJobState { - PENDING, - RESTORE_META, - DOWNLOAD, - DOWNLOADING, + PENDING, // Job is newly created. Check and prepare meta in catalog. Create replica if necessary. + // Waiting for replica creation finished synchronously, then sending snapshot tasks. + // then transfer to SNAPSHOTING + SNAPSHOTING, // Waiting for snapshot finished. Than transfer to DOWNLOAD. + DOWNLOAD, // Send download tasks. + DOWNLOADING, // Waiting for download finished. + COMMIT, // After download finished, all data is ready for taking effect. + // Send movement tasks to BE, than transfer to COMMITTING + COMMITTING, // wait all tasks finished. Transfer to FINISHED FINISHED, CANCELLED } + private String backupTimestamp; + + private BackupJobInfo jobInfo; + private boolean allowLoad; + private RestoreJobState state; - private Map> tableToPartitionNames; - private Map tableRenameMap; + private BackupMeta backupMeta; - private Map tableToCreateTableStmt; - private Map tableToRollupStmt; - private com.google.common.collect.Table tableToPartitionStmts; + private RestoreFileMapping fileMapping = new RestoreFileMapping(); - private Map tableToReplace; + private long metaPreparedTime = -1; + private long snapshotFinishedTime = -1; + private long downloadFinishedTime = -1; - private Map restoredTables; - // tableid - partition name - partition - private com.google.common.collect.Table restoredPartitions; + private int restoreReplicationNum; - private long metaRestoredTime; - private long downloadFinishedTime; + // this 2 members is to save all newly restored objs + // tbl name -> part + private List> restoredPartitions = Lists.newArrayList(); + private List restoredTbls = Lists.newArrayList(); + + // save all restored partitions' version info which are already exist in catalog + // table id -> partition id -> (version, version hash) + private com.google.common.collect.Table> restoredVersionInfo = HashBasedTable.create(); + // tablet id->(be id -> snapshot info) + private com.google.common.collect.Table snapshotInfos = HashBasedTable.create(); + + private Map unfinishedSignatureToId = Maps.newConcurrentMap(); public RestoreJob() { - super(); + super(JobType.RESTORE); } - public RestoreJob(long jobId, long dbId, LabelName labelName, String restorePath, - Map remoteProperties, Map> tableToPartitionNames, - Map tableRenameMap) { - super(jobId, dbId, labelName, restorePath, remoteProperties); - state = RestoreJobState.PENDING; - - this.tableToPartitionNames = tableToPartitionNames; - this.tableRenameMap = tableRenameMap; - - this.tableToCreateTableStmt = Maps.newHashMap(); - this.tableToRollupStmt = Maps.newHashMap(); - this.tableToPartitionStmts = HashBasedTable.create(); - - this.tableToReplace = Maps.newHashMap(); - this.restoredTables = Maps.newHashMap(); - this.restoredPartitions = HashBasedTable.create(); - - this.metaRestoredTime = -1L; - this.downloadFinishedTime = -1L; - } - - public void setState(RestoreJobState state) { - this.state = state; + public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, + boolean allowLoad, int restoreReplicationNum, long timeoutMs, + Catalog catalog, long repoId) { + super(JobType.RESTORE, label, dbId, dbName, timeoutMs, catalog, repoId); + this.backupTimestamp = backupTs; + this.jobInfo = jobInfo; + this.allowLoad = allowLoad; + this.restoreReplicationNum = restoreReplicationNum; + this.state = RestoreJobState.PENDING; } public RestoreJobState getState() { return state; } - public long getMetaRestoredTime() { - return metaRestoredTime; + public RestoreFileMapping getFileMapping() { + return fileMapping; } - public long getDownloadFinishedTime() { - return downloadFinishedTime; + public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { + Preconditions.checkState(task.getJobId() == jobId); + Preconditions.checkState(task.getDbId() == dbId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + return false; + } + + Preconditions.checkState(request.isSetSnapshot_path()); + + // snapshot path does not contains last 'tablet_id' and 'schema_hash' dir + // eg: + // /path/to/your/be/data/snapshot/20180410102311.0/ + // Full path will look like: + // /path/to/your/be/data/snapshot/20180410102311.0/10006/352781111/ + SnapshotInfo info = new SnapshotInfo(task.getDbId(), task.getTableId(), task.getPartitionId(), + task.getIndexId(), task.getTabletId(), task.getBackendId(), + task.getSchemaHash(), request.getSnapshot_path(), Lists.newArrayList()); + + snapshotInfos.put(task.getTabletId(), task.getBackendId(), info); + Long removedTabletId = unfinishedSignatureToId.remove(task.getSignature()); + if (removedTabletId != null) { + taskErrMsg.remove(task.getSignature()); + Preconditions.checkState(task.getTabletId() == removedTabletId, removedTabletId); + LOG.debug("get finished snapshot info: {}, unfinished tasks num: {}, remove result: {}. {}", + info, unfinishedSignatureToId.size(), this); + return true; + } + + return false; } - public Map> getTableToPartitionNames() { - return tableToPartitionNames; + public synchronized boolean finishTabletDownloadTask(DownloadTask task, TFinishTaskRequest request) { + Preconditions.checkState(task.getJobId() == jobId); + Preconditions.checkState(task.getDbId() == dbId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + return false; + } + + Preconditions.checkState(request.isSetDownloaded_tablet_ids()); + + for (Long tabletId : request.getDownloaded_tablet_ids()) { + SnapshotInfo info = snapshotInfos.get(tabletId, task.getBackendId()); + if (info == null) { + LOG.error("failed to find snapshot infos of tablet {} in be {}, {}", + tabletId, task.getBackendId(), this); + return false; + } + } + + Long beId = unfinishedSignatureToId.remove(task.getSignature()); + if (beId == null || beId != task.getBackendId()) { + LOG.error("invalid download task: {}. {}", task, this); + return false; + } + + taskErrMsg.remove(task.getSignature()); + return true; + } + + public synchronized boolean finishDirMoveTask(DirMoveTask task, TFinishTaskRequest request) { + Preconditions.checkState(task.getJobId() == jobId); + Preconditions.checkState(task.getDbId() == dbId); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + return false; + } + + Long tabletId = unfinishedSignatureToId.remove(task.getSignature()); + if (tabletId == null || tabletId != task.getTabletId()) { + LOG.error("invalid dir move task: {}. {}", task, this); + return false; + } + + taskErrMsg.remove(task.getSignature()); + return true; } @Override - public List getJobInfo() { - List jobInfo = Lists.newArrayList(); - jobInfo.add(jobId); - jobInfo.add(getLabel()); - jobInfo.add(state.name()); - jobInfo.add(TimeUtils.longToTimeString(createTime)); - jobInfo.add(TimeUtils.longToTimeString(metaRestoredTime)); - jobInfo.add(TimeUtils.longToTimeString(downloadFinishedTime)); - jobInfo.add(TimeUtils.longToTimeString(finishedTime)); - jobInfo.add(errMsg); - jobInfo.add(remotePath); - jobInfo.add(getLeftTasksNum()); - return jobInfo; + public synchronized void replayRun() { + LOG.info("replay run restore job: {}", this); + switch (state) { + case DOWNLOAD: + replayCheckAndPrepareMeta(); + break; + case FINISHED: + replayWaitingAllTabletsCommitted(); + break; + default: + break; + } } @Override - public void runOnce() { - LOG.debug("begin to run restore job: {}, state: {}", jobId, state.name()); + public synchronized void replayCancel() { + cancelInternal(true /* is replay */); + } + + @Override + public boolean isPending() { + return state == RestoreJobState.PENDING; + } + + @Override + public boolean isCancelled() { + return state == RestoreJobState.CANCELLED; + } + + @Override + public void run() { + if (state == RestoreJobState.FINISHED || state == RestoreJobState.CANCELLED) { + return; + } + + if (System.currentTimeMillis() - createTime > timeoutMs) { + status = new Status(ErrCode.TIMEOUT, ""); + cancel(); + return; + } + + // get repo if not set + if (repo == null) { + repo = catalog.getBackupHandler().getRepoMgr().getRepo(repoId); + if (repo == null) { + status = new Status(ErrCode.COMMON_ERROR, "failed to get repository: " + repoId); + cancelInternal(false); + return; + } + } + + LOG.info("run restore job: {}", this); + + switch (state) { + case PENDING: + checkAndPrepareMeta(); + break; + case SNAPSHOTING: + waitingAllSnapshotsFinished(); + break; + case DOWNLOAD: + downloadSnapshots(); + break; + case DOWNLOADING: + waitingAllDownloadFinished(); + break; + case COMMIT: + commit(); + break; + case COMMITTING: + waitingAllTabletsCommitted(); + break; + default: + break; + } + + if (!status.ok()) { + cancelInternal(false); + } + } + + /* + * Restore rules as follow: + * A. Table already exist + * A1. Partition already exist, generate file mapping + * A2. Partition does not exist, add restored partition to the table. + * Reset all index/tablet/replica id, and create replica on BE outside the db lock. + * B. Table does not exist + * B1. Add table to the db, reset all table/index/tablet/replica id, + * and create replica on BE outside the db lock. + * + * All newly created table/partition/index/tablet/replica should be saved for rolling back. + * + * Step: + * 1. download and deserialize backup meta from repository. + * 2. set all existing restored table's state to RESTORE. + * 3. check if the expected restore objs are valid. + * 4. create replicas if necessary. + * 5. add restored objs to catalog. + * 6. make snapshot for all replicas for incremental download later. + */ + private void checkAndPrepareMeta() { + Database db = catalog.getDb(dbId); + if (db == null) { + status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); + return; + } + + // generate job id + jobId = catalog.getNextId(); + + // deserialize meta + if (!downloadAndDeserializeMetaInfo()) { + return; + } + Preconditions.checkNotNull(backupMeta); + + // Set all restored tbls‘ state to RESTORE + // Table's origin state must be NORMAL and does not have unfinished load job. + db.writeLock(); try { + for (BackupTableInfo tblInfo : jobInfo.tables.values()) { + Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + if (tbl == null) { + continue; + } + + if (tbl.getType() != TableType.OLAP) { + status = new Status(ErrCode.COMMON_ERROR, "Only support retore olap table: " + tbl.getName()); + return; + } + + OlapTable olapTbl = (OlapTable) tbl; + if (olapTbl.getState() != OlapTableState.NORMAL) { + status = new Status(ErrCode.COMMON_ERROR, + "Table " + tbl.getName() + "'s state is not NORMAL: " + olapTbl.getState().name()); + return; + } + + for (Partition partition : olapTbl.getPartitions()) { + if (!catalog.getLoadInstance().checkPartitionLoadFinished(partition.getId(), null)) { + status = new Status(ErrCode.COMMON_ERROR, + "Table " + tbl.getName() + "'s has unfinished load job"); + return; + } + } + + olapTbl.setState(OlapTableState.RESTORE); + } + } finally { + db.writeUnlock(); + } + + // Check and prepare meta objects. + AgentBatchTask batchTask = new AgentBatchTask(); + db.readLock(); + try { + for (BackupTableInfo tblInfo : jobInfo.tables.values()) { + Table remoteTbl = backupMeta.getTable(tblInfo.name); + Preconditions.checkNotNull(remoteTbl); + Table localTbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + if (localTbl != null) { + // table already exist, check schema + if (localTbl.getType() != TableType.OLAP) { + status = new Status(ErrCode.COMMON_ERROR, + "Only support retore olap table: " + localTbl.getName()); + return; + } + OlapTable localOlapTbl = (OlapTable) localTbl; + OlapTable remoteOlapTbl = (OlapTable) remoteTbl; + + List intersectPartNames = Lists.newArrayList(); + Status st = localOlapTbl.getIntersectPartNamesWith(remoteOlapTbl, intersectPartNames); + if (!st.ok()) { + status = st; + return; + } + LOG.debug("get intersect part names: {}, job: {}", intersectPartNames, this); + if (localOlapTbl.getSignature(BackupHandler.SIGNATURE_VERSION, intersectPartNames) + != remoteOlapTbl.getSignature(BackupHandler.SIGNATURE_VERSION, intersectPartNames)) { + status = new Status(ErrCode.COMMON_ERROR, "Table " + jobInfo.getAliasByOriginNameIfSet(tblInfo.name) + + " already exist but with different schema"); + return; + } + + // Table with same name and has same schema. Check partition + for (BackupPartitionInfo backupPartInfo : tblInfo.partitions.values()) { + Partition localPartition = localOlapTbl.getPartition(backupPartInfo.name); + if (localPartition != null) { + // Partition already exist. + PartitionInfo localPartInfo = localOlapTbl.getPartitionInfo(); + if (localPartInfo.getType() == PartitionType.RANGE) { + // If this is a range partition, check range + RangePartitionInfo localRangePartInfo = (RangePartitionInfo) localPartInfo; + RangePartitionInfo remoteRangePartInfo + = (RangePartitionInfo) remoteOlapTbl.getPartitionInfo(); + Range localRange = localRangePartInfo.getRange(localPartition.getId()); + Range remoteRange = remoteRangePartInfo.getRange(backupPartInfo.id); + if (localRange.equals(remoteRange)) { + // Same partition, same range + if (localRangePartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { + status = new Status(ErrCode.COMMON_ERROR, "Parition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has different replication num '" + + localRangePartInfo.getReplicationNum(localPartition.getId()) + + "' with parition in repository"); + return; + } + genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, + true /* overwrite when commit */); + restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), + Pair.create(backupPartInfo.version, + backupPartInfo.versionHash)); + } else { + // Same partition name, different range + status = new Status(ErrCode.COMMON_ERROR, "Parition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has different range with parition in repository"); + return; + } + } else { + // If this is a single partitioned table. + if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { + status = new Status(ErrCode.COMMON_ERROR, "Parition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has different replication num '" + + localPartInfo.getReplicationNum(localPartition.getId()) + + "' with parition in repository"); + return; + } + + // No need to check range, just generate file mapping + genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, + true /* overwrite when commit */); + restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), + Pair.create(backupPartInfo.version, + backupPartInfo.versionHash)); + } + } else { + // partitions does not exist + PartitionInfo localPartitionInfo = localOlapTbl.getPartitionInfo(); + if (localPartitionInfo.getType() == PartitionType.RANGE) { + // Check if the partition range can be added to the table + RangePartitionInfo localRangePartitionInfo = (RangePartitionInfo) localPartitionInfo; + RangePartitionInfo remoteRangePartitionInfo + = (RangePartitionInfo) remoteOlapTbl.getPartitionInfo(); + Range remoteRange = remoteRangePartitionInfo.getRange(backupPartInfo.id); + if (!localRangePartitionInfo.checkRange(remoteRange)) { + status = new Status(ErrCode.COMMON_ERROR, "Parition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has conflict range with existing ranges"); + return; + } else { + // this partition can be added to this table, set ids + Partition restorePart = resetPartitionForRestore(localOlapTbl, remoteOlapTbl, + backupPartInfo.name, + db.getClusterName(), + restoreReplicationNum); + if (restorePart == null) { + return; + } + restoredPartitions.add(Pair.create(localOlapTbl.getName(), restorePart)); + } + } else { + // It is impossible that a single partitioned table exist without any existing partition + status = new Status(ErrCode.COMMON_ERROR, + "No partition exist in single partitioned table " + localOlapTbl.getName()); + return; + } + } + } + } else { + // Table does not exist + OlapTable remoteOlapTbl = (OlapTable) remoteTbl; + + // Retain only expected restore partitions in this table; + Set allPartNames = remoteOlapTbl.getPartitionNames(); + for (String partName : allPartNames) { + if (!tblInfo.containsPart(partName)) { + remoteOlapTbl.dropPartition(-1 /* db id is useless here */, partName, + true /* act like replay to disable recycle bin action */); + } + } + + // reset all ids in this table + Status st = remoteOlapTbl.resetIdsForRestore(catalog, db, restoreReplicationNum); + if (!st.ok()) { + status = st; + return; + } + + // DO NOT set remote table's new name here, cause we will still need the origin name later + // remoteOlapTbl.setName(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + remoteOlapTbl.setState(allowLoad ? OlapTableState.RESTORE_WITH_LOAD : OlapTableState.RESTORE); + LOG.debug("put remote table {} to restoredTbls", remoteOlapTbl.getName()); + restoredTbls.add(remoteOlapTbl); + } + } // end of all restore tables + + LOG.debug("finished to prepare restored partitions and tables. {}", this); + // for now, nothing is modified in catalog + + // generate create replica tasks for all restored partitions + for (Pair entry : restoredPartitions) { + OlapTable localTbl = (OlapTable) db.getTable(entry.first); + Preconditions.checkNotNull(localTbl, localTbl.getName()); + Partition restorePart = entry.second; + OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); + BackupPartitionInfo backupPartitionInfo + = jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName()); + + Set bfColumns = localTbl.getCopiedBfColumns(); + double bfFpp = localTbl.getBfFpp(); + for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices()) { + short shortKeyColumnCount = localTbl.getShortKeyColumnCountByIndexId(restoredIdx.getId()); + int schemaHash = localTbl.getSchemaHashByIndexId(restoredIdx.getId()); + KeysType keysType = localTbl.getKeysType(); + List columns = localTbl.getSchemaByIndexId(restoredIdx.getId()); + for (Tablet restoreTablet : restoredIdx.getTablets()) { + TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), + restorePart.getId(), restoredIdx.getId(), schemaHash); + Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + for (Replica restoreReplica : restoreTablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); + CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), schemaHash, + localTbl.getId(), restorePart.getId(), restoredIdx.getId(), + restoreTablet.getId(), shortKeyColumnCount, + schemaHash, restoreReplica.getVersion(), restoreReplica.getVersionHash(), + keysType, TStorageType.COLUMN, + TStorageMedium.HDD /* all restored replicas will be saved to HDD */, + columns, bfColumns, bfFpp, null); + task.setInRestoreMode(true); + batchTask.addTask(task); + } + } + } + + genFileMapping(localTbl, restorePart, remoteTbl.getId(), backupPartitionInfo, + allowLoad ? false : true /* if allow load, do not overwrite when commit */); + } + + // generate create replica task for all restored tables + for (OlapTable restoreTbl : restoredTbls) { + PartitionInfo partInfo = restoreTbl.getPartitionInfo(); + for (Partition restorePart : restoreTbl.getPartitions()) { + TStorageMedium storageMedium = partInfo.getDataProperty(restorePart.getId()).getStorageMedium(); + Set bfColumns = restoreTbl.getCopiedBfColumns(); + double bfFpp = restoreTbl.getBfFpp(); + for (MaterializedIndex index : restorePart.getMaterializedIndices()) { + short shortKeyColumnCount = restoreTbl.getShortKeyColumnCountByIndexId(index.getId()); + int schemaHash = restoreTbl.getSchemaHashByIndexId(index.getId()); + KeysType keysType = restoreTbl.getKeysType(); + List columns = restoreTbl.getSchemaByIndexId(index.getId()); + for (Tablet tablet : index.getTablets()) { + TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), + restorePart.getId(), index.getId(), schemaHash); + Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta); + for (Replica replica : tablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica); + CreateReplicaTask task = new CreateReplicaTask(replica.getBackendId(), schemaHash, + restoreTbl.getId(), restorePart.getId(), index.getId(), tablet.getId(), + shortKeyColumnCount, schemaHash, replica.getVersion(), replica.getVersionHash(), + keysType, TStorageType.COLUMN, storageMedium, columns, + bfColumns, bfFpp, null); + task.setInRestoreMode(true); + batchTask.addTask(task); + } + } + } + BackupTableInfo backupTableInfo = jobInfo.getTableInfo(restoreTbl.getName()); + genFileMapping(restoreTbl, restorePart, backupTableInfo.id, + backupTableInfo.getPartInfo(restorePart.getName()), + allowLoad ? false : true /* if allow load, do not overwrite when commit */); + } + // set restored table's new name after all 'genFileMapping' + restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); + } + + LOG.debug("finished to generate create replica tasks. {}", this); + } finally { + db.readUnlock(); + } + + // Send create replica task to BE outside the db lock + if (batchTask.getTaskNum() > 0) { + MarkedCountDownLatch latch = new MarkedCountDownLatch(batchTask.getTaskNum()); + for (AgentTask task : batchTask.getAllTasks()) { + latch.addMark(((CreateReplicaTask) task).getBackendId(), ((CreateReplicaTask) task).getTabletId()); + ((CreateReplicaTask) task).setLatch(latch); + AgentTaskQueue.addTask(task); + } + AgentTaskExecutor.submit(batchTask); + + // estimate timeout, at most 10 seconds + long timeout = Config.tablet_create_timeout_second * 1000L * batchTask.getTaskNum(); + timeout = Math.min(10 * 1000, timeout); + boolean ok = false; + try { + LOG.info("begin to send create replica tasks to BE for restore. total {} tasks. timeout: {}", + batchTask.getTaskNum(), timeout); + ok = latch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOG.warn("InterruptedException: ", e); + ok = false; + } + + if (ok) { + LOG.debug("finished to create all restored replcias. {}", this); + // add all restored partition and tbls to catalog + db.writeLock(); + try { + // add restored partitions. + // table should be in State RESTORE, so no other partitions can be + // added to or removed from this table during the restore process. + for (Pair entry : restoredPartitions) { + OlapTable localTbl = (OlapTable) db.getTable(entry.first); + Partition restoredPart = entry.second; + OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); + RangePartitionInfo localPartitionInfo = (RangePartitionInfo) localTbl.getPartitionInfo(); + RangePartitionInfo remotePartitionInfo = (RangePartitionInfo) remoteTbl.getPartitionInfo(); + BackupPartitionInfo backupPartitionInfo + = jobInfo.getTableInfo(entry.first).getPartInfo(restoredPart.getName()); + long remotePartId = backupPartitionInfo.id; + Range remoteRange = remotePartitionInfo.getRange(remotePartId); + DataProperty remoteDataProperty = remotePartitionInfo.getDataProperty(remotePartId); + localPartitionInfo.addPartitionForRestore(restoredPart.getId(), remoteRange, + remoteDataProperty, (short) restoreReplicationNum); + localTbl.addPartition(restoredPart); + } + + // add restored tables + for (OlapTable tbl : restoredTbls) { + if (!db.createTable(tbl)) { + status = new Status(ErrCode.COMMON_ERROR, "Table " + tbl.getName() + + " already exist in db: " + db.getFullName()); + return; + } + } + } finally { + db.writeUnlock(); + } + } else { + Collection> unfinishedMarks = latch.getLeftMarks(); + String idStr = Joiner.on(", ").join(unfinishedMarks); + status = new Status(ErrCode.COMMON_ERROR, + "Failed to create replicas for restore. unfinished marks: " + idStr); + return; + } + } + + LOG.info("finished to prepare meta. begin to make snapshot. {}", this); + + // begin to make snapshots for all replicas + // snapshot is for incremental download + unfinishedSignatureToId.clear(); + taskErrMsg.clear(); + batchTask = new AgentBatchTask(); + db.readLock(); + try { + for (IdChain idChain : fileMapping.getMapping().keySet()) { + OlapTable tbl = (OlapTable) db.getTable(idChain.getTblId()); + Partition part = tbl.getPartition(idChain.getPartId()); + MaterializedIndex index = part.getIndex(idChain.getIdxId()); + Tablet tablet = index.getTablet(idChain.getTabletId()); + Replica replica = tablet.getReplicaById(idChain.getReplicaId()); + long signature = catalog.getNextId(); + SnapshotTask task = new SnapshotTask(null, replica.getBackendId(), signature, + jobId, db.getId(), + tbl.getId(), part.getId(), index.getId(), tablet.getId(), + part.getCommittedVersion(), part.getCommittedVersionHash(), + tbl.getSchemaHashByIndexId(index.getId()), timeoutMs, + true /* is restore task*/); + batchTask.addTask(task); + unfinishedSignatureToId.put(signature, tablet.getId()); + } + } finally { + db.readUnlock(); + } + + // send tasks + for (AgentTask task : batchTask.getAllTasks()) { + AgentTaskQueue.addTask(task); + } + AgentTaskExecutor.submit(batchTask); + + metaPreparedTime = System.currentTimeMillis(); + state = RestoreJobState.SNAPSHOTING; + + // No log here, PENDING state restore job will redo this method + LOG.info("finished to prepare meta and send snapshot tasks, num: {}. {}", + batchTask.getTaskNum(), this); + return; + } + + // reset remote partition. + // reset all id in remote partition, but DO NOT modify any exist catalog objects. + private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteTbl, String partName, + String clusterName, int restoreReplicationNum) { + Preconditions.checkState(localTbl.getPartition(partName) == null); + Partition remotePart = remoteTbl.getPartition(partName); + Preconditions.checkNotNull(remotePart); + PartitionInfo localPartitionInfo = localTbl.getPartitionInfo(); + Preconditions.checkState(localPartitionInfo.getType() == PartitionType.RANGE); + + // generate new partition id + long newPartId = catalog.getNextId(); + remotePart.setIdForRestore(newPartId); + + // indexes + Map localIdxNameToId = localTbl.getIndexNameToId(); + for (String localidxName : localIdxNameToId.keySet()) { + // set ids of indexes in remote partition to the local index ids + long remoteIdxId = remoteTbl.getIndexIdByName(localidxName); + MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId); + long localIdxId = localIdxNameToId.get(localidxName); + remoteIdx.setIdForRestore(localIdxId); + if (localIdxId != localTbl.getId()) { + // not base table, reset + remotePart.deleteRollupIndex(remoteIdxId); + remotePart.createRollupIndex(remoteIdx); + } + } + + // save version info for creating replicas + long committedVersion = remotePart.getCommittedVersion(); + long committedVersionHash = remotePart.getCommittedVersionHash(); + + // tablets + for (MaterializedIndex remoteIdx : remotePart.getMaterializedIndices()) { + int remotetabletSize = remoteIdx.getTablets().size(); + remoteIdx.clearTabletsForRestore(); + for (int i = 0; i < remotetabletSize; i++) { + // generate new tablet id + long newTabletId = catalog.getNextId(); + Tablet newTablet = new Tablet(newTabletId); + // add tablet to index, but not add to TabletInvertedIndex + remoteIdx.addTablet(newTablet, null /* tablet meta */, true /* is restore */); + + // replicas + List beIds = Catalog.getCurrentSystemInfo().seqChooseBackendIds(restoreReplicationNum, true, + true, clusterName); + if (beIds == null) { + status = new Status(ErrCode.COMMON_ERROR, + "failed to get enough backends for creating replica of tablet " + + newTabletId + ". need: " + restoreReplicationNum); + return null; + } + for (Long beId : beIds) { + long newReplicaId = catalog.getNextId(); + Replica newReplica = new Replica(newReplicaId, beId, ReplicaState.NORMAL, + committedVersion, committedVersionHash); + newTablet.addReplica(newReplica, true /* is restore */); + } + } + } + return remotePart; + } + + // files in repo to files in local + private void genFileMapping(OlapTable localTbl, Partition localPartition, Long remoteTblId, + BackupPartitionInfo backupPartInfo, boolean overwrite) { + for (MaterializedIndex localIdx : localPartition.getMaterializedIndices()) { + LOG.debug("get index id: {}, index name: {}", localIdx.getId(), + localTbl.getIndexNameById(localIdx.getId())); + BackupIndexInfo backupIdxInfo = backupPartInfo.getIdx(localTbl.getIndexNameById(localIdx.getId())); + Preconditions.checkState(backupIdxInfo.tablets.size() == localIdx.getTablets().size()); + for (int i = 0; i < localIdx.getTablets().size(); i++) { + Tablet localTablet = localIdx.getTablets().get(i); + BackupTabletInfo backupTabletInfo = backupIdxInfo.tablets.get(i); + for (Replica localReplica : localTablet.getReplicas()) { + IdChain src = new IdChain(remoteTblId, backupPartInfo.id, backupIdxInfo.id, backupTabletInfo.id, + -1L /* no replica id */); + IdChain dest = new IdChain(localTbl.getId(), localPartition.getId(), + localIdx.getId(), localTablet.getId(), localReplica.getId()); + fileMapping.putMapping(dest, src, overwrite); + } + } + } + } + + private boolean downloadAndDeserializeMetaInfo() { + List backupMetas = Lists.newArrayList(); + Status st = repo.getSnapshotMetaFile(jobInfo.name, backupMetas); + if (!st.ok()) { + status = st; + return false; + } + Preconditions.checkState(backupMetas.size() == 1); + backupMeta = backupMetas.get(0); + return true; + } + + private void replayCheckAndPrepareMeta() { + Database db = catalog.getDb(dbId); + db.writeLock(); + try { + // replay set all existing tables's state to RESTORE + for (BackupTableInfo tblInfo : jobInfo.tables.values()) { + Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + if (tbl == null) { + continue; + } + OlapTable olapTbl = (OlapTable) tbl; + olapTbl.setState(OlapTableState.RESTORE); + } + + // restored partitions + for (Pair entry : restoredPartitions) { + OlapTable localTbl = (OlapTable) db.getTable(entry.first); + Partition restorePart = entry.second; + OlapTable remoteTbl = (OlapTable) backupMeta.getTable(entry.first); + RangePartitionInfo localPartitionInfo = (RangePartitionInfo) localTbl.getPartitionInfo(); + RangePartitionInfo remotePartitionInfo = (RangePartitionInfo) remoteTbl.getPartitionInfo(); + BackupPartitionInfo backupPartitionInfo = jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName()); + long remotePartId = backupPartitionInfo.id; + Range remoteRange = remotePartitionInfo.getRange(remotePartId); + DataProperty remoteDataProperty = remotePartitionInfo.getDataProperty(remotePartId); + localPartitionInfo.addPartitionForRestore(restorePart.getId(), remoteRange, + remoteDataProperty, (short) restoreReplicationNum); + localTbl.addPartition(restorePart); + + // modify tablet inverted index + for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices()) { + int schemaHash = localTbl.getSchemaHashByIndexId(restoreIdx.getId()); + for (Tablet restoreTablet : restoreIdx.getTablets()) { + TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), + restorePart.getId(), restoreIdx.getId(), schemaHash); + Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + for (Replica restoreReplica : restoreTablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); + } + } + } + } + + // restored tables + for (OlapTable restoreTbl : restoredTbls) { + db.createTable(restoreTbl); + // modify tablet inverted index + for (Partition restorePart : restoreTbl.getPartitions()) { + for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices()) { + int schemaHash = restoreTbl.getSchemaHashByIndexId(restoreIdx.getId()); + for (Tablet restoreTablet : restoreIdx.getTablets()) { + TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), + restorePart.getId(), restoreIdx.getId(), schemaHash); + Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + for (Replica restoreReplica : restoreTablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); + } + } + } + } + } + } finally { + db.writeUnlock(); + } + + LOG.info("replay check and prepare meta. {}", this); + } + + private void waitingAllSnapshotsFinished() { + if (unfinishedSignatureToId.isEmpty()) { + snapshotFinishedTime = System.currentTimeMillis(); + state = RestoreJobState.DOWNLOAD; + + catalog.getEditLog().logRestoreJob(this); + LOG.info("finished making snapshots. {}", this); + return; + } + + LOG.info("waiting {} replicas to make snapshot: [{}]. {}", + unfinishedSignatureToId.size(), unfinishedSignatureToId, this); + return; + } + + private void downloadSnapshots() { + // Categorize snapshot infos by db id. + ArrayListMultimap dbToSnapshotInfos = ArrayListMultimap.create(); + for (SnapshotInfo info : snapshotInfos.values()) { + dbToSnapshotInfos.put(info.getDbId(), info); + } + + // Send download tasks + unfinishedSignatureToId.clear(); + taskErrMsg.clear(); + AgentBatchTask batchTask = new AgentBatchTask(); + for (long dbId : dbToSnapshotInfos.keySet()) { + List infos = dbToSnapshotInfos.get(dbId); + + Database db = catalog.getDb(dbId); + if (db == null) { + status = new Status(ErrCode.NOT_FOUND, "db " + dbId + " does not exist"); + return; + } + + // We classify the snapshot info by backend + ArrayListMultimap beToSnapshots = ArrayListMultimap.create(); + for (SnapshotInfo info : infos) { + beToSnapshots.put(info.getBeId(), info); + } + + db.readLock(); + try { + for (Long beId : beToSnapshots.keySet()) { + List beSnapshotInfos = beToSnapshots.get(beId); + int totalNum = beSnapshotInfos.size(); + // each backend allot at most 3 tasks + int batchNum = Math.min(totalNum, 3); + // each task contains several upload sub tasks + int taskNumPerBatch = Math.max(totalNum / batchNum, 1); + LOG.debug("backend {} has {} batch, total {} tasks, {}", + beId, batchNum, totalNum, this); + + List brokerAddrs = Lists.newArrayList(); + Status st = repo.getBrokerAddress(beId, catalog, brokerAddrs); + if (!st.ok()) { + status = st; + return; + } + Preconditions.checkState(brokerAddrs.size() == 1); + + // allot tasks + int index = 0; + for (int batch = 0; batch < batchNum; batch++) { + Map srcToDest = Maps.newHashMap(); + int currentBatchTaskNum = (batch == batchNum - 1) ? totalNum - index : taskNumPerBatch; + for (int j = 0; j < currentBatchTaskNum; j++) { + SnapshotInfo info = beSnapshotInfos.get(index++); + Table tbl = db.getTable(info.getTblId()); + if (tbl == null) { + status = new Status(ErrCode.NOT_FOUND, "restored table " + + info.getTabletId() + " does not exist"); + return; + } + OlapTable olapTbl = (OlapTable) tbl; + + Partition part = olapTbl.getPartition(info.getPartitionId()); + if (part == null) { + status = new Status(ErrCode.NOT_FOUND, "partition " + + info.getPartitionId() + " does not exist in restored table: " + + tbl.getName()); + return; + } + + MaterializedIndex idx = part.getIndex(info.getIndexId()); + if (idx == null) { + status = new Status(ErrCode.NOT_FOUND, + "index " + info.getIndexId() + " does not exist in partion " + part.getName() + + "of restored table " + tbl.getName()); + return; + } + + Tablet tablet = idx.getTablet(info.getTabletId()); + if (tablet == null) { + status = new Status(ErrCode.NOT_FOUND, + "tablet " + info.getTabletId() + " does not exist in restored table " + + tbl.getName()); + return; + } + + Replica replica = tablet.getReplicaByBackendId(info.getBeId()); + if (replica == null) { + status = new Status(ErrCode.NOT_FOUND, + "replica in be " + info.getBeId() + " of tablet " + + tablet.getId() + " does not exist in restored table " + + tbl.getName()); + return; + } + + IdChain catalogIds = new IdChain(tbl.getId(), part.getId(), idx.getId(), + info.getTabletId(), replica.getId()); + IdChain repoIds = fileMapping.get(catalogIds); + if (repoIds == null) { + status = new Status(ErrCode.NOT_FOUND, + "failed to get id mapping of catalog ids: " + catalogIds.toString()); + LOG.info("current file mapping: {}", fileMapping); + return; + } + + String repoTabletPath = jobInfo.getFilePath(repoIds); + + // eg: + // bos://location/__palo_repository_my_repo/_ss_my_ss/_ss_content/__db_10000/ + // __tbl_10001/__part_10002/_idx_10001/__10003 + String src = repo.getRepoPath(label, repoTabletPath); + SnapshotInfo snapshotInfo = snapshotInfos.get(info.getTabletId(), info.getBeId()); + Preconditions.checkNotNull(snapshotInfo, info.getTabletId() + "-" + info.getBeId()); + // download to previous exist snapshot dir + String dest = snapshotInfo.getTabletPath(); + srcToDest.put(src, dest); + LOG.debug("create download src path: {}, dest path: {}", src, dest); + } + long signature = catalog.getNextId(); + DownloadTask task = new DownloadTask(null, beId, signature, jobId, dbId, + srcToDest, brokerAddrs.get(0), repo.getStorage().getProperties()); + batchTask.addTask(task); + unfinishedSignatureToId.put(signature, beId); + } + } + } finally { + db.readUnlock(); + } + } + + // send task + for (AgentTask task : batchTask.getAllTasks()) { + AgentTaskQueue.addTask(task); + } + AgentTaskExecutor.submit(batchTask); + + state = RestoreJobState.DOWNLOADING; + + // No log here + LOG.info("finished to send download tasks to BE. num: {}. {}", batchTask.getTaskNum(), this); + return; + } + + private void waitingAllDownloadFinished() { + if (unfinishedSignatureToId.isEmpty()) { + downloadFinishedTime = System.currentTimeMillis(); + state = RestoreJobState.COMMIT; + + // backupMeta is useless now + backupMeta = null; + + catalog.getEditLog().logRestoreJob(this); + LOG.info("finished to download. {}", this); + } + + LOG.info("waiting {} tasks to finish downloading from repo. {}", unfinishedSignatureToId.size(), this); + } + + private void commit() { + // Send task to move the download dir + unfinishedSignatureToId.clear(); + taskErrMsg.clear(); + AgentBatchTask batchTask = new AgentBatchTask(); + // tablet id->(be id -> download info) + for (Cell cell : snapshotInfos.cellSet()) { + SnapshotInfo info = cell.getValue(); + long signature = catalog.getNextId(); + DirMoveTask task = new DirMoveTask(null, cell.getColumnKey(), signature, jobId, dbId, + info.getTblId(), info.getPartitionId(), info.getTabletId(), + cell.getRowKey(), info.getTabletPath(), info.getSchemaHash(), + true /* need reload tablet header */); + batchTask.addTask(task); + unfinishedSignatureToId.put(signature, info.getTabletId()); + } + + // send task + for (AgentTask task : batchTask.getAllTasks()) { + AgentTaskQueue.addTask(task); + } + AgentTaskExecutor.submit(batchTask); + + state = RestoreJobState.COMMITTING; + + // No log here + LOG.info("finished to send move dir tasks. num: {}. {}", batchTask.getTaskNum(), this); + return; + } + + private void waitingAllTabletsCommitted() { + if (unfinishedSignatureToId.isEmpty()) { + LOG.info("finished to commit all tablet. {}", this); + Status st = allTabletCommitted(false /* not replay */); + if (!st.ok()) { + status = st; + } + return; + } + LOG.info("waiting {} tablets to commit. {}", unfinishedSignatureToId.size(), this); + } + + private Status allTabletCommitted(boolean isReplay) { + Database db = catalog.getDb(dbId); + if (db == null) { + return new Status(ErrCode.NOT_FOUND, "database " + dbId + " does not exist"); + } + + // set all restored partition version and version hash + db.writeLock(); + try { + // set all tables' state to NORMAL + setTableStateToNormal(db); + + for (long tblId : restoredVersionInfo.rowKeySet()) { + Table tbl = db.getTable(tblId); + if (tbl == null) { + continue; + } + OlapTable olapTbl = (OlapTable) tbl; + Map> map = restoredVersionInfo.rowMap().get(tblId); + for (Map.Entry> entry : map.entrySet()) { + long partId = entry.getKey(); + Partition part = olapTbl.getPartition(partId); + if (part == null) { + continue; + } + + // update partition committed version + part.setCommittedVersion(entry.getValue().first); + part.setCommittedVersionHash(entry.getValue().second); + + // we also need to update the replica version of these overwritten restored partitions + for (MaterializedIndex idx : part.getMaterializedIndices()) { + for (Tablet tablet : idx.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + if (!replica.checkVersionCatchUp(part.getCommittedVersion(), + part.getCommittedVersionHash())) { + replica.updateInfo(part.getCommittedVersion(), part.getCommittedVersionHash(), + replica.getDataSize(), replica.getRowCount()); + } + } + } + } + + LOG.debug("restore set partition {} version in table {}, version: {}, version hash: {}", + partId, tblId, entry.getValue().first, entry.getValue().second); + } + } + } finally { + db.writeUnlock(); + } + + if (!isReplay) { + restoredPartitions.clear(); + restoredTbls.clear(); + snapshotInfos.clear(); + + finishedTime = System.currentTimeMillis(); + state = RestoreJobState.FINISHED; + + catalog.getEditLog().logRestoreJob(this); + } + + LOG.info("job is finished. is replay: {}. {}", isReplay, this); + return Status.OK; + } + + private void replayWaitingAllTabletsCommitted() { + allTabletCommitted(true /* is replay */); + } + + public List getInfo() { + List info = Lists.newArrayList(); + info.add(String.valueOf(jobId)); + info.add(label); + info.add(backupTimestamp); + info.add(dbName); + info.add(state.name()); + info.add(String.valueOf(allowLoad)); + info.add(String.valueOf(restoreReplicationNum)); + info.add(getRestoreObjs()); + info.add(TimeUtils.longToTimeString(createTime)); + info.add(TimeUtils.longToTimeString(metaPreparedTime)); + info.add(TimeUtils.longToTimeString(snapshotFinishedTime)); + info.add(TimeUtils.longToTimeString(downloadFinishedTime)); + info.add(TimeUtils.longToTimeString(finishedTime)); + info.add(Joiner.on(", ").join(unfinishedSignatureToId.keySet())); + List msgs = taskErrMsg.entrySet().stream().map(n -> "[" + n.getKey() + ": " + n.getValue() + + "]").collect(Collectors.toList()); + info.add(Joiner.on(", ").join(msgs)); + info.add(status.toString()); + info.add(String.valueOf(timeoutMs / 1000)); + return info; + } + + private String getRestoreObjs() { + Preconditions.checkState(jobInfo != null); + return jobInfo.getInfo(); + } + + @Override + public boolean isDone() { + if (state == RestoreJobState.FINISHED || state == RestoreJobState.CANCELLED) { + return true; + } + return false; + } + + // cancel by user + @Override + public synchronized Status cancel() { + if (isDone()) { + return new Status(ErrCode.COMMON_ERROR, + "Job with label " + label + " can not be cancelled. state: " + state); + } + + status = new Status(ErrCode.COMMON_ERROR, "user cancelled, current state: " + state.name()); + cancelInternal(false); + return Status.OK; + } + + public void cancelInternal(boolean isReplay) { + // We need to clean the residual due to current state + if (!isReplay) { switch (state) { - case PENDING: - downloadBackupMeta(); - break; - case RESTORE_META: - restoreMeta(); - break; - case DOWNLOAD: - download(); + case SNAPSHOTING: + // remove all snapshot tasks in AgentTaskQueue + for (Long taskId : unfinishedSignatureToId.keySet()) { + AgentTaskQueue.removeTaskOfType(TTaskType.MAKE_SNAPSHOT, taskId); + } break; case DOWNLOADING: - waitDownload(); + // remove all down tasks in AgentTaskQueue + for (Long taskId : unfinishedSignatureToId.keySet()) { + AgentTaskQueue.removeTaskOfType(TTaskType.DOWNLOAD, taskId); + } + break; + case COMMITTING: + // remove all dir move tasks in AgentTaskQueue + for (Long taskId : unfinishedSignatureToId.keySet()) { + AgentTaskQueue.removeTaskOfType(TTaskType.MOVE, taskId); + } break; default: break; } - } catch (Exception e) { - errMsg = Strings.nullToEmpty(e.getMessage()); - LOG.warn("failed to restore: [" + errMsg + "], job[" + jobId + "]", e); - state = RestoreJobState.CANCELLED; } - if (state == RestoreJobState.FINISHED || state == RestoreJobState.CANCELLED) { - end(Catalog.getInstance(), false); - } - } - - private void downloadBackupMeta() throws DdlException, IOException, AnalysisException, InterruptedException, - ExecutionException { - Catalog catalog = Catalog.getInstance(); + // clean restored objs Database db = catalog.getDb(dbId); - if (db == null) { - throw new DdlException("Database[" + getDbName() + "] does not exist"); - } + if (db != null) { + db.writeLock(); + try { + // rollback table's state to NORMAL + setTableStateToNormal(db); - if (pathBuilder == null) { - pathBuilder = PathBuilder.createPathBuilder(getLocalDirName()); - } - - if (commandBuilder == null) { - String remotePropFilePath = pathBuilder.remoteProperties(); - commandBuilder = CommandBuilder.create(remotePropFilePath, remoteProperties); - } - - if (future == null) { - // 1. download manifest - LOG.info("begin to submit download backup meta. job: {}", jobId); - MetaDownloadTask task = new MetaDownloadTask(jobId, getDbName(), getLabel(), getLocalDirName(), remotePath, - pathBuilder, commandBuilder, - tableToPartitionNames, tableToCreateTableStmt, - tableToRollupStmt, tableToPartitionStmts, tableToReplace, - tableRenameMap); - future = Catalog.getInstance().getBackupHandler().getAsynchronousCmdExecutor().submit(task); - } else { - boolean finished = checkFuture("download backup meta"); - if (!finished) { - return; - } - - future = null; - state = RestoreJobState.RESTORE_META; - } - } - - private void restoreMeta() throws DdlException { - Catalog catalog = Catalog.getInstance(); - Database db = catalog.getDb(dbId); - if (db == null) { - throw new DdlException("Database[" + getDbName() + "] does not exist"); - } - for (Map.Entry entry : tableToCreateTableStmt.entrySet()) { - String newTableName = entry.getKey(); - CreateTableStmt createTableStmt = entry.getValue(); - Boolean replace = tableToReplace.get(newTableName); - if (replace) { - // 1. create table - Table restoredTable = catalog.createTable(createTableStmt, true); - restoredTables.put(newTableName, restoredTable); - - if (restoredTable.getType() != TableType.OLAP) { - continue; - } - - OlapTable restoredOlapTable = (OlapTable) restoredTable; - - // 2. create rollup - RollupHandler rollupHandler = catalog.getRollupHandler(); - AlterTableStmt rollupStmt = tableToRollupStmt.get(newTableName); - if (rollupStmt != null) { - // check if new table name conflicts with rollup index name - for (AlterClause clause : rollupStmt.getOps()) { - Preconditions.checkState(clause instanceof AddRollupClause); - String rollupName = ((AddRollupClause) clause).getRollupName(); - if (rollupName.equals(newTableName)) { - throw new DdlException("New table name[" + newTableName - + "] conflicts with rollup index name"); - } - } - - rollupHandler.process(rollupStmt.getOps(), db, restoredOlapTable, true); - } - - // 3. create partition - Map partitionStmts = tableToPartitionStmts.row(newTableName); - if (partitionStmts.isEmpty()) { - continue; - } - - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) restoredOlapTable.getPartitionInfo(); - for (Map.Entry entry2 : partitionStmts.entrySet()) { - AlterTableStmt stmt = entry2.getValue(); - AddPartitionClause clause = (AddPartitionClause) stmt.getOps().get(0); - Pair res = catalog.addPartition(db, newTableName, restoredOlapTable, clause, true); - Partition partition = res.second; - rangePartitionInfo.handleNewSinglePartitionDesc(clause.getSingeRangePartitionDesc(), - partition.getId()); - restoredOlapTable.addPartition(partition); - } - } else { - Map partitionStmts = tableToPartitionStmts.row(newTableName); - for (Map.Entry entry2 : partitionStmts.entrySet()) { - AlterTableStmt stmt = entry2.getValue(); - Pair res = catalog.addPartition(db, newTableName, null, - (AddPartitionClause) stmt.getOps().get(0), true); - long tableId = res.first; - Partition partition = res.second; - restoredPartitions.put(tableId, partition.getName(), partition); - } - } - } - - metaRestoredTime = System.currentTimeMillis(); - state = RestoreJobState.DOWNLOAD; - LOG.info("finished restore tables. job[{}]", jobId); - } - - private void download() { - for (Map.Entry entry : restoredTables.entrySet()) { - String newTableName = entry.getKey(); - String tableName = tableRenameMap.get(newTableName); - Table table = entry.getValue(); - if (table.getType() != TableType.OLAP) { - continue; - } - - AgentBatchTask batchTask = new AgentBatchTask(); - OlapTable olapTable = (OlapTable) table; - long tableId = olapTable.getId(); - for (Partition partition : olapTable.getPartitions()) { - String partitionName = partition.getName(); - if (olapTable.getPartitionInfo().getType() == PartitionType.UNPARTITIONED) { - // single partition table - partitionName = tableName; - } - long partitionId = partition.getId(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { - long indexId = index.getId(); - String indexName = olapTable.getIndexNameById(index.getId()); - if (indexName.equals(newTableName)) { - // base index - indexName = tableName; - } - - List orderedBackupedTabletIdList = getRestoredTabletInfo(tableName, partitionName, indexName); - - int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); - List tablets = index.getTablets(); - for (int i = 0; i < tablets.size(); i++) { - Tablet tablet = tablets.get(i); - Long backupedTabletId = orderedBackupedTabletIdList.get(i); - String remoteFilePath = PathBuilder.createPath(remotePath, getDbName(), tableName, - partitionName, indexName, - backupedTabletId.toString()); - for (Replica replica : tablet.getReplicas()) { - RestoreTask task = new RestoreTask(null, replica.getBackendId(), jobId, dbId, - tableId, partitionId, indexId, tablet.getId(), - schemaHash, remoteFilePath, remoteProperties); - batchTask.addTask(task); - } - } // end for tablets - } // end for indices - } // end for partitions - - synchronized (unfinishedTabletIds) { - for (AgentTask task : batchTask.getAllTasks()) { - AgentTaskQueue.addTask(task); - unfinishedTabletIds.put(task.getTabletId(), task.getBackendId()); - } - } - AgentTaskExecutor.submit(batchTask); - - LOG.info("finished send restore tasks for table: {}, job: {}", tableName, jobId); - } // end for tables - - state = RestoreJobState.DOWNLOADING; - LOG.info("finished send all restore tasks. job: {}", jobId); - } - - private List getRestoredTabletInfo(String tableName, String partitionName, String indexName) { - // pathBuilder.getRoot().print("\t"); - DirSaver indexDir = (DirSaver) pathBuilder.getRoot().getChild(getDbName()).getChild(tableName) - .getChild(partitionName).getChild(indexName); - Collection tabletNames = indexDir.getChildrenName(); - Set orderedBackupedTabletIds = Sets.newTreeSet(); - for (String tabletName : tabletNames) { - orderedBackupedTabletIds.add(Long.valueOf(tabletName)); - } - - List orderedBackupedTabletIdList = Lists.newArrayList(orderedBackupedTabletIds); - return orderedBackupedTabletIdList; - } - - private void waitDownload() throws DdlException { - synchronized (unfinishedTabletIds) { - if (!unfinishedTabletIds.isEmpty()) { - LOG.debug("waiting for unfinished download task. size: {}", unfinishedTabletIds.size()); - return; - } - } - - downloadFinishedTime = System.currentTimeMillis(); - LOG.info("all tablets restore finished. job: {}", jobId); - - finishing(Catalog.getInstance(), false); - - state = RestoreJobState.FINISHED; - } - - public void finishing(Catalog catalog, boolean isReplay) throws DdlException { - Database db = catalog.getDb(dbId); - if (db == null && !isReplay) { - throw new DdlException("Database[{}] does not exist"); - } - - db.writeLock(); - try { - // check again if table or partition already exist - for (Map.Entry entry : restoredTables.entrySet()) { - String tableName = entry.getKey(); - - Table currentTable = db.getTable(tableName); - if (currentTable != null) { - throw new DdlException("Table[" + tableName + "]' already exist. " - + "Drop table first or restore to another table"); - } - } - - for (long tableId : restoredPartitions.rowKeySet()) { - Table table = db.getTable(tableId); - if (table == null || table.getType() != TableType.OLAP) { - throw new DdlException("Table[" + tableId + "]' does not exist."); - } - - Map partitions = restoredPartitions.row(tableId); - OlapTable olapTable = (OlapTable) table; - for (Map.Entry entry : partitions.entrySet()) { - String partitionName = entry.getKey(); - Partition currentPartition = olapTable.getPartition(partitionName); - if (currentPartition != null) { - throw new DdlException("Partition[" + partitionName + "]' already exist in table[" - + tableId + "]. Drop partition first or restore to another table"); - } - } - } - - // add tables - for (Map.Entry entry : restoredTables.entrySet()) { - String tableName = entry.getKey(); - Table restoredTable = entry.getValue(); - - if (restoredTable.getType() == TableType.OLAP) { - OlapTable olapTable = (OlapTable) restoredTable; - olapTable.setState(OlapTableState.NORMAL); - if (isReplay) { - // add inverted index - TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - long tableId = olapTable.getId(); - for (Partition partition : olapTable.getPartitions()) { - long partitionId = partition.getId(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { - long indexId = index.getId(); - int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, - schemaHash); - invertedIndex.addTablet(tabletId, tabletMeta); - for (Replica replica : tablet.getReplicas()) { - invertedIndex.addReplica(tabletId, replica); - } - } + // remove restored tbls + for (OlapTable restoreTbl : restoredTbls) { + LOG.info("remove restored table when cancelled: {}", restoreTbl.getName()); + for (Partition part : restoreTbl.getPartitions()) { + for (MaterializedIndex idx : part.getMaterializedIndices()) { + for (Tablet tablet : idx.getTablets()) { + Catalog.getCurrentInvertedIndex().deleteTablet(tablet.getId()); } } } + db.dropTable(restoreTbl.getName()); } - db.createTable(restoredTable); - LOG.info("finished add table: {}, job: {}, replay: {}", tableName, jobId, isReplay); - } - // add partitions - for (long tableId : restoredPartitions.rowKeySet()) { - Table table = db.getTable(tableId); - String tableName = table.getName(); - Preconditions.checkState(table != null, tableName); - Preconditions.checkState(table.getType() == TableType.OLAP, tableName); - OlapTable olapTable = (OlapTable) table; - - PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - - Map partitions = restoredPartitions.row(tableId); - - for (Map.Entry entry : partitions.entrySet()) { - String partitionName = entry.getKey(); - Partition partition = entry.getValue(); - long partitionId = partition.getId(); - - // add restored partition - AlterTableStmt stmt = tableToPartitionStmts.get(tableName, partitionName); - AddPartitionClause clause = (AddPartitionClause) stmt.getOps().get(0); - rangePartitionInfo.handleNewSinglePartitionDesc(clause.getSingeRangePartitionDesc(), partitionId); - olapTable.addPartition(partition); - - // add inverted index - if (isReplay) { - TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { - long indexId = index.getId(); - int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - for (Replica replica : tablet.getReplicas()) { - invertedIndex.addReplica(tabletId, replica); - } - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, - schemaHash); - invertedIndex.addTablet(tabletId, tabletMeta); - } + // remove restored partitions + for (Pair entry : restoredPartitions) { + OlapTable restoreTbl = (OlapTable) db.getTable(entry.first); + if (restoreTbl == null) { + continue; + } + LOG.info("remove restored partition in table {} when cancelled: {}", + restoreTbl.getName(), entry.second.getName()); + for (MaterializedIndex idx : entry.second.getMaterializedIndices()) { + for (Tablet tablet : idx.getTablets()) { + Catalog.getCurrentInvertedIndex().deleteTablet(tablet.getId()); } } - LOG.info("finished add partition: {}, table: {}, job: {}, replay: {}", - partitionName, tableName, jobId, isReplay); - } // end for partitions - - olapTable.setState(OlapTableState.NORMAL); - } // end for tables - } finally { - db.writeUnlock(); - } - } - - public void handleFinishedRestore(long tabletId, long backendId) { - synchronized (unfinishedTabletIds) { - if (unfinishedTabletIds.remove(tabletId, backendId)) { - LOG.debug("finished restore tablet[{}], backend[{}]", tabletId, backendId); + restoreTbl.dropPartition(dbId, entry.second.getName(), true /* is restore */); + } + } finally { + db.writeUnlock(); } } - } - - @Override - public void end(Catalog catalog, boolean isReplay) { - if (state == RestoreJobState.CANCELLED) { - rollback(catalog); - } - - // 2. set table state - // restoreTableState(catalog); if (!isReplay) { - // 3. remove agent tasks if left - removeLeftTasks(); - - // 4. remove local file - String labelDir = pathBuilder.getRoot().getFullPath(); - Util.deleteDirectory(new File(labelDir)); - LOG.debug("delete local dir: {}", labelDir); - - // 5. remove unused tablet in tablet inverted index - clearInvertedIndex(); + // backupMeta is useless + backupMeta = null; + RestoreJobState curState = state; finishedTime = System.currentTimeMillis(); + state = RestoreJobState.CANCELLED; // log - Catalog.getInstance().getEditLog().logRestoreFinish(this); - } + catalog.getEditLog().logRestoreJob(this); - // clear for saving memory - clearJob(); - - LOG.info("finished end job[{}]. state: {}, replay: {}", jobId, state.name(), isReplay); - } - - private void clearInvertedIndex() { - TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - if (state == RestoreJobState.CANCELLED) { - // clear restored table tablets - for (Table restoredTable : restoredTables.values()) { - if (restoredTable.getType() != TableType.OLAP) { - continue; - } - - OlapTable olapTable = (OlapTable) restoredTable; - for (Partition partition : olapTable.getPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices()) { - for (Tablet tablet : index.getTablets()) { - invertedIndex.deleteTablet(tablet.getId()); - } - } - } - } - - // partition - for (Partition partition : restoredPartitions.values()) { - for (MaterializedIndex index : partition.getMaterializedIndices()) { - for (Tablet tablet : index.getTablets()) { - invertedIndex.deleteTablet(tablet.getId()); - } - } - } - } - } - - @Override - protected void clearJob() { - tableRenameMap = null; - - tableToCreateTableStmt = null; - tableToRollupStmt = null; - tableToPartitionStmts = null; - - tableToReplace = null; - restoredTables = null; - restoredPartitions = null; - - unfinishedTabletIds = null; - remoteProperties = null; - pathBuilder = null; - commandBuilder = null; - LOG.info("job[{}] cleared for saving memory", jobId); - } - - private void rollback(Catalog catalog) { - Database db = catalog.getDb(dbId); - if (db == null) { - errMsg = "Database does not exist[" + getDbName() + "]"; - LOG.info("{}. finished restore old meta. job: {}", errMsg, jobId); + LOG.info("finished to cancel restore job. current state: {}. is replay: {}. {}", + curState.name(), isReplay, this); return; } - db.writeLock(); - try { - // tables - for (Table restoredTable : restoredTables.values()) { - String tableName = restoredTable.getName(); - // use table id rather than table name. - // because table with same name may be created when doing restore. - // find table by name may get unexpected one. - Table currentTable = db.getTable(restoredTable.getId()); - // drop restored table - if (currentTable != null) { - db.dropTable(tableName); - LOG.info("drop restored table[{}] in db[{}]", tableName, dbId); - } + LOG.info("finished to cancel restore job. is replay: {}. {}", isReplay, this); + } + + private void setTableStateToNormal(Database db) { + for (BackupTableInfo tblInfo : jobInfo.tables.values()) { + Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + if (tbl == null) { + continue; } - // partitions - for (long tableId : restoredPartitions.rowKeySet()) { - OlapTable currentTable = (OlapTable) db.getTable(tableId); - if (currentTable == null) { - // table may be dropped during FINISHING phase - continue; - } - - // drop restored partitions - for (String partitionName : restoredPartitions.row(tableId).keySet()) { - Partition currentPartition = currentTable.getPartition(partitionName); - if (currentPartition != null) { - currentTable.dropPartition(dbId, partitionName, true); - LOG.info("drop restored partition[{}] in table[{}] in db[{}]", - partitionName, tableId, dbId); - } - - currentTable.setState(OlapTableState.NORMAL); - } + if (tbl.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTbl = (OlapTable) tbl; + if (olapTbl.getState() == OlapTableState.RESTORE + || olapTbl.getState() == OlapTableState.RESTORE_WITH_LOAD) { + olapTbl.setState(OlapTableState.NORMAL); } - } finally { - db.writeUnlock(); } } - private void removeLeftTasks() { - for (Map.Entry entry : unfinishedTabletIds.entries()) { - AgentTaskQueue.removeTask(entry.getValue(), TTaskType.RESTORE, entry.getKey()); - } + public static RestoreJob read(DataInput in) throws IOException { + RestoreJob job = new RestoreJob(); + job.readFields(in); + return job; } @Override public void write(DataOutput out) throws IOException { super.write(out); + + Text.writeString(out, backupTimestamp); + jobInfo.write(out); + out.writeBoolean(allowLoad); + Text.writeString(out, state.name()); - if (tableToPartitionNames == null) { - out.writeBoolean(false); - } else { + if (backupMeta != null) { out.writeBoolean(true); - int size = tableToPartitionNames.size(); - out.writeInt(size); - for (Map.Entry> entry : tableToPartitionNames.entrySet()) { - Text.writeString(out, entry.getKey()); - Set partitionNames = entry.getValue(); - size = partitionNames.size(); - out.writeInt(size); - for (String partitionName : partitionNames) { - Text.writeString(out, partitionName); - } - } + backupMeta.write(out); + } else { + out.writeBoolean(false); } - if (tableRenameMap == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tableRenameMap.size(); - out.writeInt(size); - for (Map.Entry entry : tableRenameMap.entrySet()) { - Text.writeString(out, entry.getKey()); - Text.writeString(out, entry.getValue()); - } - } + fileMapping.write(out); - if (tableToCreateTableStmt == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tableToCreateTableStmt.size(); - out.writeInt(size); - for (Map.Entry entry : tableToCreateTableStmt.entrySet()) { - Text.writeString(out, entry.getKey()); - entry.getValue().write(out); - } - } - - if (tableToRollupStmt == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tableToRollupStmt.size(); - out.writeInt(size); - for (Map.Entry entry : tableToRollupStmt.entrySet()) { - Text.writeString(out, entry.getKey()); - entry.getValue().write(out); - } - } - - if (tableToPartitionStmts == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tableToPartitionStmts.rowKeySet().size(); - out.writeInt(size); - for (String tableName : tableToPartitionStmts.rowKeySet()) { - Text.writeString(out, tableName); - Map row = tableToPartitionStmts.row(tableName); - size = row.size(); - out.writeInt(size); - for (Map.Entry entry : row.entrySet()) { - Text.writeString(out, entry.getKey()); - entry.getValue().write(out); - } - } - } - - if (tableToReplace == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = tableToReplace.size(); - out.writeInt(size); - for (Map.Entry entry : tableToReplace.entrySet()) { - Text.writeString(out, entry.getKey()); - out.writeBoolean(entry.getValue()); - } - } - - if (restoredTables == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = restoredTables.size(); - out.writeInt(size); - for (Map.Entry entry : restoredTables.entrySet()) { - Text.writeString(out, entry.getKey()); - entry.getValue().write(out); - } - } - - if (restoredPartitions == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int size = restoredPartitions.size(); - out.writeInt(size); - for (long tableId : restoredPartitions.rowKeySet()) { - out.writeLong(tableId); - Map row = restoredPartitions.row(tableId); - size = row.size(); - out.writeInt(size); - for (Map.Entry entry : row.entrySet()) { - Text.writeString(out, entry.getKey()); - entry.getValue().write(out); - } - } - } - - out.writeLong(metaRestoredTime); + out.writeLong(metaPreparedTime); + out.writeLong(snapshotFinishedTime); out.writeLong(downloadFinishedTime); + + out.writeInt(restoreReplicationNum); + + out.writeInt(restoredPartitions.size()); + for (Pair entry : restoredPartitions) { + Text.writeString(out, entry.first); + entry.second.write(out); + } + + out.writeInt(restoredTbls.size()); + for (OlapTable tbl : restoredTbls) { + tbl.write(out); + } + + out.writeInt(restoredVersionInfo.rowKeySet().size()); + for (long tblId : restoredVersionInfo.rowKeySet()) { + out.writeLong(tblId); + out.writeInt(restoredVersionInfo.row(tblId).size()); + for (Map.Entry> entry : restoredVersionInfo.row(tblId).entrySet()) { + out.writeLong(entry.getKey()); + out.writeLong(entry.getValue().first); + out.writeLong(entry.getValue().second); + } + } + + out.writeInt(snapshotInfos.rowKeySet().size()); + for (long tabletId : snapshotInfos.rowKeySet()) { + out.writeLong(tabletId); + Map map = snapshotInfos.row(tabletId); + out.writeInt(map.size()); + for (Map.Entry entry : map.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + } } @Override public void readFields(DataInput in) throws IOException { super.readFields(in); + backupTimestamp = Text.readString(in); + jobInfo = BackupJobInfo.read(in); + allowLoad = in.readBoolean(); + state = RestoreJobState.valueOf(Text.readString(in)); if (in.readBoolean()) { - tableToPartitionNames = Maps.newHashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String tableName = Text.readString(in); - int count = in.readInt(); - Set partitionNames = Sets.newHashSet(); - for (int j = 0; j < count; j++) { - String partitionName = Text.readString(in); - partitionNames.add(partitionName); - } - tableToPartitionNames.put(tableName, partitionNames); - } + backupMeta = BackupMeta.read(in); } - if (in.readBoolean()) { - tableRenameMap = Maps.newHashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String newTableName = Text.readString(in); - String tableName = Text.readString(in); - tableRenameMap.put(newTableName, tableName); - } - } + fileMapping = RestoreFileMapping.read(in); - if (in.readBoolean()) { - tableToCreateTableStmt = Maps.newHashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String tableName = Text.readString(in); - CreateTableStmt stmt = CreateTableStmt.read(in); - tableToCreateTableStmt.put(tableName, stmt); - } - } - - if (in.readBoolean()) { - tableToRollupStmt = Maps.newHashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String tableName = Text.readString(in); - AlterTableStmt stmt = new AlterTableStmt(); - stmt.readFields(in); - tableToRollupStmt.put(tableName, stmt); - } - } - - if (in.readBoolean()) { - tableToPartitionStmts = HashBasedTable.create(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String tableName = Text.readString(in); - int count = in.readInt(); - for (int j = 0; j < count; j++) { - String partitionName = Text.readString(in); - AlterTableStmt stmt = new AlterTableStmt(); - stmt.readFields(in); - tableToPartitionStmts.put(tableName, partitionName, stmt); - } - } - } - - if (in.readBoolean()) { - tableToReplace = Maps.newHashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String tableName = Text.readString(in); - Boolean replace = in.readBoolean(); - tableToReplace.put(tableName, replace); - } - } - - if (in.readBoolean()) { - restoredTables = Maps.newHashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - String tableName = Text.readString(in); - Table table = Table.read(in); - restoredTables.put(tableName, table); - } - } - - if (in.readBoolean()) { - restoredPartitions = HashBasedTable.create(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - long tableId = in.readLong(); - int count = in.readInt(); - for (int j = 0; j < count; j++) { - String partitionName = Text.readString(in); - Partition partition = Partition.read(in); - restoredPartitions.put(tableId, partitionName, partition); - } - } - } - - metaRestoredTime = in.readLong(); + metaPreparedTime = in.readLong(); + snapshotFinishedTime = in.readLong(); downloadFinishedTime = in.readLong(); + + restoreReplicationNum = in.readInt(); + + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tblName = Text.readString(in); + Partition part = Partition.read(in); + restoredPartitions.add(Pair.create(tblName, part)); + } + + size = in.readInt(); + for (int i = 0; i < size; i++) { + restoredTbls.add((OlapTable) Table.read(in)); + } + + size = in.readInt(); + for (int i = 0; i < size; i++) { + long tblId = in.readLong(); + int innerSize = in.readInt(); + for (int j = 0; j < innerSize; j++) { + long partId = in.readLong(); + long version = in.readLong(); + long versionHash = in.readLong(); + restoredVersionInfo.put(tblId, partId, Pair.create(version, versionHash)); + } + } + + size = in.readInt(); + for (int i = 0; i < size; i++) { + long tabletId = in.readLong(); + int innerSize = in.readInt(); + for (int j = 0; j < innerSize; j++) { + long beId = in.readLong(); + SnapshotInfo info = SnapshotInfo.read(in); + snapshotInfos.put(tabletId, beId, info); + } + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.append(", backup ts: ").append(backupTimestamp); + sb.append(", state: ").append(state.name()); + return sb.toString(); } } + diff --git a/fe/src/com/baidu/palo/backup/RestoreJob_D.java b/fe/src/com/baidu/palo/backup/RestoreJob_D.java new file mode 100644 index 0000000000..4cfe9620c1 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/RestoreJob_D.java @@ -0,0 +1,885 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.alter.RollupHandler; +import com.baidu.palo.analysis.AddPartitionClause; +import com.baidu.palo.analysis.AddRollupClause; +import com.baidu.palo.analysis.AlterClause; +import com.baidu.palo.analysis.AlterTableStmt; +import com.baidu.palo.analysis.CreateTableStmt; +import com.baidu.palo.analysis.LabelName; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.OlapTable.OlapTableState; +import com.baidu.palo.catalog.Partition; +import com.baidu.palo.catalog.PartitionInfo; +import com.baidu.palo.catalog.PartitionType; +import com.baidu.palo.catalog.RangePartitionInfo; +import com.baidu.palo.catalog.Replica; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Table.TableType; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.catalog.TabletInvertedIndex; +import com.baidu.palo.catalog.TabletMeta; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.Pair; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.common.util.Util; +import com.baidu.palo.task.AgentBatchTask; +import com.baidu.palo.task.AgentTask; +import com.baidu.palo.task.AgentTaskExecutor; +import com.baidu.palo.task.AgentTaskQueue; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; + +@Deprecated +public class RestoreJob_D extends AbstractBackupJob_D { + private static final Logger LOG = LogManager.getLogger(RestoreJob_D.class); + + public enum RestoreJobState { + PENDING, + RESTORE_META, + DOWNLOAD, + DOWNLOADING, + FINISHED, + CANCELLED + } + + private RestoreJobState state; + + private Map> tableToPartitionNames; + private Map tableRenameMap; + + private Map tableToCreateTableStmt; + private Map tableToRollupStmt; + private com.google.common.collect.Table tableToPartitionStmts; + + private Map tableToReplace; + + private Map restoredTables; + // tableid - partition name - partition + private com.google.common.collect.Table restoredPartitions; + + private long metaRestoredTime; + private long downloadFinishedTime; + + public RestoreJob_D() { + super(); + } + + public RestoreJob_D(long jobId, long dbId, LabelName labelName, String restorePath, + Map remoteProperties, Map> tableToPartitionNames, + Map tableRenameMap) { + super(jobId, dbId, labelName, restorePath, remoteProperties); + state = RestoreJobState.PENDING; + + this.tableToPartitionNames = tableToPartitionNames; + this.tableRenameMap = tableRenameMap; + + this.tableToCreateTableStmt = Maps.newHashMap(); + this.tableToRollupStmt = Maps.newHashMap(); + this.tableToPartitionStmts = HashBasedTable.create(); + + this.tableToReplace = Maps.newHashMap(); + this.restoredTables = Maps.newHashMap(); + this.restoredPartitions = HashBasedTable.create(); + + this.metaRestoredTime = -1L; + this.downloadFinishedTime = -1L; + } + + public void setState(RestoreJobState state) { + this.state = state; + } + + public RestoreJobState getState() { + return state; + } + + public long getMetaRestoredTime() { + return metaRestoredTime; + } + + public long getDownloadFinishedTime() { + return downloadFinishedTime; + } + + public Map> getTableToPartitionNames() { + return tableToPartitionNames; + } + + @Override + public List getJobInfo() { + List jobInfo = Lists.newArrayList(); + jobInfo.add(jobId); + jobInfo.add(getLabel()); + jobInfo.add(state.name()); + jobInfo.add(TimeUtils.longToTimeString(createTime)); + jobInfo.add(TimeUtils.longToTimeString(metaRestoredTime)); + jobInfo.add(TimeUtils.longToTimeString(downloadFinishedTime)); + jobInfo.add(TimeUtils.longToTimeString(finishedTime)); + jobInfo.add(errMsg); + jobInfo.add(remotePath); + jobInfo.add(getLeftTasksNum()); + return jobInfo; + } + + @Override + public void runOnce() { + LOG.debug("begin to run restore job: {}, state: {}", jobId, state.name()); + try { + switch (state) { + case PENDING: + downloadBackupMeta(); + break; + case RESTORE_META: + restoreMeta(); + break; + case DOWNLOAD: + download(); + break; + case DOWNLOADING: + waitDownload(); + break; + default: + break; + } + } catch (Exception e) { + errMsg = Strings.nullToEmpty(e.getMessage()); + LOG.warn("failed to restore: [" + errMsg + "], job[" + jobId + "]", e); + state = RestoreJobState.CANCELLED; + } + + if (state == RestoreJobState.FINISHED || state == RestoreJobState.CANCELLED) { + end(Catalog.getInstance(), false); + } + } + + private void downloadBackupMeta() throws DdlException, IOException, AnalysisException, InterruptedException, + ExecutionException { + Catalog catalog = Catalog.getInstance(); + Database db = catalog.getDb(dbId); + if (db == null) { + throw new DdlException("Database[" + getDbName() + "] does not exist"); + } + + if (pathBuilder == null) { + pathBuilder = PathBuilder.createPathBuilder(getLocalDirName()); + } + + if (commandBuilder == null) { + String remotePropFilePath = pathBuilder.remoteProperties(); + commandBuilder = CommandBuilder.create(remotePropFilePath, remoteProperties); + } + + if (future == null) { + // 1. download manifest + LOG.info("begin to submit download backup meta. job: {}", jobId); + MetaDownloadTask task = new MetaDownloadTask(jobId, getDbName(), getLabel(), getLocalDirName(), remotePath, + pathBuilder, commandBuilder, + tableToPartitionNames, tableToCreateTableStmt, + tableToRollupStmt, tableToPartitionStmts, tableToReplace, + tableRenameMap); + // future = Catalog.getInstance().getBackupHandler().getAsynchronousCmdExecutor().submit(task); + } else { + boolean finished = checkFuture("download backup meta"); + if (!finished) { + return; + } + + future = null; + state = RestoreJobState.RESTORE_META; + } + } + + private void restoreMeta() throws DdlException { + Catalog catalog = Catalog.getInstance(); + Database db = catalog.getDb(dbId); + if (db == null) { + throw new DdlException("Database[" + getDbName() + "] does not exist"); + } + for (Map.Entry entry : tableToCreateTableStmt.entrySet()) { + String newTableName = entry.getKey(); + CreateTableStmt createTableStmt = entry.getValue(); + Boolean replace = tableToReplace.get(newTableName); + if (replace) { + // 1. create table + Table restoredTable = catalog.createTable(createTableStmt, true); + restoredTables.put(newTableName, restoredTable); + + if (restoredTable.getType() != TableType.OLAP) { + continue; + } + + OlapTable restoredOlapTable = (OlapTable) restoredTable; + + // 2. create rollup + RollupHandler rollupHandler = catalog.getRollupHandler(); + AlterTableStmt rollupStmt = tableToRollupStmt.get(newTableName); + if (rollupStmt != null) { + // check if new table name conflicts with rollup index name + for (AlterClause clause : rollupStmt.getOps()) { + Preconditions.checkState(clause instanceof AddRollupClause); + String rollupName = ((AddRollupClause) clause).getRollupName(); + if (rollupName.equals(newTableName)) { + throw new DdlException("New table name[" + newTableName + + "] conflicts with rollup index name"); + } + } + + rollupHandler.process(rollupStmt.getOps(), db, restoredOlapTable, true); + } + + // 3. create partition + Map partitionStmts = tableToPartitionStmts.row(newTableName); + if (partitionStmts.isEmpty()) { + continue; + } + + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) restoredOlapTable.getPartitionInfo(); + for (Map.Entry entry2 : partitionStmts.entrySet()) { + AlterTableStmt stmt = entry2.getValue(); + AddPartitionClause clause = (AddPartitionClause) stmt.getOps().get(0); + Pair res = catalog.addPartition(db, newTableName, restoredOlapTable, clause, true); + Partition partition = res.second; + rangePartitionInfo.handleNewSinglePartitionDesc(clause.getSingeRangePartitionDesc(), + partition.getId()); + restoredOlapTable.addPartition(partition); + } + } else { + Map partitionStmts = tableToPartitionStmts.row(newTableName); + for (Map.Entry entry2 : partitionStmts.entrySet()) { + AlterTableStmt stmt = entry2.getValue(); + Pair res = catalog.addPartition(db, newTableName, null, + (AddPartitionClause) stmt.getOps().get(0), true); + long tableId = res.first; + Partition partition = res.second; + restoredPartitions.put(tableId, partition.getName(), partition); + } + } + } + + metaRestoredTime = System.currentTimeMillis(); + state = RestoreJobState.DOWNLOAD; + LOG.info("finished restore tables. job[{}]", jobId); + } + + private void download() { + for (Map.Entry entry : restoredTables.entrySet()) { + String newTableName = entry.getKey(); + String tableName = tableRenameMap.get(newTableName); + Table table = entry.getValue(); + if (table.getType() != TableType.OLAP) { + continue; + } + + AgentBatchTask batchTask = new AgentBatchTask(); + OlapTable olapTable = (OlapTable) table; + long tableId = olapTable.getId(); + for (Partition partition : olapTable.getPartitions()) { + String partitionName = partition.getName(); + if (olapTable.getPartitionInfo().getType() == PartitionType.UNPARTITIONED) { + // single partition table + partitionName = tableName; + } + long partitionId = partition.getId(); + for (MaterializedIndex index : partition.getMaterializedIndices()) { + long indexId = index.getId(); + String indexName = olapTable.getIndexNameById(index.getId()); + if (indexName.equals(newTableName)) { + // base index + indexName = tableName; + } + + List orderedBackupedTabletIdList = getRestoredTabletInfo(tableName, partitionName, indexName); + + int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); + List tablets = index.getTablets(); + for (int i = 0; i < tablets.size(); i++) { + Tablet tablet = tablets.get(i); + Long backupedTabletId = orderedBackupedTabletIdList.get(i); + String remoteFilePath = PathBuilder.createPath(remotePath, getDbName(), tableName, + partitionName, indexName, + backupedTabletId.toString()); + for (Replica replica : tablet.getReplicas()) { + + } + } // end for tablets + } // end for indices + } // end for partitions + + synchronized (unfinishedTabletIds) { + for (AgentTask task : batchTask.getAllTasks()) { + AgentTaskQueue.addTask(task); + unfinishedTabletIds.put(task.getTabletId(), task.getBackendId()); + } + } + AgentTaskExecutor.submit(batchTask); + + LOG.info("finished send restore tasks for table: {}, job: {}", tableName, jobId); + } // end for tables + + state = RestoreJobState.DOWNLOADING; + LOG.info("finished send all restore tasks. job: {}", jobId); + } + + private List getRestoredTabletInfo(String tableName, String partitionName, String indexName) { + // pathBuilder.getRoot().print("\t"); + DirSaver indexDir = (DirSaver) pathBuilder.getRoot().getChild(getDbName()).getChild(tableName) + .getChild(partitionName).getChild(indexName); + Collection tabletNames = indexDir.getChildrenName(); + Set orderedBackupedTabletIds = Sets.newTreeSet(); + for (String tabletName : tabletNames) { + orderedBackupedTabletIds.add(Long.valueOf(tabletName)); + } + + List orderedBackupedTabletIdList = Lists.newArrayList(orderedBackupedTabletIds); + return orderedBackupedTabletIdList; + } + + private void waitDownload() throws DdlException { + synchronized (unfinishedTabletIds) { + if (!unfinishedTabletIds.isEmpty()) { + LOG.debug("waiting for unfinished download task. size: {}", unfinishedTabletIds.size()); + return; + } + } + + downloadFinishedTime = System.currentTimeMillis(); + LOG.info("all tablets restore finished. job: {}", jobId); + + finishing(Catalog.getInstance(), false); + + state = RestoreJobState.FINISHED; + } + + public void finishing(Catalog catalog, boolean isReplay) throws DdlException { + Database db = catalog.getDb(dbId); + if (db == null && !isReplay) { + throw new DdlException("Database[{}] does not exist"); + } + + db.writeLock(); + try { + // check again if table or partition already exist + for (Map.Entry entry : restoredTables.entrySet()) { + String tableName = entry.getKey(); + + Table currentTable = db.getTable(tableName); + if (currentTable != null) { + throw new DdlException("Table[" + tableName + "]' already exist. " + + "Drop table first or restore to another table"); + } + } + + for (long tableId : restoredPartitions.rowKeySet()) { + Table table = db.getTable(tableId); + if (table == null || table.getType() != TableType.OLAP) { + throw new DdlException("Table[" + tableId + "]' does not exist."); + } + + Map partitions = restoredPartitions.row(tableId); + OlapTable olapTable = (OlapTable) table; + for (Map.Entry entry : partitions.entrySet()) { + String partitionName = entry.getKey(); + Partition currentPartition = olapTable.getPartition(partitionName); + if (currentPartition != null) { + throw new DdlException("Partition[" + partitionName + "]' already exist in table[" + + tableId + "]. Drop partition first or restore to another table"); + } + } + } + + // add tables + for (Map.Entry entry : restoredTables.entrySet()) { + String tableName = entry.getKey(); + Table restoredTable = entry.getValue(); + + if (restoredTable.getType() == TableType.OLAP) { + OlapTable olapTable = (OlapTable) restoredTable; + olapTable.setState(OlapTableState.NORMAL); + if (isReplay) { + // add inverted index + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + long tableId = olapTable.getId(); + for (Partition partition : olapTable.getPartitions()) { + long partitionId = partition.getId(); + for (MaterializedIndex index : partition.getMaterializedIndices()) { + long indexId = index.getId(); + int schemaHash = olapTable.getSchemaHashByIndexId(indexId); + for (Tablet tablet : index.getTablets()) { + long tabletId = tablet.getId(); + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, + schemaHash); + invertedIndex.addTablet(tabletId, tabletMeta); + for (Replica replica : tablet.getReplicas()) { + invertedIndex.addReplica(tabletId, replica); + } + } + } + } + } + } + db.createTable(restoredTable); + LOG.info("finished add table: {}, job: {}, replay: {}", tableName, jobId, isReplay); + } + + // add partitions + for (long tableId : restoredPartitions.rowKeySet()) { + Table table = db.getTable(tableId); + String tableName = table.getName(); + Preconditions.checkState(table != null, tableName); + Preconditions.checkState(table.getType() == TableType.OLAP, tableName); + OlapTable olapTable = (OlapTable) table; + + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + + Map partitions = restoredPartitions.row(tableId); + + for (Map.Entry entry : partitions.entrySet()) { + String partitionName = entry.getKey(); + Partition partition = entry.getValue(); + long partitionId = partition.getId(); + + // add restored partition + AlterTableStmt stmt = tableToPartitionStmts.get(tableName, partitionName); + AddPartitionClause clause = (AddPartitionClause) stmt.getOps().get(0); + rangePartitionInfo.handleNewSinglePartitionDesc(clause.getSingeRangePartitionDesc(), partitionId); + olapTable.addPartition(partition); + + // add inverted index + if (isReplay) { + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + for (MaterializedIndex index : partition.getMaterializedIndices()) { + long indexId = index.getId(); + int schemaHash = olapTable.getSchemaHashByIndexId(indexId); + for (Tablet tablet : index.getTablets()) { + long tabletId = tablet.getId(); + for (Replica replica : tablet.getReplicas()) { + invertedIndex.addReplica(tabletId, replica); + } + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, + schemaHash); + invertedIndex.addTablet(tabletId, tabletMeta); + } + } + } + + LOG.info("finished add partition: {}, table: {}, job: {}, replay: {}", + partitionName, tableName, jobId, isReplay); + } // end for partitions + + olapTable.setState(OlapTableState.NORMAL); + } // end for tables + } finally { + db.writeUnlock(); + } + } + + public void handleFinishedRestore(long tabletId, long backendId) { + synchronized (unfinishedTabletIds) { + if (unfinishedTabletIds.remove(tabletId, backendId)) { + LOG.debug("finished restore tablet[{}], backend[{}]", tabletId, backendId); + } + } + } + + @Override + public void end(Catalog catalog, boolean isReplay) { + if (state == RestoreJobState.CANCELLED) { + rollback(catalog); + } + + // 2. set table state + // restoreTableState(catalog); + + if (!isReplay) { + // 3. remove agent tasks if left + removeLeftTasks(); + + // 4. remove local file + String labelDir = pathBuilder.getRoot().getFullPath(); + Util.deleteDirectory(new File(labelDir)); + LOG.debug("delete local dir: {}", labelDir); + + // 5. remove unused tablet in tablet inverted index + clearInvertedIndex(); + + finishedTime = System.currentTimeMillis(); + // log + Catalog.getInstance().getEditLog().logRestoreFinish(this); + } + + // clear for saving memory + clearJob(); + + LOG.info("finished end job[{}]. state: {}, replay: {}", jobId, state.name(), isReplay); + } + + private void clearInvertedIndex() { + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + if (state == RestoreJobState.CANCELLED) { + // clear restored table tablets + for (Table restoredTable : restoredTables.values()) { + if (restoredTable.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTable = (OlapTable) restoredTable; + for (Partition partition : olapTable.getPartitions()) { + for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (Tablet tablet : index.getTablets()) { + invertedIndex.deleteTablet(tablet.getId()); + } + } + } + } + + // partition + for (Partition partition : restoredPartitions.values()) { + for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (Tablet tablet : index.getTablets()) { + invertedIndex.deleteTablet(tablet.getId()); + } + } + } + } + } + + @Override + protected void clearJob() { + tableRenameMap = null; + + tableToCreateTableStmt = null; + tableToRollupStmt = null; + tableToPartitionStmts = null; + + tableToReplace = null; + restoredTables = null; + restoredPartitions = null; + + unfinishedTabletIds = null; + remoteProperties = null; + pathBuilder = null; + commandBuilder = null; + LOG.info("job[{}] cleared for saving memory", jobId); + } + + private void rollback(Catalog catalog) { + Database db = catalog.getDb(dbId); + if (db == null) { + errMsg = "Database does not exist[" + getDbName() + "]"; + LOG.info("{}. finished restore old meta. job: {}", errMsg, jobId); + return; + } + + db.writeLock(); + try { + // tables + for (Table restoredTable : restoredTables.values()) { + String tableName = restoredTable.getName(); + // use table id rather than table name. + // because table with same name may be created when doing restore. + // find table by name may get unexpected one. + Table currentTable = db.getTable(restoredTable.getId()); + // drop restored table + if (currentTable != null) { + db.dropTable(tableName); + LOG.info("drop restored table[{}] in db[{}]", tableName, dbId); + } + } + + // partitions + for (long tableId : restoredPartitions.rowKeySet()) { + OlapTable currentTable = (OlapTable) db.getTable(tableId); + if (currentTable == null) { + // table may be dropped during FINISHING phase + continue; + } + + // drop restored partitions + for (String partitionName : restoredPartitions.row(tableId).keySet()) { + Partition currentPartition = currentTable.getPartition(partitionName); + if (currentPartition != null) { + currentTable.dropPartition(dbId, partitionName, true); + LOG.info("drop restored partition[{}] in table[{}] in db[{}]", + partitionName, tableId, dbId); + } + + currentTable.setState(OlapTableState.NORMAL); + } + } + } finally { + db.writeUnlock(); + } + } + + private void removeLeftTasks() { + + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + Text.writeString(out, state.name()); + + if (tableToPartitionNames == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableToPartitionNames.size(); + out.writeInt(size); + for (Map.Entry> entry : tableToPartitionNames.entrySet()) { + Text.writeString(out, entry.getKey()); + Set partitionNames = entry.getValue(); + size = partitionNames.size(); + out.writeInt(size); + for (String partitionName : partitionNames) { + Text.writeString(out, partitionName); + } + } + } + + if (tableRenameMap == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableRenameMap.size(); + out.writeInt(size); + for (Map.Entry entry : tableRenameMap.entrySet()) { + Text.writeString(out, entry.getKey()); + Text.writeString(out, entry.getValue()); + } + } + + if (tableToCreateTableStmt == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableToCreateTableStmt.size(); + out.writeInt(size); + for (Map.Entry entry : tableToCreateTableStmt.entrySet()) { + Text.writeString(out, entry.getKey()); + entry.getValue().write(out); + } + } + + if (tableToRollupStmt == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableToRollupStmt.size(); + out.writeInt(size); + for (Map.Entry entry : tableToRollupStmt.entrySet()) { + Text.writeString(out, entry.getKey()); + entry.getValue().write(out); + } + } + + if (tableToPartitionStmts == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableToPartitionStmts.rowKeySet().size(); + out.writeInt(size); + for (String tableName : tableToPartitionStmts.rowKeySet()) { + Text.writeString(out, tableName); + Map row = tableToPartitionStmts.row(tableName); + size = row.size(); + out.writeInt(size); + for (Map.Entry entry : row.entrySet()) { + Text.writeString(out, entry.getKey()); + entry.getValue().write(out); + } + } + } + + if (tableToReplace == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = tableToReplace.size(); + out.writeInt(size); + for (Map.Entry entry : tableToReplace.entrySet()) { + Text.writeString(out, entry.getKey()); + out.writeBoolean(entry.getValue()); + } + } + + if (restoredTables == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = restoredTables.size(); + out.writeInt(size); + for (Map.Entry entry : restoredTables.entrySet()) { + Text.writeString(out, entry.getKey()); + entry.getValue().write(out); + } + } + + if (restoredPartitions == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + int size = restoredPartitions.size(); + out.writeInt(size); + for (long tableId : restoredPartitions.rowKeySet()) { + out.writeLong(tableId); + Map row = restoredPartitions.row(tableId); + size = row.size(); + out.writeInt(size); + for (Map.Entry entry : row.entrySet()) { + Text.writeString(out, entry.getKey()); + entry.getValue().write(out); + } + } + } + + out.writeLong(metaRestoredTime); + out.writeLong(downloadFinishedTime); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + state = RestoreJobState.valueOf(Text.readString(in)); + + if (in.readBoolean()) { + tableToPartitionNames = Maps.newHashMap(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tableName = Text.readString(in); + int count = in.readInt(); + Set partitionNames = Sets.newHashSet(); + for (int j = 0; j < count; j++) { + String partitionName = Text.readString(in); + partitionNames.add(partitionName); + } + tableToPartitionNames.put(tableName, partitionNames); + } + } + + if (in.readBoolean()) { + tableRenameMap = Maps.newHashMap(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String newTableName = Text.readString(in); + String tableName = Text.readString(in); + tableRenameMap.put(newTableName, tableName); + } + } + + if (in.readBoolean()) { + tableToCreateTableStmt = Maps.newHashMap(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tableName = Text.readString(in); + CreateTableStmt stmt = CreateTableStmt.read(in); + tableToCreateTableStmt.put(tableName, stmt); + } + } + + if (in.readBoolean()) { + tableToRollupStmt = Maps.newHashMap(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tableName = Text.readString(in); + AlterTableStmt stmt = new AlterTableStmt(); + stmt.readFields(in); + tableToRollupStmt.put(tableName, stmt); + } + } + + if (in.readBoolean()) { + tableToPartitionStmts = HashBasedTable.create(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tableName = Text.readString(in); + int count = in.readInt(); + for (int j = 0; j < count; j++) { + String partitionName = Text.readString(in); + AlterTableStmt stmt = new AlterTableStmt(); + stmt.readFields(in); + tableToPartitionStmts.put(tableName, partitionName, stmt); + } + } + } + + if (in.readBoolean()) { + tableToReplace = Maps.newHashMap(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tableName = Text.readString(in); + Boolean replace = in.readBoolean(); + tableToReplace.put(tableName, replace); + } + } + + if (in.readBoolean()) { + restoredTables = Maps.newHashMap(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String tableName = Text.readString(in); + Table table = Table.read(in); + restoredTables.put(tableName, table); + } + } + + if (in.readBoolean()) { + restoredPartitions = HashBasedTable.create(); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long tableId = in.readLong(); + int count = in.readInt(); + for (int j = 0; j < count; j++) { + String partitionName = Text.readString(in); + Partition partition = Partition.read(in); + restoredPartitions.put(tableId, partitionName, partition); + } + } + } + + metaRestoredTime = in.readLong(); + downloadFinishedTime = in.readLong(); + } +} diff --git a/fe/src/com/baidu/palo/backup/SnapshotInfo.java b/fe/src/com/baidu/palo/backup/SnapshotInfo.java new file mode 100644 index 0000000000..1fe110be70 --- /dev/null +++ b/fe/src/com/baidu/palo/backup/SnapshotInfo.java @@ -0,0 +1,161 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +public class SnapshotInfo implements Writable { + private long dbId; + private long tblId; + private long partitionId; + private long indexId; + private long tabletId; + private long beId; + private int schemaHash; + // eg: /path/to/your/be/data/snapshot/20180410102311.0/ + private String path; + // eg: + // 10006_0_1_0_0.dat + // 10006_2_2_0_0.idx + // 10006.hdr + private List files = Lists.newArrayList(); + + public SnapshotInfo() { + // for persist + } + + public SnapshotInfo(long dbId, long tblId, long partitionId, long indexId, long tabletId, + long beId, int schemaHash, String path, List files) { + this.dbId = dbId; + this.tblId = tblId; + this.partitionId = partitionId; + this.indexId = indexId; + this.tabletId = tabletId; + this.beId = beId; + this.schemaHash = schemaHash; + this.path = path; + this.files = files; + } + + public long getDbId() { + return dbId; + } + + public long getTblId() { + return tblId; + } + + public long getPartitionId() { + return partitionId; + } + + public long getIndexId() { + return indexId; + } + + public long getTabletId() { + return tabletId; + } + + public long getBeId() { + return beId; + } + + public int getSchemaHash() { + return schemaHash; + } + + public String getPath() { + return path; + } + + public List getFiles() { + return files; + } + + public void setFiles(List files) { + this.files = files; + } + + public String getTabletPath() { + String basePath = Joiner.on("/").join(path, tabletId, schemaHash); + return basePath; + } + + public static SnapshotInfo read(DataInput in) throws IOException { + SnapshotInfo info = new SnapshotInfo(); + info.readFields(in); + return info; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeLong(dbId); + out.writeLong(tblId); + out.writeLong(partitionId); + out.writeLong(indexId); + out.writeLong(tabletId); + out.writeLong(beId); + out.writeInt(schemaHash); + Text.writeString(out, path); + + out.writeInt(files.size()); + for (String file : files) { + Text.writeString(out, file); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + dbId = in.readLong(); + tblId = in.readLong(); + partitionId = in.readLong(); + indexId = in.readLong(); + tabletId = in.readLong(); + beId = in.readLong(); + schemaHash = in.readInt(); + path = Text.readString(in); + + int size = in.readInt(); + for (int i = 0; i < size; i++) { + files.add(Text.readString(in)); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("tablet id: ").append(tabletId); + sb.append(", be id: ").append(beId); + sb.append(", path: ").append(path); + sb.append(", files:").append(files); + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/backup/Status.java b/fe/src/com/baidu/palo/backup/Status.java new file mode 100644 index 0000000000..8da9b0acdd --- /dev/null +++ b/fe/src/com/baidu/palo/backup/Status.java @@ -0,0 +1,68 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.backup; + +public class Status { + public enum ErrCode { + OK, + NOT_FOUND, + BAD_FILE, + CREATE_REMOTE_PATH_FAILED, + IS_DIR, + IS_FILE, + TIMEOUT, + BAD_CONNECTION, + COMMON_ERROR + } + + private ErrCode errCode; + private String errMsg; + + public static final Status OK = new Status(ErrCode.OK, ""); + + public Status(ErrCode errCode, String errMsg) { + this.errCode = errCode; + this.errMsg = errMsg; + } + + public ErrCode getErrCode() { + return errCode; + } + + public String getErrMsg() { + return errMsg; + } + + public boolean ok() { + return errCode == ErrCode.OK; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(errCode.name()); + if (!ok()) { + sb.append(", msg: ").append(errMsg); + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/catalog/AccessPrivilege.java b/fe/src/com/baidu/palo/catalog/AccessPrivilege.java index 0ebd8fb511..0ccff2ab16 100644 --- a/fe/src/com/baidu/palo/catalog/AccessPrivilege.java +++ b/fe/src/com/baidu/palo/catalog/AccessPrivilege.java @@ -20,7 +20,10 @@ package com.baidu.palo.catalog; -import com.google.common.collect.ImmutableSortedMap; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; + +import com.google.common.base.Preconditions; import java.util.List; @@ -29,23 +32,52 @@ import java.util.List; public enum AccessPrivilege { READ_ONLY(1, "READ_ONLY"), READ_WRITE(2, "READ_WRITE"), - ALL(3, "ALL"); + ALL(3, "ALL"), + NODE_PRIV(4, "Privilege for cluster node operations"), + GRANT_PRIV(5, "Privilege for granting privlege"), + SELECT_PRIV(6, "Privilege for select data in tables"), + LOAD_PRIV(7, "Privilege for loading data into tables"), + ALTER_PRIV(8, "Privilege for alter database or table"), + CREATE_PRIV(9, "Privilege for createing database or table"), + DROP_PRIV(10, "Privilege for dropping database or table"); private int flag; private String desc; - private static final ImmutableSortedMap NAME_MAP = - ImmutableSortedMap.orderedBy(String.CASE_INSENSITIVE_ORDER) - .put("READ_ONLY", READ_ONLY) - .put("READ_WRITE", READ_WRITE) - .put("ALL", ALL) - .build(); - private AccessPrivilege(int flag, String desc) { this.flag = flag; this.desc = desc; } + public PrivBitSet toPaloPrivilege() { + Preconditions.checkState(flag > 0 && flag < 11); + switch (flag) { + case 1: + return PrivBitSet.of(PaloPrivilege.SELECT_PRIV); + case 2: + case 3: + return PrivBitSet.of(PaloPrivilege.SELECT_PRIV, PaloPrivilege.LOAD_PRIV, + PaloPrivilege.ALTER_PRIV, PaloPrivilege.CREATE_PRIV, + PaloPrivilege.DROP_PRIV); + case 4: + return PrivBitSet.of(PaloPrivilege.NODE_PRIV); + case 5: + return PrivBitSet.of(PaloPrivilege.GRANT_PRIV); + case 6: + return PrivBitSet.of(PaloPrivilege.SELECT_PRIV); + case 7: + return PrivBitSet.of(PaloPrivilege.LOAD_PRIV); + case 8: + return PrivBitSet.of(PaloPrivilege.ALTER_PRIV); + case 9: + return PrivBitSet.of(PaloPrivilege.CREATE_PRIV); + case 10: + return PrivBitSet.of(PaloPrivilege.DROP_PRIV); + default: + return null; + } + } + public static boolean contains(AccessPrivilege p1, AccessPrivilege p2) { return p1.flag >= p2.flag; } @@ -55,7 +87,11 @@ public enum AccessPrivilege { } public static AccessPrivilege fromName(String privStr) { - return NAME_MAP.get(privStr); + try { + return AccessPrivilege.valueOf(privStr.toUpperCase()); + } catch (Exception e) { + return null; + } } public static AccessPrivilege merge(List privileges) { diff --git a/fe/src/com/baidu/palo/catalog/BrokerMgr.java b/fe/src/com/baidu/palo/catalog/BrokerMgr.java index 83dbff0ee3..c77333a3b1 100644 --- a/fe/src/com/baidu/palo/catalog/BrokerMgr.java +++ b/fe/src/com/baidu/palo/catalog/BrokerMgr.java @@ -25,7 +25,6 @@ import com.baidu.palo.common.proc.BaseProcResult; import com.baidu.palo.common.proc.ProcNodeInterface; import com.baidu.palo.common.proc.ProcResult; -import com.google.common.base.Joiner; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -119,7 +118,7 @@ public class BrokerMgr { } } - // we need IP to find the colocation broker. + // we need IP to find the co-location broker. // { BrokerName -> { IP -> [BrokerAddress] } } private final Map> brokersMap = Maps.newHashMap(); private final Map> addressListMap = Maps.newHashMap(); @@ -159,6 +158,15 @@ public class BrokerMgr { } } + public boolean contaisnBroker(String brokerName) { + lock.lock(); + try { + return brokersMap.containsKey(brokerName); + } finally { + lock.unlock(); + } + } + public BrokerAddress getAnyBroker(String name) { lock.lock(); try { @@ -364,11 +372,12 @@ public class BrokerMgr { try { for (Map.Entry> entry : brokersMap.entrySet()) { String brokerName = entry.getKey(); - List brokerAddrs = Lists.newArrayList(); for (BrokerAddress address : entry.getValue().values()) { - brokerAddrs.add(address.toString()); + List row = Lists.newArrayList(); + row.add(brokerName); + row.add(address.toString()); + result.addRow(row); } - result.addRow(Lists.newArrayList(brokerName, Joiner.on(", ").join(brokerAddrs))); } } finally { lock.unlock(); @@ -415,3 +424,4 @@ public class BrokerMgr { } } } + diff --git a/fe/src/com/baidu/palo/catalog/Catalog.java b/fe/src/com/baidu/palo/catalog/Catalog.java index c1e8f700c3..636c734c46 100644 --- a/fe/src/com/baidu/palo/catalog/Catalog.java +++ b/fe/src/com/baidu/palo/catalog/Catalog.java @@ -33,7 +33,6 @@ import com.baidu.palo.analysis.AlterDatabaseQuotaStmt; import com.baidu.palo.analysis.AlterDatabaseRename; import com.baidu.palo.analysis.AlterSystemStmt; import com.baidu.palo.analysis.AlterTableStmt; -import com.baidu.palo.analysis.AlterUserStmt; import com.baidu.palo.analysis.BackupStmt; import com.baidu.palo.analysis.CancelAlterSystemStmt; import com.baidu.palo.analysis.CancelAlterTableStmt; @@ -42,6 +41,7 @@ import com.baidu.palo.analysis.ColumnRenameClause; import com.baidu.palo.analysis.CreateClusterStmt; import com.baidu.palo.analysis.CreateDbStmt; import com.baidu.palo.analysis.CreateTableStmt; +import com.baidu.palo.analysis.CreateUserStmt; import com.baidu.palo.analysis.CreateViewStmt; import com.baidu.palo.analysis.DecommissionBackendClause; import com.baidu.palo.analysis.DistributionDesc; @@ -65,10 +65,12 @@ import com.baidu.palo.analysis.RollupRenameClause; import com.baidu.palo.analysis.ShowAlterStmt.AlterType; import com.baidu.palo.analysis.SingleRangePartitionDesc; import com.baidu.palo.analysis.TableRenameClause; -import com.baidu.palo.backup.AbstractBackupJob; +import com.baidu.palo.analysis.UserDesc; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.backup.AbstractBackupJob_D; import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.backup.BackupJob; -import com.baidu.palo.backup.RestoreJob; +import com.baidu.palo.backup.BackupJob_D; +import com.baidu.palo.backup.RestoreJob_D; import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; import com.baidu.palo.catalog.Database.DbState; import com.baidu.palo.catalog.DistributionInfo.DistributionInfoType; @@ -123,6 +125,9 @@ import com.baidu.palo.load.LoadJob.JobState; import com.baidu.palo.master.Checkpoint; import com.baidu.palo.master.MetaHelper; import com.baidu.palo.metric.MetricRepo; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.mysql.privilege.UserPropertyMgr; import com.baidu.palo.persist.BackendIdsUpdateInfo; import com.baidu.palo.persist.ClusterInfo; import com.baidu.palo.persist.DatabaseInfo; @@ -232,6 +237,7 @@ public class Catalog { private ConsistencyChecker consistencyChecker; private BackupHandler backupHandler; + @Deprecated private UserPropertyMgr userPropertyMgr; private Daemon cleaner; // To clean old LabelInfo, ExportJobInfos @@ -298,6 +304,10 @@ public class Catalog { private DeployManager deployManager; + private PaloAuth auth; + + private DomainResolver domainResolver; + public List getFrontends(FrontendNodeType nodeType) { List result = Lists.newArrayList(); readLock(); @@ -359,7 +369,7 @@ public class Catalog { this.clone = new Clone(); this.alter = new Alter(); this.consistencyChecker = new ConsistencyChecker(); - this.backupHandler = new BackupHandler(); + this.backupHandler = new BackupHandler(this); this.lock = new ReentrantReadWriteLock(true); this.metaDir = Config.meta_dir; this.userPropertyMgr = new UserPropertyMgr(); @@ -397,6 +407,10 @@ public class Catalog { this.pullLoadJobMgr = new PullLoadJobMgr(); this.brokerMgr = new BrokerMgr(); + + this.auth = new PaloAuth(); + this.domainResolver = new DomainResolver(auth); + this.domainResolver.start(); } public static void destroyCheckpoint() { @@ -434,6 +448,10 @@ public class Catalog { return brokerMgr; } + public PaloAuth getAuth() { + return auth; + } + // use this to get correct ClusterInfoService instance public static SystemInfoService getCurrentSystemInfo() { return getCurrentCatalog().getClusterInfo(); @@ -504,7 +522,6 @@ public class Catalog { this.editLog = new EditLog(nodeName); loadImage(IMAGE_DIR); // load image file editLog.open(); // open bdb env or local output stream - this.userPropertyMgr.setEditLog(editLog); // 4. start load label cleaner thread createCleaner(); @@ -517,8 +534,6 @@ public class Catalog { listener.setName("stateListener"); listener.setInterval(STATE_CHANGE_CHECK_INTERVAL_MS); listener.start(); - - userPropertyMgr.setUp(); } private void getClusterIdAndRole() throws IOException { @@ -730,6 +745,12 @@ public class Catalog { + "/role?host=" + selfNode.first + "&port=" + selfNode.second); HttpURLConnection conn = null; conn = (HttpURLConnection) url.openConnection(); + if (conn.getResponseCode() != 200) { + LOG.warn("failed to get fe node type from helper node: {}. response code: {}", + helperNode, conn.getResponseCode()); + continue; + } + String type = conn.getHeaderField("role"); if (type == null) { LOG.warn("failed to get fe node type from helper node: {}.", helperNode); @@ -1127,13 +1148,15 @@ public class Catalog { checksum = loadLoadJob(dis, checksum); checksum = loadAlterJob(dis, checksum); - checksum = loadBackupAndRestoreJob(dis, checksum); + checksum = loadBackupAndRestoreJob_D(dis, checksum); checksum = loadAccessService(dis, checksum); checksum = loadRecycleBin(dis, checksum); checksum = loadGlobalVariable(dis, checksum); checksum = loadCluster(dis, checksum); checksum = loadBrokers(dis, checksum); checksum = loadExportJob(dis, checksum); + checksum = loadBackupHandler(dis, checksum); + checksum = loadPaloAuth(dis, checksum); long remoteChecksum = dis.readLong(); Preconditions.checkState(remoteChecksum == checksum, remoteChecksum + " vs. " + checksum); @@ -1385,45 +1408,48 @@ public class Catalog { return newChecksum; } - public long loadBackupAndRestoreJob(DataInputStream dis, long checksum) throws IOException { + public long loadBackupHandler(DataInputStream dis, long checksum) throws IOException { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_42) { + getBackupHandler().readFields(dis); + } + getBackupHandler().setCatalog(this); + return checksum; + } + + public long saveBackupHandler(DataOutputStream dos, long checksum) throws IOException { + getBackupHandler().write(dos); + return checksum; + } + + // This method is deprecated, we keep it because we need to consume the old image + // which contains old backup and restore jobs + @Deprecated + public long loadBackupAndRestoreJob_D(DataInputStream dis, long checksum) throws IOException { long newChecksum = checksum; - if (getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_22) { - newChecksum = loadBackupAndRestoreJob(dis, newChecksum, BackupJob.class); - newChecksum = loadBackupAndRestoreJob(dis, newChecksum, RestoreJob.class); - newChecksum = loadBackupAndRestoreLabel(dis, newChecksum); + if (getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_22 + && getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_42) { + newChecksum = loadBackupAndRestoreJob_D(dis, newChecksum, BackupJob_D.class); + newChecksum = loadBackupAndRestoreJob_D(dis, newChecksum, RestoreJob_D.class); + newChecksum = loadBackupAndRestoreLabel_D(dis, newChecksum); } return newChecksum; } - private long loadBackupAndRestoreJob(DataInputStream dis, long checksum, - Class jobClass) throws IOException { - Map jobs = null; - List finishedOrCancelledJobs = null; - if (jobClass == BackupJob.class) { - jobs = getBackupHandler().unprotectedGetBackupJobs(); - finishedOrCancelledJobs = getBackupHandler().unprotectedGetFinishedOrCancelledBackupJobs(); - } else if (jobClass == RestoreJob.class) { - jobs = getBackupHandler().unprotectedGetRestoreJobs(); - finishedOrCancelledJobs = getBackupHandler().unprotectedGetFinishedOrCancelledRestoreJobs(); - } else { - Preconditions.checkState(false); - } - + @Deprecated + private long loadBackupAndRestoreJob_D(DataInputStream dis, long checksum, + Class jobClass) throws IOException { int size = dis.readInt(); long newChecksum = checksum ^ size; for (int i = 0; i < size; i++) { long dbId = dis.readLong(); newChecksum ^= dbId; - if (jobClass == BackupJob.class) { - BackupJob job = new BackupJob(); + if (jobClass == BackupJob_D.class) { + BackupJob_D job = new BackupJob_D(); job.readFields(dis); - jobs.put(dbId, job); } else { - RestoreJob job = new RestoreJob(); + RestoreJob_D job = new RestoreJob_D(); job.readFields(dis); - jobs.put(dbId, job); } - LOG.debug("put {} job to map", dbId); } // finished or cancelled @@ -1432,40 +1458,51 @@ public class Catalog { for (int i = 0; i < size; i++) { long dbId = dis.readLong(); newChecksum ^= dbId; - if (jobClass == BackupJob.class) { - BackupJob job = new BackupJob(); + if (jobClass == BackupJob_D.class) { + BackupJob_D job = new BackupJob_D(); job.readFields(dis); - finishedOrCancelledJobs.add(job); } else { - RestoreJob job = new RestoreJob(); + RestoreJob_D job = new RestoreJob_D(); job.readFields(dis); - finishedOrCancelledJobs.add(job); } } return newChecksum; } - private long loadBackupAndRestoreLabel(DataInputStream dis, long checksum) throws IOException { + @Deprecated + private long loadBackupAndRestoreLabel_D(DataInputStream dis, long checksum) throws IOException { int size = dis.readInt(); long newChecksum = checksum ^ size; - - Multimap dbIdtoLabels = getBackupHandler().unprotectedGetDbIdToLabels(); - for (int i = 0; i < size; i++) { long dbId = dis.readLong(); newChecksum ^= dbId; - String label = Text.readString(dis); - dbIdtoLabels.put(dbId, label); + Text.readString(dis); // label } return newChecksum; } + public long loadPaloAuth(DataInputStream dis, long checksum) throws IOException { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_43) { + // CAN NOT use PaloAuth.read(), cause this auth instance is already passed to DomainResolver + auth.readFields(dis); + } + return checksum; + } + + @Deprecated public long loadAccessService(DataInputStream dis, long checksum) throws IOException { - int size = dis.readInt(); - long newChecksum = checksum ^ size; - userPropertyMgr.readFields(dis); - return newChecksum; + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43) { + int size = dis.readInt(); + long newChecksum = checksum ^ size; + UserPropertyMgr tmpUserPropertyMgr = new UserPropertyMgr(); + tmpUserPropertyMgr.readFields(dis); + + // transform it. the old UserPropertyMgr is deprecated + tmpUserPropertyMgr.transform(auth); + return newChecksum; + } + return checksum; } public long loadRecycleBin(DataInputStream dis, long checksum) throws IOException { @@ -1473,7 +1510,7 @@ public class Catalog { Catalog.getCurrentRecycleBin().readFields(dis); if (!isCheckpointThread()) { - // add tablet in Recyclebin to TabletInvertedIndex + // add tablet in Recycle bin to TabletInvertedIndex Catalog.getCurrentRecycleBin().addTabletToInvertedIndex(); } } @@ -1516,13 +1553,13 @@ public class Catalog { checksum = saveDb(dos, checksum); checksum = saveLoadJob(dos, checksum); checksum = saveAlterJob(dos, checksum); - checksum = saveBackupAndRestoreJob(dos, checksum); - checksum = saveAccessService(dos, checksum); checksum = saveRecycleBin(dos, checksum); checksum = saveGlobalVariable(dos, checksum); checksum = saveCluster(dos, checksum); checksum = saveBrokers(dos, checksum); checksum = saveExportJob(dos, checksum); + checksum = saveBackupHandler(dos, checksum); + checksum = savePaloAuth(dos, checksum); dos.writeLong(checksum); } finally { dos.close(); @@ -1713,75 +1750,8 @@ public class Catalog { return checksum; } - private long saveBackupAndRestoreJob(DataOutputStream dos, long checksum) throws IOException { - checksum = saveBackupAndRestoreJob(dos, checksum, BackupJob.class); - checksum = saveBackupAndRestoreJob(dos, checksum, RestoreJob.class); - checksum = saveBackupAndRestoreLabel(dos, checksum); - return checksum; - } - - private long saveBackupAndRestoreJob(DataOutputStream dos, long checksum, - Class jobClass) throws IOException { - Map jobs = null; - List finishedOrCancelledJobs = null; - if (jobClass == BackupJob.class) { - jobs = getBackupHandler().unprotectedGetBackupJobs(); - finishedOrCancelledJobs = getBackupHandler().unprotectedGetFinishedOrCancelledBackupJobs(); - } else if (jobClass == RestoreJob.class) { - jobs = getBackupHandler().unprotectedGetRestoreJobs(); - finishedOrCancelledJobs = getBackupHandler().unprotectedGetFinishedOrCancelledRestoreJobs(); - } else { - Preconditions.checkState(false); - } - - // jobs - int size = jobs.size(); - checksum ^= size; - dos.writeInt(size); - for (Entry entry : jobs.entrySet()) { - long dbId = entry.getKey(); - checksum ^= dbId; - dos.writeLong(dbId); - entry.getValue().write(dos); - LOG.debug("save {} job", dbId); - } - - // finished or cancelled jobs - size = finishedOrCancelledJobs.size(); - checksum ^= size; - dos.writeInt(size); - for (AbstractBackupJob job : finishedOrCancelledJobs) { - long dbId = job.getDbId(); - checksum ^= dbId; - dos.writeLong(dbId); - job.write(dos); - } - - return checksum; - } - - private long saveBackupAndRestoreLabel(DataOutputStream dos, long checksum) throws IOException { - Multimap dbIdtoLabels = getBackupHandler().unprotectedGetDbIdToLabels(); - Collection> entries = dbIdtoLabels.entries(); - int size = entries.size(); - checksum ^= size; - dos.writeInt(size); - for (Map.Entry entry : entries) { - long dbId = entry.getKey(); - String label = entry.getValue(); - checksum ^= dbId; - dos.writeLong(dbId); - Text.writeString(dos, label); - } - - return checksum; - } - - public long saveAccessService(DataOutputStream dos, long checksum) throws IOException { - int size = userPropertyMgr.getUserMapSize(); - checksum ^= size; - dos.writeInt(size); - userPropertyMgr.write(dos); + public long savePaloAuth(DataOutputStream dos, long checksum) throws IOException { + auth.write(dos); return checksum; } @@ -3668,7 +3638,7 @@ public class Catalog { List chosenBackendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIds(replicationNum, true, true, clusterName); if (chosenBackendIds == null) { - throw new DdlException("Failed to find enough host in all backends. need: " + replicationNum); + throw new DdlException("Failed to find " + replicationNum + " different hosts to create table"); } Preconditions.checkState(chosenBackendIds.size() == replicationNum); for (long backendId : chosenBackendIds) { @@ -4113,10 +4083,6 @@ public class Catalog { return this.backupHandler; } - public UserPropertyMgr getUserMgr() { - return this.userPropertyMgr; - } - public Load getLoadInstance() { return this.load; } @@ -4573,14 +4539,16 @@ public class Catalog { } // Change current database of this session. - public void changeDb(ConnectContext ctx, String dbName) throws DdlException { - if (getDb(dbName) == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + public void changeDb(ConnectContext ctx, String qualifiedDb) throws DdlException { + if (!auth.checkDbPriv(ctx, qualifiedDb, PrivPredicate.SHOW)) { + ErrorReport.reportDdlException(ErrorCode.ERR_DB_ACCESS_DENIED, ctx.getQualifiedUser(), qualifiedDb); } - if (!userPropertyMgr.checkAccess(ctx.getUser(), dbName, AccessPrivilege.READ_ONLY)) { - ErrorReport.reportDdlException(ErrorCode.ERR_DB_ACCESS_DENIED, ctx.getUser(), dbName); + + if (getDb(qualifiedDb) == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, qualifiedDb); } - ctx.setDatabase(dbName); + + ctx.setDatabase(qualifiedDb); } // for test only @@ -4658,18 +4626,6 @@ public class Catalog { return functionSet.getFunction(desc, mode); } - public void alterUser(AlterUserStmt stmt) throws DdlException { - getUserMgr().alterUser(stmt); - } - - public boolean checkWhiteList(String user, String remoteIp) { - return getUserMgr().checkWhiltListAccess(user, remoteIp); - } - - public List> showWhiteList(String user) { - return getUserMgr().showWhiteList(user); - } - /** * create cluster * @@ -4709,6 +4665,15 @@ public class Catalog { } finally { writeUnlock(); } + + // create super user for this cluster + UserIdentity adminUser = new UserIdentity(PaloAuth.ADMIN_USER, "%"); + try { + adminUser.analyze(stmt.getClusterName()); + } catch (AnalysisException e) { + LOG.error("should not happen", e); + } + auth.createUser(new CreateUserStmt(new UserDesc(adminUser, "", true))); } private void unprotectCreateCluster(Cluster cluster) { @@ -4782,6 +4747,9 @@ public class Catalog { writeUnlock(); } + // drop user of this cluster + // set is replay to true, not write log + auth.dropUserOfCluster(stmt.getClusterName(), true /* is replay */); } private void unprotectDropCluster(ClusterInfo info, boolean isReplay) { @@ -4800,6 +4768,8 @@ public class Catalog { } finally { writeUnlock(); } + + auth.dropUserOfCluster(info.getClusterName(), true /* is replay */); } public void replayExpandCluster(ClusterInfo info) { @@ -4901,12 +4871,15 @@ public class Catalog { * @throws DdlException */ public void changeCluster(ConnectContext ctx, String clusterName) throws DdlException { + if (!Catalog.getCurrentCatalog().getAuth().checkCanEnterCluster(ConnectContext.get(), clusterName)) { + ErrorReport.reportDdlException(ErrorCode.ERR_CLUSTER_NO_AUTHORITY, + ConnectContext.get().getQualifiedUser(), "enter"); + } + if (!nameToCluster.containsKey(clusterName)) { ErrorReport.reportDdlException(ErrorCode.ERR_CLUSTER_NO_EXISTS, clusterName); } - if (!userPropertyMgr.isAdmin(ctx.getUser())) { - ErrorReport.reportDdlException(ErrorCode.ERR_CLUSTER_NO_AUTHORITY); - } + ctx.setCluster(clusterName); } @@ -5354,3 +5327,4 @@ public class Catalog { } } } + diff --git a/fe/src/com/baidu/palo/catalog/DomainResolver.java b/fe/src/com/baidu/palo/catalog/DomainResolver.java new file mode 100644 index 0000000000..9d607d9e46 --- /dev/null +++ b/fe/src/com/baidu/palo/catalog/DomainResolver.java @@ -0,0 +1,166 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.catalog; + +import com.baidu.palo.common.util.Daemon; +import com.baidu.palo.mysql.privilege.PaloAuth; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import java.util.Set; + +public class DomainResolver extends Daemon { + private static final Logger LOG = LogManager.getLogger(DomainResolver.class); + private static final String BNS_RESOLVER_TOOLS_PATH = "/usr/bin/get_instance_by_service"; + + private PaloAuth auth; + + public DomainResolver(PaloAuth auth) { + super("domain resolver", 10 * 1000); + this.auth = auth; + } + + @Override + public void runOneCycle() { + // qualified user name -> domain name + Map> userMap = Maps.newHashMap(); + auth.getCopiedWhiteList(userMap); + LOG.info("begin to resolve domain: {}", userMap); + + // get unique domain names + Set domainSet = Sets.newHashSet(); + for (Map.Entry> entry : userMap.entrySet()) { + domainSet.addAll(entry.getValue()); + } + + // resolve domain name + for (String domain : domainSet) { + Set resolvedIPs = Sets.newHashSet(); + if (!resolveWithBNS(domain, resolvedIPs) && !resolveWithBNS(domain, resolvedIPs)) { + continue; + } + LOG.debug("get resolved ip of domain {}: {}", domain, resolvedIPs); + + for (Map.Entry> userEntry : userMap.entrySet()) { + if (!userEntry.getValue().contains(domain)) { + continue; + } + + auth.updateResolovedIps(userEntry.getKey(), domain, resolvedIPs); + } + } + } + + /** + * Check if domain name is valid + * + * @param host: + * currently is the user's whitelist bns or dns name + * @return true of false + */ + public boolean isValidDomain(String domainName) { + if (Strings.isNullOrEmpty(domainName)) { + LOG.warn("Domain name is null or empty"); + return false; + } + Set ipSet = Sets.newHashSet(); + if (!resolveWithDNS(domainName, ipSet) && !resolveWithBNS(domainName, ipSet)) { + return false; + } + return true; + } + + /** + * resolve domain name with dns + */ + public boolean resolveWithDNS(String domainName, Set resolvedIPs) { + InetAddress[] address; + try { + address = InetAddress.getAllByName(domainName); + } catch (UnknownHostException e) { + LOG.warn("unknown domain name " + domainName + " with dns: " + e.getMessage()); + return false; + } + + for (InetAddress addr : address) { + resolvedIPs.add(addr.getHostAddress()); + } + return true; + } + + public boolean resolveWithBNS(String domainName, Set resolvedIPs) { + File binaryFile = new File(BNS_RESOLVER_TOOLS_PATH); + if (!binaryFile.exists()) { + LOG.warn("{} does not exist", BNS_RESOLVER_TOOLS_PATH); + return false; + } + + final StringBuilder cmdBuilder = new StringBuilder(); + cmdBuilder.append(BNS_RESOLVER_TOOLS_PATH).append(" -a ").append(domainName); + Process process = null; + BufferedReader bufferedReader = null; + String str = null; + String ip = null; + try { + process = Runtime.getRuntime().exec(cmdBuilder.toString()); + bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + while ((str = bufferedReader.readLine()) != null) { + ip = str.split(" ")[1]; + resolvedIPs.add(ip); + } + final int exitCode = process.waitFor(); + // mean something error + if (exitCode != 0) { + LOG.warn("failed to execute cmd: {}, exit code: {}", cmdBuilder.toString(), exitCode); + resolvedIPs.clear(); + return false; + } + return true; + } catch (IOException e) { + LOG.warn("failed to revole domain with BNS", e); + resolvedIPs.clear(); + return false; + } catch (InterruptedException e) { + LOG.warn("failed to revole domain with BNS", e); + resolvedIPs.clear(); + return false; + } finally { + if (process != null) { + process.destroy(); + } + try { + if (bufferedReader != null) { + bufferedReader.close(); + } + } catch (IOException e) { + LOG.error("Close bufferedReader error! " + e); + } + } + } + +} diff --git a/fe/src/com/baidu/palo/catalog/DomainResolverServer.java b/fe/src/com/baidu/palo/catalog/DomainResolverServer.java deleted file mode 100644 index 085c275035..0000000000 --- a/fe/src/com/baidu/palo/catalog/DomainResolverServer.java +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.catalog; - -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * This class is responseble for resolving domain name, to resolve domain name - * , first register your domain with username, and so get domain name's ips with - * call getUserDomainToIps，There may be delays, because domain name resolution - * is an asynchronous process. - *

- * - * @author chenhao - * - */ -public final class DomainResolverServer { - private static final Logger LOG = LogManager.getLogger(DomainResolverServer.class); - private static final int RESOLVING_INTERVAL = 10000; - private static final String BNS_RESOLVER_TOOLS_PATH = "/usr/bin/get_instance_by_service"; - private static final int RESOLVING_RETRY_COUNT = 2; - - private static DomainResolverServer instance; - // User to domain name, domains to be resolved - private Map> userToDomainName = Maps.newHashMap(); - // User to domain name, domains have been resolved successfully - private Map>> userToDomainNameToIpSet = Maps.newHashMap(); - // Lock for userToDomainName and userToDomainNameToIpSet - private Lock cloneLock = new ReentrantLock(); - private Thread server; - - private DomainResolverServer() { - server = new Thread(new ResolverServer()); - server.start(); - } - - public static DomainResolverServer getInstance() { - if (instance == null) { - synchronized (DomainResolverServer.class) { - if (instance == null) { - instance = new DomainResolverServer(); - } - } - } - return instance; - } - - //for test - public Collection getRegisteredUserDomain(String user) { - return userToDomainName.get(user); - } - - /** - * @param domainNameCollection - * @return - */ - private boolean isNullOrEmptyCollection(Collection domainNameCollection) { - return domainNameCollection == null || domainNameCollection.isEmpty(); - } - - /** - * Register domain name with username, - * - * @param user: usually a user account in palo - * @param domainNameList: currently is the user's whitelist domain name - * @return true or false - */ - public boolean register(String user, Collection domainNameCollection) { - if (Strings.isNullOrEmpty(user) || isNullOrEmptyCollection(domainNameCollection)) { - LOG.warn("Register param error user[{}]", user); - return false; - } - - if (LOG.isDebugEnabled()) { - final StringBuilder sb = new StringBuilder(); - final Iterator iterator = domainNameCollection.iterator(); - while (iterator.hasNext()) { - sb.append(iterator.next()); - if (iterator.hasNext()) { - sb.append(","); - } - } - LOG.debug("Register user[{}], domain[{}] ...", user, sb.toString()); - } - - cloneLock.lock(); - try { - Set domainNameSet = userToDomainName.get(user); - if (domainNameSet == null) { - domainNameSet = Sets.newHashSet(); - userToDomainName.put(user, domainNameSet); - } - - boolean needUpdate = false; - for (String domainName : domainNameCollection) { - if (Strings.isNullOrEmpty(domainName)) { - LOG.warn("Register param error user[{}] domain[null]", user); - continue; - } - if (!domainNameSet.contains(domainName)) { - domainNameSet.add(domainName); - needUpdate = true; - } - } - - if (needUpdate) { - server.interrupt(); - } - } finally { - cloneLock.unlock(); - } - - return true; - } - - /** - * Unregister domain name with username - * - * @param user: usually a user account in palo - * @param domainNameList: currently is the user's whitelist domain name - */ - public void unregister(String user, Collection domainNameCollection) { - if (Strings.isNullOrEmpty(user) || isNullOrEmptyCollection(domainNameCollection)) { - LOG.warn("Unregister param error"); - return; - } - if (LOG.isDebugEnabled()) { - final StringBuilder sb = new StringBuilder(); - final Iterator iterator = domainNameCollection.iterator(); - while (iterator.hasNext()) { - sb.append(iterator.next()); - if (iterator.hasNext()) { - sb.append(","); - } - } - LOG.debug("Unregister user[{}], domain[{}] ...", user, sb.toString()); - } - - cloneLock.lock(); - try { - final Set domainNameSet = userToDomainName.get(user); - if (domainNameSet == null) { - return; - } - final Map> resolvedDomainNameMap = - userToDomainNameToIpSet.get(user); - for (String domainName : domainNameCollection) { - domainNameSet.remove(domainName); - if (resolvedDomainNameMap != null) { - resolvedDomainNameMap.remove(domainName); - } - } - - if (domainNameSet.isEmpty()) { - userToDomainName.remove(user); - } - - if (resolvedDomainNameMap != null && resolvedDomainNameMap.isEmpty()) { - userToDomainNameToIpSet.remove(user); - } - } finally { - cloneLock.unlock(); - } - } - - /** - * Utils for clone - * @param srcMap - * @return cloneMap - */ - private Map> cloneMap(Map> srcMap) { - final Map> copyOfMap = Maps.newHashMap(); - for (String key : srcMap.keySet()) { - final Set sets = Sets.newHashSet(); - for (String value : srcMap.get(key)) { - sets.add(value); - } - copyOfMap.put(key, sets); - } - return copyOfMap; - } - - /** - * Get user's ips - * - * @param user: usually a user account in palo - * @return map domain name to ips - */ - public Map> getUserDomainToIps(String user) { - Map> copyOfDomainToIpSet = null; - cloneLock.lock(); - try { - final Map> domainNameToIpSet = userToDomainNameToIpSet.get(user); - if (domainNameToIpSet == null || domainNameToIpSet.isEmpty()) { - LOG.debug("GetUserDomainToIps error, user[{}]", user); - return null; - } - copyOfDomainToIpSet = cloneMap(domainNameToIpSet); - } finally { - cloneLock.unlock(); - } - return copyOfDomainToIpSet; - } - - /** - * - * @param domainName: currently is the user's whitelist domain name - * @return ips - * @throws UnknownHostException - */ - private Set getDNSIps(String domainName) throws UnknownHostException { - final Set hostIpSet = Sets.newHashSet(); - final InetAddress[] address = InetAddress.getAllByName(domainName); - for (InetAddress addr : address) { - hostIpSet.add(addr.getHostAddress()); - } - return hostIpSet; - } - - /** - * Synchronous resolve domain name with dns - * - * @param domainName: currently is the user's whitelist domain name - * @return ips - */ - private Set resolveWithDNS(String domainName) { - try { - for (int i = 0; i < RESOLVING_RETRY_COUNT; i++) { - final Set resolvedIpSet = getDNSIps(domainName); - if (resolvedIpSet.size() > 0) { - return resolvedIpSet; - } - // avoid last unused wait - if (i < (RESOLVING_RETRY_COUNT - 1)) { - // sleep 5ms for retry - try { - Thread.sleep(5); - } catch (InterruptedException e2) { - LOG.warn("Sleep encounter InterruptedException"); - } - } - } - } catch (UnknownHostException e) { - LOG.warn("Resolve domain name[{}] with dns error: {}", domainName, e.getMessage()); - return null; - } - - LOG.warn("Resolve domain name[{}] with dns unknown error.", domainName); - return null; - } - - /** - * - * @param domainName: currently is the user's whitelist domain name - * @return - * @throws Exception - */ - private Set getBNSIps(String domainName) throws Exception { - final Set resolvedIpSet = Sets.newHashSet(); - final StringBuilder cmdBuilder = new StringBuilder(); - cmdBuilder.append(BNS_RESOLVER_TOOLS_PATH).append(" -a ").append(domainName); - Process process = null; - BufferedReader bufferedReader = null; - String str = null; - String ip = null; - try { - process = Runtime.getRuntime().exec(cmdBuilder.toString()); - bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - while ((str = bufferedReader.readLine()) != null) { - ip = str.split(" ")[1]; - resolvedIpSet.add(ip); - } - final int exitCode = process.waitFor(); - // mean something error - if (exitCode != 0) { - LOG.warn("GetBNSIps error code:{}", exitCode); - resolvedIpSet.clear(); - } - } finally { - if (process != null) { - process.destroy(); - } - try { - if (bufferedReader != null) { - bufferedReader.close(); - } - } catch (IOException e) { - LOG.error("Close bufferedReader error! " + e); - } - } - return resolvedIpSet; - } - - /** - * synchronous resolve domain name with bns - * - * @param domainName: currently is the user's whitelist domain name - * @return ips - */ - private Set resolveWithBNS(String domainName) { - try { - for (int i = 0; i < RESOLVING_RETRY_COUNT; i++) { - final Set resolvedIpSet = getBNSIps(domainName); - if (resolvedIpSet.size() > 0) { - return resolvedIpSet; - } - // avoid last unused wait - if (i < (RESOLVING_RETRY_COUNT - 1)) { - // sleep 5ms for retry - try { - Thread.sleep(5); - } catch (InterruptedException e2) { - LOG.warn("Sleep encounter InterruptedException"); - } - } - } - } catch (Exception e) { - LOG.warn("Resolve domain name[{}] with bns error: {}", domainName, e.getMessage()); - return null; - } - - LOG.warn("Resolve domain name[{}] with bns unknown error", domainName); - return null; - } - - /** - * Check if domain name is valid - * - * @param host: currently is the user's whitelist bns or dns name - * @return true of false - */ - public boolean isAvaliableDomain(String domainName) { - if (Strings.isNullOrEmpty(domainName)) { - LOG.warn("Domain name is null or empty"); - return false; - } - Set ips = resolveWithDNS(domainName); - if (isNullOrEmptyCollection(ips)) { - ips = resolveWithBNS(domainName); - if (isNullOrEmptyCollection(ips)) { - return false; - } - } - return true; - } - - /** - * Clone userToDomainName - * - * @return userToHost copy - */ - private Map> cloneUserToDomainName() { - cloneLock.lock(); - final Map> copyMaps = cloneMap(userToDomainName); - cloneLock.unlock(); - return copyMaps; - } - - // Resolve domain name at intervals, when new domain name are registered - // calling register() , server will immediately start a new asynchronous - // resolvation. - private class ResolverServer implements Runnable { - - public ResolverServer() { - } - - @Override - public void run() { - LOG.info("DomainResolverServer start"); - while (true) { - // avoid lock userToDomainName in resolvation - final Map> userToDomainNameCopy = cloneUserToDomainName(); - LOG.debug("Start a new resolvation"); - final Map>> newUserToDomainNameToIpSet = Maps.newHashMap(); - for (String user : userToDomainNameCopy.keySet()) { - LOG.debug("Start resolve user[{}]", user); - final Set domainNameWithDNSSet = userToDomainNameCopy.get(user); - final Map> domainNameToIpSet = Maps.newHashMap(); - final Set domainNameWithBNSSet = Sets.newHashSet(); - - // 1. check ipWhiteList if contains domain name with dns - for (String domainName : domainNameWithDNSSet) { - Set ipSet = resolveWithDNS(domainName); - if (ipSet == null || ipSet.isEmpty()) { - domainNameWithBNSSet.add(domainName); - continue; - } - LOG.debug("DNS: domain[{}] ip[{}]", domainName, ipSet); - domainNameToIpSet.put(domainName, ipSet); - } - - // 2. check ipWhiteList if contains domain name with bns - for (String domainName : domainNameWithBNSSet) { - final Set ipSet = resolveWithBNS(domainName); - if (ipSet == null || ipSet.isEmpty()) { - continue; - } - LOG.debug("BNS: domain[{}] ip[{}]", domainName, ipSet); - domainNameToIpSet.put(domainName, ipSet); - } - newUserToDomainNameToIpSet.put(user, domainNameToIpSet); - } - cloneLock.lock(); - userToDomainNameToIpSet.clear(); - userToDomainNameToIpSet.putAll(newUserToDomainNameToIpSet); - cloneLock.unlock(); - try { - Thread.sleep(RESOLVING_INTERVAL); - } catch (InterruptedException e) { - LOG.info("Sleep interrupted"); - } - } - } - - } -} diff --git a/fe/src/com/baidu/palo/catalog/MaterializedIndex.java b/fe/src/com/baidu/palo/catalog/MaterializedIndex.java index 34ea1ee925..e05666d496 100644 --- a/fe/src/com/baidu/palo/catalog/MaterializedIndex.java +++ b/fe/src/com/baidu/palo/catalog/MaterializedIndex.java @@ -20,18 +20,18 @@ package com.baidu.palo.catalog; -import com.baidu.palo.common.io.Text; -import com.baidu.palo.common.io.Writable; - -import com.google.common.collect.Lists; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import com.google.common.collect.Lists; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; /** @@ -94,13 +94,27 @@ public class MaterializedIndex extends MetaObject implements Writable { public Tablet getTablet(long tabletId) { return idToTablets.get(tabletId); + } + + public void clearTabletsForRestore() { + idToTablets.clear(); + tablets.clear(); } public void addTablet(Tablet tablet, TabletMeta tabletMeta) { - idToTablets.put(tablet.getId(), tablet); - tablets.add(tablet); - - Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta); + addTablet(tablet, tabletMeta, false); + } + + public void addTablet(Tablet tablet, TabletMeta tabletMeta, boolean isRestore) { + idToTablets.put(tablet.getId(), tablet); + tablets.add(tablet); + if (!isRestore) { + Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta); + } + } + + public void setIdForRestore(long idxId) { + this.id = idxId; } public long getId() { diff --git a/fe/src/com/baidu/palo/catalog/OlapTable.java b/fe/src/com/baidu/palo/catalog/OlapTable.java index 87faba8e83..b94f9ffb99 100644 --- a/fe/src/com/baidu/palo/catalog/OlapTable.java +++ b/fe/src/com/baidu/palo/catalog/OlapTable.java @@ -13,8 +13,8 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.catalog; - +package com.baidu.palo.catalog; + import com.baidu.palo.analysis.AddPartitionClause; import com.baidu.palo.analysis.AddRollupClause; import com.baidu.palo.analysis.AlterClause; @@ -26,8 +26,12 @@ import com.baidu.palo.analysis.PartitionDesc; import com.baidu.palo.analysis.RangePartitionDesc; import com.baidu.palo.analysis.SingleRangePartitionDesc; import com.baidu.palo.analysis.TableName; +import com.baidu.palo.backup.Status; +import com.baidu.palo.backup.Status.ErrCode; import com.baidu.palo.catalog.DistributionInfo.DistributionInfoType; +import com.baidu.palo.catalog.Replica.ReplicaState; import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.io.DeepCopy; import com.baidu.palo.common.io.Text; import com.baidu.palo.common.util.PropertyAnalyzer; import com.baidu.palo.common.util.Util; @@ -51,686 +55,872 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.Adler32; - -/** - * Internal representation of tableFamilyGroup-related metadata. A OlaptableFamilyGroup contains several tableFamily. - */ -public class OlapTable extends Table { - private static final Logger LOG = LogManager.getLogger(OlapTable.class); - - public enum OlapTableState { - NORMAL, - ROLLUP, - SCHEMA_CHANGE, - BACKUP, - RESTORE - } - - private OlapTableState state; - // index id -> table's schema - private Map> indexIdToSchema; - // index id -> table's schema version - private Map indexIdToSchemaVersion; - // index id -> table's schema hash - private Map indexIdToSchemaHash; - // index id -> table's short key column count - private Map indexIdToShortKeyColumnCount; - // index id -> table's storage type - private Map indexIdToStorageType; - // index name -> index id - private Map indexNameToId; - - private KeysType keysType; - private PartitionInfo partitionInfo; - private DistributionInfo defaultDistributionInfo; - - private Map idToPartition; - private Map nameToPartition; - - // bloom filter columns - private Set bfColumns; - private double bfFpp; - - public OlapTable() { - // for persist - super(TableType.OLAP); - this.indexIdToSchema = new HashMap>(); - this.indexIdToSchemaHash = new HashMap(); - this.indexIdToSchemaVersion = new HashMap(); - - this.indexIdToShortKeyColumnCount = new HashMap(); - this.indexIdToStorageType = new HashMap(); - - this.indexNameToId = new HashMap(); - - this.idToPartition = new HashMap(); - this.nameToPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - - this.bfColumns = null; - this.bfFpp = 0; - } - - public OlapTable(long id, String tableName, List baseSchema, - KeysType keysType, PartitionInfo partitionInfo, DistributionInfo defaultDistributionInfo) { - super(id, tableName, TableType.OLAP, baseSchema); - - this.state = OlapTableState.NORMAL; - - this.indexIdToSchema = new HashMap>(); - this.indexIdToSchemaHash = new HashMap(); - this.indexIdToSchemaVersion = new HashMap(); - - this.indexIdToShortKeyColumnCount = new HashMap(); - this.indexIdToStorageType = new HashMap(); - - this.indexNameToId = new HashMap(); - - this.idToPartition = new HashMap(); - this.nameToPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - - this.keysType = keysType; - this.partitionInfo = partitionInfo; - this.defaultDistributionInfo = defaultDistributionInfo; - - this.bfColumns = null; - this.bfFpp = 0; - } - - public void setState(OlapTableState state) { - this.state = state; - } - - public OlapTableState getState() { - return state; - } - - public void setName(String newName) { - // change name in indexNameToId - long baseIndexId = indexNameToId.remove(this.name); - indexNameToId.put(newName, baseIndexId); - - // change name - this.name = newName; - - // change single partition name - if (this.partitionInfo.getType() == PartitionType.UNPARTITIONED) { - // use for loop, because if we use getPartition(partitionName), - // we may not be able to get partition because this is a bug fix - for (Partition partition : getPartitions()) { - partition.setName(newName); - nameToPartition.clear(); - nameToPartition.put(newName, partition); - break; - } - } - } - - public boolean hasMaterializedIndex(String indexName) { - return indexNameToId.containsKey(indexName); - } - - public void setIndexSchemaInfo(Long indexId, String indexName, List schema, int schemaVersion, - int schemaHash, short shortKeyColumnCount) { - if (indexName == null) { - Preconditions.checkState(indexNameToId.containsValue(indexId)); - } else { - indexNameToId.put(indexName, indexId); - } - indexIdToSchema.put(indexId, schema); - indexIdToSchemaVersion.put(indexId, schemaVersion); - indexIdToSchemaHash.put(indexId, schemaHash); +import java.util.zip.Adler32; + +/** + * Internal representation of tableFamilyGroup-related metadata. A OlaptableFamilyGroup contains several tableFamily. + */ +public class OlapTable extends Table { + private static final Logger LOG = LogManager.getLogger(OlapTable.class); + + public enum OlapTableState { + NORMAL, + ROLLUP, + SCHEMA_CHANGE, + @Deprecated + BACKUP, + RESTORE, + RESTORE_WITH_LOAD + } + + private OlapTableState state; + // index id -> table's schema + private Map> indexIdToSchema; + // index id -> table's schema version + private Map indexIdToSchemaVersion; + // index id -> table's schema hash + private Map indexIdToSchemaHash; + // index id -> table's short key column count + private Map indexIdToShortKeyColumnCount; + // index id -> table's storage type + private Map indexIdToStorageType; + // index name -> index id + private Map indexNameToId; + + private KeysType keysType; + private PartitionInfo partitionInfo; + private DistributionInfo defaultDistributionInfo; + + private Map idToPartition; + private Map nameToPartition; + + // bloom filter columns + private Set bfColumns; + private double bfFpp; + + public OlapTable() { + // for persist + super(TableType.OLAP); + this.indexIdToSchema = new HashMap>(); + this.indexIdToSchemaHash = new HashMap(); + this.indexIdToSchemaVersion = new HashMap(); + + this.indexIdToShortKeyColumnCount = new HashMap(); + this.indexIdToStorageType = new HashMap(); + + this.indexNameToId = new HashMap(); + + this.idToPartition = new HashMap(); + this.nameToPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + + this.bfColumns = null; + this.bfFpp = 0; + } + + public OlapTable(long id, String tableName, List baseSchema, + KeysType keysType, PartitionInfo partitionInfo, DistributionInfo defaultDistributionInfo) { + super(id, tableName, TableType.OLAP, baseSchema); + + this.state = OlapTableState.NORMAL; + + this.indexIdToSchema = new HashMap>(); + this.indexIdToSchemaHash = new HashMap(); + this.indexIdToSchemaVersion = new HashMap(); + + this.indexIdToShortKeyColumnCount = new HashMap(); + this.indexIdToStorageType = new HashMap(); + + this.indexNameToId = new HashMap(); + + this.idToPartition = new HashMap(); + this.nameToPartition = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + + this.keysType = keysType; + this.partitionInfo = partitionInfo; + this.defaultDistributionInfo = defaultDistributionInfo; + + this.bfColumns = null; + this.bfFpp = 0; + } + + public void setState(OlapTableState state) { + this.state = state; + } + + public OlapTableState getState() { + return state; + } + + public void setName(String newName) { + // change name in indexNameToId + long baseIndexId = indexNameToId.remove(this.name); + indexNameToId.put(newName, baseIndexId); + + // change name + this.name = newName; + + // change single partition name + if (this.partitionInfo.getType() == PartitionType.UNPARTITIONED) { + // use for loop, because if we use getPartition(partitionName), + // we may not be able to get partition because this is a bug fix + for (Partition partition : getPartitions()) { + partition.setName(newName); + nameToPartition.clear(); + nameToPartition.put(newName, partition); + break; + } + } + } + + public boolean hasMaterializedIndex(String indexName) { + return indexNameToId.containsKey(indexName); + } + + public void setIndexSchemaInfo(Long indexId, String indexName, List schema, int schemaVersion, + int schemaHash, short shortKeyColumnCount) { + if (indexName == null) { + Preconditions.checkState(indexNameToId.containsValue(indexId)); + } else { + indexNameToId.put(indexName, indexId); + } + indexIdToSchema.put(indexId, schema); + indexIdToSchemaVersion.put(indexId, schemaVersion); + indexIdToSchemaHash.put(indexId, schemaHash); indexIdToShortKeyColumnCount.put(indexId, shortKeyColumnCount); } public void setIndexStorageType(Long indexId, TStorageType newStorageType) { Preconditions.checkState(newStorageType == TStorageType.COLUMN); indexIdToStorageType.put(indexId, newStorageType); - } - - public void deleteIndexInfo(String indexName) { - long indexId = this.indexNameToId.remove(indexName); - - indexIdToSchema.remove(indexId); - indexIdToSchemaVersion.remove(indexId); - indexIdToSchemaHash.remove(indexId); - indexIdToShortKeyColumnCount.remove(indexId); - indexIdToStorageType.remove(indexId); - } - - public Map getIndexNameToId() { - return indexNameToId; - } - - public Long getIndexIdByName(String indexName) { - return indexNameToId.get(indexName); - } - - public String getIndexNameById(long indexId) { - for (Map.Entry entry : indexNameToId.entrySet()) { - if (entry.getValue() == indexId) { - return entry.getKey(); - } - } - return null; - } - - // schema - public Map> getIndexIdToSchema() { - return indexIdToSchema; - } - - public Map> getCopiedIndexIdToSchema() { - Map> copiedIndexIdToSchema = new HashMap>(); - copiedIndexIdToSchema.putAll(indexIdToSchema); - return copiedIndexIdToSchema; - } - - public List getSchemaByIndexId(Long indexId) { - return indexIdToSchema.get(indexId); - } - - public List getKeyColumnsByIndexId(Long indexId) { - ArrayList keyColumns = Lists.newArrayList(); - List allColumns = this.getSchemaByIndexId(indexId); - for (Column column : allColumns) { - if (column.isKey()) { - keyColumns.add(column); - } - } - - return keyColumns; - } - - // schema version - public int getSchemaVersionByIndexId(Long indexId) { - if (indexIdToSchemaVersion.containsKey(indexId)) { - return indexIdToSchemaVersion.get(indexId); - } - return -1; - } - - // schemaHash - public Map getIndexIdToSchemaHash() { - return indexIdToSchemaHash; - } - - public Map getCopiedIndexIdToSchemaHash() { - Map copiedIndexIdToSchemaHash = new HashMap(); - copiedIndexIdToSchemaHash.putAll(indexIdToSchemaHash); - return copiedIndexIdToSchemaHash; - } - - public int getSchemaHashByIndexId(Long indexId) { - if (indexIdToSchemaHash.containsKey(indexId)) { - return indexIdToSchemaHash.get(indexId); - } - return -1; - } - - // short key - public Map getIndexIdToShortKeyColumnCount() { - return indexIdToShortKeyColumnCount; - } - - public Map getCopiedIndexIdToShortKeyColumnCount() { - Map copiedIndexIdToShortKeyColumnCount = new HashMap(); - copiedIndexIdToShortKeyColumnCount.putAll(indexIdToShortKeyColumnCount); - return copiedIndexIdToShortKeyColumnCount; - } - - public short getShortKeyColumnCountByIndexId(Long indexId) { - if (indexIdToShortKeyColumnCount.containsKey(indexId)) { - return indexIdToShortKeyColumnCount.get(indexId); - } - return (short) -1; - } - - // storage type - public Map getIndexIdToStorageType() { - return indexIdToStorageType; - } - - public Map getCopiedIndexIdToStorageType() { - Map copiedIndexIdToStorageType = new HashMap(); - copiedIndexIdToStorageType.putAll(indexIdToStorageType); - return copiedIndexIdToStorageType; - } - - public void setStorageTypeToIndex(Long indexId, TStorageType storageType) { - indexIdToStorageType.put(indexId, storageType); - } - - public TStorageType getStorageTypeByIndexId(Long indexId) { - return indexIdToStorageType.get(indexId); - } - - public KeysType getKeysType() { - return keysType; - } - - public PartitionInfo getPartitionInfo() { - return partitionInfo; - } - - public DistributionInfo getDefaultDistributionInfo() { - return defaultDistributionInfo; - } - - public void renamePartition(String partitionName, String newPartitionName) { - if (partitionInfo.getType() == PartitionType.UNPARTITIONED) { - // bug fix - for (Partition partition : idToPartition.values()) { - partition.setName(newPartitionName); - nameToPartition.clear(); - nameToPartition.put(newPartitionName, partition); - LOG.info("rename patition {} in table {}", newPartitionName, name); - break; - } - } else { - Partition partition = nameToPartition.remove(partitionName); - partition.setName(newPartitionName); - nameToPartition.put(newPartitionName, partition); - } - } - - public void addPartition(Partition partition) { - idToPartition.put(partition.getId(), partition); - nameToPartition.put(partition.getName(), partition); - } - - public Partition dropPartition(long dbId, String partitionName) { - return dropPartition(dbId, partitionName, false); - } - - public Partition dropPartition(long dbId, String partitionName, boolean isRestore) { - Partition partition = nameToPartition.get(partitionName); - if (partition != null) { - idToPartition.remove(partition.getId()); - nameToPartition.remove(partitionName); - - Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - - if (!isRestore) { - // recycle partition - Catalog.getCurrentRecycleBin().recyclePartition(dbId, id, partition, - rangePartitionInfo.getRange(partition.getId()), - rangePartitionInfo.getDataProperty(partition.getId()), - rangePartitionInfo.getReplicationNum(partition.getId())); - } - - // drop partition info - rangePartitionInfo.dropPartition(partition.getId()); - } - return partition; - } - - public Collection getPartitions() { - return idToPartition.values(); - } - - public Partition getPartition(long partitionId) { - return idToPartition.get(partitionId); - } - - public Partition getPartition(String partitionName) { - return nameToPartition.get(partitionName); - } - - public Set getCopiedBfColumns() { - if (bfColumns == null) { - return null; - } - - return Sets.newHashSet(bfColumns); - } - - public double getBfFpp() { - return bfFpp; - } - - public void setBloomFilterInfo(Set bfColumns, double bfFpp) { - this.bfColumns = bfColumns; - this.bfFpp = bfFpp; - } - - public TTableDescriptor toThrift() { - TOlapTable tOlapTable = new TOlapTable(getName()); - TTableDescriptor tTableDescriptor = new TTableDescriptor(id, TTableType.OLAP_TABLE, - baseSchema.size(), 0, getName(), ""); - tTableDescriptor.setOlapTable(tOlapTable); - return tTableDescriptor; - } - - public long getRowCount() { - long rowCount = 0; - for (Map.Entry entry : idToPartition.entrySet()) { - rowCount += ((Partition) entry.getValue()).getBaseIndex().getRowCount(); - } - return rowCount; - } - - public AlterTableStmt toAddRollupStmt(String dbName, Collection indexIds) { - List alterClauses = Lists.newArrayList(); - for (Map.Entry entry : indexNameToId.entrySet()) { - String indexName = entry.getKey(); - long indexId = entry.getValue(); - if (!indexIds.contains(indexId)) { - continue; - } - - // cols - List columnNames = Lists.newArrayList(); - for (Column column : indexIdToSchema.get(indexId)) { - columnNames.add(column.getName()); - } - - // properties - Map properties = Maps.newHashMap(); - properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_TYPE, indexIdToStorageType.get(indexId).name()); - properties.put(PropertyAnalyzer.PROPERTIES_SHORT_KEY, indexIdToShortKeyColumnCount.get(indexId).toString()); - properties.put(PropertyAnalyzer.PROPERTIES_SCHEMA_VERSION, indexIdToSchemaVersion.get(indexId).toString()); - - AddRollupClause addRollupClause = new AddRollupClause(indexName, columnNames, null, null, properties); - alterClauses.add(addRollupClause); - } - - AlterTableStmt alterTableStmt = new AlterTableStmt(new TableName(dbName, name), alterClauses); - return alterTableStmt; - } - - public AlterTableStmt toAddPartitionStmt(String dbName, String partitionName) { - Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - List alterClauses = Lists.newArrayList(); - - Partition partition = nameToPartition.get(partitionName); - Map properties = Maps.newHashMap(); - long version = partition.getCommittedVersion(); - long versionHash = partition.getCommittedVersionHash(); - properties.put(PropertyAnalyzer.PROPERTIES_VERSION_INFO, version + "," + versionHash); - properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, - String.valueOf(partitionInfo.getReplicationNum(partition.getId()))); - - SingleRangePartitionDesc singleDesc = - rangePartitionInfo.toSingleRangePartitionDesc(partition.getId(), partitionName, properties); - DistributionDesc distributionDesc = partition.getDistributionInfo().toDistributionDesc(); - - AddPartitionClause addPartitionClause = new AddPartitionClause(singleDesc, distributionDesc, null); - alterClauses.add(addPartitionClause); - AlterTableStmt stmt = new AlterTableStmt(new TableName(dbName, name), alterClauses); - return stmt; - } - - @Override - public CreateTableStmt toCreateTableStmt(String dbName) { - Map properties = Maps.newHashMap(); - - // partition - PartitionDesc partitionDesc = null; - if (partitionInfo.getType() == PartitionType.RANGE) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - List partitionColumns = rangePartitionInfo.getPartitionColumns(); - List partitionColNames = Lists.newArrayList(); - for (Column partCol : partitionColumns) { - partitionColNames.add(partCol.getName()); - } - - List singlePartitionDescs = Lists.newArrayList(); - partitionDesc = new RangePartitionDesc(partitionColNames, singlePartitionDescs); - } else { - Short replicationNum = partitionInfo.getReplicationNum(nameToPartition.get(name).getId()); - properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, replicationNum.toString()); - // and partition version info here for non-partitioned table - Partition partition = getPartition(name); - Preconditions.checkNotNull(partition); - long version = partition.getCommittedVersion(); - long versionHash = partition.getCommittedVersionHash(); - String versionProp = Joiner.on(",").join(version, versionHash); - properties.put(PropertyAnalyzer.PROPERTIES_VERSION_INFO, versionProp); - } - - // keys - List keysColumnNames = Lists.newArrayList(); - for (Column column : baseSchema) { - if (column.isKey()) { - keysColumnNames.add(column.getName()); - } - } - KeysDesc keysDesc = new KeysDesc(keysType, keysColumnNames); - - // distribution - DistributionDesc distributionDesc = defaultDistributionInfo.toDistributionDesc(); - - // other properties - properties.put(PropertyAnalyzer.PROPERTIES_SHORT_KEY, indexIdToShortKeyColumnCount.get(id).toString()); - properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_TYPE, indexIdToStorageType.get(id).name()); - if (bfColumns != null) { - String bfCols = Joiner.on(",").join(bfColumns); - properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, bfCols); - properties.put(PropertyAnalyzer.PROPERTIES_BF_FPP, String.valueOf(bfFpp)); - } - properties.put(PropertyAnalyzer.PROPERTIES_SCHEMA_VERSION, indexIdToSchemaVersion.get(id).toString()); - - CreateTableStmt stmt = new CreateTableStmt(false, false, new TableName(dbName, name), baseSchema, - type.name(), keysDesc, partitionDesc, distributionDesc, - properties, null); - return stmt; - } - - @Override - public int getSignature(int signatureVersion) { - Adler32 adler32 = new Adler32(); - adler32.update(signatureVersion); - final String charsetName = "UTF-8"; - - try { - // ignore table name - // adler32.update(name.getBytes(charsetName)); - // type - adler32.update(type.name().getBytes(charsetName)); - - // all indices(should be in order) - Set indexNames = Sets.newTreeSet(); - indexNames.addAll(indexNameToId.keySet()); - for (String indexName : indexNames) { - long indexId = indexNameToId.get(indexName); - if (!indexName.equals(name)) { - // index name(ignore base index name. base index name maybe changed) - adler32.update(indexName.getBytes(charsetName)); - } - // schema hash - adler32.update(indexIdToSchemaHash.get(indexId)); - // short key column count - adler32.update(indexIdToShortKeyColumnCount.get(indexId)); - // storage type - adler32.update(indexIdToStorageType.get(indexId).name().getBytes(charsetName)); - } - - // partition type - adler32.update(partitionInfo.getType().name().getBytes(charsetName)); - // partition columns - if (partitionInfo.getType() == PartitionType.RANGE) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - List partitionColumns = rangePartitionInfo.getPartitionColumns(); - adler32.update(Util.schemaHash(0, partitionColumns, null, 0)); - } - - } catch (UnsupportedEncodingException e) { - LOG.error("encoding error", e); - return -1; - } - - return Math.abs((int) adler32.getValue()); - } - - @Override - public boolean isPartitioned() { - int numSegs = 0; - for (Partition part : getPartitions()) { - numSegs += part.getDistributionInfo().getBucketNum(); - if (numSegs > 1) { - return true; - } - } - return false; - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - - // state - Text.writeString(out, state.name()); - - // indices' schema - int counter = indexNameToId.size(); - out.writeInt(counter); - for (Map.Entry entry : indexNameToId.entrySet()) { - String indexName = entry.getKey(); - long indexId = entry.getValue(); - Text.writeString(out, indexName); - out.writeLong(indexId); - // schema - out.writeInt(indexIdToSchema.get(indexId).size()); - for (Column column : indexIdToSchema.get(indexId)) { - column.write(out); - } - - // storage type - Text.writeString(out, indexIdToStorageType.get(indexId).name()); - - // indices's schema version - out.writeInt(indexIdToSchemaVersion.get(indexId)); - - // indices's schema hash - out.writeInt(indexIdToSchemaHash.get(indexId)); - - // indices's short key column count - out.writeShort(indexIdToShortKeyColumnCount.get(indexId)); - } - - Text.writeString(out, keysType.name()); - Text.writeString(out, partitionInfo.getType().name()); - partitionInfo.write(out); - Text.writeString(out, defaultDistributionInfo.getType().name()); - defaultDistributionInfo.write(out); - - // partitions - int partitionCount = idToPartition.size(); - out.writeInt(partitionCount); - for (Partition partition : idToPartition.values()) { - partition.write(out); - } - - // bloom filter columns - if (bfColumns == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeInt(bfColumns.size()); - for (String bfColumn : bfColumns) { - Text.writeString(out, bfColumn); - } - out.writeDouble(bfFpp); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - - this.state = OlapTableState.valueOf(Text.readString(in)); - - // indices's schema - int counter = in.readInt(); - for (int i = 0; i < counter; i++) { - String indexName = Text.readString(in); - long indexId = in.readLong(); - this.indexNameToId.put(indexName, indexId); - - // schema - int colCount = in.readInt(); - List schema = new LinkedList(); - for (int j = 0; j < colCount; j++) { - Column column = Column.read(in); - schema.add(column); - } - this.indexIdToSchema.put(indexId, schema); - - // storage type - TStorageType type = TStorageType.valueOf(Text.readString(in)); - this.indexIdToStorageType.put(indexId, type); - - // indices's schema version - this.indexIdToSchemaVersion.put(indexId, in.readInt()); - - // indices's schema hash - this.indexIdToSchemaHash.put(indexId, in.readInt()); - - // indices's short key column count - this.indexIdToShortKeyColumnCount.put(indexId, in.readShort()); - } - - // partition and distribution info - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_30) { - keysType = KeysType.valueOf(Text.readString(in)); - } else { - keysType = KeysType.AGG_KEYS; - } - - PartitionType partType = PartitionType.valueOf(Text.readString(in)); - if (partType == PartitionType.UNPARTITIONED) { - partitionInfo = PartitionInfo.read(in); - } else if (partType == PartitionType.RANGE) { - partitionInfo = RangePartitionInfo.read(in); - } else { - throw new IOException("invalid partition type: " + partType); - } - - DistributionInfoType distriType = DistributionInfoType.valueOf(Text.readString(in)); - if (distriType == DistributionInfoType.HASH) { - defaultDistributionInfo = HashDistributionInfo.read(in); - } else if (distriType == DistributionInfoType.RANDOM) { - defaultDistributionInfo = RandomDistributionInfo.read(in); - } else { - throw new IOException("invalid distribution type: " + distriType); - } - - int partitionCount = in.readInt(); - for (int i = 0; i < partitionCount; ++i) { - Partition partition = Partition.read(in); - idToPartition.put(partition.getId(), partition); - nameToPartition.put(partition.getName(), partition); - } - - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_9) { - if (in.readBoolean()) { - int bfColumnCount = in.readInt(); - bfColumns = Sets.newHashSet(); - for (int i = 0; i < bfColumnCount; i++) { - bfColumns.add(Text.readString(in)); - } - - bfFpp = in.readDouble(); - } - } - } - - public boolean equals(Table table) { - if (this == table) { - return true; - } - if (!(table instanceof OlapTable)) { - return false; - } - - return true; - } -} + } + + public void deleteIndexInfo(String indexName) { + long indexId = this.indexNameToId.remove(indexName); + + indexIdToSchema.remove(indexId); + indexIdToSchemaVersion.remove(indexId); + indexIdToSchemaHash.remove(indexId); + indexIdToShortKeyColumnCount.remove(indexId); + indexIdToStorageType.remove(indexId); + } + + public Map getIndexNameToId() { + return indexNameToId; + } + + public Long getIndexIdByName(String indexName) { + return indexNameToId.get(indexName); + } + + public String getIndexNameById(long indexId) { + for (Map.Entry entry : indexNameToId.entrySet()) { + if (entry.getValue() == indexId) { + return entry.getKey(); + } + } + return null; + } + + public Status resetIdsForRestore(Catalog catalog, Database db, int restoreReplicationNum) { + // table id + id = catalog.getNextId(); + + // copy an origin index id to name map + Map origIdxIdToName = Maps.newHashMap(); + for (Map.Entry entry : indexNameToId.entrySet()) { + origIdxIdToName.put(entry.getValue(), entry.getKey()); + } + + // reset all 'indexIdToXXX' map + for (Map.Entry entry : origIdxIdToName.entrySet()) { + long newIdxId = 0; + if (entry.getValue().equals(name)) { + // base index + newIdxId = id; + } else { + newIdxId = catalog.getNextId(); + } + indexIdToSchema.put(newIdxId, indexIdToSchema.remove(entry.getKey())); + indexIdToSchemaHash.put(newIdxId, indexIdToSchemaHash.remove(entry.getKey())); + indexIdToSchemaVersion.put(newIdxId, indexIdToSchemaVersion.remove(entry.getKey())); + indexIdToShortKeyColumnCount.put(newIdxId, indexIdToShortKeyColumnCount.remove(entry.getKey())); + indexIdToStorageType.put(newIdxId, indexIdToStorageType.remove(entry.getKey())); + indexNameToId.put(entry.getValue(), newIdxId); + } + + // generate a partition name to id map + Map origPartNameToId = Maps.newHashMap(); + for (Partition partition : idToPartition.values()) { + origPartNameToId.put(partition.getName(), partition.getId()); + } + + // reset partition info and idToPartition map + if (partitionInfo.getType() == PartitionType.RANGE) { + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + for (Map.Entry entry : origPartNameToId.entrySet()) { + long newPartId = catalog.getNextId(); + rangePartitionInfo.idToDataProperty.put(newPartId, + rangePartitionInfo.idToDataProperty.remove(entry.getValue())); + rangePartitionInfo.idToReplicationNum.remove(entry.getValue()); + rangePartitionInfo.idToReplicationNum.put(newPartId, + (short) restoreReplicationNum); + rangePartitionInfo.getIdToRange().put(newPartId, + rangePartitionInfo.getIdToRange().remove(entry.getValue())); + + idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); + } + } else { + // Single partitioned + long newPartId = catalog.getNextId(); + for (Map.Entry entry : origPartNameToId.entrySet()) { + partitionInfo.idToDataProperty.put(newPartId, partitionInfo.idToDataProperty.remove(entry.getValue())); + partitionInfo.idToReplicationNum.remove(entry.getValue()); + partitionInfo.idToReplicationNum.put(newPartId, (short) restoreReplicationNum); + idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); + } + } + + // for each partition, reset rollup index map + for (Map.Entry entry : idToPartition.entrySet()) { + Partition partition = entry.getValue(); + for (Map.Entry entry2 : origIdxIdToName.entrySet()) { + MaterializedIndex idx = partition.getIndex(entry2.getKey()); + long newIdxId = indexNameToId.get(entry2.getValue()); + idx.setIdForRestore(newIdxId); + if (newIdxId != id) { + // not base table, reset + partition.deleteRollupIndex(entry2.getKey()); + partition.createRollupIndex(idx); + } + + // generate new tablets in origin tablet order + int tabletNum = idx.getTablets().size(); + idx.clearTabletsForRestore(); + for (int i = 0; i < tabletNum; i++) { + long newTabletId = catalog.getNextId(); + Tablet newTablet = new Tablet(newTabletId); + idx.addTablet(newTablet, null /* tablet meta */, true /* is restore */); + + // replicas + List beIds = Catalog.getCurrentSystemInfo().seqChooseBackendIds(partitionInfo.getReplicationNum(entry.getKey()), + true, true, + db.getClusterName()); + if (beIds == null) { + return new Status(ErrCode.COMMON_ERROR, "failed to find " + + partitionInfo.getReplicationNum(entry.getKey()) + + " different hosts to create table: " + name); + } + for (Long beId : beIds) { + long newReplicaId = catalog.getNextId(); + Replica replica = new Replica(newReplicaId, beId, ReplicaState.NORMAL, + partition.getCommittedVersion(), partition.getCommittedVersionHash()); + newTablet.addReplica(replica, true /* is restore */); + } + } + } + + // reset partition id + partition.setIdForRestore(entry.getKey()); + } + + return Status.OK; + } + + // schema + public Map> getIndexIdToSchema() { + return indexIdToSchema; + } + + public Map> getCopiedIndexIdToSchema() { + Map> copiedIndexIdToSchema = new HashMap>(); + copiedIndexIdToSchema.putAll(indexIdToSchema); + return copiedIndexIdToSchema; + } + + public List getSchemaByIndexId(Long indexId) { + return indexIdToSchema.get(indexId); + } + + public List getKeyColumnsByIndexId(Long indexId) { + ArrayList keyColumns = Lists.newArrayList(); + List allColumns = this.getSchemaByIndexId(indexId); + for (Column column : allColumns) { + if (column.isKey()) { + keyColumns.add(column); + } + } + + return keyColumns; + } + + // schema version + public int getSchemaVersionByIndexId(Long indexId) { + if (indexIdToSchemaVersion.containsKey(indexId)) { + return indexIdToSchemaVersion.get(indexId); + } + return -1; + } + + // schemaHash + public Map getIndexIdToSchemaHash() { + return indexIdToSchemaHash; + } + + public Map getCopiedIndexIdToSchemaHash() { + Map copiedIndexIdToSchemaHash = new HashMap(); + copiedIndexIdToSchemaHash.putAll(indexIdToSchemaHash); + return copiedIndexIdToSchemaHash; + } + + public int getSchemaHashByIndexId(Long indexId) { + if (indexIdToSchemaHash.containsKey(indexId)) { + return indexIdToSchemaHash.get(indexId); + } + return -1; + } + + // short key + public Map getIndexIdToShortKeyColumnCount() { + return indexIdToShortKeyColumnCount; + } + + public Map getCopiedIndexIdToShortKeyColumnCount() { + Map copiedIndexIdToShortKeyColumnCount = new HashMap(); + copiedIndexIdToShortKeyColumnCount.putAll(indexIdToShortKeyColumnCount); + return copiedIndexIdToShortKeyColumnCount; + } + + public short getShortKeyColumnCountByIndexId(Long indexId) { + if (indexIdToShortKeyColumnCount.containsKey(indexId)) { + return indexIdToShortKeyColumnCount.get(indexId); + } + return (short) -1; + } + + // storage type + public Map getIndexIdToStorageType() { + return indexIdToStorageType; + } + + public Map getCopiedIndexIdToStorageType() { + Map copiedIndexIdToStorageType = new HashMap(); + copiedIndexIdToStorageType.putAll(indexIdToStorageType); + return copiedIndexIdToStorageType; + } + + public void setStorageTypeToIndex(Long indexId, TStorageType storageType) { + indexIdToStorageType.put(indexId, storageType); + } + + public TStorageType getStorageTypeByIndexId(Long indexId) { + return indexIdToStorageType.get(indexId); + } + + public KeysType getKeysType() { + return keysType; + } + + public PartitionInfo getPartitionInfo() { + return partitionInfo; + } + + public DistributionInfo getDefaultDistributionInfo() { + return defaultDistributionInfo; + } + + public void renamePartition(String partitionName, String newPartitionName) { + if (partitionInfo.getType() == PartitionType.UNPARTITIONED) { + // bug fix + for (Partition partition : idToPartition.values()) { + partition.setName(newPartitionName); + nameToPartition.clear(); + nameToPartition.put(newPartitionName, partition); + LOG.info("rename patition {} in table {}", newPartitionName, name); + break; + } + } else { + Partition partition = nameToPartition.remove(partitionName); + partition.setName(newPartitionName); + nameToPartition.put(newPartitionName, partition); + } + } + + public void addPartition(Partition partition) { + idToPartition.put(partition.getId(), partition); + nameToPartition.put(partition.getName(), partition); + } + + public Partition dropPartition(long dbId, String partitionName) { + return dropPartition(dbId, partitionName, false); + } + + public Partition dropPartition(long dbId, String partitionName, boolean isRestore) { + Partition partition = nameToPartition.get(partitionName); + if (partition != null) { + idToPartition.remove(partition.getId()); + nameToPartition.remove(partitionName); + + Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + + if (!isRestore) { + // recycle partition + Catalog.getCurrentRecycleBin().recyclePartition(dbId, id, partition, + rangePartitionInfo.getRange(partition.getId()), + rangePartitionInfo.getDataProperty(partition.getId()), + rangePartitionInfo.getReplicationNum(partition.getId())); + } + + // drop partition info + rangePartitionInfo.dropPartition(partition.getId()); + } + return partition; + } + + public Partition dropPartitionForBackup(String partitionName) { + return dropPartition(-1, partitionName, true); + } + + public Collection getPartitions() { + return idToPartition.values(); + } + + public Partition getPartition(long partitionId) { + return idToPartition.get(partitionId); + } + + public Partition getPartition(String partitionName) { + return nameToPartition.get(partitionName); + } + + public Set getPartitionNames() { + return Sets.newHashSet(nameToPartition.keySet()); + } + + public Set getCopiedBfColumns() { + if (bfColumns == null) { + return null; + } + + return Sets.newHashSet(bfColumns); + } + + public double getBfFpp() { + return bfFpp; + } + + public void setBloomFilterInfo(Set bfColumns, double bfFpp) { + this.bfColumns = bfColumns; + this.bfFpp = bfFpp; + } + + public TTableDescriptor toThrift() { + TOlapTable tOlapTable = new TOlapTable(getName()); + TTableDescriptor tTableDescriptor = new TTableDescriptor(id, TTableType.OLAP_TABLE, + baseSchema.size(), 0, getName(), ""); + tTableDescriptor.setOlapTable(tOlapTable); + return tTableDescriptor; + } + + public long getRowCount() { + long rowCount = 0; + for (Map.Entry entry : idToPartition.entrySet()) { + rowCount += ((Partition) entry.getValue()).getBaseIndex().getRowCount(); + } + return rowCount; + } + + public AlterTableStmt toAddRollupStmt(String dbName, Collection indexIds) { + List alterClauses = Lists.newArrayList(); + for (Map.Entry entry : indexNameToId.entrySet()) { + String indexName = entry.getKey(); + long indexId = entry.getValue(); + if (!indexIds.contains(indexId)) { + continue; + } + + // cols + List columnNames = Lists.newArrayList(); + for (Column column : indexIdToSchema.get(indexId)) { + columnNames.add(column.getName()); + } + + // properties + Map properties = Maps.newHashMap(); + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_TYPE, indexIdToStorageType.get(indexId).name()); + properties.put(PropertyAnalyzer.PROPERTIES_SHORT_KEY, indexIdToShortKeyColumnCount.get(indexId).toString()); + properties.put(PropertyAnalyzer.PROPERTIES_SCHEMA_VERSION, indexIdToSchemaVersion.get(indexId).toString()); + + AddRollupClause addRollupClause = new AddRollupClause(indexName, columnNames, null, null, properties); + alterClauses.add(addRollupClause); + } + + AlterTableStmt alterTableStmt = new AlterTableStmt(new TableName(dbName, name), alterClauses); + return alterTableStmt; + } + + public AlterTableStmt toAddPartitionStmt(String dbName, String partitionName) { + Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + List alterClauses = Lists.newArrayList(); + + Partition partition = nameToPartition.get(partitionName); + Map properties = Maps.newHashMap(); + long version = partition.getCommittedVersion(); + long versionHash = partition.getCommittedVersionHash(); + properties.put(PropertyAnalyzer.PROPERTIES_VERSION_INFO, version + "," + versionHash); + properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, + String.valueOf(partitionInfo.getReplicationNum(partition.getId()))); + + SingleRangePartitionDesc singleDesc = + rangePartitionInfo.toSingleRangePartitionDesc(partition.getId(), partitionName, properties); + DistributionDesc distributionDesc = partition.getDistributionInfo().toDistributionDesc(); + + AddPartitionClause addPartitionClause = new AddPartitionClause(singleDesc, distributionDesc, null); + alterClauses.add(addPartitionClause); + AlterTableStmt stmt = new AlterTableStmt(new TableName(dbName, name), alterClauses); + return stmt; + } + + @Override + public CreateTableStmt toCreateTableStmt(String dbName) { + Map properties = Maps.newHashMap(); + + // partition + PartitionDesc partitionDesc = null; + if (partitionInfo.getType() == PartitionType.RANGE) { + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + List partitionColumns = rangePartitionInfo.getPartitionColumns(); + List partitionColNames = Lists.newArrayList(); + for (Column partCol : partitionColumns) { + partitionColNames.add(partCol.getName()); + } + + List singlePartitionDescs = Lists.newArrayList(); + partitionDesc = new RangePartitionDesc(partitionColNames, singlePartitionDescs); + } else { + Short replicationNum = partitionInfo.getReplicationNum(nameToPartition.get(name).getId()); + properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, replicationNum.toString()); + // and partition version info here for non-partitioned table + Partition partition = getPartition(name); + Preconditions.checkNotNull(partition); + long version = partition.getCommittedVersion(); + long versionHash = partition.getCommittedVersionHash(); + String versionProp = Joiner.on(",").join(version, versionHash); + properties.put(PropertyAnalyzer.PROPERTIES_VERSION_INFO, versionProp); + } + + // keys + List keysColumnNames = Lists.newArrayList(); + for (Column column : baseSchema) { + if (column.isKey()) { + keysColumnNames.add(column.getName()); + } + } + KeysDesc keysDesc = new KeysDesc(keysType, keysColumnNames); + + // distribution + DistributionDesc distributionDesc = defaultDistributionInfo.toDistributionDesc(); + + // other properties + properties.put(PropertyAnalyzer.PROPERTIES_SHORT_KEY, indexIdToShortKeyColumnCount.get(id).toString()); + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_TYPE, indexIdToStorageType.get(id).name()); + if (bfColumns != null) { + String bfCols = Joiner.on(",").join(bfColumns); + properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, bfCols); + properties.put(PropertyAnalyzer.PROPERTIES_BF_FPP, String.valueOf(bfFpp)); + } + properties.put(PropertyAnalyzer.PROPERTIES_SCHEMA_VERSION, indexIdToSchemaVersion.get(id).toString()); + + CreateTableStmt stmt = new CreateTableStmt(false, false, new TableName(dbName, name), baseSchema, + type.name(), keysDesc, partitionDesc, distributionDesc, + properties, null); + return stmt; + } + + public int getSignature(int signatureVersion, List partNames) { + Adler32 adler32 = new Adler32(); + adler32.update(signatureVersion); + final String charsetName = "UTF-8"; + + try { + // table name + adler32.update(name.getBytes(charsetName)); + LOG.debug("signature. table name: {}", name); + // type + adler32.update(type.name().getBytes(charsetName)); + LOG.debug("signature. table type: {}", type.name()); + + // all indices(should be in order) + Set indexNames = Sets.newTreeSet(); + indexNames.addAll(indexNameToId.keySet()); + for (String indexName : indexNames) { + long indexId = indexNameToId.get(indexName); + adler32.update(indexName.getBytes(charsetName)); + LOG.debug("signature. index name: {}", indexName); + // schema hash + adler32.update(indexIdToSchemaHash.get(indexId)); + LOG.debug("signature. index schema hash: {}", indexIdToSchemaHash.get(indexId)); + // short key column count + adler32.update(indexIdToShortKeyColumnCount.get(indexId)); + LOG.debug("signature. index short key: {}", indexIdToShortKeyColumnCount.get(indexId)); + // storage type + adler32.update(indexIdToStorageType.get(indexId).name().getBytes(charsetName)); + LOG.debug("signature. index storage type: {}", indexIdToStorageType.get(indexId)); + } + + // bloom filter + if (bfColumns != null && !bfColumns.isEmpty()) { + for (String bfCol : bfColumns) { + adler32.update(bfCol.getBytes()); + LOG.debug("signature. bf col: {}", bfCol); + } + adler32.update(String.valueOf(bfFpp).getBytes()); + LOG.debug("signature. bf fpp: {}", bfFpp); + } + + // partition type + adler32.update(partitionInfo.getType().name().getBytes(charsetName)); + LOG.debug("signature. partition type: {}", partitionInfo.getType().name()); + // partition columns + if (partitionInfo.getType() == PartitionType.RANGE) { + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; + List partitionColumns = rangePartitionInfo.getPartitionColumns(); + adler32.update(Util.schemaHash(0, partitionColumns, null, 0)); + LOG.debug("signature. partition col hash: {}", Util.schemaHash(0, partitionColumns, null, 0)); + } + + // partition and distribution + Collections.sort(partNames, String.CASE_INSENSITIVE_ORDER); + for (String partName : partNames) { + Partition partition = getPartition(partName); + Preconditions.checkNotNull(partition, partName); + adler32.update(partName.getBytes(charsetName)); + LOG.debug("signature. partition name: {}", partName); + DistributionInfo distributionInfo = partition.getDistributionInfo(); + adler32.update(distributionInfo.getType().name().getBytes(charsetName)); + if (distributionInfo.getType() == DistributionInfoType.HASH) { + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + adler32.update(Util.schemaHash(0, hashDistributionInfo.getDistributionColumns(), null, 0)); + LOG.debug("signature. distribution col hash: {}", + Util.schemaHash(0, hashDistributionInfo.getDistributionColumns(), null, 0)); + adler32.update(hashDistributionInfo.getBucketNum()); + LOG.debug("signature. bucket num: {}", hashDistributionInfo.getBucketNum()); + } + } + + } catch (UnsupportedEncodingException e) { + LOG.error("encoding error", e); + return -1; + } + + LOG.debug("signature: {}", Math.abs((int) adler32.getValue())); + return Math.abs((int) adler32.getValue()); + } + + public Status getIntersectPartNamesWith(OlapTable anotherTbl, List intersectPartNames) { + if (this.getPartitionInfo().getType() != anotherTbl.getPartitionInfo().getType()) { + return new Status(ErrCode.COMMON_ERROR, "Table's partition type is different"); + } + + Set intersect = this.getPartitionNames(); + intersect.retainAll(anotherTbl.getPartitionNames()); + intersectPartNames.addAll(intersect); + return Status.OK; + } + + @Override + public boolean isPartitioned() { + int numSegs = 0; + for (Partition part : getPartitions()) { + numSegs += part.getDistributionInfo().getBucketNum(); + if (numSegs > 1) { + return true; + } + } + return false; + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + + // state + Text.writeString(out, state.name()); + + // indices' schema + int counter = indexNameToId.size(); + out.writeInt(counter); + for (Map.Entry entry : indexNameToId.entrySet()) { + String indexName = entry.getKey(); + long indexId = entry.getValue(); + Text.writeString(out, indexName); + out.writeLong(indexId); + // schema + out.writeInt(indexIdToSchema.get(indexId).size()); + for (Column column : indexIdToSchema.get(indexId)) { + column.write(out); + } + + // storage type + Text.writeString(out, indexIdToStorageType.get(indexId).name()); + + // indices's schema version + out.writeInt(indexIdToSchemaVersion.get(indexId)); + + // indices's schema hash + out.writeInt(indexIdToSchemaHash.get(indexId)); + + // indices's short key column count + out.writeShort(indexIdToShortKeyColumnCount.get(indexId)); + } + + Text.writeString(out, keysType.name()); + Text.writeString(out, partitionInfo.getType().name()); + partitionInfo.write(out); + Text.writeString(out, defaultDistributionInfo.getType().name()); + defaultDistributionInfo.write(out); + + // partitions + int partitionCount = idToPartition.size(); + out.writeInt(partitionCount); + for (Partition partition : idToPartition.values()) { + partition.write(out); + } + + // bloom filter columns + if (bfColumns == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeInt(bfColumns.size()); + for (String bfColumn : bfColumns) { + Text.writeString(out, bfColumn); + } + out.writeDouble(bfFpp); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + this.state = OlapTableState.valueOf(Text.readString(in)); + + // indices's schema + int counter = in.readInt(); + for (int i = 0; i < counter; i++) { + String indexName = Text.readString(in); + long indexId = in.readLong(); + this.indexNameToId.put(indexName, indexId); + + // schema + int colCount = in.readInt(); + List schema = new LinkedList(); + for (int j = 0; j < colCount; j++) { + Column column = Column.read(in); + schema.add(column); + } + this.indexIdToSchema.put(indexId, schema); + + // storage type + TStorageType type = TStorageType.valueOf(Text.readString(in)); + this.indexIdToStorageType.put(indexId, type); + + // indices's schema version + this.indexIdToSchemaVersion.put(indexId, in.readInt()); + + // indices's schema hash + this.indexIdToSchemaHash.put(indexId, in.readInt()); + + // indices's short key column count + this.indexIdToShortKeyColumnCount.put(indexId, in.readShort()); + } + + // partition and distribution info + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_30) { + keysType = KeysType.valueOf(Text.readString(in)); + } else { + keysType = KeysType.AGG_KEYS; + } + + PartitionType partType = PartitionType.valueOf(Text.readString(in)); + if (partType == PartitionType.UNPARTITIONED) { + partitionInfo = PartitionInfo.read(in); + } else if (partType == PartitionType.RANGE) { + partitionInfo = RangePartitionInfo.read(in); + } else { + throw new IOException("invalid partition type: " + partType); + } + + DistributionInfoType distriType = DistributionInfoType.valueOf(Text.readString(in)); + if (distriType == DistributionInfoType.HASH) { + defaultDistributionInfo = HashDistributionInfo.read(in); + } else if (distriType == DistributionInfoType.RANDOM) { + defaultDistributionInfo = RandomDistributionInfo.read(in); + } else { + throw new IOException("invalid distribution type: " + distriType); + } + + int partitionCount = in.readInt(); + for (int i = 0; i < partitionCount; ++i) { + Partition partition = Partition.read(in); + idToPartition.put(partition.getId(), partition); + nameToPartition.put(partition.getName(), partition); + } + + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_9) { + if (in.readBoolean()) { + int bfColumnCount = in.readInt(); + bfColumns = Sets.newHashSet(); + for (int i = 0; i < bfColumnCount; i++) { + bfColumns.add(Text.readString(in)); + } + + bfFpp = in.readDouble(); + } + } + } + + public boolean equals(Table table) { + if (this == table) { + return true; + } + if (!(table instanceof OlapTable)) { + return false; + } + + return true; + } + + public OlapTable selectiveCopy(Collection reservedPartNames) { + OlapTable copied = new OlapTable(); + if (!DeepCopy.copy(this, copied)) { + LOG.warn("failed to copy olap table: " + getName()); + return null; + } + + if (reservedPartNames == null || reservedPartNames.isEmpty()) { + // reserve all + return copied; + } + + Set partNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + partNames.addAll(copied.getPartitionNames()); + + for (String partName : partNames) { + if (!reservedPartNames.contains(partName)) { + copied.dropPartitionForBackup(partName); + } + } + + return copied; + } +} + diff --git a/fe/src/com/baidu/palo/catalog/Partition.java b/fe/src/com/baidu/palo/catalog/Partition.java index 4e22a9d224..6384ff5db9 100644 --- a/fe/src/com/baidu/palo/catalog/Partition.java +++ b/fe/src/com/baidu/palo/catalog/Partition.java @@ -15,17 +15,17 @@ package com.baidu.palo.catalog; -import com.baidu.palo.catalog.DistributionInfo.DistributionInfoType; -import com.baidu.palo.common.io.Text; -import com.baidu.palo.common.io.Writable; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.List; -import java.util.ArrayList; +import com.baidu.palo.catalog.DistributionInfo.DistributionInfoType; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; /** @@ -67,6 +67,10 @@ public class Partition extends MetaObject implements Writable { this.committedVersion = PARTITION_INIT_VERSION; this.committedVersionHash = PARTITION_INIT_VERSION_HASH; this.distributionInfo = distributionInfo; + } + + public void setIdForRestore(long id) { + this.id = id; } public long getId() { diff --git a/fe/src/com/baidu/palo/catalog/RangePartitionInfo.java b/fe/src/com/baidu/palo/catalog/RangePartitionInfo.java index 76aea2c98d..c350ebaf45 100644 --- a/fe/src/com/baidu/palo/catalog/RangePartitionInfo.java +++ b/fe/src/com/baidu/palo/catalog/RangePartitionInfo.java @@ -13,403 +13,423 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.catalog; - -import com.baidu.palo.analysis.PartitionKeyDesc; -import com.baidu.palo.analysis.SingleRangePartitionDesc; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.FeNameFormat; -import com.baidu.palo.common.util.PropertyAnalyzer; - -import com.google.common.base.Preconditions; -import com.google.common.collect.BoundType; -import com.google.common.collect.Lists; -import com.google.common.collect.Range; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class RangePartitionInfo extends PartitionInfo { - private static final Logger LOG = LogManager.getLogger(RangePartitionInfo.class); - - private List partitionColumns; - // partition id -> partition range - private Map> idToRange; - - private static final Comparator>> RANGE_MAP_ENTRY_COMPARATOR; - - static { - RANGE_MAP_ENTRY_COMPARATOR = new Comparator>>() { - @Override - public int compare(Map.Entry> o1, - Map.Entry> o2) { - return o1.getValue().lowerEndpoint().compareTo(o2.getValue().lowerEndpoint()); - } - }; - } - - public RangePartitionInfo() { - // for persist - super(); - this.partitionColumns = new LinkedList(); - this.idToRange = new HashMap>(); - } - - public RangePartitionInfo(List partitionColumns) { - super(PartitionType.RANGE); - this.partitionColumns = partitionColumns; - this.idToRange = new HashMap>(); - } - - public List getPartitionColumns() { - return partitionColumns; - } - - public void dropPartition(long partitionId) { - idToRange.remove(partitionId); - idToDataProperty.remove(partitionId); - idToReplicationNum.remove(partitionId); - } - - public Range checkAndCreateRange(SingleRangePartitionDesc desc) throws DdlException { - Range newRange = null; - // check range - try { - // create single value partition key - PartitionKeyDesc partKeyDesc = desc.getPartitionKeyDesc(); - PartitionKey singlePartitionKey = null; - if (partKeyDesc.isMax()) { - singlePartitionKey = PartitionKey.createInfinityPartitionKey(partitionColumns, true); - } else { - singlePartitionKey = PartitionKey.createPartitionKey(partKeyDesc.getUpperValues(), partitionColumns); - } - - if (singlePartitionKey.isMinValue()) { - throw new DdlException("Partition value should not be MIN VALUE: " + singlePartitionKey.toSql()); - } - - List>> entries = - new ArrayList>>(this.idToRange.entrySet()); - Collections.sort(entries, RANGE_MAP_ENTRY_COMPARATOR); - - Range lastRange = null; - Range nextRange = null; - for (Map.Entry> entry : entries) { - nextRange = entry.getValue(); - - // check if equals to upper bound - PartitionKey upperKey = nextRange.upperEndpoint(); - if (upperKey.compareTo(singlePartitionKey) >= 0) { - PartitionKey lowKey = null; - if (!partKeyDesc.getLowerValues().isEmpty()) { - lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); - } else { - if (lastRange == null) { - lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); - } else { - lowKey = lastRange.upperEndpoint(); - } - } - - newRange = Range.closedOpen(lowKey, singlePartitionKey); - - // check if range intersected - checkRangeIntersect(newRange, nextRange); - break; - } - lastRange = nextRange; - } // end for ranges - - if (newRange == null) { - PartitionKey lowKey = null; - if (!partKeyDesc.getLowerValues().isEmpty()) { - lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); - } else { - if (lastRange == null) { - // add first partition to this table. so the lower key is MIN - lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); - } else { - lowKey = lastRange.upperEndpoint(); - } - } - - newRange = Range.closedOpen(lowKey, singlePartitionKey); - } - } catch (AnalysisException e) { - throw new DdlException("Invalid range value format： " + e.getMessage()); - } - - Preconditions.checkNotNull(newRange); - return newRange; - } - - public static void checkRangeIntersect(Range range1, Range range2) throws DdlException { - if (range2.isConnected(range1)) { - if (!range2.intersection(range1).isEmpty()) { - throw new DdlException("Range " + range1 + " is intersected with range: " + range2); - } - } - } - - public void handleNewSinglePartitionDesc(SingleRangePartitionDesc desc, long partitionId) throws DdlException { - Preconditions.checkArgument(desc.isAnalyzed()); - try { - Range range = checkAndCreateRange(desc); - idToRange.put(partitionId, range); - } catch (IllegalArgumentException e) { - // Range.closedOpen may throw this if (lower > upper) - throw new DdlException("Invalid key range", e); - } - idToDataProperty.put(partitionId, desc.getPartitionDataProperty()); - idToReplicationNum.put(partitionId, desc.getReplicationNum()); - } - - // for catalog restore - public void unprotectHandleNewSinglePartitionDesc(long partitionId, Range range, - DataProperty dataProperty, short replicationNum) - throws DdlException { - idToRange.put(partitionId, range); - idToDataProperty.put(partitionId, dataProperty); - idToReplicationNum.put(partitionId, replicationNum); - } - - public void setRange(long partitionId, Range range) { - idToRange.put(partitionId, range); - } - - public Map> getIdToRange() { - return idToRange; - } - - public Range getRange(long partitionId) { - return idToRange.get(partitionId); - } - - public static void checkRangeColumnType(Column column) throws AnalysisException { - PrimitiveType type = column.getDataType(); - if (!type.isFixedPointType() && !type.isDateType()) { - throw new AnalysisException("Column[" + column.getName() + "] type[" + type - + "] cannot be a range partition key."); - } - } - - public List>> getSortedRangeMap() { - List>> sortedList = Lists.newArrayList(this.idToRange.entrySet()); - Collections.sort(sortedList, RANGE_MAP_ENTRY_COMPARATOR); - return sortedList; - } - - public static void writeRange(DataOutput out, Range range) throws IOException { - boolean hasLowerBound = false; - boolean hasUpperBound = false; - - // write lower bound if lower bound exists - hasLowerBound = range.hasLowerBound(); - out.writeBoolean(hasLowerBound); - if (hasLowerBound) { - PartitionKey lowerBound = range.lowerEndpoint(); - out.writeBoolean(range.lowerBoundType() == BoundType.CLOSED); - lowerBound.write(out); - } - - // write upper bound if upper bound exists - hasUpperBound = range.hasUpperBound(); - out.writeBoolean(hasUpperBound); - if (hasUpperBound) { - PartitionKey upperBound = range.upperEndpoint(); - out.writeBoolean(range.upperBoundType() == BoundType.CLOSED); - upperBound.write(out); - } - } - - public static Range readRange(DataInput in) throws IOException { - boolean hasLowerBound = false; - boolean hasUpperBound = false; - boolean lowerBoundClosed = false; - boolean upperBoundClosed = false; - PartitionKey lowerBound = null; - PartitionKey upperBound = null; - - hasLowerBound = in.readBoolean(); - if (hasLowerBound) { - lowerBoundClosed = in.readBoolean(); - lowerBound = PartitionKey.read(in); - } - - hasUpperBound = in.readBoolean(); - if (hasUpperBound) { - upperBoundClosed = in.readBoolean(); - upperBound = PartitionKey.read(in); - } - - // Totally 9 cases. Both lower bound and upper bound could be open, closed or not exist - if (hasLowerBound && lowerBoundClosed && hasUpperBound && upperBoundClosed) { - return Range.closed(lowerBound, upperBound); - } - if (hasLowerBound && lowerBoundClosed && hasUpperBound && !upperBoundClosed) { - return Range.closedOpen(lowerBound, upperBound); - } - if (hasLowerBound && !lowerBoundClosed && hasUpperBound && upperBoundClosed) { - return Range.openClosed(lowerBound, upperBound); - } - if (hasLowerBound && !lowerBoundClosed && hasUpperBound && !upperBoundClosed) { - return Range.open(lowerBound, upperBound); - } - if (hasLowerBound && lowerBoundClosed && !hasUpperBound) { - return Range.atLeast(lowerBound); - } - if (hasLowerBound && !lowerBoundClosed && !hasUpperBound) { - return Range.greaterThan(lowerBound); - } - if (!hasLowerBound && hasUpperBound && upperBoundClosed) { - return Range.atMost(upperBound); - } - if (!hasLowerBound && hasUpperBound && !upperBoundClosed) { - return Range.lessThan(upperBound); - } - // Neither lower bound nor upper bound exists, return null. This means just one partition - return null; - } - - public SingleRangePartitionDesc toSingleRangePartitionDesc(long partitionId, String partitionName, - Map properties) { - Range range = idToRange.get(partitionId); - List upperValues = Lists.newArrayList(); - List lowerValues = Lists.newArrayList(); - // FIXME(cmy): check here(getStringValue) - lowerValues.add(range.lowerEndpoint().getKeys().get(0).getStringValue()); - - PartitionKey upperKey = range.upperEndpoint(); - PartitionKeyDesc keyDesc = null; - if (upperKey.isMaxValue()) { - keyDesc = PartitionKeyDesc.createMaxKeyDesc(); - keyDesc.setLowerValues(lowerValues); - } else { - upperValues.add(range.upperEndpoint().getKeys().get(0).getStringValue()); - keyDesc = new PartitionKeyDesc(lowerValues, upperValues); - } - - SingleRangePartitionDesc singleDesc = new SingleRangePartitionDesc(false, partitionName, keyDesc, properties); - - if (properties != null) { - // properties - Short replicationNum = getReplicationNum(partitionId); - properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, replicationNum.toString()); - } - return singleDesc; - } - - public static PartitionInfo read(DataInput in) throws IOException { - PartitionInfo partitionInfo = new RangePartitionInfo(); - partitionInfo.readFields(in); - return partitionInfo; - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - - // partition columns - out.writeInt(partitionColumns.size()); - for (Column column : partitionColumns) { - column.write(out); - } - - out.writeInt(idToRange.size()); - for (Map.Entry> entry : idToRange.entrySet()) { - out.writeLong(entry.getKey()); - RangePartitionInfo.writeRange(out, entry.getValue()); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - - int counter = in.readInt(); - for (int i = 0; i < counter; i++) { - Column column = Column.read(in); - partitionColumns.add(column); - } - - counter = in.readInt(); - for (int i = 0; i < counter; i++) { - long partitionId = in.readLong(); - Range range = RangePartitionInfo.readRange(in); - idToRange.put(partitionId, range); - } - } - - @Override - public String toSql(OlapTable table, List partitionId) { - StringBuilder sb = new StringBuilder(); - sb.append("PARTITION BY RANGE("); - int idx = 0; - for (Column column : partitionColumns) { - if (idx != 0) { - sb.append(", "); - } - sb.append("`").append(column.getName()).append("`"); - idx++; - } - sb.append(")\n("); - - // sort range - List>> entries = - new ArrayList>>(this.idToRange.entrySet()); - Collections.sort(entries, RANGE_MAP_ENTRY_COMPARATOR); - - Range lastRange = null; - idx = 0; - for (Map.Entry> entry : entries) { - Partition partition = table.getPartition(entry.getKey()); - String partitionName = partition.getName(); - Range range = entry.getValue(); - - if (idx == 0) { - // first partition - if (!range.lowerEndpoint().isMinValue()) { - sb.append("PARTITION ").append(FeNameFormat.FORBIDDEN_PARTITION_NAME).append(idx) - .append(" VALUES LESS THAN ").append(range.lowerEndpoint().toSql()); - sb.append(",\n"); - } - } else { - Preconditions.checkNotNull(lastRange); - if (!lastRange.upperEndpoint().equals(range.lowerEndpoint())) { - sb.append("PARTITION ").append(FeNameFormat.FORBIDDEN_PARTITION_NAME).append(idx) - .append(" VALUES LESS THAN ").append(range.lowerEndpoint().toSql()); - sb.append(",\n"); - } - } - - sb.append("PARTITION ").append(partitionName).append(" VALUES LESS THAN "); - sb.append(range.upperEndpoint().toSql()); - - if (partitionId != null) { - partitionId.add(entry.getKey()); - break; - } - - if (idx != entries.size() - 1) { - sb.append(",\n"); - } - idx++; - - lastRange = range; - } - sb.append(")"); - return sb.toString(); - } -} +package com.baidu.palo.catalog; + +import com.baidu.palo.analysis.PartitionKeyDesc; +import com.baidu.palo.analysis.SingleRangePartitionDesc; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.FeNameFormat; +import com.baidu.palo.common.util.PropertyAnalyzer; + +import com.google.common.base.Preconditions; +import com.google.common.collect.BoundType; +import com.google.common.collect.Lists; +import com.google.common.collect.Range; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class RangePartitionInfo extends PartitionInfo { + private static final Logger LOG = LogManager.getLogger(RangePartitionInfo.class); + + private List partitionColumns; + // partition id -> partition range + private Map> idToRange; + + private static final Comparator>> RANGE_MAP_ENTRY_COMPARATOR; + + static { + RANGE_MAP_ENTRY_COMPARATOR = new Comparator>>() { + @Override + public int compare(Map.Entry> o1, + Map.Entry> o2) { + return o1.getValue().lowerEndpoint().compareTo(o2.getValue().lowerEndpoint()); + } + }; + } + + public RangePartitionInfo() { + // for persist + super(); + this.partitionColumns = new LinkedList(); + this.idToRange = new HashMap>(); + } + + public RangePartitionInfo(List partitionColumns) { + super(PartitionType.RANGE); + this.partitionColumns = partitionColumns; + this.idToRange = new HashMap>(); + } + + public List getPartitionColumns() { + return partitionColumns; + } + + public void dropPartition(long partitionId) { + idToRange.remove(partitionId); + idToDataProperty.remove(partitionId); + idToReplicationNum.remove(partitionId); + } + + public void addPartitionForRestore(long partitionId, Range range, DataProperty dataProperty, + short replicationNum) { + idToRange.put(partitionId, range); + idToDataProperty.put(partitionId, dataProperty); + idToReplicationNum.put(partitionId, replicationNum); + } + + public Range checkAndCreateRange(SingleRangePartitionDesc desc) throws DdlException { + Range newRange = null; + // check range + try { + // create single value partition key + PartitionKeyDesc partKeyDesc = desc.getPartitionKeyDesc(); + PartitionKey singlePartitionKey = null; + if (partKeyDesc.isMax()) { + singlePartitionKey = PartitionKey.createInfinityPartitionKey(partitionColumns, true); + } else { + singlePartitionKey = PartitionKey.createPartitionKey(partKeyDesc.getUpperValues(), partitionColumns); + } + + if (singlePartitionKey.isMinValue()) { + throw new DdlException("Partition value should not be MIN VALUE: " + singlePartitionKey.toSql()); + } + + List>> entries = + new ArrayList>>(this.idToRange.entrySet()); + Collections.sort(entries, RANGE_MAP_ENTRY_COMPARATOR); + + Range lastRange = null; + Range nextRange = null; + for (Map.Entry> entry : entries) { + nextRange = entry.getValue(); + + // check if equals to upper bound + PartitionKey upperKey = nextRange.upperEndpoint(); + if (upperKey.compareTo(singlePartitionKey) >= 0) { + PartitionKey lowKey = null; + if (!partKeyDesc.getLowerValues().isEmpty()) { + lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); + } else { + if (lastRange == null) { + lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); + } else { + lowKey = lastRange.upperEndpoint(); + } + } + + newRange = Range.closedOpen(lowKey, singlePartitionKey); + + // check if range intersected + checkRangeIntersect(newRange, nextRange); + break; + } + lastRange = nextRange; + } // end for ranges + + if (newRange == null) { + PartitionKey lowKey = null; + if (!partKeyDesc.getLowerValues().isEmpty()) { + lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); + } else { + if (lastRange == null) { + // add first partition to this table. so the lower key is MIN + lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); + } else { + lowKey = lastRange.upperEndpoint(); + } + } + + newRange = Range.closedOpen(lowKey, singlePartitionKey); + } + } catch (AnalysisException e) { + throw new DdlException("Invalid range value format： " + e.getMessage()); + } + + Preconditions.checkNotNull(newRange); + return newRange; + } + + public static void checkRangeIntersect(Range range1, Range range2) throws DdlException { + if (range2.isConnected(range1)) { + if (!range2.intersection(range1).isEmpty()) { + throw new DdlException("Range " + range1 + " is intersected with range: " + range2); + } + } + } + + public void handleNewSinglePartitionDesc(SingleRangePartitionDesc desc, long partitionId) throws DdlException { + Preconditions.checkArgument(desc.isAnalyzed()); + try { + Range range = checkAndCreateRange(desc); + idToRange.put(partitionId, range); + } catch (IllegalArgumentException e) { + // Range.closedOpen may throw this if (lower > upper) + throw new DdlException("Invalid key range", e); + } + idToDataProperty.put(partitionId, desc.getPartitionDataProperty()); + idToReplicationNum.put(partitionId, desc.getReplicationNum()); + } + + // for catalog restore + public void unprotectHandleNewSinglePartitionDesc(long partitionId, Range range, + DataProperty dataProperty, short replicationNum) + throws DdlException { + idToRange.put(partitionId, range); + idToDataProperty.put(partitionId, dataProperty); + idToReplicationNum.put(partitionId, replicationNum); + } + + public void setRange(long partitionId, Range range) { + idToRange.put(partitionId, range); + } + + public Map> getIdToRange() { + return idToRange; + } + + public Range getRange(long partitionId) { + return idToRange.get(partitionId); + } + + public static void checkRangeColumnType(Column column) throws AnalysisException { + PrimitiveType type = column.getDataType(); + if (!type.isFixedPointType() && !type.isDateType()) { + throw new AnalysisException("Column[" + column.getName() + "] type[" + type + + "] cannot be a range partition key."); + } + } + + public List>> getSortedRangeMap() { + List>> sortedList = Lists.newArrayList(this.idToRange.entrySet()); + Collections.sort(sortedList, RANGE_MAP_ENTRY_COMPARATOR); + return sortedList; + } + + public static void writeRange(DataOutput out, Range range) throws IOException { + boolean hasLowerBound = false; + boolean hasUpperBound = false; + + // write lower bound if lower bound exists + hasLowerBound = range.hasLowerBound(); + out.writeBoolean(hasLowerBound); + if (hasLowerBound) { + PartitionKey lowerBound = range.lowerEndpoint(); + out.writeBoolean(range.lowerBoundType() == BoundType.CLOSED); + lowerBound.write(out); + } + + // write upper bound if upper bound exists + hasUpperBound = range.hasUpperBound(); + out.writeBoolean(hasUpperBound); + if (hasUpperBound) { + PartitionKey upperBound = range.upperEndpoint(); + out.writeBoolean(range.upperBoundType() == BoundType.CLOSED); + upperBound.write(out); + } + } + + public static Range readRange(DataInput in) throws IOException { + boolean hasLowerBound = false; + boolean hasUpperBound = false; + boolean lowerBoundClosed = false; + boolean upperBoundClosed = false; + PartitionKey lowerBound = null; + PartitionKey upperBound = null; + + hasLowerBound = in.readBoolean(); + if (hasLowerBound) { + lowerBoundClosed = in.readBoolean(); + lowerBound = PartitionKey.read(in); + } + + hasUpperBound = in.readBoolean(); + if (hasUpperBound) { + upperBoundClosed = in.readBoolean(); + upperBound = PartitionKey.read(in); + } + + // Totally 9 cases. Both lower bound and upper bound could be open, closed or not exist + if (hasLowerBound && lowerBoundClosed && hasUpperBound && upperBoundClosed) { + return Range.closed(lowerBound, upperBound); + } + if (hasLowerBound && lowerBoundClosed && hasUpperBound && !upperBoundClosed) { + return Range.closedOpen(lowerBound, upperBound); + } + if (hasLowerBound && !lowerBoundClosed && hasUpperBound && upperBoundClosed) { + return Range.openClosed(lowerBound, upperBound); + } + if (hasLowerBound && !lowerBoundClosed && hasUpperBound && !upperBoundClosed) { + return Range.open(lowerBound, upperBound); + } + if (hasLowerBound && lowerBoundClosed && !hasUpperBound) { + return Range.atLeast(lowerBound); + } + if (hasLowerBound && !lowerBoundClosed && !hasUpperBound) { + return Range.greaterThan(lowerBound); + } + if (!hasLowerBound && hasUpperBound && upperBoundClosed) { + return Range.atMost(upperBound); + } + if (!hasLowerBound && hasUpperBound && !upperBoundClosed) { + return Range.lessThan(upperBound); + } + // Neither lower bound nor upper bound exists, return null. This means just one partition + return null; + } + + public SingleRangePartitionDesc toSingleRangePartitionDesc(long partitionId, String partitionName, + Map properties) { + Range range = idToRange.get(partitionId); + List upperValues = Lists.newArrayList(); + List lowerValues = Lists.newArrayList(); + // FIXME(cmy): check here(getStringValue) + lowerValues.add(range.lowerEndpoint().getKeys().get(0).getStringValue()); + + PartitionKey upperKey = range.upperEndpoint(); + PartitionKeyDesc keyDesc = null; + if (upperKey.isMaxValue()) { + keyDesc = PartitionKeyDesc.createMaxKeyDesc(); + keyDesc.setLowerValues(lowerValues); + } else { + upperValues.add(range.upperEndpoint().getKeys().get(0).getStringValue()); + keyDesc = new PartitionKeyDesc(lowerValues, upperValues); + } + + SingleRangePartitionDesc singleDesc = new SingleRangePartitionDesc(false, partitionName, keyDesc, properties); + + if (properties != null) { + // properties + Short replicationNum = getReplicationNum(partitionId); + properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, replicationNum.toString()); + } + return singleDesc; + } + + public boolean checkRange(Range newRange) { + for (Range range : idToRange.values()) { + if (range.isConnected(newRange)) { + Range intersection = range.intersection(newRange); + if (!intersection.isEmpty()) { + return false; + } + } + } + return true; + } + + public static PartitionInfo read(DataInput in) throws IOException { + PartitionInfo partitionInfo = new RangePartitionInfo(); + partitionInfo.readFields(in); + return partitionInfo; + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + + // partition columns + out.writeInt(partitionColumns.size()); + for (Column column : partitionColumns) { + column.write(out); + } + + out.writeInt(idToRange.size()); + for (Map.Entry> entry : idToRange.entrySet()) { + out.writeLong(entry.getKey()); + RangePartitionInfo.writeRange(out, entry.getValue()); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + int counter = in.readInt(); + for (int i = 0; i < counter; i++) { + Column column = Column.read(in); + partitionColumns.add(column); + } + + counter = in.readInt(); + for (int i = 0; i < counter; i++) { + long partitionId = in.readLong(); + Range range = RangePartitionInfo.readRange(in); + idToRange.put(partitionId, range); + } + } + + @Override + public String toSql(OlapTable table, List partitionId) { + StringBuilder sb = new StringBuilder(); + sb.append("PARTITION BY RANGE("); + int idx = 0; + for (Column column : partitionColumns) { + if (idx != 0) { + sb.append(", "); + } + sb.append("`").append(column.getName()).append("`"); + idx++; + } + sb.append(")\n("); + + // sort range + List>> entries = + new ArrayList>>(this.idToRange.entrySet()); + Collections.sort(entries, RANGE_MAP_ENTRY_COMPARATOR); + + Range lastRange = null; + idx = 0; + for (Map.Entry> entry : entries) { + Partition partition = table.getPartition(entry.getKey()); + String partitionName = partition.getName(); + Range range = entry.getValue(); + + if (idx == 0) { + // first partition + if (!range.lowerEndpoint().isMinValue()) { + sb.append("PARTITION ").append(FeNameFormat.FORBIDDEN_PARTITION_NAME).append(idx) + .append(" VALUES LESS THAN ").append(range.lowerEndpoint().toSql()); + sb.append(",\n"); + } + } else { + Preconditions.checkNotNull(lastRange); + if (!lastRange.upperEndpoint().equals(range.lowerEndpoint())) { + sb.append("PARTITION ").append(FeNameFormat.FORBIDDEN_PARTITION_NAME).append(idx) + .append(" VALUES LESS THAN ").append(range.lowerEndpoint().toSql()); + sb.append(",\n"); + } + } + + sb.append("PARTITION ").append(partitionName).append(" VALUES LESS THAN "); + sb.append(range.upperEndpoint().toSql()); + + if (partitionId != null) { + partitionId.add(entry.getKey()); + break; + } + + if (idx != entries.size() - 1) { + sb.append(",\n"); + } + idx++; + + lastRange = range; + } + sb.append(")"); + return sb.toString(); + } +} + diff --git a/fe/src/com/baidu/palo/catalog/Replica.java b/fe/src/com/baidu/palo/catalog/Replica.java index 1f91c80df3..4b7eee861f 100644 --- a/fe/src/com/baidu/palo/catalog/Replica.java +++ b/fe/src/com/baidu/palo/catalog/Replica.java @@ -13,194 +13,205 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.catalog; - -import com.baidu.palo.common.io.Text; -import com.baidu.palo.common.io.Writable; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Comparator; - -/** - * This class represents the olap replica related metadata. - */ -public class Replica implements Writable { - private static final Logger LOG = LogManager.getLogger(Replica.class); - public static final VersionComparator VERSION_DESC_COMPARATOR = new VersionComparator(); - - public enum ReplicaState { - NORMAL, - ROLLUP, - SCHEMA_CHANGE, - CLONE - } - - private long id; - private long backendId; - private long version; - private long versionHash; - private long dataSize; - private long rowCount; - private ReplicaState state; - - public Replica() { - } - - public Replica(long replicaId, long backendId, ReplicaState state) { - this(replicaId, backendId, -1, 0, -1, -1, state); - } - - public Replica(long replicaId, long backendId, ReplicaState state, long version, long versionHash) { - this(replicaId, backendId, version, versionHash, -1, -1, state); - } - - public Replica(long replicaId, long backendId, long version, long versionHash, - long dataSize, long rowCount, ReplicaState state) { - this.id = replicaId; - this.backendId = backendId; - this.version = version; - this.versionHash = versionHash; - this.dataSize = dataSize; - this.rowCount = rowCount; - this.state = state; - if (this.state == null) { - this.state = ReplicaState.NORMAL; - } - } - - public long getVersion() { - return this.version; - } - - public long getVersionHash() { - return this.versionHash; - } - - public long getId() { - return this.id; - } - - public long getBackendId() { - return this.backendId; - } - - public long getDataSize() { - return dataSize; - } - - public long getRowCount() { - return rowCount; - } - - public void updateInfo(long newVersion, long newVersionHash, long newDataSize, long newRowCount) { - if (newVersion < this.version) { - LOG.warn("replica[" + id + "] new version is lower than meta version. " + newVersion + " vs " + version); - } - this.version = newVersion; - this.versionHash = newVersionHash; - this.dataSize = newDataSize; - this.rowCount = newRowCount; - - LOG.debug("update {}", this.toString()); - } - - public boolean checkVersionCatchUp(long committedVersion, long committedVersionHash) { - if (this.version < committedVersion - || (this.version == committedVersion && this.versionHash != committedVersionHash)) { - LOG.debug("replica version does not catch up with version: {}-{}. replica: {}", - committedVersion, committedVersionHash, this); - return false; - } - return true; - } - - public void setState(ReplicaState replicaState) { - this.state = replicaState; - } - - public ReplicaState getState() { - return this.state; - } - - @Override - public String toString() { - StringBuffer strBuffer = new StringBuffer("replicaId="); - strBuffer.append(id); - strBuffer.append(", BackendId="); - strBuffer.append(backendId); - strBuffer.append(", version="); - strBuffer.append(version); - strBuffer.append(", versionHash="); - strBuffer.append(versionHash); - strBuffer.append(", dataSize="); - strBuffer.append(dataSize); - strBuffer.append(", rowCount="); - strBuffer.append(rowCount); - return strBuffer.toString(); - } - - public void write(DataOutput out) throws IOException { - out.writeLong(id); - out.writeLong(backendId); - out.writeLong(version); - out.writeLong(versionHash); - out.writeLong(dataSize); - out.writeLong(rowCount); - Text.writeString(out, state.name()); - } - - public void readFields(DataInput in) throws IOException { - id = in.readLong(); - backendId = in.readLong(); - version = in.readLong(); - versionHash = in.readLong(); - dataSize = in.readLong(); - rowCount = in.readLong(); - state = ReplicaState.valueOf(Text.readString(in)); - } - - public static Replica read(DataInput in) throws IOException { - Replica replica = new Replica(); - replica.readFields(in); - return replica; - } - - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof Replica)) { - return false; - } - - Replica replica = (Replica) obj; - return (id == replica.id) - && (backendId == replica.backendId) - && (version == replica.version) - && (versionHash == replica.versionHash) - && (dataSize == replica.dataSize) - && (rowCount == replica.rowCount) - && (state.equals(replica.state)); - } - - private static class VersionComparator implements Comparator { - public VersionComparator() { - } - - @Override - public int compare(T replica1, T replica2) { - if (replica1.getVersion() < replica2.getVersion()) { - return 1; - } else if (replica1.getVersion() == replica2.getVersion()) { - return 0; - } else { - return -1; - } - } - } -} +package com.baidu.palo.catalog; + +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicLong; + +/** + * This class represents the olap replica related metadata. + */ +public class Replica implements Writable { + private static final Logger LOG = LogManager.getLogger(Replica.class); + public static final VersionComparator VERSION_DESC_COMPARATOR = new VersionComparator(); + + public enum ReplicaState { + NORMAL, + ROLLUP, + SCHEMA_CHANGE, + CLONE + } + + private long id; + private long backendId; + private long version; + private long versionHash; + private long dataSize; + private long rowCount; + private ReplicaState state; + private AtomicLong versionCount = new AtomicLong(-1); + + public Replica() { + } + + public Replica(long replicaId, long backendId, ReplicaState state) { + this(replicaId, backendId, -1, 0, -1, -1, state); + } + + public Replica(long replicaId, long backendId, ReplicaState state, long version, long versionHash) { + this(replicaId, backendId, version, versionHash, -1, -1, state); + } + + public Replica(long replicaId, long backendId, long version, long versionHash, + long dataSize, long rowCount, ReplicaState state) { + this.id = replicaId; + this.backendId = backendId; + this.version = version; + this.versionHash = versionHash; + this.dataSize = dataSize; + this.rowCount = rowCount; + this.state = state; + if (this.state == null) { + this.state = ReplicaState.NORMAL; + } + } + + public long getVersion() { + return this.version; + } + + public long getVersionHash() { + return this.versionHash; + } + + public long getId() { + return this.id; + } + + public long getBackendId() { + return this.backendId; + } + + public long getDataSize() { + return dataSize; + } + + public long getRowCount() { + return rowCount; + } + + public void updateInfo(long newVersion, long newVersionHash, long newDataSize, long newRowCount) { + if (newVersion < this.version) { + LOG.warn("replica[" + id + "] new version is lower than meta version. " + newVersion + " vs " + version); + } + this.version = newVersion; + this.versionHash = newVersionHash; + this.dataSize = newDataSize; + this.rowCount = newRowCount; + + LOG.debug("update {}", this.toString()); + } + + public boolean checkVersionCatchUp(long committedVersion, long committedVersionHash) { + if (this.version < committedVersion + || (this.version == committedVersion && this.versionHash != committedVersionHash)) { + LOG.debug("replica version does not catch up with version: {}-{}. replica: {}", + committedVersion, committedVersionHash, this); + return false; + } + return true; + } + + public void setState(ReplicaState replicaState) { + this.state = replicaState; + } + + public ReplicaState getState() { + return this.state; + } + + public long getVersionCount() { + return versionCount.get(); + } + + public void setVersionCount(long versionCount) { + this.versionCount.set(versionCount); + } + + @Override + public String toString() { + StringBuffer strBuffer = new StringBuffer("replicaId="); + strBuffer.append(id); + strBuffer.append(", BackendId="); + strBuffer.append(backendId); + strBuffer.append(", version="); + strBuffer.append(version); + strBuffer.append(", versionHash="); + strBuffer.append(versionHash); + strBuffer.append(", dataSize="); + strBuffer.append(dataSize); + strBuffer.append(", rowCount="); + strBuffer.append(rowCount); + return strBuffer.toString(); + } + + public void write(DataOutput out) throws IOException { + out.writeLong(id); + out.writeLong(backendId); + out.writeLong(version); + out.writeLong(versionHash); + out.writeLong(dataSize); + out.writeLong(rowCount); + Text.writeString(out, state.name()); + } + + public void readFields(DataInput in) throws IOException { + id = in.readLong(); + backendId = in.readLong(); + version = in.readLong(); + versionHash = in.readLong(); + dataSize = in.readLong(); + rowCount = in.readLong(); + state = ReplicaState.valueOf(Text.readString(in)); + } + + public static Replica read(DataInput in) throws IOException { + Replica replica = new Replica(); + replica.readFields(in); + return replica; + } + + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Replica)) { + return false; + } + + Replica replica = (Replica) obj; + return (id == replica.id) + && (backendId == replica.backendId) + && (version == replica.version) + && (versionHash == replica.versionHash) + && (dataSize == replica.dataSize) + && (rowCount == replica.rowCount) + && (state.equals(replica.state)); + } + + private static class VersionComparator implements Comparator { + public VersionComparator() { + } + + @Override + public int compare(T replica1, T replica2) { + if (replica1.getVersion() < replica2.getVersion()) { + return 1; + } else if (replica1.getVersion() == replica2.getVersion()) { + return 0; + } else { + return -1; + } + } + } +} + diff --git a/fe/src/com/baidu/palo/catalog/ResourceGroup.java b/fe/src/com/baidu/palo/catalog/ResourceGroup.java index 8ebf10d464..7968088322 100644 --- a/fe/src/com/baidu/palo/catalog/ResourceGroup.java +++ b/fe/src/com/baidu/palo/catalog/ResourceGroup.java @@ -140,7 +140,7 @@ public class ResourceGroup implements Writable { return new ResourceGroup(cpuShare); } - Builder cpuShare(int share) { + public Builder cpuShare(int share) { this.cpuShare = share; return this; } diff --git a/fe/src/com/baidu/palo/catalog/Table.java b/fe/src/com/baidu/palo/catalog/Table.java index 4809ba0c3a..06a2b83e06 100644 --- a/fe/src/com/baidu/palo/catalog/Table.java +++ b/fe/src/com/baidu/palo/catalog/Table.java @@ -59,6 +59,9 @@ public class Table extends MetaObject implements Writable { // tree map for case-insensitive lookup protected Map nameToColumn; + // DO NOT persist this variable. + protected boolean isTypeRead = false; + public Table(TableType type) { this.type = type; this.baseSchema = new LinkedList(); @@ -82,6 +85,14 @@ public class Table extends MetaObject implements Writable { } } + public boolean isTypeRead() { + return isTypeRead; + } + + public void setTypeRead(boolean isTypeRead) { + this.isTypeRead = isTypeRead; + } + public long getId() { return id; } @@ -129,27 +140,27 @@ public class Table extends MetaObject implements Writable { TableType type = TableType.valueOf(Text.readString(in)); if (type == TableType.OLAP) { table = new OlapTable(); - table.readFields(in); } else if (type == TableType.MYSQL) { table = new MysqlTable(); - table.readFields(in); } else if (type == TableType.VIEW) { - View view = new View(); - view.readFields(in); + table = new View(); + } else if (type == TableType.KUDU) { + table = new KuduTable(); + } else if (type == TableType.BROKER) { + table = new BrokerTable(); + } else { + throw new IOException("Unknown table type: " + type.name()); + } + + table.setTypeRead(true); + table.readFields(in); + if (type == TableType.VIEW) { + View view = (View) table; try { view.init(); } catch (InternalException e) { throw new IOException(e.getMessage()); } - table = view; - } else if (type == TableType.KUDU) { - table = new KuduTable(); - table.readFields(in); - } else if (type == TableType.BROKER) { - table = new BrokerTable(); - table.readFields(in); - } else { - throw new IOException("Unknown table type: " + type.name()); } return table; @@ -176,6 +187,11 @@ public class Table extends MetaObject implements Writable { @Override public void readFields(DataInput in) throws IOException { + if (!isTypeRead) { + type = TableType.valueOf(Text.readString(in)); + isTypeRead = true; + } + super.readFields(in); this.id = in.readLong(); diff --git a/fe/src/com/baidu/palo/catalog/Tablet.java b/fe/src/com/baidu/palo/catalog/Tablet.java index 5c840c96ec..de96b6a548 100644 --- a/fe/src/com/baidu/palo/catalog/Tablet.java +++ b/fe/src/com/baidu/palo/catalog/Tablet.java @@ -15,22 +15,22 @@ package com.baidu.palo.catalog; -import com.baidu.palo.catalog.Replica.ReplicaState; -import com.baidu.palo.common.io.Writable; - -import com.google.common.collect.Sets; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; +import com.baidu.palo.catalog.Replica.ReplicaState; +import com.baidu.palo.common.io.Writable; + +import com.google.common.collect.Sets; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.Set; /** @@ -66,8 +66,12 @@ public class Tablet extends MetaObject implements Writable { checkedVersionHash = -1L; isConsistent = true; - } - + } + + public void setIdForRestore(long tabletId) { + this.id = tabletId; + } + public long getId() { return this.id; } @@ -111,13 +115,19 @@ public class Tablet extends MetaObject implements Writable { return delete || !hasBackend; } - public void addReplica(Replica replica) { + public void addReplica(Replica replica, boolean isRestore) { if (deleteRedundantReplica(replica.getBackendId(), replica.getVersion())) { - replicas.add(replica); - Catalog.getCurrentInvertedIndex().addReplica(id, replica); + replicas.add(replica); + if (!isRestore) { + Catalog.getCurrentInvertedIndex().addReplica(id, replica); + } } - } - + } + + public void addReplica(Replica replica) { + addReplica(replica, false); + } + public List getReplicas() { return this.replicas; } diff --git a/fe/src/com/baidu/palo/catalog/TabletInvertedIndex.java b/fe/src/com/baidu/palo/catalog/TabletInvertedIndex.java index c1474b271d..dcf0f1f18b 100644 --- a/fe/src/com/baidu/palo/catalog/TabletInvertedIndex.java +++ b/fe/src/com/baidu/palo/catalog/TabletInvertedIndex.java @@ -49,31 +49,30 @@ public class TabletInvertedIndex { public static final int NOT_EXIST_VALUE = -1; - private ReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); // tablet id -> tablet meta - private Map tabletMetaMap; + private Map tabletMetaMap = Maps.newHashMap(); /* * we use this to save memory. - * we do not need create TabletMeta intance for each tablet, + * we do not need create TabletMeta instance for each tablet, * cause tablets in one (Partition-MaterializedIndex) has same parent info * (dbId, tableId, partitionId, indexId, schemaHash) * we use 'tabletMetaTable' to do the update things * (eg. update schema hash in TabletMeta) - * partitionid -> (index id -> tablet meta) + * partition id -> (index id -> tablet meta) */ - private Table tabletMetaTable; + private Table tabletMetaTable = HashBasedTable.create(); // tablet id -> (backend id -> replica) - private Table replicaMetaTable; + private Table replicaMetaTable = HashBasedTable.create(); + // backing replica table, for visiting backend replicas faster. + // backend id -> (tablet id -> replica) + private Table backingReplicaMetaTable = HashBasedTable.create(); public TabletInvertedIndex() { - lock = new ReentrantReadWriteLock(); - tabletMetaMap = Maps.newHashMap(); - tabletMetaTable = HashBasedTable.create(); - replicaMetaTable = HashBasedTable.create(); } private final void readLock() { @@ -105,7 +104,7 @@ public class TabletInvertedIndex { try { LOG.info("begin to do tablet diff with backend[{}]. num: {}", backendId, backendTablets.size()); start = System.currentTimeMillis(); - Map replicaMetaWithBackend = replicaMetaTable.column(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); if (replicaMetaWithBackend != null) { // traverse replicas in meta with this backend for (Map.Entry entry : replicaMetaWithBackend.entrySet()) { @@ -134,6 +133,12 @@ public class TabletInvertedIndex { tabletMigrationMap.put(storageMedium, tabletId); } } + + // update replicas's version count + // no need to write log, and no need to get db lock. + if (backendTabletInfo.isSetVersion_count()) { + replica.setVersionCount(backendTabletInfo.getVersion_count()); + } } else { // tablet with invalid schemahash foundTabletsWithInvalidSchema.put(tabletId, backendTabletInfo); @@ -262,7 +267,12 @@ public class TabletInvertedIndex { } writeLock(); try { - replicaMetaTable.rowMap().remove(tabletId); + Map replicas = replicaMetaTable.rowMap().remove(tabletId); + if (replicas != null) { + for (long backendId : replicas.keySet()) { + backingReplicaMetaTable.remove(backendId, tabletId); + } + } TabletMeta tabletMeta = tabletMetaMap.remove(tabletId); if (tabletMeta != null) { tabletMetaTable.remove(tabletMeta.getPartitionId(), tabletMeta.getIndexId()); @@ -280,6 +290,7 @@ public class TabletInvertedIndex { try { Preconditions.checkState(tabletMetaMap.containsKey(tabletId)); replicaMetaTable.put(tabletId, replica.getBackendId(), replica); + backingReplicaMetaTable.put(replica.getBackendId(), tabletId, replica); } finally { writeUnlock(); } @@ -295,6 +306,7 @@ public class TabletInvertedIndex { // Preconditions.checkState(replicaMetaTable.containsRow(tabletId)); if (replicaMetaTable.containsRow(tabletId)) { replicaMetaTable.remove(tabletId, backendId); + backingReplicaMetaTable.remove(backendId, tabletId); LOG.debug("delete tablet[{}] in backend[{}]", tabletId, backendId); } else { // this may happend when fe restart after tablet is empty(bug cause) @@ -329,7 +341,6 @@ public class TabletInvertedIndex { } finally { writeUnlock(); } - } public void updateToNewSchemaHash(long partitionId, long indexId) { @@ -364,7 +375,7 @@ public class TabletInvertedIndex { List tabletIds = Lists.newArrayList(); readLock(); try { - Map replicaMetaWithBackend = replicaMetaTable.column(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); if (replicaMetaWithBackend != null) { tabletIds.addAll(replicaMetaWithBackend.keySet()); } @@ -377,7 +388,7 @@ public class TabletInvertedIndex { public int getTabletNumByBackendId(long backendId) { readLock(); try { - Map replicaMetaWithBackend = replicaMetaTable.column(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); if (replicaMetaWithBackend != null) { return replicaMetaWithBackend.size(); } @@ -394,8 +405,10 @@ public class TabletInvertedIndex { tabletMetaMap.clear(); tabletMetaTable.clear(); replicaMetaTable.clear(); + backingReplicaMetaTable.clear(); } finally { writeUnlock(); } } } + diff --git a/fe/src/com/baidu/palo/catalog/UserPropertyMgr.java b/fe/src/com/baidu/palo/catalog/UserPropertyMgr.java deleted file mode 100644 index 1942e341a4..0000000000 --- a/fe/src/com/baidu/palo/catalog/UserPropertyMgr.java +++ /dev/null @@ -1,655 +0,0 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.catalog; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.baidu.palo.analysis.AlterUserStmt; -import com.baidu.palo.analysis.AlterUserType; -import com.baidu.palo.analysis.SetUserPropertyStmt; -import com.baidu.palo.cluster.ClusterNamespace; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.Config; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.Pair; -import com.baidu.palo.common.publish.FixedTimePublisher; -import com.baidu.palo.common.publish.Listener; -import com.baidu.palo.common.publish.TopicUpdate; -import com.baidu.palo.load.DppConfig; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.thrift.TAgentServiceVersion; -import com.baidu.palo.thrift.TFetchResourceResult; -import com.baidu.palo.thrift.TTopicItem; -import com.baidu.palo.thrift.TTopicType; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -// TODO(dhc): we don't consider "drop database" -public class UserPropertyMgr { - private static final Logger LOG = LogManager.getLogger(UserPropertyMgr.class); - - private EditLog editLog; - protected Map userMap; - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); - public static final String ROOT_USER = "root"; - public static final String SYSTEM_RESOURCE_USER = "system"; - private AtomicLong resourceVersion; - - public UserPropertyMgr() { - userMap = Maps.newConcurrentMap(); - // there is no problem to init the root's password in constructor. - // because during the log replay, the real root password will cover the - // initial password. - unprotectAddUser("", ROOT_USER, new byte[0]); - unprotectAddUser("", SYSTEM_RESOURCE_USER, new byte[0]); - resourceVersion = new AtomicLong(0); - } - - public static String getRootName() { - return ROOT_USER; - } - - private void readLock() { - this.lock.readLock().lock(); - } - - private void readUnlock() { - this.lock.readLock().unlock(); - } - - private void writeLock() { - this.lock.writeLock().lock(); - } - - private void writeUnlock() { - this.lock.writeLock().unlock(); - } - - public void setEditLog(EditLog editLog) { - this.editLog = editLog; - } - - // Register callback to FixedTimePublisher - public void setUp() { - FixedTimePublisher.getInstance().register(new FixedTimePublisher.Callback() { - @Override - public TopicUpdate getTopicUpdate() { - TopicUpdate update = new TopicUpdate(TTopicType.RESOURCE); - TTopicItem tTopicItem = new TTopicItem("version"); - tTopicItem.setInt_value(resourceVersion.get()); - update.addUpdates(tTopicItem); - return update; - } - - @Override - public Listener getListener() { - return null; - } - }, Config.meta_resource_publish_interval_ms); - } - - public List> fetchAccessResourceResult(String user) { - List> result = Lists.newArrayList(); - readLock(); - try { - LOG.debug("get user name: {}", user); - UserProperty userProperty = userMap.get(user); - if (userProperty == null) { - // no such user - return result; - } - - boolean isAdmin = userProperty.isAdmin(); - boolean isSuperuser = userProperty.isSuperuser(); - - if (!isAdmin && !isSuperuser) { - // just a normal user, show its own property - result.add(Arrays.asList(userProperty.getUser(), new String(userProperty.getPassword()), - String.valueOf(userProperty.isAdmin()), String.valueOf(userProperty.isSuperuser()), - String.valueOf(userProperty.getMaxConn()), userProperty.fetchPrivilegeResult())); - return result; - } - - if (isAdmin) { - // If this is admin user(root), show all users' property. - for (Map.Entry entry : userMap.entrySet()) { - UserProperty oneProp = entry.getValue(); - - result.add(Arrays.asList(oneProp.getUser(), new String(oneProp.getPassword()), - String.valueOf(oneProp.isAdmin()), String.valueOf(oneProp.isSuperuser()), - String.valueOf(oneProp.getMaxConn()), oneProp.fetchPrivilegeResult())); - } - } else if (isSuperuser) { - // If this is a superuser, show property of all users' who are belong to the - // cluster. - String clusterName = ClusterNamespace.getClusterNameFromFullName(user); - LOG.debug("cluster name: {}", clusterName); - - for (Map.Entry entry : userMap.entrySet()) { - UserProperty oneProp = entry.getValue(); - if (!oneProp.getClusterName().equals(clusterName)) { - continue; - } - - result.add(Arrays.asList(oneProp.getUser(), new String(oneProp.getPassword()), - String.valueOf(oneProp.isAdmin()), String.valueOf(oneProp.isSuperuser()), - String.valueOf(oneProp.getMaxConn()), oneProp.fetchPrivilegeResult())); - } - } - - return result; - } finally { - readUnlock(); - } - } - - // we provide four function to support ddl stmt: addUser, setPasswd, grant, - // dropUser - // we provide two funciton to support search stmt: hasAccess(), - // getIsAdmin(), getPassword() - public void addUser(String cluster, String user, byte[] password, boolean isSuperuser) throws DdlException { - writeLock(); - try { - checkUserNotExists(user); - UserProperty userProperty = unprotectAddUser(cluster, user, password); - userProperty.setIsSuperuser(isSuperuser); - // all user has READ_ONLY privilege to InfoSchemaDb - this.getAccessResource(user).setAccess(ClusterNamespace.getFullName(cluster, InfoSchemaDb.DATABASE_NAME), - AccessPrivilege.READ_ONLY); - String msg = "addUser username=" + user + " password='" + password; - writeEditsOfAlterAccess(userProperty, msg); - } finally { - writeUnlock(); - } - } - - public void setPasswd(String user, byte[] password) throws DdlException { - writeLock(); - try { - checkUserExists(user); - this.getAccessResource(user).setPassword(password); - writeEditsOfAlterAccess(this.getAccessResource(user), "set password"); - } finally { - writeUnlock(); - } - } - - public void grant(String user, String db, AccessPrivilege privilege) throws DdlException { - if (Catalog.getInstance().getDb(db) == null) { - throw new DdlException("db[" + db + "] does not exist"); - } - - writeLock(); - try { - checkUserExists(user); - this.getAccessResource(user).setAccess(db, privilege); - - String msg = "grant user " + user + " db " + db + " privilege " + privilege; - writeEditsOfAlterAccess(this.getAccessResource(user), msg); - } finally { - writeUnlock(); - } - } - - public void revoke(String user, String db) throws DdlException { - // we do not check if db is exist in catalog - // db may be dropped or renamed - - writeLock(); - try { - checkUserExists(user); - this.getAccessResource(user).revokeAccess(db); - - String msg = "revoke user " + user + " db " + db; - writeEditsOfAlterAccess(this.getAccessResource(user), msg); - } finally { - writeUnlock(); - } - } - - public void dropUser(String user) throws DdlException { - writeLock(); - try { - checkUserExists(user); - unprotectDropUser(user); - editLog.logDropUser(user); - resourceVersion.incrementAndGet(); - } finally { - writeUnlock(); - } - } - - // functions bellow are used to get user information: hasAccess() IsAdmin(), - // getPassword() - // TODO(add functionget and set maxCount of user) - public boolean checkAccess(String user, String db, AccessPrivilege priv) { - readLock(); - try { - if (!isUserExists(user)) { - return false; - } - return this.getAccessResource(user).checkAccess(db, priv); - } finally { - readUnlock(); - } - } - - public boolean checkUserAccess(String opUser, String user) { - if (Strings.isNullOrEmpty(opUser) || Strings.isNullOrEmpty(user)) { - return false; - } - if (isAdmin(opUser)) { - return true; - } - if (isSuperuser(opUser) && !isSuperuser(user)) { - return true; - } - if (opUser.equals(user)) { - return true; - } - return false; - } - - public byte[] getPassword(String user) { - readLock(); - try { - if (!isUserExists(user)) { - return null; - } - return this.getAccessResource(user).getPassword(); - } finally { - readUnlock(); - } - } - - public boolean isAdmin(String user) { - readLock(); - try { - if (!isUserExists(user)) { - return false; - } - return this.getAccessResource(user).isAdmin(); - } finally { - readUnlock(); - } - } - - public boolean isSuperuser(String user) { - readLock(); - try { - if (!isUserExists(user)) { - return false; - } - return this.getAccessResource(user).isSuperuser(); - } finally { - readUnlock(); - } - } - - public long getMaxConn(String user) { - readLock(); - try { - if (!isUserExists(user)) { - return 0; - } - return this.getAccessResource(user).getMaxConn(); - } finally { - readUnlock(); - } - } - - private boolean isUserExists(String user) { - if (Strings.isNullOrEmpty(user)) { - return false; - } - if (this.getAccessResource(user) == null) { - return false; - } - return true; - } - - // this two function used to read snapshot or write snapshot - public void write(DataOutput out) throws IOException { - int numUsers = userMap.size(); - out.writeInt(numUsers); - - for (Map.Entry entry : userMap.entrySet()) { - entry.getValue().write(out); - } - // Write resource version - out.writeLong(resourceVersion.get()); - } - - public void readFields(DataInput in) throws IOException { - int numUsers = in.readInt(); - - for (int i = 0; i < numUsers; ++i) { - UserProperty userProperty = UserProperty.read(in); - userMap.put(userProperty.getUser(), userProperty); - } - - // Read resource - resourceVersion = new AtomicLong(in.readLong()); - } - - public int getUserMapSize() { - return userMap.size(); - } - - // Editlog will call this four function to playback journal - public void unprotectDropUser(String user) { - userMap.remove(user); - // TODO(zhaochun): Now we add resource version every time. - resourceVersion.incrementAndGet(); - } - - public void replayDropUser(String user) { - writeLock(); - try { - unprotectDropUser(user); - } finally { - writeUnlock(); - } - } - - public void unprotectAlterAccess(UserProperty userProperty) { - userMap.put(userProperty.getUser(), userProperty); - // TODO(zhaochun): Now we add resource version every time. - resourceVersion.incrementAndGet(); - } - - public void replayAlterAccess(UserProperty userProperty) { - writeLock(); - try { - unprotectAlterAccess(userProperty); - } finally { - writeUnlock(); - } - } - - // private function which used to support four public function - private void checkUserExists(String user) throws DdlException { - if (Strings.isNullOrEmpty(user)) { - throw new DdlException(new String("user is null")); - } - if (userMap.get(user) == null) { - throw new DdlException(new String("user dosn't exists")); - } - } - - public void checkUserIfExist(String user) throws DdlException { - readLock(); - try { - checkUserExists(user); - } finally { - readUnlock(); - } - } - - private void checkUserNotExists(String user) throws DdlException { - if (Strings.isNullOrEmpty(user)) { - throw new DdlException(new String("user is null")); - } - if (userMap.get(user) != null) { - throw new DdlException(new String("user has existed")); - } - } - - private UserProperty unprotectAddUser(String cluster, String user, byte[] password) { - UserProperty userProperty = new UserProperty(); - userProperty.setUser(user); - userProperty.setPassword(password); - userProperty.setClusterName(cluster); - // 默认“root”用户是管理员，其他需要其他接口 - if (user.equals(ROOT_USER)) { - userProperty.setIsAdmin(true); - } - try { - if (user.equals(SYSTEM_RESOURCE_USER)) { - setSystemUserDefaultResource(userProperty); - } - if (!user.equals(SYSTEM_RESOURCE_USER) && !user.equals(ROOT_USER)) { - setNormalUserDefaultResource(userProperty); - } - } catch (DdlException e) { - // this should not happen, because the value is set by us!! - } - userMap.put(user, userProperty); - return userProperty; - } - - private void setSystemUserDefaultResource(UserProperty user) throws DdlException { - UserResource userResource = user.getResource(); - userResource.updateResource("CPU_SHARE", 100); - userResource.updateResource("IO_SHARE", 100); - userResource.updateResource("SSD_READ_MBPS", 30); - userResource.updateResource("SSD_WRITE_MBPS", 30); - userResource.updateResource("HDD_READ_MBPS", 30); - userResource.updateResource("HDD_WRITE_MBPS", 30); - } - - private void setNormalUserDefaultResource(UserProperty user) throws DdlException { - UserResource userResource = user.getResource(); - userResource.updateResource("CPU_SHARE", 1000); - userResource.updateResource("IO_SHARE", 1000); - userResource.updateResource("SSD_READ_IOPS", 1000); - userResource.updateResource("HDD_READ_IOPS", 80); - userResource.updateResource("SSD_READ_MBPS", 30); - userResource.updateResource("HDD_READ_MBPS", 30); - } - - private void writeEditsOfAlterAccess(UserProperty userProperty, String msg) { - editLog.logAlterAccess(userProperty); - resourceVersion.incrementAndGet(); - } - - private UserProperty getAccessResource(String user) { - return userMap.get(user); - } - - public void updateUserProperty(SetUserPropertyStmt stmt) throws DdlException { - writeLock(); - try { - UserProperty property = userMap.get(stmt.getUser()); - if (property == null) { - throw new DdlException("Unknown user(" + stmt.getUser() + ")"); - } - - property.update(stmt.getPropertyList()); - - writeEditsOfAlterAccess(property, "update user property"); - } finally { - writeUnlock(); - } - } - - public TFetchResourceResult toResourceThrift() { - TFetchResourceResult tResult = new TFetchResourceResult(); - tResult.setProtocolVersion(TAgentServiceVersion.V1); - tResult.setResourceVersion(resourceVersion.get()); - readLock(); - try { - for (Map.Entry entry : userMap.entrySet()) { - tResult.putToResourceByUser(entry.getKey(), entry.getValue().getResource().toThrift()); - } - } finally { - readUnlock(); - } - return tResult; - } - - public Pair getClusterInfo(String user, String cluster) throws DdlException { - Pair clusterInfo = null; - - readLock(); - try { - if (!userMap.containsKey(user)) { - throw new DdlException("User[" + user + "] does not exist"); - } - - UserProperty property = userMap.get(user); - clusterInfo = property.getClusterInfo(cluster); - } finally { - readUnlock(); - } - - return clusterInfo; - } - - public List> fetchUserProperty(String user) throws AnalysisException { - readLock(); - try { - if (!userMap.containsKey(user)) { - throw new AnalysisException("User[" + user + "] does not exist"); - } - - UserProperty property = userMap.get(user); - return property.fetchProperty(); - } finally { - readUnlock(); - } - } - - public void alterUser(AlterUserStmt stmt) throws DdlException { - List ips = stmt.getIps(); - List starIps = stmt.getStarIps(); - List hosts = stmt.getHosts(); - String user = stmt.getUser(); - AlterUserType type = stmt.getAlterUserType(); - - // check host if can dns - if (type == AlterUserType.ADD_USER_WHITELIST) { - for (String host : hosts) { - boolean isAvaliable = DomainResolverServer.getInstance().isAvaliableDomain(host); - if (!isAvaliable) { - String msg = "May be error hostname. host=" + host; - LOG.warn("alter user={} stmt={} occur dns Exception msg={}", stmt.getUser(), stmt, msg); - throw new DdlException(msg); - } - } - } - - writeLock(); - try { - UserProperty property = userMap.get(user); - if (property == null) { - throw new DdlException("use dosn't exists user=" + user); - } - - WhiteList whiteList = property.getWhiteList(); - String msg = type.toString(); - - switch (type) { - case ADD_USER_WHITELIST: { - whiteList.addWhiteList(ips, starIps, hosts); - break; - } - case DELETE_USER_WHITELIST: { - whiteList.deleteWhiteList(ips, starIps, hosts); - break; - } - default: { - LOG.warn("alterUser occur unkown type = {}", type); - throw new RuntimeException("unkown type"); - } - } - // write editlog - writeEditsOfAlterAccess(this.getAccessResource(user), msg); - } finally { - writeUnlock(); - } - } - - public boolean checkWhiltListAccess(String user, String remoteIp) { - if (user.equals("root") && remoteIp.equals("127.0.0.1")) { - return true; - } - readLock(); - try { - UserProperty property = userMap.get(user); - if (property == null) { - return false; - } - return property.getWhiteList().hasAccess(remoteIp); - } finally { - readUnlock(); - } - } - - public List> showWhiteList(String user) { - List> result = Lists.newArrayList(); - readLock(); - try { - // ordinary user - if (!isSuperuser(user)) { - WhiteList whitelist = userMap.get(user).getWhiteList(); - List row = Lists.newArrayList(); - row.add(user); - row.add(whitelist.toString()); - result.add(row); - return result; - } else { - for (Map.Entry entry : userMap.entrySet()) { - String candidateUser = entry.getKey(); - boolean addRow = false; - // admin can see every one's whitelist - // superuse can see own and odinary user's whitelist - // ordinary can see own whitelist - if (isAdmin(user)) { - addRow = true; - } else if (user.equals(candidateUser)) { - addRow = true; - } else if (!isSuperuser(candidateUser)) { - addRow = true; - } - if (addRow) { - WhiteList whitelist = userMap.get(candidateUser).getWhiteList(); - List row = Lists.newArrayList(); - row.add(candidateUser); - row.add(whitelist.toString()); - result.add(row); - } - } - } - - } finally { - readUnlock(); - } - return result; - } - - public int getWhiteListSize(String userName) throws DdlException { - readLock(); - try { - checkUserExists(userName); - return userMap.get(userName).getWhiteList().getSize(); - } finally { - readUnlock(); - } - } -} - diff --git a/fe/src/com/baidu/palo/catalog/WhiteList.java b/fe/src/com/baidu/palo/catalog/WhiteList.java deleted file mode 100644 index 32c7ef01a4..0000000000 --- a/fe/src/com/baidu/palo/catalog/WhiteList.java +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.catalog; - -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.io.Text; -import com.baidu.palo.common.io.Writable; - -import com.google.common.collect.Sets; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class WhiteList implements Writable { - private static final Logger LOG = LogManager.getLogger(WhiteList.class); - - // Ip (123.123.1.1) - protected Set ipWhiteLists = Sets.newHashSet(); - // startIp (123.123.*.*) - protected Set starIpWhiteLists = Sets.newHashSet(); - // hostName(www.baidu.com), which need to dns analysis - protected Set hostWhiteLists = Sets.newHashSet(); - protected Map> ipOfHostWhiteLists; - private String user; - - public WhiteList() { - } - - // for limit the max whiteListsize - public int getSize() { - return ipWhiteLists.size() + starIpWhiteLists.size() + hostWhiteLists.size(); - } - - public boolean hasAccess(String ip) { - // whileList is null, all people can access - if (getSize() == 0) { - return true; - } - - // 1. check if specified ip in white list - if (ipWhiteLists.contains(ip)) { - return true; - } - - // 2. check if specified ip in start white list - for (String starIp : starIpWhiteLists) { - String[] splittedStarIp = starIp.split("\\."); - String[] splittedSpecifiedIp = ip.split("\\."); - int starIpLen = splittedStarIp.length; - int specifiedIpLen = splittedSpecifiedIp.length; - if (!(specifiedIpLen == 4 && starIpLen == 4)) { - String msg = String.format("Invalid IP format: %s", ip); - LOG.warn(msg); - throw new RuntimeException(msg); - } - - boolean hit = true; - for (int i = 0; i < 4; i++) { - if (splittedSpecifiedIp[i].equals(splittedStarIp[i])) { - continue; - } else if (splittedStarIp[i].equals("*")) { - continue; - } else { - hit = false; - break; - } - } - - if (hit) { - return true; - } - } - - ipOfHostWhiteLists = DomainResolverServer.getInstance().getUserDomainToIps(user); - // 3. check ipWhiteList - if (ipOfHostWhiteLists != null) { - for (String entryIp : ipOfHostWhiteLists.keySet()) { - Set ipSet = ipOfHostWhiteLists.get(entryIp); - if (ipSet == null || ipSet.size() == 0) { - LOG.debug("dns error ip={}", entryIp); - continue; - } - if (ipSet.contains(ip)) { - return true; - } - } - } - LOG.debug("can't match whitelist ip={}", ip); - return false; - } - - public void addWhiteList(List ips, List starIps, List hosts) throws DdlException { - ipWhiteLists.addAll(ips); - starIpWhiteLists.addAll(starIps); - hostWhiteLists.addAll(hosts); - DomainResolverServer.getInstance().register(user, hosts); - } - - public void deleteWhiteList(List ips, List starIps, List hosts) { - if (ips != null && ips.size() > 0) { - ipWhiteLists.removeAll(ips); - } - if (starIps != null && starIps.size() > 0) { - starIpWhiteLists.removeAll(starIps); - } - if (hosts != null && hosts.size() > 0) { - hostWhiteLists.removeAll(hosts); - } - if (hosts != null && hosts.size() > 0) { - DomainResolverServer.getInstance().unregister(user, hosts); - } - - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - for (String ip : ipWhiteLists) { - builder.append(ip); - builder.append(","); - } - for (String ip : starIpWhiteLists) { - builder.append(ip); - builder.append(","); - } - for (String ip : hostWhiteLists) { - builder.append(ip); - builder.append(","); - } - String result = builder.toString(); - String newResult = result; - // del the last , - if (result.length() > 0) { - newResult = result.substring(0, result.length() - 1); - } - return newResult; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(ipWhiteLists.size()); - for (String ip : ipWhiteLists) { - Text.writeString(out, ip); - } - out.writeInt(starIpWhiteLists.size()); - for (String ip : starIpWhiteLists) { - Text.writeString(out, ip); - } - out.writeInt(hostWhiteLists.size()); - for (String ip : hostWhiteLists) { - Text.writeString(out, ip); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - int ipWhiteListsLen = in.readInt(); - for (int i = 0; i < ipWhiteListsLen; i++) { - String ip = Text.readString(in); - ipWhiteLists.add(ip); - } - int starIpWhiteListsLen = in.readInt(); - for (int i = 0; i < starIpWhiteListsLen; i++) { - String ip = Text.readString(in); - starIpWhiteLists.add(ip); - } - int hostWhiteListsLen = in.readInt(); - for (int i = 0; i < hostWhiteListsLen; i++) { - String ip = Text.readString(in); - hostWhiteLists.add(ip); - } - - if (hostWhiteLists != null && hostWhiteLists.size() > 0) { - DomainResolverServer.getInstance().register(user, hostWhiteLists); - } - } - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } -} diff --git a/fe/src/com/baidu/palo/cluster/ClusterNamespace.java b/fe/src/com/baidu/palo/cluster/ClusterNamespace.java index 14978ccb88..5f561fb957 100644 --- a/fe/src/com/baidu/palo/cluster/ClusterNamespace.java +++ b/fe/src/com/baidu/palo/cluster/ClusterNamespace.java @@ -15,9 +15,9 @@ package com.baidu.palo.cluster; -import com.google.common.base.Strings; +import com.baidu.palo.mysql.privilege.PaloAuth; -import com.baidu.palo.catalog.UserPropertyMgr; +import com.google.common.base.Strings; /** * used to isolate the use for the database name and user name in the catalog, @@ -62,7 +62,8 @@ public class ClusterNamespace { if (Strings.isNullOrEmpty(cluster) || Strings.isNullOrEmpty(name)) { return null; } - if (name.contains(CLUSTER_DELIMITER) || name.equals(UserPropertyMgr.getRootName())) { + if (name.contains(CLUSTER_DELIMITER) || name.equalsIgnoreCase(PaloAuth.ROOT_USER) + || name.equalsIgnoreCase(PaloAuth.ADMIN_USER)) { return name; } final StringBuilder sb = new StringBuilder(cluster); diff --git a/fe/src/com/baidu/palo/common/CaseSensibility.java b/fe/src/com/baidu/palo/common/CaseSensibility.java new file mode 100644 index 0000000000..8cae59aaf1 --- /dev/null +++ b/fe/src/com/baidu/palo/common/CaseSensibility.java @@ -0,0 +1,41 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.common; + +public enum CaseSensibility { + CLUSTER(true), + DATABASE(true), + TABLE(true), + ROLUP(true), + PARTITION(true), + COLUMN(true), + USER(true), + ROLE(false), + HOST(false), + LABEL(false), + VARIABLES(true); + + private boolean caseSensitive; + + private CaseSensibility(boolean caseSensitive) { + this.caseSensitive = caseSensitive; + } + + public boolean getCaseSensibility() { + return caseSensitive; + } + +} diff --git a/fe/src/com/baidu/palo/common/Config.java b/fe/src/com/baidu/palo/common/Config.java index e7a629dfb5..cab715d092 100644 --- a/fe/src/com/baidu/palo/common/Config.java +++ b/fe/src/com/baidu/palo/common/Config.java @@ -82,6 +82,13 @@ public class Config extends ConfigBase { * 2. Safe (RAID) */ @ConfField public static String meta_dir = System.getenv("PALO_HOME") + "/palo-meta"; + + /* + * temp dir is used to save intermediate results of some process, such as backup and restore process. + * file in this dir will be cleaned after these process is finished. + */ + @ConfField public static String tmp_dir = System.getenv("PALO_HOME") + "/temp_dir"; + /* * Edit log type. * BDB: write log to bdbje @@ -97,7 +104,7 @@ public class Config extends ConfigBase { /* * Master FE will save image every *edit_log_roll_num* meta journals. */ - @ConfField public static int edit_log_roll_num = 100000; + @ConfField public static int edit_log_roll_num = 50000; /* * Non-master FE will stop offering service * if meta data delay gap exceeds *meta_delay_toleration_second* @@ -503,6 +510,9 @@ public class Config extends ConfigBase { // Set runtime locale when exec some cmds @ConfField public static String locale = "zh_CN.UTF-8"; + + // default timeout of backup job + @ConfField public static int backup_job_default_timeout_ms = 86400 * 1000; // 1 day /* * storage_high_watermark_usage_percent limit the max capacity usage percent of a Backend storage path. @@ -512,4 +522,16 @@ public class Config extends ConfigBase { */ @ConfField public static double storage_high_watermark_usage_percent = 0.85; @ConfField public static double storage_min_left_capacity_bytes = 1000 * 1024 * 1024; // 1G + + // May be necessary to modify the following BRPC configurations in high concurrency scenarios. + // The number of concurrent requests BRPC can processed + @ConfField public static int brpc_number_of_concurrent_requests_processed = 4096; + + // BRPC idle wait time (ms) + @ConfField public static int brpc_idle_wait_max_time = 10000; + + /* + * if set to false, auth check will be disable, in case some goes wrong with the new privilege system. + */ + @ConfField public static boolean enable_auth_check = true; } diff --git a/fe/src/com/baidu/palo/common/ErrorCode.java b/fe/src/com/baidu/palo/common/ErrorCode.java index 50417b5c17..673a6289c6 100644 --- a/fe/src/com/baidu/palo/common/ErrorCode.java +++ b/fe/src/com/baidu/palo/common/ErrorCode.java @@ -59,6 +59,10 @@ public enum ErrorCode { ERR_PASSWORD_NOT_ALLOWED(1132, new byte[] {'4', '2', '0', '0', '0'}, "You must have privileges to " + "update tables in the mysql database to be able to change passwords for others"), + ERR_NONEXISTING_GRANT(1141, new byte[] { '4', '2', '0', '0', '0' }, + "There is no such grant defined for user '%s' on host '%s'"), + ERR_TABLEACCESS_DENIED_ERROR(1142, new byte[] { '4', '2', '0', '0', '0' }, + "%s command denied to user '%s'@'%s' for table '%s'"), ERR_WRONG_COLUMN_NAME(1166, new byte[] {'4', '2', '0', '0', '0'}, "Incorrect column name '%s'"), ERR_UNKNOWN_SYSTEM_VARIABLE(1193, new byte[] {'H', 'Y', '0', '0', '0'}, "Unknown system variable '%s'"), ERR_TOO_MANY_USER_CONNECTIONS(1203, new byte[] {'4', '2', '0', '0', '0'}, @@ -154,8 +158,6 @@ public enum ErrorCode { "All datbases in cluster must be dropped before dropping cluster"), ERR_CLUSTER_DELETE_BE_ID_ERROR(5037, new byte[] { 'H', 'Y', '0', '0', '0' }, "There is no be's id in the System"), ERR_CLUSTER_NO_CLUSTER_NAME(5038, new byte[] { 'H', 'Y', '0', '0', '0' }, "There is no cluster name"), - ERR_CLUSTER_SHOW_ACCESS_DENIED(5039, new byte[] {'4', '2', '0', '0', '0'}, - "Access denied for user '%s' to show cluster"), ERR_CLUSTER_UNKNOWN_ERROR(5040, new byte[] {'4', '2', '0', '0', '0'}, "Unknown cluster '%s'"), ERR_CLUSTER_NAME_NULL(5041, new byte[] {'4', '2', '0', '0', '0'}, "No cluster name"), ERR_CLUSTER_NO_PERMISSIONS(5042, new byte[] {'4', '2', '0', '0', '0'}, "No permissions"), @@ -193,7 +195,11 @@ public enum ErrorCode { ERR_KUDU_NOT_SUPPORT_VALUE_TYPE(5061, new byte[] { '4', '2', '0', '0', '0' }, "Kudu does not support value type '%s'"), ERR_WRONG_CLUSTER_NAME(5062, new byte[] { '4', '2', '0', '0', '0' }, - "Incorrect cluster name '%s'(name 'default_cluster' is a reserved name)"); + "Incorrect cluster name '%s'(name 'default_cluster' is a reserved name)"), + ERR_WRONG_NAME_FORMAT(5063, new byte[] { '4', '2', '0', '0', '0' }, + "Incorrect %s name '%s'"), + ERR_COMMON_ERROR(5064, new byte[] { '4', '2', '0', '0', '0' }, + "%s"); ErrorCode(int code, byte[] sqlState, String errorMsg) { diff --git a/fe/src/com/baidu/palo/common/FeConstants.java b/fe/src/com/baidu/palo/common/FeConstants.java index 02bd290252..c11423e34c 100644 --- a/fe/src/com/baidu/palo/common/FeConstants.java +++ b/fe/src/com/baidu/palo/common/FeConstants.java @@ -38,5 +38,5 @@ public class FeConstants { // general model // Current meta data version. Use this version to write journals and image - public static int meta_version = FeMetaVersion.VERSION_41; + public static int meta_version = FeMetaVersion.VERSION_43; } diff --git a/fe/src/com/baidu/palo/common/FeMetaVersion.java b/fe/src/com/baidu/palo/common/FeMetaVersion.java index 85ccfa634a..43d097909e 100644 --- a/fe/src/com/baidu/palo/common/FeMetaVersion.java +++ b/fe/src/com/baidu/palo/common/FeMetaVersion.java @@ -91,4 +91,10 @@ public final class FeMetaVersion { // change the way to name Frontend public static final int VERSION_41 = 41; + + // new backup and restore + public static final int VERSION_42 = 42; + + // new privilege management + public static final int VERSION_43 = 43; } diff --git a/fe/src/com/baidu/palo/common/FeNameFormat.java b/fe/src/com/baidu/palo/common/FeNameFormat.java index 3f97ef273b..d446e90bd7 100644 --- a/fe/src/com/baidu/palo/common/FeNameFormat.java +++ b/fe/src/com/baidu/palo/common/FeNameFormat.java @@ -20,23 +20,19 @@ package com.baidu.palo.common; +import com.baidu.palo.mysql.privilege.PaloRole; import com.baidu.palo.system.SystemInfoService; import com.google.common.base.Strings; public class FeNameFormat { - private static final String CLUSTER_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; - private static final String DB_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; - private static final String TABLE_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; - private static final String PARTITION_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; - private static final String COLUMN_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; - private static final String USER_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; private static final String LABEL_REGEX = "^[-_A-Za-z0-9]{1,128}$"; + private static final String COMMON_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; public static final String FORBIDDEN_PARTITION_NAME = "placeholder_"; public static void checkClusterName(String clusterName) throws AnalysisException { - if (Strings.isNullOrEmpty(clusterName) || !clusterName.matches(CLUSTER_REGEX)) { + if (Strings.isNullOrEmpty(clusterName) || !clusterName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_CLUSTER_NAME, clusterName); } if (clusterName.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { @@ -45,19 +41,19 @@ public class FeNameFormat { } public static void checkDbName(String dbName) throws AnalysisException { - if (Strings.isNullOrEmpty(dbName) || !dbName.matches(DB_REGEX)) { + if (Strings.isNullOrEmpty(dbName) || !dbName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName); } } public static void checkTableName(String tableName) throws AnalysisException { - if (Strings.isNullOrEmpty(tableName) || !tableName.matches(TABLE_REGEX)) { + if (Strings.isNullOrEmpty(tableName) || !tableName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_TABLE_NAME, tableName); } } public static void checkPartitionName(String partitionName) throws AnalysisException { - if (Strings.isNullOrEmpty(partitionName) || !partitionName.matches(PARTITION_REGEX)) { + if (Strings.isNullOrEmpty(partitionName) || !partitionName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_PARTITION_NAME, partitionName); } @@ -67,7 +63,7 @@ public class FeNameFormat { } public static void checkColumnName(String columnName) throws AnalysisException { - if (Strings.isNullOrEmpty(columnName) || !columnName.matches(COLUMN_REGEX)) { + if (Strings.isNullOrEmpty(columnName) || !columnName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName); } } @@ -79,8 +75,32 @@ public class FeNameFormat { } public static void checkUserName(String userName) throws AnalysisException { - if (Strings.isNullOrEmpty(userName) || !userName.matches(USER_REGEX)) { + if (Strings.isNullOrEmpty(userName) || !userName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CANNOT_USER, "CREATE USER", userName); } } + + public static void checkRoleName(String role, boolean canBeSuperuser) throws AnalysisException { + if (Strings.isNullOrEmpty(role) || !role.matches(COMMON_NAME_REGEX)) { + throw new AnalysisException("invalid role format: " + role); + } + + boolean res = false; + if (CaseSensibility.ROLE.getCaseSensibility()) { + res = role.equals(PaloRole.OPERATOR_ROLE) || (!canBeSuperuser && role.equals(PaloRole.ADMIN_ROLE)); + } else { + res = role.equalsIgnoreCase(PaloRole.OPERATOR_ROLE) + || (!canBeSuperuser && role.equalsIgnoreCase(PaloRole.ADMIN_ROLE)); + } + + if (res) { + throw new AnalysisException("Can not create role with name: " + role); + } + } + + public static void checkCommonName(String type, String name) throws AnalysisException { + if (Strings.isNullOrEmpty(name) || !name.matches(COMMON_NAME_REGEX)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_NAME_FORMAT, type, name); + } + } } diff --git a/fe/src/com/baidu/palo/common/PatternMatcher.java b/fe/src/com/baidu/palo/common/PatternMatcher.java index 4e69625a3b..5745cfd984 100644 --- a/fe/src/com/baidu/palo/common/PatternMatcher.java +++ b/fe/src/com/baidu/palo/common/PatternMatcher.java @@ -21,13 +21,19 @@ package com.baidu.palo.common; import com.google.common.base.Strings; +import com.google.common.collect.Sets; +import java.util.Set; import java.util.regex.Pattern; // Wrap for Java pattern and matcher public class PatternMatcher { private Pattern pattern; + private static final Set FORBIDDEN_CHARS = Sets.newHashSet('<', '(', '[', '{', '^', '=', + '$', '!', '|', ']', '}', ')', + '?', '*', '+', '>', '@'); + public boolean match(String candidate) { if (pattern == null || candidate == null) { // No pattern, how can I explain this? Return false now. @@ -40,24 +46,64 @@ public class PatternMatcher { return false; } - private static String convertMysqlPattern(String mysqlPattern) { + /* + * Mysql has only 2 patterns. + * '%' to match any character sequence + * '_' to master any single character. + * So we convert '%' to '.*', and '_' to '.' + * + * eg: + * abc% -> abc.* + * ab_c -> ab.c + * + * We also need to handle escape character '\'. + * User use '\' to escape reserved words like '%', '_', or '\' it self + * + * eg: + * ab\%c = ab%c + * ab\_c = ab_c + * ab\\c = ab\c + * + * We also have to ignore meaningless '\' like：'ab\c', convert it to 'abc'. + * The following characters are not permitted: + * <([{^=$!|]})?*+> + */ + private static String convertMysqlPattern(String mysqlPattern) throws AnalysisException { String newMysqlPattern = mysqlPattern; StringBuilder sb = new StringBuilder(); for (int i = 0; i < newMysqlPattern.length(); ++i) { char ch = newMysqlPattern.charAt(i); + checkPermittedCharactor(ch); switch (ch) { case '%': sb.append(".*"); break; + case '.': + sb.append("\\."); + break; case '_': sb.append("."); break; case '\\': { if (i == newMysqlPattern.length() - 1) { - // Last character of this pattern. - sb.append(newMysqlPattern.charAt(i)); - } else { - sb.append(newMysqlPattern.charAt(++i)); + // last character of this pattern. leave this '\' as it is + sb.append('\\'); + break; + } + // we need to look ahead the next character + // to decide ignore this '\' or treat it as escape character. + char nextChar = newMysqlPattern.charAt(i + 1); + switch (nextChar) { + case '%': + case '_': + case '\\': + // this is a escape character, eat this '\' and get next character. + sb.append(nextChar); + ++i; + break; + default: + // ignore this '\' and continue; + break; } break; } @@ -66,13 +112,26 @@ public class PatternMatcher { break; } } - // Replace all the '\' to '\'.'\' in Java pattern + + // Replace all the '\' to '\\' in Java pattern newMysqlPattern = sb.toString(); sb = new StringBuilder(); for (int i = 0; i < newMysqlPattern.length(); ++i) { char ch = newMysqlPattern.charAt(i); switch (ch) { case '\\': + if (i == newMysqlPattern.length() - 1) { + // last character of this pattern. leave this '\' as it is + sb.append('\\').append('\\'); + break; + } + // look ahead + if (newMysqlPattern.charAt(i + 1) == '.') { + // leave '\.' as it is. + sb.append('\\').append('.'); + i++; + break; + } sb.append('\\').append('\\'); break; default: @@ -81,10 +140,18 @@ public class PatternMatcher { } } + // System.out.println("result: " + sb.toString()); return sb.toString(); } - public static PatternMatcher createMysqlPattern(String mysqlPattern) throws AnalysisException { + private static void checkPermittedCharactor(char c) throws AnalysisException { + if (FORBIDDEN_CHARS.contains(c)) { + throw new AnalysisException("Forbidden charactor: '" + c + "'"); + } + } + + public static PatternMatcher createMysqlPattern(String mysqlPattern, boolean caseSensitive) + throws AnalysisException { PatternMatcher matcher = new PatternMatcher(); // Match nothing @@ -92,9 +159,13 @@ public class PatternMatcher { String javaPattern = convertMysqlPattern(newMysqlPattern); try { - matcher.pattern = Pattern.compile(javaPattern, Pattern.CASE_INSENSITIVE); + if (caseSensitive) { + matcher.pattern = Pattern.compile(javaPattern); + } else { + matcher.pattern = Pattern.compile(javaPattern, Pattern.CASE_INSENSITIVE); + } } catch (Exception e) { - throw new AnalysisException("Bad pattern in SQL."); + throw new AnalysisException("Bad pattern in SQL: " + e.getMessage()); } return matcher; } diff --git a/fe/src/com/baidu/palo/common/io/DeepCopy.java b/fe/src/com/baidu/palo/common/io/DeepCopy.java new file mode 100644 index 0000000000..6e31b212f4 --- /dev/null +++ b/fe/src/com/baidu/palo/common/io/DeepCopy.java @@ -0,0 +1,50 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.common.io; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInputStream; +import java.io.DataOutputStream; + +public class DeepCopy { + private static final Logger LOG = LogManager.getLogger(DeepCopy.class); + + public static boolean copy(Writable orig, Writable copied) { + FastByteArrayOutputStream byteArrayOutputStream = new FastByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteArrayOutputStream); + try { + orig.write(out); + out.flush(); + out.close(); + + DataInputStream in = new DataInputStream(byteArrayOutputStream.getInputStream()); + copied.readFields(in); + in.close(); + } catch (Exception e) { + e.printStackTrace(); + LOG.warn("failed to copy object.", e); + return false; + } + return true; + } +} diff --git a/fe/src/com/baidu/palo/common/io/FastByteArrayInputStream.java b/fe/src/com/baidu/palo/common/io/FastByteArrayInputStream.java new file mode 100644 index 0000000000..a3b941e080 --- /dev/null +++ b/fe/src/com/baidu/palo/common/io/FastByteArrayInputStream.java @@ -0,0 +1,82 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.common.io; + +import java.io.InputStream; + +/** + * ByteArrayInputStream implementation that does not synchronize methods. + * http://javatechniques.com/blog/faster-deep-copies-of-java-objects/ + */ +public class FastByteArrayInputStream extends InputStream { + /** + * Our byte buffer + */ + protected byte[] buf = null; + + /** + * Number of bytes that we can read from the buffer + */ + protected int count = 0; + + /** + * Number of bytes that have been read from the buffer + */ + protected int pos = 0; + + public FastByteArrayInputStream(byte[] buf, int count) { + this.buf = buf; + this.count = count; + } + + public final int available() { + return count - pos; + } + + public final int read() { + return (pos < count) ? (buf[pos++] & 0xff) : -1; + } + + public final int read(byte[] b, int off, int len) { + if (pos >= count) + return -1; + + if ((pos + len) > count) { + len = (count - pos); + } + + System.arraycopy(buf, pos, b, off, len); + pos += len; + return len; + } + + public final long skip(long n) { + if ((pos + n) > count) { + n = count - pos; + } + if (n < 0) { + return 0; + } + pos += n; + return n; + } + +} diff --git a/fe/src/com/baidu/palo/common/io/FastByteArrayOutputStream.java b/fe/src/com/baidu/palo/common/io/FastByteArrayOutputStream.java new file mode 100644 index 0000000000..9fedb084d8 --- /dev/null +++ b/fe/src/com/baidu/palo/common/io/FastByteArrayOutputStream.java @@ -0,0 +1,106 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.common.io; + +import java.io.InputStream; +import java.io.OutputStream; + +/** + * ByteArrayOutputStream implementation that doesn't synchronize methods + * and doesn't copy the data on toByteArray(). + * http://javatechniques.com/blog/faster-deep-copies-of-java-objects/ + */ +public class FastByteArrayOutputStream extends OutputStream { + /** + * Buffer and size + */ + protected byte[] buf = null; + protected int size = 0; + + /** + * Constructs a stream with buffer capacity size 5K + */ + public FastByteArrayOutputStream() { + this(5 * 1024); + } + + /** + * Constructs a stream with the given initial size + */ + public FastByteArrayOutputStream(int initSize) { + this.size = 0; + this.buf = new byte[initSize]; + } + + /** + * Ensures that we have a large enough buffer for the given size. + */ + private void verifyBufferSize(int sz) { + if (sz > buf.length) { + byte[] old = buf; + buf = new byte[Math.max(sz, 2 * buf.length)]; + System.arraycopy(old, 0, buf, 0, old.length); + old = null; + } + } + + public int getSize() { + return size; + } + + /** + * Returns the byte array containing the written data. Note that this + * array will almost always be larger than the amount of data actually + * written. + */ + public byte[] getByteArray() { + return buf; + } + + public final void write(byte[] b) { + verifyBufferSize(size + b.length); + System.arraycopy(b, 0, buf, size, b.length); + size += b.length; + } + + public final void write(byte[] b, int off, int len) { + verifyBufferSize(size + len); + System.arraycopy(b, off, buf, size, len); + size += len; + } + + public final void write(int b) { + verifyBufferSize(size + 1); + buf[size++] = (byte) b; + } + + public void reset() { + size = 0; + } + + /** + * Returns a ByteArrayInputStream for reading back the written data + */ + public InputStream getInputStream() { + return new FastByteArrayInputStream(buf, size); + } + +} diff --git a/fe/src/com/baidu/palo/common/proc/AccessResourceProcDir.java b/fe/src/com/baidu/palo/common/proc/AuthProcDir.java similarity index 61% rename from fe/src/com/baidu/palo/common/proc/AccessResourceProcDir.java rename to fe/src/com/baidu/palo/common/proc/AuthProcDir.java index b03595467e..09dc8907c5 100644 --- a/fe/src/com/baidu/palo/common/proc/AccessResourceProcDir.java +++ b/fe/src/com/baidu/palo/common/proc/AuthProcDir.java @@ -20,24 +20,27 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.catalog.UserPropertyMgr; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.mysql.privilege.PaloAuth; + import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; /* * It describes the information about the authorization(privilege) and the authentication(user) - * SHOW PROC /access_resource/ + * SHOW PROC /auth/ */ -public class AccessResourceProcDir implements ProcDirInterface { +public class AuthProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("UserName").add("Password").add("IsAdmin").add("IsSuperuser") - .add("MaxConn").add("Privilege").build(); + .add("UserIdentity").add("Password").add("GlobalPrivs").add("DatabasePrivs") + .add("TablePrivs").build(); - private UserPropertyMgr userPropertyMgr; + private PaloAuth auth; - public AccessResourceProcDir(UserPropertyMgr userPropertyMgr) { - this.userPropertyMgr = userPropertyMgr; + public AuthProcDir(PaloAuth auth) { + this.auth = auth; } @Override @@ -46,19 +49,24 @@ public class AccessResourceProcDir implements ProcDirInterface { } @Override - public ProcNodeInterface lookup(String user) throws AnalysisException { - if (Strings.isNullOrEmpty(user)) { - throw new AnalysisException("User[" + user + "] is null"); + public ProcNodeInterface lookup(String userIdent) throws AnalysisException { + if (Strings.isNullOrEmpty(userIdent)) { + throw new AnalysisException("User is not specified"); } - return new UserPropertyProcNode(userPropertyMgr, user); + UserIdentity userIdentity = UserIdentity.fromString(userIdent); + if (userIdentity == null) { + throw new AnalysisException("Invalid user ident: " + userIdent); + } + + return new UserPropertyProcNode(auth, userIdentity.getQualifiedUser()); } @Override public ProcResult fetchResult() throws AnalysisException { BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); - result.setRows(userPropertyMgr.fetchAccessResourceResult(UserPropertyMgr.ROOT_USER)); + result.setRows(Catalog.getCurrentCatalog().getAuth().getAuthInfo(null, true /* is all */)); return result; } } diff --git a/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java b/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java index d2e79e6d24..47b85aa9ce 100644 --- a/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java +++ b/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java @@ -74,7 +74,7 @@ public class BackendsProcDir implements ProcDirInterface { BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); - final List> backendInfos = getBackendInfos(); + final List> backendInfos = getClusterBackendInfos(null); for (List backendInfo : backendInfos) { List oneInfo = new ArrayList(backendInfo.size()); for (String info : backendInfo) { @@ -84,14 +84,6 @@ public class BackendsProcDir implements ProcDirInterface { } return result; } - - /** - * get all backends of system - * @return - */ - public static List> getBackendInfos() { - return getClusterBackendInfos(null); - } /** * get backends of cluster @@ -177,9 +169,9 @@ public class BackendsProcDir implements ProcDirInterface { free = (double) backend.getDataUsedCapacityB() * 100 / backend.getTotalCapacityB(); } backendInfo.add(String.format("%.2f", free) + " %"); - comparableBackendInfos.add(backendInfo); } + // backends proc node get result too slow, add log to observer. LOG.info("backends proc get tablet num cost: {}, total cost: {}", watch.elapsed(TimeUnit.MILLISECONDS), (System.currentTimeMillis() - start)); @@ -226,3 +218,4 @@ public class BackendsProcDir implements ProcDirInterface { } } + diff --git a/fe/src/com/baidu/palo/common/proc/BackupJobProcNode.java b/fe/src/com/baidu/palo/common/proc/BackupJobProcNode.java deleted file mode 100644 index 158bf1b8cf..0000000000 --- a/fe/src/com/baidu/palo/common/proc/BackupJobProcNode.java +++ /dev/null @@ -1,62 +0,0 @@ -// Modifications copyright (C) 2017, Baidu.com, Inc. -// Copyright 2017 The Apache Software Foundation - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.common.proc; - -import com.baidu.palo.backup.AbstractBackupJob; -import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.common.AnalysisException; - -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -public class BackupJobProcNode implements ProcNodeInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("TabletId").add("UnfinishedBackends") - .build(); - - private BackupHandler backupHandler; - private long dbId; - private Class jobClass; - - public BackupJobProcNode(BackupHandler backupHandler, long dbId, Class jobClass) { - this.backupHandler = backupHandler; - this.dbId = dbId; - this.jobClass = jobClass; - } - - @Override - public ProcResult fetchResult() throws AnalysisException { - BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - - List> infos = backupHandler.getJobUnfinishedTablet(dbId, jobClass); - for (List info : infos) { - List oneInfo = new ArrayList(TITLE_NAMES.size()); - for (Comparable element : info) { - oneInfo.add(element.toString()); - } - result.addRow(oneInfo); - } - return result; - } -} diff --git a/fe/src/com/baidu/palo/common/proc/BackupProcNode.java b/fe/src/com/baidu/palo/common/proc/BackupProcNode.java deleted file mode 100644 index ed3eb58876..0000000000 --- a/fe/src/com/baidu/palo/common/proc/BackupProcNode.java +++ /dev/null @@ -1,84 +0,0 @@ -// Modifications copyright (C) 2017, Baidu.com, Inc. -// Copyright 2017 The Apache Software Foundation - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.common.proc; - -import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.backup.BackupJob; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.common.AnalysisException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -public class BackupProcNode implements ProcDirInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("JobId").add("Lable").add("State").add("CreateTime") - .add("MetaSavedTime").add("SnapshotFinishedTime").add("UploadFinishedTime").add("FinishedTime") - .add("ErrMsg").add("BackupPath").add("Manifest").add("LeftTaskNum").add("LatestLoadLabel") - .build(); - - private BackupHandler backupHandler; - private Database db; - - public BackupProcNode(BackupHandler backupHandler, Database db) { - this.backupHandler = backupHandler; - this.db = db; - } - - @Override - public ProcResult fetchResult() throws AnalysisException { - Preconditions.checkNotNull(db); - Preconditions.checkNotNull(backupHandler); - - BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - - List> backupJobInfos = backupHandler.getJobInfosByDb(db.getId(), BackupJob.class, null); - for (List infoStr : backupJobInfos) { - List oneInfo = new ArrayList(TITLE_NAMES.size()); - for (Comparable element : infoStr) { - oneInfo.add(element.toString()); - } - result.addRow(oneInfo); - } - return result; - } - - @Override - public boolean register(String name, ProcNodeInterface node) { - return false; - } - - @Override - public ProcNodeInterface lookup(String jobIdStr) throws AnalysisException { - try { - Long.valueOf(jobIdStr); - } catch (NumberFormatException e) { - throw new AnalysisException("Invalid job id format: " + jobIdStr); - } - - return new BackupJobProcNode(backupHandler, db.getId(), BackupJob.class); - } - -} diff --git a/fe/src/com/baidu/palo/common/proc/FrontendsProcNode.java b/fe/src/com/baidu/palo/common/proc/FrontendsProcNode.java index 9b8f2368e7..1810816e8f 100644 --- a/fe/src/com/baidu/palo/common/proc/FrontendsProcNode.java +++ b/fe/src/com/baidu/palo/common/proc/FrontendsProcNode.java @@ -37,7 +37,7 @@ import java.util.List; */ public class FrontendsProcNode implements ProcNodeInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("name").add("Host").add("Port").add("Role").add("IsMaster").add("ClusterId").add("Join") + .add("name").add("Host").add("EditLogPort").add("Role").add("IsMaster").add("ClusterId").add("Join") .build(); private Catalog catalog; @@ -51,6 +51,18 @@ public class FrontendsProcNode implements ProcNodeInterface { BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); + List> infos = Lists.newArrayList(); + + getFrontendsInfo(catalog, infos); + + for (List info : infos) { + result.addRow(info); + } + + return result; + } + + public static void getFrontendsInfo(Catalog catalog, List> infos) { InetSocketAddress master = catalog.getHaProtocol().getLeader(); String masterIp = master.getAddress().getHostAddress(); int masterPort = master.getPort(); @@ -79,12 +91,11 @@ public class FrontendsProcNode implements ProcNodeInterface { info.add("true"); } - result.addRow(info); + infos.add(info); } - return result; } - private boolean isJoin(List> allFeHosts, Frontend fe) { + private static boolean isJoin(List> allFeHosts, Frontend fe) { for (Pair pair : allFeHosts) { if (fe.getHost().equals(pair.first) && fe.getEditLogPort() == pair.second) { return true; @@ -93,7 +104,7 @@ public class FrontendsProcNode implements ProcNodeInterface { return false; } - private List> convertToHostPortPair(List addrs) { + private static List> convertToHostPortPair(List addrs) { List> hostPortPair = Lists.newArrayList(); for (InetSocketAddress addr : addrs) { hostPortPair.add(Pair.create(addr.getAddress().getHostAddress(), addr.getPort())); diff --git a/fe/src/com/baidu/palo/common/proc/JobsProcDir.java b/fe/src/com/baidu/palo/common/proc/JobsProcDir.java index f8922baf18..73bf34ca1b 100644 --- a/fe/src/com/baidu/palo/common/proc/JobsProcDir.java +++ b/fe/src/com/baidu/palo/common/proc/JobsProcDir.java @@ -22,9 +22,6 @@ package com.baidu.palo.common.proc; import com.baidu.palo.alter.RollupHandler; import com.baidu.palo.alter.SchemaChangeHandler; -import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.backup.BackupJob.BackupJobState; -import com.baidu.palo.backup.RestoreJob.RestoreJobState; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Database; import com.baidu.palo.clone.CloneJob.JobState; @@ -53,8 +50,6 @@ public class JobsProcDir implements ProcDirInterface { private static final String DELETE = "delete"; private static final String ROLLUP = "rollup"; private static final String SCHEMA_CHANGE = "schema_change"; - private static final String BACKUP = "backup"; - private static final String RESTORE = "restore"; private static final String EXPORT = "export"; private Catalog catalog; @@ -86,10 +81,6 @@ public class JobsProcDir implements ProcDirInterface { return new RollupProcDir(catalog.getRollupHandler(), db); } else if (jobTypeName.equals(SCHEMA_CHANGE)) { return new SchemaChangeProcNode(catalog.getSchemaChangeHandler(), db); - } else if (jobTypeName.equals(BACKUP)) { - return new BackupProcNode(catalog.getBackupHandler(), db); - } else if (jobTypeName.equals(RESTORE)) { - return new RestoreProcNode(catalog.getBackupHandler(), db); } else if (jobTypeName.equals(EXPORT)) { return new ExportProcNode(catalog.getExportMgr(), db); } else { @@ -156,6 +147,7 @@ public class JobsProcDir implements ProcDirInterface { result.addRow(Lists.newArrayList(SCHEMA_CHANGE, pendingNum.toString(), runningNum.toString(), finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); + /* // backup BackupHandler backupHandler = Catalog.getInstance().getBackupHandler(); pendingNum = backupHandler.getBackupJobNum(BackupJobState.PENDING, dbId); @@ -168,7 +160,7 @@ public class JobsProcDir implements ProcDirInterface { totalNum = pendingNum + runningNum + finishedNum + cancelledNum; result.addRow(Lists.newArrayList(BACKUP, pendingNum.toString(), runningNum.toString(), finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); - + // restore pendingNum = backupHandler.getRestoreJobNum(RestoreJobState.PENDING, dbId); runningNum = backupHandler.getRestoreJobNum(RestoreJobState.RESTORE_META, dbId) @@ -179,6 +171,7 @@ public class JobsProcDir implements ProcDirInterface { totalNum = pendingNum + runningNum + finishedNum + cancelledNum; result.addRow(Lists.newArrayList(RESTORE, pendingNum.toString(), runningNum.toString(), finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); + */ // export ExportMgr exportMgr = Catalog.getInstance().getExportMgr(); diff --git a/fe/src/com/baidu/palo/common/proc/LoadProcDir.java b/fe/src/com/baidu/palo/common/proc/LoadProcDir.java index ef7852153b..dca0d330ca 100644 --- a/fe/src/com/baidu/palo/common/proc/LoadProcDir.java +++ b/fe/src/com/baidu/palo/common/proc/LoadProcDir.java @@ -20,16 +20,16 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.load.Load; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.load.Load; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; public class LoadProcDir implements ProcDirInterface { @@ -62,7 +62,8 @@ public class LoadProcDir implements ProcDirInterface { BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); - LinkedList> loadJobInfos = load.getLoadJobInfosByDb(db.getId(), null, false, null, null); + LinkedList> loadJobInfos = load.getLoadJobInfosByDb(db.getId(), db.getFullName(), + null, false, null, null); int counter = 0; Iterator> iterator = loadJobInfos.descendingIterator(); while (iterator.hasNext()) { diff --git a/fe/src/com/baidu/palo/common/proc/ProcService.java b/fe/src/com/baidu/palo/common/proc/ProcService.java index 2cc71e6571..ff1c1a6903 100644 --- a/fe/src/com/baidu/palo/common/proc/ProcService.java +++ b/fe/src/com/baidu/palo/common/proc/ProcService.java @@ -37,7 +37,7 @@ public final class ProcService { private ProcService() { root = new BaseProcDir(); - root.register("access_resource", new AccessResourceProcDir(Catalog.getInstance().getUserMgr())); + root.register("auth", new AuthProcDir(Catalog.getCurrentCatalog().getAuth())); root.register("backends", new BackendsProcDir(Catalog.getCurrentSystemInfo())); root.register("dbs", new DbsProcDir(Catalog.getInstance())); root.register("jobs", new JobsDbProcDir(Catalog.getInstance())); diff --git a/fe/src/com/baidu/palo/common/proc/ReplicasProcNode.java b/fe/src/com/baidu/palo/common/proc/ReplicasProcNode.java index 7aee14f623..b717a212d6 100644 --- a/fe/src/com/baidu/palo/common/proc/ReplicasProcNode.java +++ b/fe/src/com/baidu/palo/common/proc/ReplicasProcNode.java @@ -18,57 +18,59 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.common.proc; - -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.Replica; -import com.baidu.palo.catalog.Tablet; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.Arrays; - -/* - * SHOW PROC /dbs/dbId/tableId/partitions/partitionId/indexId/tabletId - * show replicas' detail info within a tablet - */ -public class ReplicasProcNode implements ProcNodeInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("ReplicaId").add("BackendId").add("Version").add("VersionHash") - .add("DataSize").add("RowCount").add("State") - .build(); - - private Database db; - private Tablet tablet; - - public ReplicasProcNode(Database db, Tablet tablet) { - this.db = db; - this.tablet = tablet; - } - - @Override - public ProcResult fetchResult() { - Preconditions.checkNotNull(db); - Preconditions.checkNotNull(tablet); - - db.readLock(); - try { - BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - for (Replica replica : tablet.getReplicas()) { - // id -- backendId -- version -- versionHash -- dataSize -- rowCount -- state - result.addRow(Arrays.asList(String.valueOf(replica.getId()), - String.valueOf(replica.getBackendId()), - String.valueOf(replica.getVersion()), - String.valueOf(replica.getVersionHash()), - String.valueOf(replica.getDataSize()), - String.valueOf(replica.getRowCount()), - String.valueOf(replica.getState()))); - } - return result; - } finally { - db.readUnlock(); - } - } -} +package com.baidu.palo.common.proc; + +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.Replica; +import com.baidu.palo.catalog.Tablet; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Arrays; + +/* + * SHOW PROC /dbs/dbId/tableId/partitions/partitionId/indexId/tabletId + * show replicas' detail info within a tablet + */ +public class ReplicasProcNode implements ProcNodeInterface { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("ReplicaId").add("BackendId").add("Version").add("VersionHash") + .add("DataSize").add("RowCount").add("State").add("VersionCount") + .build(); + + private Database db; + private Tablet tablet; + + public ReplicasProcNode(Database db, Tablet tablet) { + this.db = db; + this.tablet = tablet; + } + + @Override + public ProcResult fetchResult() { + Preconditions.checkNotNull(db); + Preconditions.checkNotNull(tablet); + + db.readLock(); + try { + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + for (Replica replica : tablet.getReplicas()) { + // id -- backendId -- version -- versionHash -- dataSize -- rowCount -- state + result.addRow(Arrays.asList(String.valueOf(replica.getId()), + String.valueOf(replica.getBackendId()), + String.valueOf(replica.getVersion()), + String.valueOf(replica.getVersionHash()), + String.valueOf(replica.getDataSize()), + String.valueOf(replica.getRowCount()), + String.valueOf(replica.getState()), + String.valueOf(replica.getVersionCount()))); + } + return result; + } finally { + db.readUnlock(); + } + } +} + diff --git a/fe/src/com/baidu/palo/common/proc/RestoreProcNode.java b/fe/src/com/baidu/palo/common/proc/RestoreProcNode.java deleted file mode 100644 index 524f02cb30..0000000000 --- a/fe/src/com/baidu/palo/common/proc/RestoreProcNode.java +++ /dev/null @@ -1,84 +0,0 @@ -// Modifications copyright (C) 2017, Baidu.com, Inc. -// Copyright 2017 The Apache Software Foundation - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.common.proc; - -import com.baidu.palo.backup.BackupHandler; -import com.baidu.palo.backup.RestoreJob; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.common.AnalysisException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -public class RestoreProcNode implements ProcDirInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("JobId").add("Lable").add("State").add("CreateTime") - .add("MetaRestoredTime").add("DowloadFinishedTime").add("FinishedTime").add("ErrMsg") - .add("RestorePath").add("LeftTaskNum") - .build(); - - private BackupHandler backupHandler; - private Database db; - - public RestoreProcNode(BackupHandler backupHandler, Database db) { - this.backupHandler = backupHandler; - this.db = db; - } - - @Override - public ProcResult fetchResult() throws AnalysisException { - Preconditions.checkNotNull(db); - Preconditions.checkNotNull(backupHandler); - - BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - - List> restoreJobInfos = backupHandler.getJobInfosByDb(db.getId(), RestoreJob.class, null); - for (List infoStr : restoreJobInfos) { - List oneInfo = new ArrayList(TITLE_NAMES.size()); - for (Comparable element : infoStr) { - oneInfo.add(element.toString()); - } - result.addRow(oneInfo); - } - return result; - } - - @Override - public boolean register(String name, ProcNodeInterface node) { - return false; - } - - @Override - public ProcNodeInterface lookup(String jobIdStr) throws AnalysisException { - try { - Long.valueOf(jobIdStr); - } catch (NumberFormatException e) { - throw new AnalysisException("Invalid job id format: " + jobIdStr); - } - - return new BackupJobProcNode(backupHandler, db.getId(), RestoreJob.class); - } - -} diff --git a/fe/src/com/baidu/palo/common/proc/SchemaChangeProcNode.java b/fe/src/com/baidu/palo/common/proc/SchemaChangeProcNode.java index a23d79db49..8ae23ba3db 100644 --- a/fe/src/com/baidu/palo/common/proc/SchemaChangeProcNode.java +++ b/fe/src/com/baidu/palo/common/proc/SchemaChangeProcNode.java @@ -20,15 +20,14 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.alter.SchemaChangeHandler; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.Arrays; +import com.baidu.palo.alter.SchemaChangeHandler; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.common.AnalysisException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; import java.util.List; public class SchemaChangeProcNode implements ProcNodeInterface { diff --git a/fe/src/com/baidu/palo/common/proc/TabletsProcDir.java b/fe/src/com/baidu/palo/common/proc/TabletsProcDir.java index 26990373fd..cf8eea3a3f 100644 --- a/fe/src/com/baidu/palo/common/proc/TabletsProcDir.java +++ b/fe/src/com/baidu/palo/common/proc/TabletsProcDir.java @@ -18,157 +18,161 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.common.proc; - -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.Replica; -import com.baidu.palo.catalog.MaterializedIndex; -import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.util.ListComparator; -import com.baidu.palo.common.util.TimeUtils; -import com.baidu.palo.system.Backend; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/* - * SHOW PROC /dbs/dbId/tableId/partitions/partitionId/indexId - * show tablets' detail info within an index - */ -public class TabletsProcDir implements ProcDirInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("TabletId").add("ReplicaId").add("BackendId").add("HostName").add("Version") - .add("VersionHash").add("DataSize").add("RowCount").add("State") - .add("LastConsistencyCheckTime").add("CheckVersion").add("CheckVersionHash") - .build(); - - private Database db; - private MaterializedIndex index; - - public TabletsProcDir(Database db, MaterializedIndex index) { - this.db = db; - this.index = index; - } - - @Override - public ProcResult fetchResult() { - Preconditions.checkNotNull(db); - Preconditions.checkNotNull(index); - - List> tabletInfos = new ArrayList>(); - db.readLock(); - try { - // get infos - for (Tablet tablet : index.getTablets()) { - long tabletId = tablet.getId(); - if (tablet.getReplicas().size() == 0) { - List tabletInfo = new ArrayList(); - tabletInfo.add(tabletId); - tabletInfo.add(-1); - tabletInfo.add(-1); - tabletInfo.add(-1); - tabletInfo.add(-1); - tabletInfo.add(-1); - tabletInfo.add(-1); - tabletInfo.add("N/A"); - tabletInfo.add("N/A"); - tabletInfo.add(-1); - tabletInfo.add(-1); - - tabletInfos.add(tabletInfo); - } else { - for (Replica replica : tablet.getReplicas()) { - List tabletInfo = new ArrayList(); - // tabletId -- replicaId -- backendId -- version -- versionHash -- dataSize -- rowCount -- state - tabletInfo.add(tabletId); - tabletInfo.add(replica.getId()); - long backendId = replica.getBackendId(); - tabletInfo.add(replica.getBackendId()); - Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendId); +package com.baidu.palo.common.proc; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.Replica; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.util.ListComparator; +import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.system.Backend; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/* + * SHOW PROC /dbs/dbId/tableId/partitions/partitionId/indexId + * show tablets' detail info within an index + */ +public class TabletsProcDir implements ProcDirInterface { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("TabletId").add("ReplicaId").add("BackendId").add("HostName").add("Version") + .add("VersionHash").add("DataSize").add("RowCount").add("State") + .add("LastConsistencyCheckTime").add("CheckVersion").add("CheckVersionHash") + .add("VersionCount") + .build(); + + private Database db; + private MaterializedIndex index; + + public TabletsProcDir(Database db, MaterializedIndex index) { + this.db = db; + this.index = index; + } + + @Override + public ProcResult fetchResult() { + Preconditions.checkNotNull(db); + Preconditions.checkNotNull(index); + + List> tabletInfos = new ArrayList>(); + db.readLock(); + try { + // get infos + for (Tablet tablet : index.getTablets()) { + long tabletId = tablet.getId(); + if (tablet.getReplicas().size() == 0) { + List tabletInfo = new ArrayList(); + tabletInfo.add(tabletId); + tabletInfo.add(-1); + tabletInfo.add(-1); + tabletInfo.add(-1); + tabletInfo.add(-1); + tabletInfo.add(-1); + tabletInfo.add(-1); + tabletInfo.add("N/A"); + tabletInfo.add("N/A"); + tabletInfo.add(-1); + tabletInfo.add(-1); + tabletInfo.add(-1); + + tabletInfos.add(tabletInfo); + } else { + for (Replica replica : tablet.getReplicas()) { + List tabletInfo = new ArrayList(); + // tabletId -- replicaId -- backendId -- version -- versionHash -- dataSize -- rowCount -- state + tabletInfo.add(tabletId); + tabletInfo.add(replica.getId()); + long backendId = replica.getBackendId(); + tabletInfo.add(replica.getBackendId()); + Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendId); // backend may be dropped concurrently, ignore it. if (backend == null) { continue; } - String hostName = null; - try { - InetAddress address = InetAddress.getByName(backend.getHost()); - hostName = address.getHostName(); - } catch (UnknownHostException e) { - continue; - } - tabletInfo.add(hostName); - tabletInfo.add(replica.getVersion()); - tabletInfo.add(replica.getVersionHash()); - tabletInfo.add(replica.getDataSize()); - tabletInfo.add(replica.getRowCount()); - tabletInfo.add(replica.getState()); - - tabletInfo.add(TimeUtils.longToTimeString(tablet.getLastCheckTime())); - tabletInfo.add(tablet.getCheckedVersion()); - tabletInfo.add(tablet.getCheckedVersionHash()); - - tabletInfos.add(tabletInfo); - } - } - } - } finally { - db.readUnlock(); - } - - // sort by tabletId, replicaId - ListComparator> comparator = new ListComparator>(0, 1); - Collections.sort(tabletInfos, comparator); - - // set result - BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - - for (int i = 0; i < tabletInfos.size(); i++) { - List info = tabletInfos.get(i); - List row = new ArrayList(info.size()); - for (int j = 0; j < info.size(); j++) { - row.add(info.get(j).toString()); - } - result.addRow(row); - } - return result; - } - - @Override - public boolean register(String name, ProcNodeInterface node) { - return false; - } - - @Override - public ProcNodeInterface lookup(String tabletIdStr) throws AnalysisException { - Preconditions.checkNotNull(db); - Preconditions.checkNotNull(index); - - long tabletId = -1L; - try { - tabletId = Long.valueOf(tabletIdStr); - } catch (NumberFormatException e) { - throw new AnalysisException("Invalid tablet id format: " + tabletIdStr); - } - - db.readLock(); - try { - Tablet tablet = index.getTablet(tabletId); - if (tablet == null) { - throw new AnalysisException("Tablet[" + tabletId + "] does not exist."); - } - return new ReplicasProcNode(db, tablet); - } finally { - db.readUnlock(); - } - } - -} + String hostName = null; + try { + InetAddress address = InetAddress.getByName(backend.getHost()); + hostName = address.getHostName(); + } catch (UnknownHostException e) { + continue; + } + tabletInfo.add(hostName); + tabletInfo.add(replica.getVersion()); + tabletInfo.add(replica.getVersionHash()); + tabletInfo.add(replica.getDataSize()); + tabletInfo.add(replica.getRowCount()); + tabletInfo.add(replica.getState()); + + tabletInfo.add(TimeUtils.longToTimeString(tablet.getLastCheckTime())); + tabletInfo.add(tablet.getCheckedVersion()); + tabletInfo.add(tablet.getCheckedVersionHash()); + tabletInfo.add(replica.getVersionCount()); + + tabletInfos.add(tabletInfo); + } + } + } + } finally { + db.readUnlock(); + } + + // sort by tabletId, replicaId + ListComparator> comparator = new ListComparator>(0, 1); + Collections.sort(tabletInfos, comparator); + + // set result + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + for (int i = 0; i < tabletInfos.size(); i++) { + List info = tabletInfos.get(i); + List row = new ArrayList(info.size()); + for (int j = 0; j < info.size(); j++) { + row.add(info.get(j).toString()); + } + result.addRow(row); + } + return result; + } + + @Override + public boolean register(String name, ProcNodeInterface node) { + return false; + } + + @Override + public ProcNodeInterface lookup(String tabletIdStr) throws AnalysisException { + Preconditions.checkNotNull(db); + Preconditions.checkNotNull(index); + + long tabletId = -1L; + try { + tabletId = Long.valueOf(tabletIdStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid tablet id format: " + tabletIdStr); + } + + db.readLock(); + try { + Tablet tablet = index.getTablet(tabletId); + if (tablet == null) { + throw new AnalysisException("Tablet[" + tabletId + "] does not exist."); + } + return new ReplicasProcNode(db, tablet); + } finally { + db.readUnlock(); + } + } + +} + diff --git a/fe/src/com/baidu/palo/common/proc/UserPropertyProcNode.java b/fe/src/com/baidu/palo/common/proc/UserPropertyProcNode.java index 42604af431..c2b05bdc0c 100644 --- a/fe/src/com/baidu/palo/common/proc/UserPropertyProcNode.java +++ b/fe/src/com/baidu/palo/common/proc/UserPropertyProcNode.java @@ -20,32 +20,33 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.catalog.UserPropertyMgr; -import com.baidu.palo.common.AnalysisException; -import com.google.common.collect.ImmutableList; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.mysql.privilege.PaloAuth; + +import com.google.common.collect.ImmutableList; -/* - * SHOW PROC '/access_resource/user' +/* + * SHOW PROC '/auth/user' */ public class UserPropertyProcNode implements ProcNodeInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("Key").add("Value") .build(); - private UserPropertyMgr userPropertyMgr; - private String user; + private PaloAuth auth; + private String qualifiedUser; - public UserPropertyProcNode(UserPropertyMgr userPropertyMgr, String user) { - this.userPropertyMgr = userPropertyMgr; - this.user = user; + public UserPropertyProcNode(PaloAuth auth, String qualifiedUser) { + this.auth = auth; + this.qualifiedUser = qualifiedUser; } @Override public ProcResult fetchResult() throws AnalysisException { BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - result.setRows(userPropertyMgr.fetchUserProperty(user)); + result.setNames(TITLE_NAMES); + + result.setRows(auth.getUserProperties(qualifiedUser)); return result; } - } diff --git a/fe/src/com/baidu/palo/common/util/TimeUtils.java b/fe/src/com/baidu/palo/common/util/TimeUtils.java index 1e6c2b3aa2..0634ff8128 100644 --- a/fe/src/com/baidu/palo/common/util/TimeUtils.java +++ b/fe/src/com/baidu/palo/common/util/TimeUtils.java @@ -20,23 +20,23 @@ package com.baidu.palo.common.util; -import com.baidu.palo.catalog.PrimitiveType; -import com.baidu.palo.catalog.Type; -import com.baidu.palo.common.AnalysisException; - -import com.google.common.base.Preconditions; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import java.text.ParseException; -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.SimpleTimeZone; -import java.util.TimeZone; -import java.util.regex.Matcher; +import com.baidu.palo.catalog.PrimitiveType; +import com.baidu.palo.catalog.Type; +import com.baidu.palo.common.AnalysisException; + +import com.google.common.base.Preconditions; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.text.ParseException; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.SimpleTimeZone; +import java.util.TimeZone; +import java.util.regex.Matcher; import java.util.regex.Pattern; // TODO(dhc) add nanosecond timer for coordinator's root profile @@ -102,11 +102,15 @@ public class TimeUtils { return DATETIME_FORMAT.format(new Date()); } - public static synchronized String longToTimeString(long timeStamp) { - if (timeStamp < 0L) { - return "N/A"; - } - return DATETIME_FORMAT.format(new Date(timeStamp)); + public static String longToTimeString(long timeStamp, SimpleDateFormat dateFormat) { + if (timeStamp < 0L) { + return "N/A"; + } + return dateFormat.format(new Date(timeStamp)); + } + + public static synchronized String longToTimeString(long timeStamp) { + return longToTimeString(timeStamp, DATETIME_FORMAT); } public static synchronized Date getTimeAsDate(String timeString) { @@ -189,5 +193,15 @@ public class TimeUtils { public static long dateTransform(long time, Type type) { return dateTransform(time, type.getPrimitiveType()); + } + + public static long timeStringToLong(String timeStr) { + Date d; + try { + d = DATETIME_FORMAT.parse(timeStr); + } catch (ParseException e) { + return -1; + } + return d.getTime(); } } diff --git a/fe/src/com/baidu/palo/consistency/ConsistencyChecker.java b/fe/src/com/baidu/palo/consistency/ConsistencyChecker.java index dd7b558fab..1b52b9f941 100644 --- a/fe/src/com/baidu/palo/consistency/ConsistencyChecker.java +++ b/fe/src/com/baidu/palo/consistency/ConsistencyChecker.java @@ -242,6 +242,9 @@ public class ConsistencyChecker extends Daemon { // sort dbs List dbIds = catalog.getDbIds(); + if (dbIds.isEmpty()) { + return -1L; + } Queue dbQueue = new PriorityQueue(dbIds.size(), COMPARATOR); for (Long dbId : dbIds) { if (dbId == 0L) { diff --git a/fe/src/com/baidu/palo/deploy/DeployManager.java b/fe/src/com/baidu/palo/deploy/DeployManager.java index 732e2caebd..16aebae39b 100644 --- a/fe/src/com/baidu/palo/deploy/DeployManager.java +++ b/fe/src/com/baidu/palo/deploy/DeployManager.java @@ -630,4 +630,3 @@ public class DeployManager extends Daemon { } } } - diff --git a/fe/src/com/baidu/palo/deploy/impl/LocalFileDeployManager.java b/fe/src/com/baidu/palo/deploy/impl/LocalFileDeployManager.java index 7c85f9f5c2..4db5a30320 100644 --- a/fe/src/com/baidu/palo/deploy/impl/LocalFileDeployManager.java +++ b/fe/src/com/baidu/palo/deploy/impl/LocalFileDeployManager.java @@ -68,7 +68,7 @@ public class LocalFileDeployManager extends DeployManager { @Override public List> getGroupHostPorts(String groupName) { List> result = Lists.newArrayList(); - LOG.debug("begin to get group: {} from file: {}", groupName, clusterInfoFile); + LOG.info("begin to get group: {} from file: {}", groupName, clusterInfoFile); FileChannel channel = null; FileLock lock = null; diff --git a/fe/src/com/baidu/palo/ha/BDBHA.java b/fe/src/com/baidu/palo/ha/BDBHA.java index 3df7877b63..8ab589c2e8 100644 --- a/fe/src/com/baidu/palo/ha/BDBHA.java +++ b/fe/src/com/baidu/palo/ha/BDBHA.java @@ -120,8 +120,8 @@ public class BDBHA implements HAProtocol { return null; } List ret = new ArrayList(); - try { - ReplicationGroup replicationGroup = replicationGroupAdmin.getGroup(); + try { + ReplicationGroup replicationGroup = replicationGroupAdmin.getGroup(); for (ReplicationNode replicationNode : replicationGroup.getElectableNodes()) { if (leaderIncluded) { ret.add(replicationNode.getSocketAddress()); diff --git a/fe/src/com/baidu/palo/ha/BDBStateChangeListener.java b/fe/src/com/baidu/palo/ha/BDBStateChangeListener.java index d318e0bd57..946f7d2895 100644 --- a/fe/src/com/baidu/palo/ha/BDBStateChangeListener.java +++ b/fe/src/com/baidu/palo/ha/BDBStateChangeListener.java @@ -15,14 +15,15 @@ package com.baidu.palo.ha; -import org.apache.logging.log4j.LogManager; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.persist.EditLog; + +import com.sleepycat.je.rep.StateChangeEvent; +import com.sleepycat.je.rep.StateChangeListener; + +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.persist.EditLog; -import com.sleepycat.je.rep.StateChangeEvent; -import com.sleepycat.je.rep.StateChangeListener; - public class BDBStateChangeListener implements StateChangeListener { public static final Logger LOG = LogManager.getLogger(EditLog.class); @@ -30,7 +31,7 @@ public class BDBStateChangeListener implements StateChangeListener { } @Override - public synchronized void stateChange(StateChangeEvent sce) throws RuntimeException { + public synchronized void stateChange(StateChangeEvent sce) throws RuntimeException { FrontendNodeType originalType = Catalog.getInstance().getFeType(); switch (sce.getState()) { case MASTER: { diff --git a/fe/src/com/baidu/palo/ha/MasterInfo.java b/fe/src/com/baidu/palo/ha/MasterInfo.java index 1af4bbc9f0..0a45d4f652 100644 --- a/fe/src/com/baidu/palo/ha/MasterInfo.java +++ b/fe/src/com/baidu/palo/ha/MasterInfo.java @@ -13,25 +13,25 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.ha; - +package com.baidu.palo.ha; + import com.baidu.palo.common.io.Text; import com.baidu.palo.common.io.Writable; import java.io.DataInput; import java.io.DataOutput; -import java.io.IOException; - -public class MasterInfo implements Writable { - - private String ip; - private int httpPort; +import java.io.IOException; + +public class MasterInfo implements Writable { + + private String ip; + private int httpPort; private int rpcPort; - - public MasterInfo() { - this.ip = ""; - this.httpPort = 0; - this.rpcPort = 0; + + public MasterInfo() { + this.ip = ""; + this.httpPort = 0; + this.rpcPort = 0; } public MasterInfo(String ip, int httpPort, int rpcPort) { @@ -39,44 +39,43 @@ public class MasterInfo implements Writable { this.httpPort = httpPort; this.rpcPort = rpcPort; } - - public String getIp() { - return this.ip; - } - - public void setIp(String ip) { - this.ip = ip; - } - - public int getHttpPort() { - return this.httpPort; - } - - public void setHttpPort(int httpPort) { - this.httpPort = httpPort; - } - - public int getRpcPort() { - return this.rpcPort; - } - - public void setRpcPort(int rpcPort) { - this.rpcPort = rpcPort; - } - - @Override - public void write(DataOutput out) throws IOException { - Text.writeString(out, ip); - out.writeInt(httpPort); - out.writeInt(rpcPort); - } - - @Override - public void readFields(DataInput in) throws IOException { - ip = Text.readString(in); - httpPort = in.readInt(); - rpcPort = in.readInt(); - } - -} - + + public String getIp() { + return this.ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public int getHttpPort() { + return this.httpPort; + } + + public void setHttpPort(int httpPort) { + this.httpPort = httpPort; + } + + public int getRpcPort() { + return this.rpcPort; + } + + public void setRpcPort(int rpcPort) { + this.rpcPort = rpcPort; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, ip); + out.writeInt(httpPort); + out.writeInt(rpcPort); + } + + @Override + public void readFields(DataInput in) throws IOException { + ip = Text.readString(in); + httpPort = in.readInt(); + rpcPort = in.readInt(); + } + +} diff --git a/fe/src/com/baidu/palo/http/BaseAction.java b/fe/src/com/baidu/palo/http/BaseAction.java index 9917076bde..ff3b4c7705 100644 --- a/fe/src/com/baidu/palo/http/BaseAction.java +++ b/fe/src/com/baidu/palo/http/BaseAction.java @@ -15,11 +15,10 @@ package com.baidu.palo.http; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; -import com.baidu.palo.mysql.MysqlPassword; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.qe.QeService; import com.baidu.palo.system.SystemInfoService; @@ -90,9 +89,13 @@ public abstract class BaseAction implements IAction { try { execute(request, response); } catch (Exception e) { - LOG.warn("fail to process url={}. error={}", - request.getRequest().uri(), e); - writeResponse(request, response, HttpResponseStatus.NOT_FOUND); + LOG.warn("fail to process url: {}", request.getRequest().uri(), e); + if (e instanceof UnauthorizedException) { + response.updateHeader(HttpHeaders.Names.WWW_AUTHENTICATE, "Basic realm=\"\""); + writeResponse(request, response, HttpResponseStatus.UNAUTHORIZED); + } else { + writeResponse(request, response, HttpResponseStatus.NOT_FOUND); + } } } @@ -230,11 +233,67 @@ public abstract class BaseAction implements IAction { public static class AuthorizationInfo { public String fullUserName; + public String remoteIp; public String password; public String cluster; + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("user: ").append(fullUserName).append(", remote ip: ").append(remoteIp); + sb.append(", password: ").append(password).append(", cluster: ").append(cluster); + return sb.toString(); + } } - public boolean parseAuth(BaseRequest request, AuthorizationInfo authInfo) { + protected void checkGlobalAuth(AuthorizationInfo authInfo, PrivPredicate predicate) throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(authInfo.remoteIp, + authInfo.fullUserName, + predicate)) { + throw new UnauthorizedException("Access denied; you need (at least one of) the " + + predicate.getPrivs().toString() + " privilege(s) for this operation"); + } + } + + protected void checkDbAuth(AuthorizationInfo authInfo, String db, PrivPredicate predicate) + throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(authInfo.remoteIp, db, authInfo.fullUserName, + predicate)) { + throw new UnauthorizedException("Access denied; you need (at least one of) the " + + predicate.getPrivs().toString() + " privilege(s) for this operation"); + } + } + + protected void checkTblAuth(AuthorizationInfo authInfo, String db, String tbl, PrivPredicate predicate) + throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(authInfo.remoteIp, db, authInfo.fullUserName, + tbl, predicate)) { + throw new UnauthorizedException("Access denied; you need (at least one of) the " + + predicate.getPrivs().toString() + " privilege(s) for this operation"); + } + } + + protected void checkPassword(AuthorizationInfo authInfo) + throws UnauthorizedException { + if (!Catalog.getCurrentCatalog().getAuth().checkPlainPassword(authInfo.fullUserName, + authInfo.remoteIp, + authInfo.password)) { + throw new UnauthorizedException("Access denied for " + + authInfo.fullUserName + "@" + authInfo.remoteIp); + } + } + + public AuthorizationInfo getAuthorizationInfo(BaseRequest request) + throws UnauthorizedException { + AuthorizationInfo authInfo = new AuthorizationInfo(); + if (!parseAuthInfo(request, authInfo)) { + throw new UnauthorizedException("Need auth information."); + } + LOG.debug("get auth info: {}", authInfo); + return authInfo; + } + + private boolean parseAuthInfo(BaseRequest request, AuthorizationInfo authInfo) { String encodedAuthString = request.getAuthorizationHeader(); if (Strings.isNullOrEmpty(encodedAuthString)) { return false; @@ -265,6 +324,7 @@ public abstract class BaseAction implements IAction { authInfo.cluster = elements[1]; } authInfo.password = authString.substring(index + 1); + authInfo.remoteIp = request.getHostString(); } finally { // release the buf after using Unpooled.copiedBuffer // or it will get memory leak @@ -275,50 +335,6 @@ public abstract class BaseAction implements IAction { return true; } - // check authenticate information - private AuthorizationInfo checkAndGetUser(BaseRequest request) - throws UnauthorizedException { - AuthorizationInfo authInfo = new AuthorizationInfo(); - if (!parseAuth(request, authInfo)) { - throw new UnauthorizedException("Need auth information."); - } - byte[] hashedPasswd = catalog.getUserMgr().getPassword(authInfo.fullUserName); - if (hashedPasswd == null) { - // No such user - throw new UnauthorizedException("No such user(" + authInfo.fullUserName + ")"); - } - if (!MysqlPassword.checkPlainPass(hashedPasswd, authInfo.password)) { - throw new UnauthorizedException("Password error"); - } - return authInfo; - } - - protected void checkAdmin(BaseRequest request) throws UnauthorizedException { - final AuthorizationInfo authInfo = checkAndGetUser(request); - if (!catalog.getUserMgr().isAdmin(authInfo.fullUserName)) { - throw new UnauthorizedException("Administrator needed"); - } - } - - protected void checkReadPriv(String fullUserName, String fullDbName) - throws UnauthorizedException { - if (!catalog.getUserMgr().checkAccess(fullUserName, fullDbName, AccessPrivilege.READ_ONLY)) { - throw new UnauthorizedException("Read Privilege needed"); - } - } - - protected void checkWritePriv(String fullUserName, String fullDbName) - throws UnauthorizedException { - if (!catalog.getUserMgr().checkAccess(fullUserName, fullDbName, AccessPrivilege.READ_WRITE)) { - throw new UnauthorizedException("Write Privilege needed"); - } - } - - public AuthorizationInfo getAuthorizationInfo(BaseRequest request) - throws UnauthorizedException { - return checkAndGetUser(request); - } - protected int checkIntParam(String strParam) { return Integer.parseInt(strParam); } diff --git a/fe/src/com/baidu/palo/http/BaseRequest.java b/fe/src/com/baidu/palo/http/BaseRequest.java index 5b92263081..e99b3e3117 100644 --- a/fe/src/com/baidu/palo/http/BaseRequest.java +++ b/fe/src/com/baidu/palo/http/BaseRequest.java @@ -35,7 +35,7 @@ public class BaseRequest { protected HttpRequest request; protected Map params = Maps.newHashMap(); - private boolean isAdmin = false; + private boolean isAuthorized = false; private QueryStringDecoder decoder; public BaseRequest(ChannelHandlerContext ctx, HttpRequest request) { @@ -67,12 +67,12 @@ public class BaseRequest { this.params = params; } - public boolean isAdmin() { - return isAdmin; + public boolean isAuthorized() { + return isAuthorized; } - public void setAdmin(boolean isAdmin) { - this.isAdmin = isAdmin; + public void setAuthorized(boolean isAuthorized) { + this.isAuthorized = isAuthorized; } public Cookie getCookieByName(String cookieName) { @@ -114,7 +114,7 @@ public class BaseRequest { return params.get(key); } - // get an array patameter. + // get an array parameter. // eg. ?a=1&a=2 public List getArrayParameter(String key) { String uri = request.uri(); diff --git a/fe/src/com/baidu/palo/http/HttpAuthManager.java b/fe/src/com/baidu/palo/http/HttpAuthManager.java index e1c6be2a13..cfc6aeca1a 100755 --- a/fe/src/com/baidu/palo/http/HttpAuthManager.java +++ b/fe/src/com/baidu/palo/http/HttpAuthManager.java @@ -43,9 +43,7 @@ public final class HttpAuthManager { } public String getUsername(String sessionId) { - String username = null; - username = authSessions.getIfPresent(sessionId); - return username; + return authSessions.getIfPresent(sessionId); } public void addClient(String key, String value) { diff --git a/fe/src/com/baidu/palo/http/HttpServer.java b/fe/src/com/baidu/palo/http/HttpServer.java index aabdab1cf2..8b723a4cd0 100755 --- a/fe/src/com/baidu/palo/http/HttpServer.java +++ b/fe/src/com/baidu/palo/http/HttpServer.java @@ -55,6 +55,7 @@ import com.baidu.palo.http.rest.SetConfigAction; import com.baidu.palo.http.rest.ShowMetaInfoAction; import com.baidu.palo.http.rest.ShowProcAction; import com.baidu.palo.http.rest.ShowRuntimeInfoAction; +import com.baidu.palo.http.rest.StorageTypeCheckAction; import com.baidu.palo.master.MetaHelper; import com.baidu.palo.qe.QeService; @@ -100,6 +101,7 @@ public class HttpServer { SetConfigAction.registerAction(controller); GetDdlStmtAction.registerAction(controller); MigrationAction.registerAction(controller); + StorageTypeCheckAction.registerAction(controller); // add web action IndexAction.registerAction(controller); diff --git a/fe/src/com/baidu/palo/http/action/HaAction.java b/fe/src/com/baidu/palo/http/action/HaAction.java index f39c6fb275..3c0cab1815 100644 --- a/fe/src/com/baidu/palo/http/action/HaAction.java +++ b/fe/src/com/baidu/palo/http/action/HaAction.java @@ -169,4 +169,4 @@ public class HaAction extends WebBaseAction { buffer.append(""); } -} +} \ No newline at end of file diff --git a/fe/src/com/baidu/palo/http/action/HelpAction.java b/fe/src/com/baidu/palo/http/action/HelpAction.java index dc9c45fc62..a0b318a8f3 100755 --- a/fe/src/com/baidu/palo/http/action/HelpAction.java +++ b/fe/src/com/baidu/palo/http/action/HelpAction.java @@ -24,10 +24,10 @@ import com.baidu.palo.qe.HelpTopic; import com.google.common.base.Strings; -import io.netty.handler.codec.http.HttpMethod; - import java.util.List; +import io.netty.handler.codec.http.HttpMethod; + public class HelpAction extends WebBaseAction { private static final String DIV_BACKGROUND_COLOR = "#FCFCFC"; diff --git a/fe/src/com/baidu/palo/http/action/SystemAction.java b/fe/src/com/baidu/palo/http/action/SystemAction.java index 753fc007de..862bb5ba67 100755 --- a/fe/src/com/baidu/palo/http/action/SystemAction.java +++ b/fe/src/com/baidu/palo/http/action/SystemAction.java @@ -26,10 +26,10 @@ import com.baidu.palo.http.IllegalArgException; import com.google.common.base.Strings; -import io.netty.handler.codec.http.HttpMethod; - import java.util.List; +import io.netty.handler.codec.http.HttpMethod; + public class SystemAction extends WebBaseAction { public SystemAction(ActionController controller) { @@ -110,7 +110,8 @@ public class SystemAction extends WebBaseAction { for (String str : strList) { buffer.append("

"); if (isDir && columnIndex == 1) { - buffer.append(""); + String escapeStr = str.replace("%", "%25"); + buffer.append(""); buffer.append(str); buffer.append(""); } else { diff --git a/fe/src/com/baidu/palo/http/action/WebBaseAction.java b/fe/src/com/baidu/palo/http/action/WebBaseAction.java index eee93e2afa..48db15b1af 100644 --- a/fe/src/com/baidu/palo/http/action/WebBaseAction.java +++ b/fe/src/com/baidu/palo/http/action/WebBaseAction.java @@ -15,9 +15,10 @@ package com.baidu.palo.http.action; +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.Config; -import com.baidu.palo.common.DdlException; import com.baidu.palo.common.proc.ProcNodeInterface; import com.baidu.palo.common.proc.ProcService; import com.baidu.palo.http.ActionController; @@ -25,14 +26,18 @@ import com.baidu.palo.http.BaseAction; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.HttpAuthManager; +import com.baidu.palo.http.UnauthorizedException; import com.baidu.palo.http.rest.RestBaseResult; +import com.baidu.palo.mysql.privilege.PaloPrivilege; +import com.baidu.palo.mysql.privilege.PrivBitSet; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.net.InetSocketAddress; import java.util.List; import java.util.UUID; @@ -43,7 +48,6 @@ import io.netty.handler.codec.http.HttpResponseStatus; public class WebBaseAction extends BaseAction { private static final Logger LOG = LogManager.getLogger(WebBaseAction.class); - private static final String ADMIN_USER = "root"; protected static final String LINE_SEP = System.getProperty("line.separator"); @@ -109,7 +113,7 @@ public class WebBaseAction extends BaseAction { @Override public void execute(BaseRequest request, BaseResponse response) { - if (!checkAuth(request, response)) { + if (!checkAuthWithCookie(request, response)) { return; } @@ -119,66 +123,89 @@ public class WebBaseAction extends BaseAction { } else if (method.equals(HttpMethod.POST)) { executePost(request, response); } else { - response.appendContent(new RestBaseResult("HTTP method is not allowed.").toJson()); + response.appendContent(new RestBaseResult("HTTP method is not allowed: " + method.name()).toJson()); writeResponse(request, response, HttpResponseStatus.METHOD_NOT_ALLOWED); } } - // Sub Action class should overvide this method + // Sub Action class should override this method public void executeGet(BaseRequest request, BaseResponse response) { response.appendContent(new RestBaseResult("Not implemented").toJson()); writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED); } - // Sub Action class should overvide this method + // Sub Action class should override this method public void executePost(BaseRequest request, BaseResponse response) { response.appendContent(new RestBaseResult("Not implemented").toJson()); writeResponse(request, response, HttpResponseStatus.NOT_IMPLEMENTED); } // We first check cookie, if not admin, we check http's authority header - protected boolean checkAuth(BaseRequest request, BaseResponse response) { - if (checkAuthByCookie(request, response)) { + private boolean checkAuthWithCookie(BaseRequest request, BaseResponse response) { + if (!needPassword()) { return true; } - if (needAdmin()) { - try { - checkAdmin(request); - request.setAdmin(true); - addSession(request, response, ADMIN_USER); - return true; - } catch (DdlException e) { - response.appendContent("Authentication Failed.
" - + "You can only access '/help' page without login!"); - writeAuthResponse(request, response); - return false; - } + if (checkCookie(request, response)) { + return true; } - return true; + // cookie is invalid. + AuthorizationInfo authInfo; + try { + authInfo = getAuthorizationInfo(request); + checkPassword(authInfo); + if (needAdmin()) { + checkGlobalAuth(authInfo, PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.NODE_PRIV), + Operator.OR)); + } + request.setAuthorized(true); + addSession(request, response, authInfo.fullUserName); + + ConnectContext ctx = new ConnectContext(null); + ctx.setQualifiedUser(authInfo.fullUserName); + ctx.setRemoteIP(authInfo.remoteIp); + ctx.setThreadLocalInfo(); + + return true; + } catch (UnauthorizedException e) { + response.appendContent("Authentication Failed.
" + e.getMessage()); + writeAuthResponse(request, response); + return false; + } } - protected boolean checkAuthByCookie(BaseRequest request, BaseResponse response) { + private boolean checkCookie(BaseRequest request, BaseResponse response) { String sessionId = request.getCookieValue(PALO_SESSION_ID); HttpAuthManager authMgr = HttpAuthManager.getInstance(); - String username = ""; if (!Strings.isNullOrEmpty(sessionId)) { - username = authMgr.getUsername(sessionId); - if (!Strings.isNullOrEmpty(username)) { - if (username.equals(ADMIN_USER)) { - response.updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME); - request.setAdmin(true); - return true; - } + String username = authMgr.getUsername(sessionId); + if (Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(request.getHostString(), username, + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.NODE_PRIV), + Operator.OR))) { + response.updateCookieAge(request, PALO_SESSION_ID, PALO_SESSION_EXPIRED_TIME); + request.setAuthorized(true); + + ConnectContext ctx = new ConnectContext(null); + ctx.setQualifiedUser(username); + ctx.setRemoteIP(request.getHostString()); + ctx.setThreadLocalInfo(); + return true; } } return false; } - // ATTN: sub Action classes can override it when there is no need to check authority. - // eg. It is no need admin privileges to access to HelpAction, so we will override this - // mothod in HelpAction by returning false. + // return true if this Action need to check password. + // Currently, all sub actions need to check password except for MetaBaseAction. + // if needPassword() is false, then needAdmin() should also return false + public boolean needPassword() { + return true; + } + + // return true if this Action need Admin privilege. public boolean needAdmin() { return true; } @@ -193,9 +220,6 @@ public class WebBaseAction extends BaseAction { } protected void addSession(BaseRequest request, BaseResponse response, String value) { - // We use hashcode of client's IP and timestamp, which not only can identify users from - // different host machine, but also can improve the difficulty of forging cookie. - int clientAddrHashCode = ((InetSocketAddress) request.getContext().channel().remoteAddress()).hashCode(); String key = UUID.randomUUID().toString(); DefaultCookie cookie = new DefaultCookie(PALO_SESSION_ID, key); cookie.setMaxAge(PALO_SESSION_EXPIRED_TIME); @@ -212,7 +236,7 @@ public class WebBaseAction extends BaseAction { sb.append(NAVIGATION_BAR_PREFIX); // TODO(lingbin): maybe should change to register the menu item? - if (request.isAdmin()) { + if (request.isAuthorized()) { sb.append("
  • ") .append("system") .append("
    • "); diff --git a/fe/src/com/baidu/palo/http/meta/MetaBaseAction.java b/fe/src/com/baidu/palo/http/meta/MetaBaseAction.java index 4a1f9e4d26..6032e21d53 100644 --- a/fe/src/com/baidu/palo/http/meta/MetaBaseAction.java +++ b/fe/src/com/baidu/palo/http/meta/MetaBaseAction.java @@ -49,6 +49,11 @@ public class MetaBaseAction extends WebBaseAction { return false; } + @Override + public boolean needPassword() { + return false; + } + @Override public void execute(BaseRequest request, BaseResponse response) { if (needCheckClientIsFe()) { diff --git a/fe/src/com/baidu/palo/http/meta/MetaService.java b/fe/src/com/baidu/palo/http/meta/MetaService.java index 0b14b8ba03..40c0b8b319 100644 --- a/fe/src/com/baidu/palo/http/meta/MetaService.java +++ b/fe/src/com/baidu/palo/http/meta/MetaService.java @@ -321,7 +321,7 @@ public class MetaService { public void executeGet(BaseRequest request, BaseResponse response) { /* * Before dump, we acquired the catalog read lock and all databases' read lock and all - * the jobs' read lock. This will guarantee the consistance of database and job queues. + * the jobs' read lock. This will guarantee the consistency of database and job queues. * But Backend may still inconsistent. */ diff --git a/fe/src/com/baidu/palo/http/rest/BootstrapFinishAction.java b/fe/src/com/baidu/palo/http/rest/BootstrapFinishAction.java index 85b76f6de1..d7996b2f73 100644 --- a/fe/src/com/baidu/palo/http/rest/BootstrapFinishAction.java +++ b/fe/src/com/baidu/palo/http/rest/BootstrapFinishAction.java @@ -16,18 +16,11 @@ package com.baidu.palo.http.rest; import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.Pair; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; -import com.baidu.palo.system.SystemInfoService; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -import java.util.List; import io.netty.handler.codec.http.HttpMethod; @@ -50,7 +43,6 @@ public class BootstrapFinishAction extends RestBaseAction { @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { - boolean canRead = Catalog.getInstance().canRead(); // to json response diff --git a/fe/src/com/baidu/palo/http/rest/CheckDecommissionAction.java b/fe/src/com/baidu/palo/http/rest/CheckDecommissionAction.java index 9d03e3245e..affe0c7a2a 100644 --- a/fe/src/com/baidu/palo/http/rest/CheckDecommissionAction.java +++ b/fe/src/com/baidu/palo/http/rest/CheckDecommissionAction.java @@ -23,7 +23,9 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.system.SystemInfoService; + import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -50,8 +52,9 @@ public class CheckDecommissionAction extends RestBaseAction { } @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - checkAdmin(request); + public void executeWithoutPassword(AuthorizationInfo authInfo, BaseRequest request, BaseResponse response) + throws DdlException { + checkGlobalAuth(authInfo, PrivPredicate.OPERATOR); String hostPorts = request.getSingleParameter(HOST_PORTS); if (Strings.isNullOrEmpty(hostPorts)) { diff --git a/fe/src/com/baidu/palo/http/rest/GetDdlStmtAction.java b/fe/src/com/baidu/palo/http/rest/GetDdlStmtAction.java index f090796d8a..67af99d0dc 100644 --- a/fe/src/com/baidu/palo/http/rest/GetDdlStmtAction.java +++ b/fe/src/com/baidu/palo/http/rest/GetDdlStmtAction.java @@ -23,6 +23,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -32,11 +33,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; -import io.netty.handler.codec.http.HttpMethod; - import java.util.List; import java.util.Map; +import io.netty.handler.codec.http.HttpMethod; + /* * used to get a table's ddl stmt * eg: @@ -57,8 +58,9 @@ public class GetDdlStmtAction extends RestBaseAction { } @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - checkAdmin(request); + public void executeWithoutPassword(AuthorizationInfo authInfo, BaseRequest request, BaseResponse response) + throws DdlException { + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); String dbName = request.getSingleParameter(DB_PARAM); String tableName = request.getSingleParameter(TABLE_PARAM); diff --git a/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java b/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java index 741b76022a..18e24caab0 100644 --- a/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java +++ b/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java @@ -15,13 +15,13 @@ package com.baidu.palo.http.rest; -import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.load.Load; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.base.Strings; @@ -43,8 +43,8 @@ public class GetLoadInfoAction extends RestBaseAction { } @Override - public void execute(BaseRequest request, BaseResponse response) throws DdlException { - AuthorizationInfo authInfo = getAuthorizationInfo(request); + public void executeWithoutPassword(AuthorizationInfo authInfo, BaseRequest request, BaseResponse response) + throws DdlException { Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY), request.getSingleParameter(LABEL_KEY), @@ -59,13 +59,19 @@ public class GetLoadInfoAction extends RestBaseAction { throw new DdlException("No cluster name selected"); } - String fullDbName = ClusterNamespace.getFullName(info.clusterName, info.dbName); - checkReadPriv(authInfo.fullUserName, fullDbName); - if (redirectToMaster(request, response)) { return; } catalog.getLoadInstance().getJobInfo(info); + + if (info.tblNames.isEmpty()) { + checkDbAuth(authInfo, info.dbName, PrivPredicate.LOAD); + } else { + for (String tblName : info.tblNames) { + checkTblAuth(authInfo, info.dbName, tblName, PrivPredicate.LOAD); + } + } + sendResult(request, response, new Result(info)); } diff --git a/fe/src/com/baidu/palo/http/rest/LoadAction.java b/fe/src/com/baidu/palo/http/rest/LoadAction.java index b9e88d1b97..393dbbad73 100644 --- a/fe/src/com/baidu/palo/http/rest/LoadAction.java +++ b/fe/src/com/baidu/palo/http/rest/LoadAction.java @@ -22,6 +22,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.baidu.palo.system.Backend; import com.baidu.palo.thrift.TNetworkAddress; @@ -93,7 +94,9 @@ public class LoadAction extends RestBaseAction { throw new DdlException("No label selected."); } - checkWritePriv(authInfo.fullUserName, fullDbName); + // check auth + checkTblAuth(authInfo, fullDbName, tableName, PrivPredicate.LOAD); + // Try to redirect to master if (redirectToMaster(request, response)) { return; @@ -102,12 +105,12 @@ public class LoadAction extends RestBaseAction { // Choose a backend sequentially. List backendIds = Catalog.getCurrentSystemInfo().seqChooseBackendIds(1, true, false, clusterName); if (backendIds == null) { - throw new DdlException("No live backend."); + throw new DdlException("No backend alive."); } Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0)); if (backend == null) { - throw new DdlException("No live backend."); + throw new DdlException("No backend alive."); } TNetworkAddress redirectAddr = new TNetworkAddress(backend.getHost(), backend.getHttpPort()); @@ -116,8 +119,6 @@ public class LoadAction extends RestBaseAction { } LOG.info("mini load redirect to backend: {}, label: {}", redirectAddr.toString(), label); - LOG.info("redrect address is {}, {}", backend.getHost(), backend.getHttpPort()); - redirectTo(request, response, redirectAddr); } } diff --git a/fe/src/com/baidu/palo/http/rest/MetaReplayerCheckAction.java b/fe/src/com/baidu/palo/http/rest/MetaReplayerCheckAction.java index 7198f37cb4..9b084227d0 100644 --- a/fe/src/com/baidu/palo/http/rest/MetaReplayerCheckAction.java +++ b/fe/src/com/baidu/palo/http/rest/MetaReplayerCheckAction.java @@ -21,13 +21,14 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import org.codehaus.jackson.map.ObjectMapper; -import io.netty.handler.codec.http.HttpMethod; - import java.util.Map; +import io.netty.handler.codec.http.HttpMethod; + /* * used to get meta replay info * eg: @@ -45,7 +46,8 @@ public class MetaReplayerCheckAction extends RestBaseAction { @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { - checkAdmin(request); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); Map resultMap = Catalog.getInstance().getMetaReplayState().getInfo(); diff --git a/fe/src/com/baidu/palo/http/rest/MigrationAction.java b/fe/src/com/baidu/palo/http/rest/MigrationAction.java index adaa9ddd82..0a7d4ae9c7 100644 --- a/fe/src/com/baidu/palo/http/rest/MigrationAction.java +++ b/fe/src/com/baidu/palo/http/rest/MigrationAction.java @@ -30,6 +30,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -38,11 +39,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; -import io.netty.handler.codec.http.HttpMethod; - import java.util.Collections; import java.util.List; +import io.netty.handler.codec.http.HttpMethod; + /* * used to get table's sorted tablet info * eg: @@ -64,7 +65,8 @@ public class MigrationAction extends RestBaseAction { @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { - checkAdmin(request); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); String dbName = request.getSingleParameter(DB_PARAM); String tableName = request.getSingleParameter(TABLE_PARAM); diff --git a/fe/src/com/baidu/palo/http/rest/MultiAbort.java b/fe/src/com/baidu/palo/http/rest/MultiAbort.java index 3381a88a5e..ff5e407ab7 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiAbort.java +++ b/fe/src/com/baidu/palo/http/rest/MultiAbort.java @@ -21,6 +21,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.google.common.base.Strings; @@ -57,8 +58,8 @@ public class MultiAbort extends RestBaseAction { AuthorizationInfo authInfo = getAuthorizationInfo(request); String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, db); + checkDbAuth(authInfo, fullDbName, PrivPredicate.LOAD); - checkWritePriv(authInfo.fullUserName, fullDbName); if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/MultiCommit.java b/fe/src/com/baidu/palo/http/rest/MultiCommit.java index d9fd0c7fd3..7ef45a944e 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiCommit.java +++ b/fe/src/com/baidu/palo/http/rest/MultiCommit.java @@ -21,6 +21,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.google.common.base.Strings; @@ -57,8 +58,8 @@ public class MultiCommit extends RestBaseAction { AuthorizationInfo authInfo = getAuthorizationInfo(request); String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, db); + checkDbAuth(authInfo, fullDbName, PrivPredicate.LOAD); - checkWritePriv(authInfo.fullUserName, fullDbName); if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/MultiDesc.java b/fe/src/com/baidu/palo/http/rest/MultiDesc.java index 50247faa70..ba2dbf4d3b 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiDesc.java +++ b/fe/src/com/baidu/palo/http/rest/MultiDesc.java @@ -21,6 +21,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.google.common.base.Strings; @@ -61,7 +62,7 @@ public class MultiDesc extends RestBaseAction { AuthorizationInfo authInfo = getAuthorizationInfo(request); String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, db); - checkReadPriv(authInfo.fullUserName, fullDbName); + checkDbAuth(authInfo, fullDbName, PrivPredicate.LOAD); if (redirectToMaster(request, response)) { return; diff --git a/fe/src/com/baidu/palo/http/rest/MultiList.java b/fe/src/com/baidu/palo/http/rest/MultiList.java index c1b99f7687..329a40fcae 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiList.java +++ b/fe/src/com/baidu/palo/http/rest/MultiList.java @@ -21,6 +21,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.google.common.base.Strings; @@ -56,7 +57,7 @@ public class MultiList extends RestBaseAction { AuthorizationInfo authInfo = getAuthorizationInfo(request); String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, db); - checkReadPriv(authInfo.fullUserName, fullDbName); + checkDbAuth(authInfo, fullDbName, PrivPredicate.LOAD); if (redirectToMaster(request, response)) { return; diff --git a/fe/src/com/baidu/palo/http/rest/MultiStart.java b/fe/src/com/baidu/palo/http/rest/MultiStart.java index b8ef9286aa..f557993e47 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiStart.java +++ b/fe/src/com/baidu/palo/http/rest/MultiStart.java @@ -22,6 +22,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.google.common.base.Strings; @@ -62,7 +63,7 @@ public class MultiStart extends RestBaseAction { AuthorizationInfo authInfo = getAuthorizationInfo(request); String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, db); - checkWritePriv(authInfo.fullUserName, fullDbName); + checkDbAuth(authInfo, fullDbName, PrivPredicate.LOAD); if (redirectToMaster(request, response)) { return; diff --git a/fe/src/com/baidu/palo/http/rest/MultiUnload.java b/fe/src/com/baidu/palo/http/rest/MultiUnload.java index d1c52654ec..218d440c90 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiUnload.java +++ b/fe/src/com/baidu/palo/http/rest/MultiUnload.java @@ -21,6 +21,7 @@ import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.service.ExecuteEnv; import com.google.common.base.Strings; @@ -62,7 +63,7 @@ public class MultiUnload extends RestBaseAction { AuthorizationInfo authInfo = getAuthorizationInfo(request); String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, db); - checkWritePriv(authInfo.fullUserName, fullDbName); + checkDbAuth(authInfo, fullDbName, PrivPredicate.LOAD); if (redirectToMaster(request, response)) { return; diff --git a/fe/src/com/baidu/palo/http/rest/RestBaseAction.java b/fe/src/com/baidu/palo/http/rest/RestBaseAction.java index b81c784242..201f4b82ba 100644 --- a/fe/src/com/baidu/palo/http/rest/RestBaseAction.java +++ b/fe/src/com/baidu/palo/http/rest/RestBaseAction.java @@ -57,7 +57,16 @@ public class RestBaseAction extends BaseAction { @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { - throw new DdlException("Do not implemented."); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + // check password + checkPassword(authInfo); + executeWithoutPassword(authInfo, request, response); + } + + // all derived classed should implement this method, NOT 'execute' + protected void executeWithoutPassword(AuthorizationInfo authInfo, BaseRequest request, BaseResponse response) + throws DdlException { + throw new DdlException("Not implemented"); } public void sendResult(BaseRequest request, BaseResponse response, RestBaseResult result) { diff --git a/fe/src/com/baidu/palo/http/rest/RowCountAction.java b/fe/src/com/baidu/palo/http/rest/RowCountAction.java index 50c94fa828..d1938e6ed9 100644 --- a/fe/src/com/baidu/palo/http/rest/RowCountAction.java +++ b/fe/src/com/baidu/palo/http/rest/RowCountAction.java @@ -22,13 +22,14 @@ import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.Replica; import com.baidu.palo.catalog.Table; -import com.baidu.palo.catalog.Tablet; import com.baidu.palo.catalog.Table.TableType; +import com.baidu.palo.catalog.Tablet; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.base.Strings; import com.google.common.collect.Maps; @@ -57,6 +58,9 @@ public class RowCountAction extends RestBaseAction { @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { + AuthorizationInfo authInfo = getAuthorizationInfo(request); + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); + String dbName = request.getSingleParameter(DB_NAME_PARAM); if (Strings.isNullOrEmpty(dbName)) { throw new DdlException("No database selected."); @@ -66,8 +70,6 @@ public class RowCountAction extends RestBaseAction { if (Strings.isNullOrEmpty(tableName)) { throw new DdlException("No table selected."); } - - checkAdmin(request); Map indexRowCountMap = Maps.newHashMap(); Catalog catalog = Catalog.getInstance(); diff --git a/fe/src/com/baidu/palo/http/rest/SetConfigAction.java b/fe/src/com/baidu/palo/http/rest/SetConfigAction.java index 893f871c48..6923b44b56 100644 --- a/fe/src/com/baidu/palo/http/rest/SetConfigAction.java +++ b/fe/src/com/baidu/palo/http/rest/SetConfigAction.java @@ -16,12 +16,13 @@ package com.baidu.palo.http.rest; import com.baidu.palo.common.ConfigBase; -import com.baidu.palo.common.DdlException; import com.baidu.palo.common.ConfigBase.ConfField; +import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.collect.Maps; @@ -29,12 +30,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; -import io.netty.handler.codec.http.HttpMethod; - import java.lang.reflect.Field; import java.util.List; import java.util.Map; +import io.netty.handler.codec.http.HttpMethod; + /* * used to set fe config * eg: @@ -54,7 +55,8 @@ public class SetConfigAction extends RestBaseAction { @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { - checkAdmin(request); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); Map> configs = request.getAllParameters(); Map setConfigs = Maps.newHashMap(); diff --git a/fe/src/com/baidu/palo/http/rest/ShowProcAction.java b/fe/src/com/baidu/palo/http/rest/ShowProcAction.java index e268b8df40..c2c2ccb347 100644 --- a/fe/src/com/baidu/palo/http/rest/ShowProcAction.java +++ b/fe/src/com/baidu/palo/http/rest/ShowProcAction.java @@ -24,6 +24,7 @@ import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.http.UnauthorizedException; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.base.Strings; import com.google.gson.Gson; @@ -31,10 +32,10 @@ import com.google.gson.Gson; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import io.netty.handler.codec.http.HttpMethod; - import java.util.List; +import io.netty.handler.codec.http.HttpMethod; + // Format: // http://username:password@10.73.150.30:8138/api/show_proc?path=/ public class ShowProcAction extends RestBaseAction { @@ -52,8 +53,8 @@ public class ShowProcAction extends RestBaseAction { public void execute(BaseRequest request, BaseResponse response) { // check authority try { - checkAdmin(request); - request.setAdmin(true); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); } catch (UnauthorizedException e) { response.appendContent("Authentication Failed. " + e.getMessage()); sendResult(request, response); diff --git a/fe/src/com/baidu/palo/http/rest/StorageTypeCheckAction.java b/fe/src/com/baidu/palo/http/rest/StorageTypeCheckAction.java new file mode 100644 index 0000000000..c442c5c039 --- /dev/null +++ b/fe/src/com/baidu/palo/http/rest/StorageTypeCheckAction.java @@ -0,0 +1,82 @@ +package com.baidu.palo.http.rest; + +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Table.TableType; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.http.ActionController; +import com.baidu.palo.http.BaseRequest; +import com.baidu.palo.http.BaseResponse; +import com.baidu.palo.http.IllegalArgException; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.thrift.TStorageType; + +import com.google.common.base.Strings; + +import org.json.JSONObject; + +import java.util.List; +import java.util.Map; + +import io.netty.handler.codec.http.HttpMethod; + +public class StorageTypeCheckAction extends RestBaseAction { + public StorageTypeCheckAction(ActionController controller) { + super(controller); + } + + public static void registerAction(ActionController controller) throws IllegalArgException { + StorageTypeCheckAction action = new StorageTypeCheckAction(controller); + controller.registerHandler(HttpMethod.GET, "/api/_check_storagetype", action); + } + + @Override + public void execute(BaseRequest request, BaseResponse response) throws DdlException { + AuthorizationInfo authInfo = getAuthorizationInfo(request); + checkGlobalAuth(authInfo, PrivPredicate.ADMIN); + + String dbName = request.getSingleParameter("db"); + if (Strings.isNullOrEmpty(dbName)) { + throw new DdlException("Parameter db is missing"); + } + + String fullDbName = ClusterNamespace.getFullName(authInfo.cluster, dbName); + Database db = catalog.getDb(fullDbName); + if (db == null) { + throw new DdlException("Database " + dbName + " does not exist"); + } + + JSONObject root = new JSONObject(); + db.readLock(); + try { + List tbls = db.getTables(); + for (Table tbl : tbls) { + if (tbl.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTbl = (OlapTable) tbl; + JSONObject indexObj = new JSONObject(); + for (Map.Entry entry : olapTbl.getIndexIdToStorageType().entrySet()) { + if (entry.getValue() == TStorageType.ROW) { + String idxName = olapTbl.getIndexNameById(entry.getKey()); + indexObj.put(idxName, entry.getValue().name()); + } + } + root.put(tbl.getName(), indexObj); + } + } finally { + db.readUnlock(); + } + + // to json response + String result = root.toString(); + + // send result + response.setContentType("application/json"); + response.getContent().append(result); + sendResult(request, response); + } +} diff --git a/fe/src/com/baidu/palo/journal/JournalEntity.java b/fe/src/com/baidu/palo/journal/JournalEntity.java index 8d5a4e44c1..382b0702eb 100644 --- a/fe/src/com/baidu/palo/journal/JournalEntity.java +++ b/fe/src/com/baidu/palo/journal/JournalEntity.java @@ -16,11 +16,14 @@ package com.baidu.palo.journal; import com.baidu.palo.alter.AlterJob; +import com.baidu.palo.analysis.UserIdentity; import com.baidu.palo.backup.BackupJob; +import com.baidu.palo.backup.BackupJob_D; +import com.baidu.palo.backup.Repository; import com.baidu.palo.backup.RestoreJob; +import com.baidu.palo.backup.RestoreJob_D; import com.baidu.palo.catalog.BrokerMgr; import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.UserProperty; import com.baidu.palo.cluster.BaseParam; import com.baidu.palo.cluster.Cluster; import com.baidu.palo.common.io.Text; @@ -33,6 +36,7 @@ import com.baidu.palo.load.ExportJob; import com.baidu.palo.load.LoadErrorHub; import com.baidu.palo.load.LoadJob; import com.baidu.palo.master.Checkpoint; +import com.baidu.palo.mysql.privilege.UserProperty; import com.baidu.palo.persist.BackendIdsUpdateInfo; import com.baidu.palo.persist.CloneInfo; import com.baidu.palo.persist.ClusterInfo; @@ -45,6 +49,7 @@ import com.baidu.palo.persist.DropPartitionInfo; import com.baidu.palo.persist.ModifyPartitionInfo; import com.baidu.palo.persist.OperationType; import com.baidu.palo.persist.PartitionPersistInfo; +import com.baidu.palo.persist.PrivInfo; import com.baidu.palo.persist.RecoverInfo; import com.baidu.palo.persist.ReplicaPersistInfo; import com.baidu.palo.persist.TableInfo; @@ -176,12 +181,22 @@ public class JournalEntity implements Writable { case OperationType.OP_BACKUP_START: case OperationType.OP_BACKUP_FINISH_SNAPSHOT: case OperationType.OP_BACKUP_FINISH: { - data = new BackupJob(); + data = new BackupJob_D(); break; } case OperationType.OP_RESTORE_START: case OperationType.OP_RESTORE_FINISH: { - data = new RestoreJob(); + data = new RestoreJob_D(); + break; + } + case OperationType.OP_BACKUP_JOB: { + data = BackupJob.read(in); + needRead = false; + break; + } + case OperationType.OP_RESTORE_JOB: { + data = RestoreJob.read(in); + needRead = false; break; } case OperationType.OP_FINISH_CONSISTENCY_CHECK: { @@ -245,6 +260,21 @@ public class JournalEntity implements Writable { data = new Text(); break; } + case OperationType.OP_NEW_DROP_USER: { + data = UserIdentity.read(in); + needRead = false; + break; + } + case OperationType.OP_CREATE_USER: + case OperationType.OP_GRANT_PRIV: + case OperationType.OP_REVOKE_PRIV: + case OperationType.OP_SET_PASSWORD: + case OperationType.OP_CREATE_ROLE: + case OperationType.OP_DROP_ROLE: { + data = PrivInfo.read(in); + needRead = false; + break; + } case OperationType.OP_MASTER_INFO_CHANGE: { data = new MasterInfo(); break; @@ -306,6 +336,15 @@ public class JournalEntity implements Writable { data = new BackendIdsUpdateInfo(); break; } + case OperationType.OP_CREATE_REPOSITORY: { + data = Repository.read(in); + needRead = false; + break; + } + case OperationType.OP_DROP_REPOSITORY: { + data = new Text(); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/src/com/baidu/palo/journal/bdbje/BDBEnvironment.java b/fe/src/com/baidu/palo/journal/bdbje/BDBEnvironment.java index 4b5e019bf5..f254a5ef7d 100644 --- a/fe/src/com/baidu/palo/journal/bdbje/BDBEnvironment.java +++ b/fe/src/com/baidu/palo/journal/bdbje/BDBEnvironment.java @@ -13,8 +13,8 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.journal.bdbje; - +package com.baidu.palo.journal.bdbje; + import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.Config; import com.baidu.palo.ha.BDBHA; @@ -52,175 +52,175 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/* this class contains the reference to bdb environment. - * including all the opened databases and the replicationGroupAdmin. - * we can get the information of this bdb group through the API of replicationGroupAdmin - */ -public class BDBEnvironment { - private static final Logger LOG = LogManager.getLogger(BDBEnvironment.class); - private static final int RETRY_TIME = 3; - private static final int MEMORY_CACHE_PERCENT = 20; - - public static final String PALO_JOURNAL_GROUP = "PALO_JOURNAL_GROUP"; - - private ReplicatedEnvironment replicatedEnvironment; - private EnvironmentConfig environmentConfig; - private ReplicationConfig replicationConfig; - private DatabaseConfig dbConfig; - private Database epochDB = null; // used for fencing - private ReplicationGroupAdmin replicationGroupAdmin = null; - private ReentrantReadWriteLock lock; - private List openedDatabases; - - public BDBEnvironment() { - openedDatabases = new ArrayList(); - this.lock = new ReentrantReadWriteLock(true); - } - - // The setup() method opens the environment and database - public void setup(File envHome, String selfNodeName, String selfNodeHostPort, - String helperHostPort, boolean isElectable) { - - // Almost never used, just in case the master can not restart - if (Config.metadata_failure_recovery.equals("true")) { - if (!isElectable) { - LOG.error("Current node is not in the electable_nodes list. will exit"); - System.exit(-1); - } - DbResetRepGroup resetUtility = new DbResetRepGroup(envHome, PALO_JOURNAL_GROUP, selfNodeName, - selfNodeHostPort); - resetUtility.reset(); - LOG.info("group has been reset."); +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/* this class contains the reference to bdb environment. + * including all the opened databases and the replicationGroupAdmin. + * we can get the information of this bdb group through the API of replicationGroupAdmin + */ +public class BDBEnvironment { + private static final Logger LOG = LogManager.getLogger(BDBEnvironment.class); + private static final int RETRY_TIME = 3; + private static final int MEMORY_CACHE_PERCENT = 20; + + public static final String PALO_JOURNAL_GROUP = "PALO_JOURNAL_GROUP"; + + private ReplicatedEnvironment replicatedEnvironment; + private EnvironmentConfig environmentConfig; + private ReplicationConfig replicationConfig; + private DatabaseConfig dbConfig; + private Database epochDB = null; // used for fencing + private ReplicationGroupAdmin replicationGroupAdmin = null; + private ReentrantReadWriteLock lock; + private List openedDatabases; + + public BDBEnvironment() { + openedDatabases = new ArrayList(); + this.lock = new ReentrantReadWriteLock(true); + } + + // The setup() method opens the environment and database + public void setup(File envHome, String selfNodeName, String selfNodeHostPort, + String helperHostPort, boolean isElectable) { + + // Almost never used, just in case the master can not restart + if (Config.metadata_failure_recovery.equals("true")) { + if (!isElectable) { + LOG.error("Current node is not in the electable_nodes list. will exit"); + System.exit(-1); + } + DbResetRepGroup resetUtility = new DbResetRepGroup(envHome, PALO_JOURNAL_GROUP, selfNodeName, + selfNodeHostPort); + resetUtility.reset(); + LOG.info("group has been reset."); } - - // set replication config - replicationConfig = new ReplicationConfig(); - replicationConfig.setNodeName(selfNodeName); - replicationConfig.setNodeHostPort(selfNodeHostPort); - replicationConfig.setHelperHosts(helperHostPort); - replicationConfig.setGroupName(PALO_JOURNAL_GROUP); + + // set replication config + replicationConfig = new ReplicationConfig(); + replicationConfig.setNodeName(selfNodeName); + replicationConfig.setNodeHostPort(selfNodeHostPort); + replicationConfig.setHelperHosts(helperHostPort); + replicationConfig.setGroupName(PALO_JOURNAL_GROUP); replicationConfig.setConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT, "10"); replicationConfig.setMaxClockDelta(Config.max_bdbje_clock_delta_ms, TimeUnit.MILLISECONDS); - - if (isElectable) { - replicationConfig.setReplicaAckTimeout(2, TimeUnit.SECONDS); + + if (isElectable) { + replicationConfig.setReplicaAckTimeout(2, TimeUnit.SECONDS); replicationConfig.setConfigParam(ReplicationConfig.REPLICA_MAX_GROUP_COMMIT, "0"); - replicationConfig.setConsistencyPolicy(new NoConsistencyRequiredPolicy()); - } else { - replicationConfig.setNodeType(NodeType.SECONDARY); - replicationConfig.setConsistencyPolicy(new NoConsistencyRequiredPolicy()); - } - - // set environment config - environmentConfig = new EnvironmentConfig(); - environmentConfig.setTransactional(true); - environmentConfig.setAllowCreate(true); - environmentConfig.setCachePercent(MEMORY_CACHE_PERCENT); - if (isElectable) { - Durability durability = new Durability(getSyncPolicy(Config.master_sync_policy), - getSyncPolicy(Config.replica_sync_policy), getAckPolicy(Config.replica_ack_policy)); - environmentConfig.setDurability(durability); - } - - // set database config - dbConfig = new DatabaseConfig(); - dbConfig.setTransactional(true); - if (isElectable) { - dbConfig.setAllowCreate(true); - dbConfig.setReadOnly(false); - } else { - dbConfig.setAllowCreate(false); - dbConfig.setReadOnly(true); - } - - // open environment and epochDB - for (int i = 0; i < RETRY_TIME; i++) { - try { - // open the environment - replicatedEnvironment = new ReplicatedEnvironment(envHome, replicationConfig, environmentConfig); - - // get replicationGroupAdmin object. - Set adminNodes = new HashSet(); - // 1. add helper node - InetSocketAddress helper = new InetSocketAddress(helperHostPort.split(":")[0], - Integer.parseInt(helperHostPort.split(":")[1])); - adminNodes.add(helper); - LOG.info("add helper[{}] as ReplicationGroupAdmin", helperHostPort); - // 2. add self if is electable - if (!selfNodeHostPort.equals(helperHostPort) && Catalog.getInstance().isElectable()) { - InetSocketAddress self = new InetSocketAddress(selfNodeHostPort.split(":")[0], - Integer.parseInt(selfNodeHostPort.split(":")[1])); - adminNodes.add(self); - LOG.info("add self[{}] as ReplicationGroupAdmin", selfNodeHostPort); - } - + replicationConfig.setConsistencyPolicy(new NoConsistencyRequiredPolicy()); + } else { + replicationConfig.setNodeType(NodeType.SECONDARY); + replicationConfig.setConsistencyPolicy(new NoConsistencyRequiredPolicy()); + } + + // set environment config + environmentConfig = new EnvironmentConfig(); + environmentConfig.setTransactional(true); + environmentConfig.setAllowCreate(true); + environmentConfig.setCachePercent(MEMORY_CACHE_PERCENT); + if (isElectable) { + Durability durability = new Durability(getSyncPolicy(Config.master_sync_policy), + getSyncPolicy(Config.replica_sync_policy), getAckPolicy(Config.replica_ack_policy)); + environmentConfig.setDurability(durability); + } + + // set database config + dbConfig = new DatabaseConfig(); + dbConfig.setTransactional(true); + if (isElectable) { + dbConfig.setAllowCreate(true); + dbConfig.setReadOnly(false); + } else { + dbConfig.setAllowCreate(false); + dbConfig.setReadOnly(true); + } + + // open environment and epochDB + for (int i = 0; i < RETRY_TIME; i++) { + try { + // open the environment + replicatedEnvironment = new ReplicatedEnvironment(envHome, replicationConfig, environmentConfig); + + // get replicationGroupAdmin object. + Set adminNodes = new HashSet(); + // 1. add helper node + InetSocketAddress helper = new InetSocketAddress(helperHostPort.split(":")[0], + Integer.parseInt(helperHostPort.split(":")[1])); + adminNodes.add(helper); + LOG.info("add helper[{}] as ReplicationGroupAdmin", helperHostPort); + // 2. add self if is electable + if (!selfNodeHostPort.equals(helperHostPort) && Catalog.getInstance().isElectable()) { + InetSocketAddress self = new InetSocketAddress(selfNodeHostPort.split(":")[0], + Integer.parseInt(selfNodeHostPort.split(":")[1])); + adminNodes.add(self); + LOG.info("add self[{}] as ReplicationGroupAdmin", selfNodeHostPort); + } + replicationGroupAdmin = new ReplicationGroupAdmin(PALO_JOURNAL_GROUP, adminNodes); - - // get a BDBHA object and pass the reference to Catalog - HAProtocol protocol = new BDBHA(this, selfNodeName); - Catalog.getInstance().setHaProtocol(protocol); - - // start state change listener - StateChangeListener listener = new BDBStateChangeListener(); + + // get a BDBHA object and pass the reference to Catalog + HAProtocol protocol = new BDBHA(this, selfNodeName); + Catalog.getInstance().setHaProtocol(protocol); + + // start state change listener + StateChangeListener listener = new BDBStateChangeListener(); replicatedEnvironment.setStateChangeListener(listener); - - // open epochDB. the first parameter null means auto-commit - epochDB = replicatedEnvironment.openDatabase(null, "epochDB", dbConfig); - break; - } catch (InsufficientLogException insufficientLogEx) { - NetworkRestore restore = new NetworkRestore(); - NetworkRestoreConfig config = new NetworkRestoreConfig(); - config.setRetainLogFiles(false); // delete obsolete log files. - // Use the members returned by insufficientLogEx.getLogProviders() - // to select the desired subset of members and pass the resulting - // list as the argument to config.setLogProviders(), if the - // default selection of providers is not suitable. - restore.execute(insufficientLogEx, config); - continue; - } catch (DatabaseException e) { - if (i < RETRY_TIME - 1) { - try { - Thread.sleep(5 * 1000); - } catch (InterruptedException e1) { - e1.printStackTrace(); - } - continue; - } else { - LOG.error("error to open replicated environment. will exit.", e); - System.exit(-1); - } - } - } - } - - public ReplicationGroupAdmin getReplicationGroupAdmin() { - return this.replicationGroupAdmin; + + // open epochDB. the first parameter null means auto-commit + epochDB = replicatedEnvironment.openDatabase(null, "epochDB", dbConfig); + break; + } catch (InsufficientLogException insufficientLogEx) { + NetworkRestore restore = new NetworkRestore(); + NetworkRestoreConfig config = new NetworkRestoreConfig(); + config.setRetainLogFiles(false); // delete obsolete log files. + // Use the members returned by insufficientLogEx.getLogProviders() + // to select the desired subset of members and pass the resulting + // list as the argument to config.setLogProviders(), if the + // default selection of providers is not suitable. + restore.execute(insufficientLogEx, config); + continue; + } catch (DatabaseException e) { + if (i < RETRY_TIME - 1) { + try { + Thread.sleep(5 * 1000); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + continue; + } else { + LOG.error("error to open replicated environment. will exit.", e); + System.exit(-1); + } + } + } + } + + public ReplicationGroupAdmin getReplicationGroupAdmin() { + return this.replicationGroupAdmin; } public void setNewReplicationGroupAdmin(Set newHelperNodes) { this.replicationGroupAdmin = new ReplicationGroupAdmin(PALO_JOURNAL_GROUP, newHelperNodes); } - - // Return a handle to the epochDB - public Database getEpochDB() { - return epochDB; - } - - // Return a handle to the environment + + // Return a handle to the epochDB + public Database getEpochDB() { + return epochDB; + } + + // Return a handle to the environment public ReplicatedEnvironment getReplicatedEnvironment() { - return replicatedEnvironment; - } - + return replicatedEnvironment; + } + // return the database reference with the given name - // also try to close previous opened database. - public Database openDatabase(String dbName) { - Database db = null; - lock.writeLock().lock(); + // also try to close previous opened database. + public Database openDatabase(String dbName) { + Database db = null; + lock.writeLock().lock(); try { - // find if the specified database is already opened. find and return it. - for (java.util.Iterator iter = openedDatabases.iterator(); iter.hasNext();) { + // find if the specified database is already opened. find and return it. + for (java.util.Iterator iter = openedDatabases.iterator(); iter.hasNext();) { Database openedDb = iter.next(); try { if (openedDb.getDatabaseName() == null) { @@ -251,157 +251,156 @@ public class BDBEnvironment { iter.remove(); continue; } + + if (openedDb.getDatabaseName().equals(dbName)) { + return openedDb; + } + } - if (openedDb.getDatabaseName().equals(dbName)) { - return openedDb; - } - } - - // open the specified database. - // the first parameter null means auto-commit - try { - db = replicatedEnvironment.openDatabase(null, dbName, dbConfig); - openedDatabases.add(db); - } catch (Exception e) { - LOG.warn("catch an exception when open database {}", dbName, e); - } - } finally { - lock.writeLock().unlock(); - } - return db; - } - - // close and remove the database whose name is dbName - public void removeDatabase(String dbName) { - lock.writeLock().lock(); - try { - String targetDbName = null; - int index = 0; - for (Database db : openedDatabases) { - String name = db.getDatabaseName(); - if (dbName.equals(name)) { + // open the specified database. + // the first parameter null means auto-commit + try { + db = replicatedEnvironment.openDatabase(null, dbName, dbConfig); + openedDatabases.add(db); + } catch (Exception e) { + LOG.warn("catch an exception when open database {}", dbName, e); + } + } finally { + lock.writeLock().unlock(); + } + return db; + } + + // close and remove the database whose name is dbName + public void removeDatabase(String dbName) { + lock.writeLock().lock(); + try { + String targetDbName = null; + int index = 0; + for (Database db : openedDatabases) { + String name = db.getDatabaseName(); + if (dbName.equals(name)) { db.close(); - LOG.info("database {} has been closed", name); - targetDbName = name; - break; - } - index++; - } - if (targetDbName != null) { - LOG.info("begin to remove database {} from openedDatabases", targetDbName); - openedDatabases.remove(index); - } - try { - LOG.info("begin to remove database {} from replicatedEnviroment", dbName); - // the first parameter null means auto-commit - replicatedEnvironment.removeDatabase(null, dbName); - } catch (DatabaseNotFoundException e) { - LOG.warn("catch an exception when remove db:{}, this db does not exist", dbName, e); - } - } finally { - lock.writeLock().unlock(); - } - } - - // get journal db names and sort the names - public List getDatabaseNames() { - List ret = new ArrayList(); - List names = null; - int tried = 0; - while (true) { - try { - names = replicatedEnvironment.getDatabaseNames(); - break; - } catch (InsufficientLogException e) { - throw e; - } catch (EnvironmentFailureException e) { - tried++; - if (tried == RETRY_TIME) { - LOG.error("bdb environment failure exception.", e); - System.exit(-1); - } - LOG.warn("bdb environment failure exception. will retry", e); - try { - Thread.sleep(1000); - } catch (InterruptedException e1) { - e1.printStackTrace(); - } - continue; - } catch (DatabaseException e) { - LOG.warn("catch an exception when calling getDatabaseNames", e); - return null; - } - } - - if (names != null) { - for (String name : names) { - // We don't count epochDB - if (name.equals("epochDB")) { - continue; - } - - long db = Long.parseLong(name); - ret.add(db); - } - } - - Collections.sort(ret); - return ret; - } - - // Close the store and environment - public void close() { - for (Database db : openedDatabases) { - try { - db.close(); - } catch (DatabaseException exception) { - LOG.error("Error closing db {} will exit", db.getDatabaseName(), exception); - System.exit(-1); - } - } - openedDatabases.clear(); - - if (epochDB != null) { - try { - epochDB.close(); - } catch (DatabaseException exception) { - LOG.error("Error closing db {} will exit", epochDB.getDatabaseName(), exception); - System.exit(-1); - } - } - - if (replicatedEnvironment != null) { - try { - // Finally, close the store and environment. - replicatedEnvironment.close(); - } catch (DatabaseException exception) { - LOG.error("Error closing replicatedEnviroment", exception); - System.exit(-1); - } - } - } - - private SyncPolicy getSyncPolicy(String policy) { - if (policy.equalsIgnoreCase("SYNC")) { - return Durability.SyncPolicy.SYNC; - } - if (policy.equalsIgnoreCase("NO_SYNC")) { - return Durability.SyncPolicy.NO_SYNC; - } - // default value is WRITE_NO_SYNC - return Durability.SyncPolicy.WRITE_NO_SYNC; - } - - private ReplicaAckPolicy getAckPolicy(String policy) { - if (policy.equalsIgnoreCase("ALL")) { - return Durability.ReplicaAckPolicy.ALL; - } - if (policy.equalsIgnoreCase("NONE")) { - return Durability.ReplicaAckPolicy.NONE; - } - // default value is SIMPLE_MAJORITY - return Durability.ReplicaAckPolicy.SIMPLE_MAJORITY; - } - -} - + LOG.info("database {} has been closed", name); + targetDbName = name; + break; + } + index++; + } + if (targetDbName != null) { + LOG.info("begin to remove database {} from openedDatabases", targetDbName); + openedDatabases.remove(index); + } + try { + LOG.info("begin to remove database {} from replicatedEnviroment", dbName); + // the first parameter null means auto-commit + replicatedEnvironment.removeDatabase(null, dbName); + } catch (DatabaseNotFoundException e) { + LOG.warn("catch an exception when remove db:{}, this db does not exist", dbName, e); + } + } finally { + lock.writeLock().unlock(); + } + } + + // get journal db names and sort the names + public List getDatabaseNames() { + List ret = new ArrayList(); + List names = null; + int tried = 0; + while (true) { + try { + names = replicatedEnvironment.getDatabaseNames(); + break; + } catch (InsufficientLogException e) { + throw e; + } catch (EnvironmentFailureException e) { + tried++; + if (tried == RETRY_TIME) { + LOG.error("bdb environment failure exception.", e); + System.exit(-1); + } + LOG.warn("bdb environment failure exception. will retry", e); + try { + Thread.sleep(1000); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + continue; + } catch (DatabaseException e) { + LOG.warn("catch an exception when calling getDatabaseNames", e); + return null; + } + } + + if (names != null) { + for (String name : names) { + // We don't count epochDB + if (name.equals("epochDB")) { + continue; + } + + long db = Long.parseLong(name); + ret.add(db); + } + } + + Collections.sort(ret); + return ret; + } + + // Close the store and environment + public void close() { + for (Database db : openedDatabases) { + try { + db.close(); + } catch (DatabaseException exception) { + LOG.error("Error closing db {} will exit", db.getDatabaseName(), exception); + System.exit(-1); + } + } + openedDatabases.clear(); + + if (epochDB != null) { + try { + epochDB.close(); + } catch (DatabaseException exception) { + LOG.error("Error closing db {} will exit", epochDB.getDatabaseName(), exception); + System.exit(-1); + } + } + + if (replicatedEnvironment != null) { + try { + // Finally, close the store and environment. + replicatedEnvironment.close(); + } catch (DatabaseException exception) { + LOG.error("Error closing replicatedEnviroment", exception); + System.exit(-1); + } + } + } + + private SyncPolicy getSyncPolicy(String policy) { + if (policy.equalsIgnoreCase("SYNC")) { + return Durability.SyncPolicy.SYNC; + } + if (policy.equalsIgnoreCase("NO_SYNC")) { + return Durability.SyncPolicy.NO_SYNC; + } + // default value is WRITE_NO_SYNC + return Durability.SyncPolicy.WRITE_NO_SYNC; + } + + private ReplicaAckPolicy getAckPolicy(String policy) { + if (policy.equalsIgnoreCase("ALL")) { + return Durability.ReplicaAckPolicy.ALL; + } + if (policy.equalsIgnoreCase("NONE")) { + return Durability.ReplicaAckPolicy.NONE; + } + // default value is SIMPLE_MAJORITY + return Durability.ReplicaAckPolicy.SIMPLE_MAJORITY; + } + +} diff --git a/fe/src/com/baidu/palo/journal/bdbje/BDBJEJournal.java b/fe/src/com/baidu/palo/journal/bdbje/BDBJEJournal.java index 913d9b51e0..19557bbe3f 100644 --- a/fe/src/com/baidu/palo/journal/bdbje/BDBJEJournal.java +++ b/fe/src/com/baidu/palo/journal/bdbje/BDBJEJournal.java @@ -424,4 +424,4 @@ public class BDBJEJournal implements Journal { } return flag; } -} +} \ No newline at end of file diff --git a/fe/src/com/baidu/palo/journal/bdbje/BDBJournalCursor.java b/fe/src/com/baidu/palo/journal/bdbje/BDBJournalCursor.java index 08b4ad57b0..56119d61b5 100644 --- a/fe/src/com/baidu/palo/journal/bdbje/BDBJournalCursor.java +++ b/fe/src/com/baidu/palo/journal/bdbje/BDBJournalCursor.java @@ -13,8 +13,8 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.journal.bdbje; - +package com.baidu.palo.journal.bdbje; + import com.baidu.palo.journal.JournalCursor; import com.baidu.palo.journal.JournalEntity; @@ -29,105 +29,105 @@ import org.apache.logging.log4j.Logger; import java.io.ByteArrayInputStream; import java.io.DataInputStream; -import java.util.List; - -public class BDBJournalCursor implements JournalCursor { - private static final Logger LOG = LogManager.getLogger(JournalCursor.class); - - private long toKey; - private long currentKey; - private BDBEnvironment environment; - private List dbNames; - private Database database; - private int nextDbPositionIndex; - private final int maxTryTime = 3; - - public static BDBJournalCursor getJournalCursor(BDBEnvironment env, long fromKey, long toKey) { - if (toKey < fromKey || fromKey < 0) { - System.out.println("Invalid key range!"); - return null; - } - BDBJournalCursor cursor = null; - try { - cursor = new BDBJournalCursor(env, fromKey, toKey); - } catch (Exception e) { - LOG.error("new BDBJournalCursor error.", e); - } - return cursor; - } - - - private BDBJournalCursor(BDBEnvironment env, long fromKey, long toKey) throws Exception { - this.environment = env; - this.toKey = toKey; - this.currentKey = fromKey; - this.dbNames = env.getDatabaseNames(); - if (dbNames == null) { - throw new NullPointerException("dbNames is null."); - } - this.nextDbPositionIndex = 0; - - // find the db which may contain the fromKey - String dbName = null; - for (long db : dbNames) { - if (fromKey >= db) { - dbName = Long.toString(db); - nextDbPositionIndex++; - continue; - } else { - break; - } - } - - if (dbName == null) { - LOG.error("Can not find the key:{}, fail to get journal cursor. will exit.", fromKey); - System.exit(-1); - } - this.database = env.openDatabase(dbName); - } - - @Override - public JournalEntity next() { - JournalEntity ret = null; - if (currentKey > toKey) { - return ret; - } - Long key = new Long(currentKey); - DatabaseEntry theKey = new DatabaseEntry(); - TupleBinding myBinding = TupleBinding.getPrimitiveBinding(Long.class); - myBinding.objectToEntry(key, theKey); - - DatabaseEntry theData = new DatabaseEntry(); - // if current db does not contain any more data, then we go to search the next db - try { - // null means perform the operation without transaction protection. - // READ_COMMITTED guarantees no dirty read. - int tryTimes = 0; +import java.util.List; + +public class BDBJournalCursor implements JournalCursor { + private static final Logger LOG = LogManager.getLogger(JournalCursor.class); + + private long toKey; + private long currentKey; + private BDBEnvironment environment; + private List dbNames; + private Database database; + private int nextDbPositionIndex; + private final int maxTryTime = 3; + + public static BDBJournalCursor getJournalCursor(BDBEnvironment env, long fromKey, long toKey) { + if (toKey < fromKey || fromKey < 0) { + System.out.println("Invalid key range!"); + return null; + } + BDBJournalCursor cursor = null; + try { + cursor = new BDBJournalCursor(env, fromKey, toKey); + } catch (Exception e) { + LOG.error("new BDBJournalCursor error.", e); + } + return cursor; + } + + + private BDBJournalCursor(BDBEnvironment env, long fromKey, long toKey) throws Exception { + this.environment = env; + this.toKey = toKey; + this.currentKey = fromKey; + this.dbNames = env.getDatabaseNames(); + if (dbNames == null) { + throw new NullPointerException("dbNames is null."); + } + this.nextDbPositionIndex = 0; + + // find the db which may contain the fromKey + String dbName = null; + for (long db : dbNames) { + if (fromKey >= db) { + dbName = Long.toString(db); + nextDbPositionIndex++; + continue; + } else { + break; + } + } + + if (dbName == null) { + LOG.error("Can not find the key:{}, fail to get journal cursor. will exit.", fromKey); + System.exit(-1); + } + this.database = env.openDatabase(dbName); + } + + @Override + public JournalEntity next() { + JournalEntity ret = null; + if (currentKey > toKey) { + return ret; + } + Long key = new Long(currentKey); + DatabaseEntry theKey = new DatabaseEntry(); + TupleBinding myBinding = TupleBinding.getPrimitiveBinding(Long.class); + myBinding.objectToEntry(key, theKey); + + DatabaseEntry theData = new DatabaseEntry(); + // if current db does not contain any more data, then we go to search the next db + try { + // null means perform the operation without transaction protection. + // READ_COMMITTED guarantees no dirty read. + int tryTimes = 0; while (true) { - OperationStatus operationStatus = database.get(null, theKey, theData, LockMode.READ_COMMITTED); - if (operationStatus == OperationStatus.SUCCESS) { - // Recreate the data String. - byte[] retData = theData.getData(); - DataInputStream in = new DataInputStream(new ByteArrayInputStream(retData)); - ret = new JournalEntity(); - try { - ret.readFields(in); - } catch (Exception e) { - LOG.error("fail to read journal entity key={}, will exit", currentKey, e); - System.exit(-1); - } - currentKey++; - return ret; - } else if (nextDbPositionIndex < dbNames.size() && currentKey == dbNames.get(nextDbPositionIndex)) { - database = environment.openDatabase(dbNames.get(nextDbPositionIndex).toString()); - nextDbPositionIndex++; - tryTimes = 0; - continue; - } else if (tryTimes < maxTryTime) { - tryTimes++; - LOG.warn("fail to get journal {}, will try again. status: {}", currentKey, operationStatus); - Thread.sleep(3000); - continue; + OperationStatus operationStatus = database.get(null, theKey, theData, LockMode.READ_COMMITTED); + if (operationStatus == OperationStatus.SUCCESS) { + // Recreate the data String. + byte[] retData = theData.getData(); + DataInputStream in = new DataInputStream(new ByteArrayInputStream(retData)); + ret = new JournalEntity(); + try { + ret.readFields(in); + } catch (Exception e) { + LOG.error("fail to read journal entity key={}, will exit", currentKey, e); + System.exit(-1); + } + currentKey++; + return ret; + } else if (nextDbPositionIndex < dbNames.size() && currentKey == dbNames.get(nextDbPositionIndex)) { + database = environment.openDatabase(dbNames.get(nextDbPositionIndex).toString()); + nextDbPositionIndex++; + tryTimes = 0; + continue; + } else if (tryTimes < maxTryTime) { + tryTimes++; + LOG.warn("fail to get journal {}, will try again. status: {}", currentKey, operationStatus); + Thread.sleep(3000); + continue; } else if (operationStatus == OperationStatus.NOTFOUND) { // In the case: // On non-master FE, the replayer will first get the max journal id, @@ -143,17 +143,16 @@ public class BDBJournalCursor implements JournalCursor { } else { LOG.error("fail to get journal {}, status: {}, will exit", currentKey); System.exit(-1); - } - } - } catch (Exception e) { - LOG.warn("Catch an exception when get next JournalEntity. key:{}", currentKey, e); - return null; - } - } - - @Override - public void close() { - - } -} - + } + } + } catch (Exception e) { + LOG.warn("Catch an exception when get next JournalEntity. key:{}", currentKey, e); + return null; + } + } + + @Override + public void close() { + + } +} diff --git a/fe/src/com/baidu/palo/journal/local/LocalJournalCursor.java b/fe/src/com/baidu/palo/journal/local/LocalJournalCursor.java index a938cecb9c..df5ae99a45 100644 --- a/fe/src/com/baidu/palo/journal/local/LocalJournalCursor.java +++ b/fe/src/com/baidu/palo/journal/local/LocalJournalCursor.java @@ -17,7 +17,6 @@ package com.baidu.palo.journal.local; import com.baidu.palo.alter.AlterJob; import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.UserProperty; import com.baidu.palo.common.io.Text; import com.baidu.palo.ha.MasterInfo; import com.baidu.palo.journal.JournalCursor; @@ -27,6 +26,7 @@ import com.baidu.palo.load.AsyncDeleteJob; import com.baidu.palo.load.DeleteInfo; import com.baidu.palo.load.LoadErrorHub; import com.baidu.palo.load.LoadJob; +import com.baidu.palo.mysql.privilege.UserProperty; import com.baidu.palo.persist.CloneInfo; import com.baidu.palo.persist.ConsistencyCheckInfo; import com.baidu.palo.persist.CreateTableInfo; @@ -53,7 +53,8 @@ import java.io.EOFException; import java.io.File; import java.io.IOException; import java.util.List; - + +@Deprecated public final class LocalJournalCursor implements JournalCursor { private static final Logger LOG = LogManager.getLogger(LocalJournalCursor.class); private String imageDir; @@ -183,7 +184,8 @@ public final class LocalJournalCursor implements JournalCursor { } return ret; } - + + @Deprecated private JournalEntity getJournalEntity(DataInputStream in, short opCode) throws IOException { JournalEntity ret = new JournalEntity(); ret.setOpCode(opCode); diff --git a/fe/src/com/baidu/palo/load/ExportJob.java b/fe/src/com/baidu/palo/load/ExportJob.java index 817b139510..3cf2fcd426 100644 --- a/fe/src/com/baidu/palo/load/ExportJob.java +++ b/fe/src/com/baidu/palo/load/ExportJob.java @@ -34,11 +34,12 @@ import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.catalog.Table; import com.baidu.palo.catalog.Type; import com.baidu.palo.common.Config; +import com.baidu.palo.common.FeMetaVersion; import com.baidu.palo.common.InternalException; -import com.baidu.palo.common.io.Text; -import com.baidu.palo.common.io.Writable; import com.baidu.palo.common.Pair; import com.baidu.palo.common.Status; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; import com.baidu.palo.common.util.TimeUtils; import com.baidu.palo.planner.DataPartition; import com.baidu.palo.planner.ExportSink; @@ -54,13 +55,13 @@ import com.baidu.palo.system.Backend; import com.baidu.palo.task.AgentClient; import com.baidu.palo.thrift.TAgentResult; import com.baidu.palo.thrift.TNetworkAddress; -import com.baidu.palo.thrift.TStatusCode; import com.baidu.palo.thrift.TScanRangeLocation; import com.baidu.palo.thrift.TScanRangeLocations; +import com.baidu.palo.thrift.TStatusCode; import com.baidu.palo.thrift.TUniqueId; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -213,7 +214,7 @@ public class ExportJob implements Writable { scanNode = new OlapScanNode(new PlanNodeId(0), exportTupleDesc, "OlapScanNodeForExport"); Map columnFilters = Maps.newHashMap(); ((OlapScanNode) scanNode).setColumnFilters(columnFilters); - ((OlapScanNode) scanNode).setIsPreAggregation(false); + ((OlapScanNode) scanNode).setIsPreAggregation(false, "Export"); ((OlapScanNode) scanNode).setCanTurnOnPreAggr(false); break; case MYSQL: @@ -361,38 +362,18 @@ public class ExportJob implements Writable { return id; } - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - public long getDbId() { return dbId; } - public void setDbId(long dbId) { - this.dbId = dbId; - } - public long getTableId() { return this.tableId; } - public void setTableId(long tableId) { - this.tableId = tableId; - } - - public void setTableName(TableName tblName) { - this.tableName = tblName; - } - public JobState getState() { return state; } - public void setState(JobState state) { - this.state = state; - } - public BrokerDesc getBrokerDesc() { return brokerDesc; } @@ -405,22 +386,10 @@ public class ExportJob implements Writable { return exportPath; } - public void setExportPath(String exportPath) { - this.exportPath = exportPath; - } - - public void setColumnSeparator(String columnSeparator) { - this.columnSeparator = columnSeparator; - } - public String getColumnSeparator() { return this.columnSeparator; } - public void setLineDelimiter(String lineDelimiter) { - this.lineDelimiter = lineDelimiter; - } - public String getLineDelimiter() { return this.lineDelimiter; } @@ -429,10 +398,6 @@ public class ExportJob implements Writable { return partitions; } - public void setPartitions(List partitions) { - this.partitions = partitions; - } - public int getProgress() { return progress; } @@ -453,10 +418,6 @@ public class ExportJob implements Writable { return startTimeMs; } - public void setStartTimeMs(long startTimeMs) { - this.startTimeMs = startTimeMs; - } - public long getFinishTimeMs() { return finishTimeMs; } @@ -502,8 +463,8 @@ public class ExportJob implements Writable { return sql; } - public void setSql(String sql) { - this.sql = sql; + public TableName getTableName() { + return tableName; } public synchronized void cancel(ExportFailMsg.CancelType type, String msg) { @@ -620,6 +581,8 @@ public class ExportJob implements Writable { out.writeBoolean(true); brokerDesc.write(out); } + + tableName.write(out); } @Override @@ -652,6 +615,13 @@ public class ExportJob implements Writable { if (in.readBoolean()) { brokerDesc = BrokerDesc.read(in); } + + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_43) { + tableName = new TableName(); + tableName.readFields(in); + } else { + tableName = new TableName("DUMMY", "DUMMY"); + } } @Override diff --git a/fe/src/com/baidu/palo/load/ExportMgr.java b/fe/src/com/baidu/palo/load/ExportMgr.java index 5e79925aeb..5c558391be 100644 --- a/fe/src/com/baidu/palo/load/ExportMgr.java +++ b/fe/src/com/baidu/palo/load/ExportMgr.java @@ -17,11 +17,15 @@ package com.baidu.palo.load; import com.baidu.palo.analysis.BrokerDesc; import com.baidu.palo.analysis.ExportStmt; +import com.baidu.palo.analysis.TableName; import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; import com.baidu.palo.common.Config; import com.baidu.palo.common.util.ListComparator; import com.baidu.palo.common.util.OrderByPair; import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -135,6 +139,28 @@ public class ExportMgr { } } + // check auth + + TableName tableName = job.getTableName(); + if (tableName == null || tableName.getTbl().equals("DUMMY")) { + // forward compatibility, no table name is saved before + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + continue; + } + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), + db.getFullName(), PrivPredicate.SHOW)) { + continue; + } + } else { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), + tableName.getDb(), tableName.getTbl(), + PrivPredicate.SHOW)) { + continue; + } + } + + if (states != null) { if (!states.contains(state)) { continue; diff --git a/fe/src/com/baidu/palo/load/Load.java b/fe/src/com/baidu/palo/load/Load.java index 8479325bff..30a468b773 100644 --- a/fe/src/com/baidu/palo/load/Load.java +++ b/fe/src/com/baidu/palo/load/Load.java @@ -62,6 +62,7 @@ import com.baidu.palo.load.FailMsg.CancelType; import com.baidu.palo.load.LoadJob.EtlJobType; import com.baidu.palo.load.LoadJob.JobState; import com.baidu.palo.metric.MetricRepo; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.persist.ReplicaPersistInfo; import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.system.Backend; @@ -391,12 +392,28 @@ public class Load { addLoadJob(job, db); } + // This is a final step of all addLoadJob() methods private void addLoadJob(LoadJob job, Database db) throws DdlException { // check cluster capacity Catalog.getCurrentSystemInfo().checkClusterCapacity(db.getClusterName()); // check db quota db.checkQuota(); + // check if table is in restore process + db.readLock(); + try { + for (Long tblId : job.getIdToTableLoadInfo().keySet()) { + Table tbl = db.getTable(tblId); + if (tbl != null && tbl.getType() == TableType.OLAP + && ((OlapTable) tbl).getState() == OlapTableState.RESTORE) { + throw new DdlException("Table " + tbl.getName() + " is in restore process. " + + "Can not load into it"); + } + } + } finally { + db.readUnlock(); + } + writeLock(); try { unprotectAddLoadJob(job); @@ -481,6 +498,7 @@ public class Load { for (DataDescription dataDescription : dataDescriptions) { // create source createSource(db, dataDescription, tableToPartitionSources, job.getDeleteFlag()); + job.addTableName(dataDescription.getTableName()); } for (Entry>> tableEntry : tableToPartitionSources.entrySet()) { long tableId = tableEntry.getKey(); @@ -552,7 +570,7 @@ public class Load { cluster = properties.get(LoadStmt.CLUSTER_PROPERTY); } - Pair clusterInfo = Catalog.getInstance().getUserMgr().getClusterInfo( + Pair clusterInfo = Catalog.getInstance().getAuth().getLoadClusterInfo( stmt.getUser(), cluster); cluster = clusterInfo.first; DppConfig clusterConfig = clusterInfo.second; @@ -1033,6 +1051,25 @@ public class Load { readUnlock(); } + // check auth here, cause we need table info + Set tableNames = job.getTableNames(); + if (tableNames.isEmpty()) { + // forward compatibility + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), dbName, + PrivPredicate.LOAD)) { + ErrorReport.reportDdlException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CANCEL LOAD"); + } + } else { + for (String tblName : tableNames) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, tblName, + PrivPredicate.LOAD)) { + ErrorReport.reportDdlException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "CANCEL LOAD", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), tblName); + } + } + } + // cancel job if (!cancelLoadJob(job, CancelType.USER_CANCEL, "user cancel")) { throw new DdlException("Cancel load job fail"); @@ -1244,8 +1281,8 @@ public class Load { } } - public LinkedList> getLoadJobInfosByDb(long dbId, String labelValue, boolean accurateMatch, - Set states, ArrayList orderByPairs) { + public LinkedList> getLoadJobInfosByDb(long dbId, String dbName, String labelValue, + boolean accurateMatch, Set states, ArrayList orderByPairs) { LinkedList> loadJobInfos = new LinkedList>(); readLock(); try { @@ -1254,11 +1291,13 @@ public class Load { return loadJobInfos; } + long start = System.currentTimeMillis(); + LOG.debug("begin to get load job info, size: {}", loadJobs.size()); for (LoadJob loadJob : loadJobs) { // filter first String label = loadJob.getLabel(); JobState state = loadJob.getState(); - + if (labelValue != null) { if (accurateMatch) { if (!label.equals(labelValue)) { @@ -1270,13 +1309,35 @@ public class Load { } } } - + if (states != null) { if (!states.contains(state)) { continue; } } - + + // check auth + Set tableNames = loadJob.getTableNames(); + if (tableNames.isEmpty()) { + // forward compatibility + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), dbName, + PrivPredicate.SHOW)) { + continue; + } + } else { + boolean auth = true; + for (String tblName : tableNames) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, + tblName, PrivPredicate.SHOW)) { + auth = false; + break; + } + } + if (!auth) { + continue; + } + } + List jobInfo = new ArrayList(); // jobId @@ -1362,6 +1423,8 @@ public class Load { loadJobInfos.add(jobInfo); } // end for loadJobs + + LOG.debug("finished to get load job info, cost: {}", (System.currentTimeMillis() - start)); } finally { readUnlock(); } @@ -1378,13 +1441,14 @@ public class Load { return loadJobInfos; } - public long getLatestJobIdByLabel(long dbId, String labelValue) { + public LoadJob getLatestJobIdByLabel(long dbId, String labelValue) { + LoadJob job = null; long jobId = 0; try { readLock(); List loadJobs = this.dbToLoadJobs.get(dbId); if (loadJobs == null) { - return 0; + return null; } for (LoadJob loadJob : loadJobs) { @@ -1400,13 +1464,14 @@ public class Load { if (currJobId > jobId) { jobId = currJobId; + job = loadJob; } } } finally { readUnlock(); } - return jobId; + return job; } public List> getLoadJobUnfinishedInfo(long jobId) { @@ -1509,8 +1574,8 @@ public class Load { // Note: althrough this.loadErrorHubInfo is volatile, no need to lock. // but editlog need be locked public void changeLoadErrorHubInfo(LoadErrorHub.Param info) { + writeLock(); try { - writeLock(); this.loadErrorHubInfo = info; Catalog.getInstance().getEditLog().logSetLoadErrorHub(info); } finally { @@ -1520,6 +1585,7 @@ public class Load { public static class JobInfo { public String dbName; + public Set tblNames = Sets.newHashSet(); public String label; public String clusterName; public JobState state; @@ -1537,6 +1603,7 @@ public class Load { // result saved in info public void getJobInfo(JobInfo info) throws DdlException { String fullDbName = ClusterNamespace.getFullName(info.clusterName, info.dbName); + info.dbName = fullDbName; Database db = Catalog.getInstance().getDb(fullDbName); if (db == null) { throw new DdlException("Unknown database(" + info.dbName + ")"); @@ -1553,6 +1620,11 @@ public class Load { } // only the last one should be running LoadJob job = loadJobs.get(loadJobs.size() - 1); + + if (!job.getTableNames().isEmpty()) { + info.tblNames.addAll(job.getTableNames()); + } + info.state = job.getState(); if (info.state == JobState.QUORUM_FINISHED) { info.state = JobState.FINISHED; @@ -2887,6 +2959,12 @@ public class Load { public List> getDeleteInfosByDb(long dbId, boolean forUser) { LinkedList> infos = new LinkedList>(); + Database db = Catalog.getInstance().getDb(dbId); + if (db == null) { + return infos; + } + + String dbName = db.getFullName(); readLock(); try { List deleteInfos = dbToDeleteInfos.get(dbId); @@ -2895,6 +2973,12 @@ public class Load { } for (DeleteInfo deleteInfo : deleteInfos) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), dbName, + deleteInfo.getTableName(), + PrivPredicate.LOAD)) { + continue; + } + List info = Lists.newArrayList(); if (!forUser) { info.add(deleteInfo.getJobId()); diff --git a/fe/src/com/baidu/palo/load/LoadJob.java b/fe/src/com/baidu/palo/load/LoadJob.java index 793e5c7474..c016ec08a1 100644 --- a/fe/src/com/baidu/palo/load/LoadJob.java +++ b/fe/src/com/baidu/palo/load/LoadJob.java @@ -26,7 +26,9 @@ import com.baidu.palo.task.PushTask; import com.baidu.palo.thrift.TPriority; import com.baidu.palo.thrift.TResourceInfo; +import com.google.common.base.Strings; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -106,6 +108,9 @@ public class LoadJob implements Writable { private long execMemLimit; + // save table names for auth check + private Set tableNames; + public LoadJob() { this(""); } @@ -141,8 +146,17 @@ public class LoadJob implements Writable { this.resourceInfo = null; this.priority = TPriority.NORMAL; this.execMemLimit = DEFAULT_EXEC_MEM_LIMIT; + this.tableNames = Sets.newHashSet(); } + public void addTableName(String tableName) { + tableNames.add(tableName); + } + + public Set getTableNames() { + return tableNames; + } + public long getId() { return id; } @@ -616,7 +630,8 @@ public class LoadJob implements Writable { } // resourceInfo - if (resourceInfo == null) { + if (resourceInfo == null || Strings.isNullOrEmpty(resourceInfo.getGroup()) + || Strings.isNullOrEmpty(resourceInfo.getUser())) { out.writeBoolean(false); } else { out.writeBoolean(true); @@ -642,13 +657,20 @@ public class LoadJob implements Writable { } out.writeLong(execMemLimit); + + out.writeInt(tableNames.size()); + for (String tableName : tableNames) { + Text.writeString(out, tableName); + } } public void readFields(DataInput in) throws IOException { + long version = Catalog.getCurrentCatalogJournalVersion(); + id = in.readLong(); dbId = in.readLong(); label = Text.readString(in); - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_23) { + if (version >= FeMetaVersion.VERSION_23) { timestamp = in.readLong(); } else { timestamp = -1; @@ -657,7 +679,7 @@ public class LoadJob implements Writable { maxFilterRatio = in.readDouble(); deleteFlag = false; - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_30) { + if (version >= FeMetaVersion.VERSION_30) { deleteFlag = in.readBoolean(); } @@ -726,7 +748,6 @@ public class LoadJob implements Writable { resourceInfo = new TResourceInfo(user, group); } - long version = Catalog.getCurrentCatalogJournalVersion(); if (version >= 3 && version < 7) { // bos 3 parameters String bosEndpoint = Text.readString(in); @@ -754,6 +775,13 @@ public class LoadJob implements Writable { if (version >= FeMetaVersion.VERSION_34) { this.execMemLimit = in.readLong(); } + + if (version >= FeMetaVersion.VERSION_43) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + tableNames.add(Text.readString(in)); + } + } } @Override diff --git a/fe/src/com/baidu/palo/master/MasterImpl.java b/fe/src/com/baidu/palo/master/MasterImpl.java index 9b31fba0d9..59361f48b6 100644 --- a/fe/src/com/baidu/palo/master/MasterImpl.java +++ b/fe/src/com/baidu/palo/master/MasterImpl.java @@ -40,8 +40,9 @@ import com.baidu.palo.task.CheckConsistencyTask; import com.baidu.palo.task.CloneTask; import com.baidu.palo.task.CreateReplicaTask; import com.baidu.palo.task.CreateRollupTask; +import com.baidu.palo.task.DirMoveTask; +import com.baidu.palo.task.DownloadTask; import com.baidu.palo.task.PushTask; -import com.baidu.palo.task.RestoreTask; import com.baidu.palo.task.SchemaChangeTask; import com.baidu.palo.task.SnapshotTask; import com.baidu.palo.task.UploadTask; @@ -121,7 +122,11 @@ public class MasterImpl { } else { if (taskStatus.getStatus_code() != TStatusCode.OK) { task.failed(); - return result; + // We start to let FE perceive the task's error msg, begin with these 4 types of task. + if (taskType != TTaskType.MAKE_SNAPSHOT && taskType != TTaskType.UPLOAD + && taskType != TTaskType.DOWNLOAD && taskType != TTaskType.MOVE) { + return result; + } } } @@ -160,14 +165,16 @@ public class MasterImpl { finishConsistenctCheck(task, request); break; case MAKE_SNAPSHOT: - Preconditions.checkState(request.isSetSnapshot_path()); - finishMakeSnapshot(task, request.getSnapshot_path()); + finishMakeSnapshot(task, request); break; case UPLOAD: - finishUpload(task); + finishUpload(task, request); break; - case RESTORE: - finishRestore(task); + case DOWNLOAD: + finishDownloadTask(task, request); + break; + case MOVE: + finishMoveDirTask(task, request); break; default: break; @@ -481,22 +488,33 @@ public class MasterImpl { AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.CHECK_CONSISTENCY, task.getSignature()); } - private void finishMakeSnapshot(AgentTask task, String snapshotPath) { + private void finishMakeSnapshot(AgentTask task, TFinishTaskRequest request) { SnapshotTask snapshotTask = (SnapshotTask) task; - Catalog.getInstance().getBackupHandler().handleFinishedSnapshot(snapshotTask, snapshotPath); - AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.MAKE_SNAPSHOT, task.getSignature()); + if (Catalog.getInstance().getBackupHandler().handleFinishedSnapshotTask(snapshotTask, request)) { + AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.MAKE_SNAPSHOT, task.getSignature()); + } + } - private void finishUpload(AgentTask task) { + private void finishUpload(AgentTask task, TFinishTaskRequest request) { UploadTask uploadTask = (UploadTask) task; - Catalog.getInstance().getBackupHandler().handleFinishedUpload(uploadTask); - AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.UPLOAD, task.getSignature()); + if (Catalog.getInstance().getBackupHandler().handleFinishedSnapshotUploadTask(uploadTask, request)) { + AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.UPLOAD, task.getSignature()); + } } - private void finishRestore(AgentTask task) { - RestoreTask restoreTask = (RestoreTask) task; - Catalog.getInstance().getBackupHandler().handleFinishedRestore(restoreTask); - AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.RESTORE, task.getSignature()); + private void finishDownloadTask(AgentTask task, TFinishTaskRequest request) { + DownloadTask downloadTask = (DownloadTask) task; + if (Catalog.getInstance().getBackupHandler().handleDownloadSnapshotTask(downloadTask, request)) { + AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.DOWNLOAD, task.getSignature()); + } + } + + private void finishMoveDirTask(AgentTask task, TFinishTaskRequest request) { + DirMoveTask dirMoveTask = (DirMoveTask) task; + if (Catalog.getInstance().getBackupHandler().handleDirMoveTask(dirMoveTask, request)) { + AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.MOVE, task.getSignature()); + } } public TMasterResult report(TReportRequest request) throws TException { @@ -505,7 +523,7 @@ public class MasterImpl { } public TFetchResourceResult fetchResource() { - return Catalog.getInstance().getUserMgr().toResourceThrift(); + return Catalog.getInstance().getAuth().toResourceThrift(); } } diff --git a/fe/src/com/baidu/palo/master/ReportHandler.java b/fe/src/com/baidu/palo/master/ReportHandler.java index b00c846a7b..bdcaf89fc3 100644 --- a/fe/src/com/baidu/palo/master/ReportHandler.java +++ b/fe/src/com/baidu/palo/master/ReportHandler.java @@ -577,7 +577,8 @@ public class ReportHandler extends Daemon { for (Replica replica : replicas) { final long id = replica.getBackendId(); final Backend backend = Catalog.getCurrentSystemInfo().getBackend(id); - if (backend.isAlive() && !backend.isDecommissioned() && replica.getState() == ReplicaState.NORMAL) { + if (backend != null && backend.isAlive() && !backend.isDecommissioned() + && replica.getState() == ReplicaState.NORMAL) { replicationOnLine++; } } @@ -627,4 +628,3 @@ public class ReportHandler extends Daemon { } } } - diff --git a/fe/src/com/baidu/palo/mysql/MysqlChannel.java b/fe/src/com/baidu/palo/mysql/MysqlChannel.java index 2ef2457153..a457febfec 100644 --- a/fe/src/com/baidu/palo/mysql/MysqlChannel.java +++ b/fe/src/com/baidu/palo/mysql/MysqlChannel.java @@ -45,7 +45,8 @@ public class MysqlChannel { // default packet byte buffer for most packet private ByteBuffer defaultBuffer = ByteBuffer.allocate(16 * 1024); private ByteBuffer sendBuffer; - private String remoteHostString; + // for log and show + private String remoteHostPortString; private String remoteIp; private boolean isSend; @@ -54,7 +55,7 @@ public class MysqlChannel { this.channel = channel; this.sendBuffer = ByteBuffer.allocate(2 * 1024 * 1024); this.isSend = false; - this.remoteHostString = ""; + this.remoteHostPortString = ""; this.remoteIp = ""; if (channel != null) { @@ -62,11 +63,11 @@ public class MysqlChannel { if (channel.getRemoteAddress() instanceof InetSocketAddress) { InetSocketAddress address = (InetSocketAddress) channel.getRemoteAddress(); // avoid calling getHostName() which may trigger a name service reverse lookup - remoteHostString = address.getHostString() + ":" + address.getPort(); + remoteHostPortString = address.getHostString() + ":" + address.getPort(); remoteIp = address.getAddress().getHostAddress(); } else { // Reach here, what's it? - remoteHostString = channel.getRemoteAddress().toString(); + remoteHostPortString = channel.getRemoteAddress().toString(); remoteIp = channel.getRemoteAddress().toString(); } } catch (Exception e) { @@ -266,7 +267,7 @@ public class MysqlChannel { return isSend; } - public String getRemoteHostString() { - return remoteHostString; + public String getRemoteHostPortString() { + return remoteHostPortString; } } diff --git a/fe/src/com/baidu/palo/mysql/MysqlProto.java b/fe/src/com/baidu/palo/mysql/MysqlProto.java index af03103723..354b1ad81c 100644 --- a/fe/src/com/baidu/palo/mysql/MysqlProto.java +++ b/fe/src/com/baidu/palo/mysql/MysqlProto.java @@ -16,12 +16,12 @@ package com.baidu.palo.mysql; import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.UserResource; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.mysql.privilege.UserResource; import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.system.SystemInfoService; @@ -38,34 +38,27 @@ public class MysqlProto { private static final Logger LOG = LogManager.getLogger(MysqlProto.class); // scramble: data receive from server. - // randomString: data send by server in plugin data field + // randomString: data send by server in plug-in data field + // user_name#HIGH@cluster_name private static boolean authenticate(ConnectContext context, byte[] scramble, byte[] randomString, String user) { - String usePass = scramble.length == 0 ? "NO" : "YES"; - String clusterName = ""; + String usePasswd = scramble.length == 0 ? "NO" : "YES"; String tmpUser = user; if (tmpUser == null || tmpUser.isEmpty()) { - ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, "", usePass); + ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, "", usePasswd); return false; } - String remoteIp = ""; - // check ip - if (tmpUser.charAt(0) == '@') { - String[] strList = tmpUser.split("@", 3); - if (strList.length != 3) { - ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, strList[1]); - return false; - } - remoteIp = strList[1]; - tmpUser = strList[2]; - } - // check deploy id + // check cluster, user name may contains cluster name or cluster id. + // eg: + // user_name@cluster_name + String clusterName = ""; String[] strList = tmpUser.split("@", 2); if (strList.length > 1) { tmpUser = strList[0]; clusterName = strList[1]; try { + // if cluster does not exist and it is not a valid cluster id, authenticate failed if (Catalog.getInstance().getCluster(clusterName) == null && Integer.valueOf(strList[1]) != context.getCatalog().getClusterId()) { ErrorReport.report(ErrorCode.ERR_UNKNOWN_CLUSTER_ID, strList[1]); @@ -76,7 +69,15 @@ public class MysqlProto { return false; } } + if (Strings.isNullOrEmpty(clusterName)) { + clusterName = SystemInfoService.DEFAULT_CLUSTER; + } + context.setCluster(clusterName); + // check resource group level. user name may contains resource group level. + // eg: + // ...@user_name#HIGH + // set resource group if it is valid, or just ignore it strList = tmpUser.split("#", 2); if (strList.length > 1) { tmpUser = strList[0]; @@ -85,46 +86,17 @@ public class MysqlProto { } } - if (Strings.isNullOrEmpty(clusterName)) { - clusterName = SystemInfoService.DEFAULT_CLUSTER; - } - context.setCluster(clusterName); - - if (Catalog.getInstance().getUserMgr().getPassword(tmpUser) == null) { - tmpUser = ClusterNamespace.getFullName(clusterName, tmpUser); - } - - byte[] userPassword = Catalog.getInstance().getUserMgr().getPassword(tmpUser); - - if (userPassword == null) { - ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, tmpUser, usePass); - return false; - } + LOG.debug("parse cluster: {}", clusterName); + String qualifiedUser = ClusterNamespace.getFullName(clusterName, tmpUser); + String remoteIp = context.getMysqlChannel().getRemoteIp(); - userPassword = MysqlPassword.getSaltFromPassword(userPassword); - - // when the length of password is zero, the user has no password - if ((scramble.length == userPassword.length) - && (scramble.length == 0 || MysqlPassword.checkScramble(scramble, randomString, userPassword))) { - // authenticate success - context.setUser(tmpUser); - } else { - // password check failed. - ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, tmpUser, usePass); + if (!Catalog.getCurrentCatalog().getAuth().checkPassword(qualifiedUser, remoteIp, + scramble, randomString)) { + ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, qualifiedUser, usePasswd); return false; } - // check whitelist - if (remoteIp.equals("")) { - remoteIp = context.getMysqlChannel().getRemoteIp(); - } - boolean ok = context.getCatalog().checkWhiteList(tmpUser, remoteIp); - if (!ok) { - LOG.debug("deny by whiltList. remoteIp={} user={}", context.getMysqlChannel().getRemoteIp(), tmpUser); - ErrorReport.report(ErrorCode.ERR_IP_NOT_ALLOWED, context.getMysqlChannel().getRemoteIp()); - return false; - } - + context.setQualifiedUser(qualifiedUser); return true; } diff --git a/fe/src/com/baidu/palo/mysql/MysqlServer.java b/fe/src/com/baidu/palo/mysql/MysqlServer.java index 766776cf85..99d7c11776 100644 --- a/fe/src/com/baidu/palo/mysql/MysqlServer.java +++ b/fe/src/com/baidu/palo/mysql/MysqlServer.java @@ -110,7 +110,20 @@ public class MysqlServer { context.cleanup(); } } catch (IOException e) { - // some error when accept + // ClosedChannelException + // AsynchronousCloseException + // ClosedByInterruptException + // Other IOException, for example "to many open files" ... + LOG.warn("Query server encounter exception.", e); + try { + Thread.sleep(100); + } catch (InterruptedException e1) { + // Do nothing + } + continue; + } catch (RuntimeException e) { + // NotYetBoundException + // SecurityException LOG.warn("Query server failed when calling accept.", e); return; } diff --git a/fe/src/com/baidu/palo/mysql/privilege/DbPrivEntry.java b/fe/src/com/baidu/palo/mysql/privilege/DbPrivEntry.java new file mode 100644 index 0000000000..da7b6f2fa7 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/DbPrivEntry.java @@ -0,0 +1,155 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.catalog.InfoSchemaDb; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; +import com.baidu.palo.common.PatternMatcher; +import com.baidu.palo.common.io.Text; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class DbPrivEntry extends PrivEntry { + protected static final String ANY_DB = "*"; + + protected PatternMatcher dbPattern; + protected String origDb; + protected boolean isAnyDb; + + protected DbPrivEntry() { + } + + protected DbPrivEntry(PatternMatcher hostPattern, String origHost, PatternMatcher dbPattern, String origDb, + PatternMatcher userPattern, String user, PrivBitSet privSet) { + super(hostPattern, origHost, userPattern, user, privSet); + this.dbPattern = dbPattern; + this.origDb = origDb; + if (origDb.equals(ANY_DB)) { + isAnyDb = true; + } + } + + public static DbPrivEntry create(String host, String db, String user, PrivBitSet privs) + throws AnalysisException { + PatternMatcher hostPattern = PatternMatcher.createMysqlPattern(host, CaseSensibility.HOST.getCaseSensibility()); + + PatternMatcher dbPattern = createDbPatternMatcher(db); + + PatternMatcher userPattern = PatternMatcher.createMysqlPattern(user, CaseSensibility.USER.getCaseSensibility()); + + if (privs.containsNodeOrGrantPriv()) { + throw new AnalysisException("Db privilege can not contains global privileges: " + privs); + } + + return new DbPrivEntry(hostPattern, host, dbPattern, db, userPattern, user, privs); + } + + private static PatternMatcher createDbPatternMatcher(String db) throws AnalysisException { + // the database 'information_schema''s name is case insensibility. + boolean dbCaseSensibility = CaseSensibility.DATABASE.getCaseSensibility(); + if (ClusterNamespace.getNameFromFullName(db).equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { + dbCaseSensibility = false; + } + + PatternMatcher dbPattern = PatternMatcher.createMysqlPattern(db.equals(ANY_DB) ? "%" : db, dbCaseSensibility); + return dbPattern; + } + + public PatternMatcher getDbPattern() { + return dbPattern; + } + + public String getOrigDb() { + return origDb; + } + + public boolean isAnyDb() { + return isAnyDb; + } + + @Override + public int compareTo(PrivEntry other) { + if (!(other instanceof DbPrivEntry)) { + throw new ClassCastException("cannot cast " + other.getClass().toString() + " to " + this.getClass()); + } + + DbPrivEntry otherEntry = (DbPrivEntry) other; + int res = origHost.compareTo(otherEntry.origHost); + if (res != 0) { + return -res; + } + + res = origDb.compareTo(otherEntry.origDb); + if (res != 0) { + return -res; + } + + return -origUser.compareTo(otherEntry.origUser); + } + + @Override + public boolean keyMatch(PrivEntry other) { + if (!(other instanceof DbPrivEntry)) { + return false; + } + + DbPrivEntry otherEntry = (DbPrivEntry) other; + if (origHost.equals(otherEntry.origHost) && origUser.equals(otherEntry.origUser) + && origDb.equals(otherEntry.origDb)) { + return true; + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("db priv. host: ").append(origHost).append(", db: ").append(origDb); + sb.append(", user: ").append(origUser); + sb.append(", priv: ").append(privSet).append(", set by resolver: ").append(isSetByDomainResolver); + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = DbPrivEntry.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + super.write(out); + Text.writeString(out, origDb); + isClassNameWrote = false; + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + origDb = Text.readString(in); + try { + dbPattern = createDbPatternMatcher(origDb); + } catch (AnalysisException e) { + throw new IOException(e); + } + isAnyDb = origDb.equals(ANY_DB); + } + +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/DbPrivTable.java b/fe/src/com/baidu/palo/mysql/privilege/DbPrivTable.java new file mode 100644 index 0000000000..05ffcb8bc7 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/DbPrivTable.java @@ -0,0 +1,80 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.common.io.Text; +import com.baidu.palo.qe.ConnectContext; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataOutput; +import java.io.IOException; + +public class DbPrivTable extends PrivTable { + private static final Logger LOG = LogManager.getLogger(DbPrivTable.class); + + public void getPrivs(String host, String db, String user, PrivBitSet savedPrivs) { + DbPrivEntry matchedEntry = null; + for (PrivEntry entry : entries) { + DbPrivEntry dbPrivEntry = (DbPrivEntry) entry; + + // check host + if (!dbPrivEntry.isAnyHost() && !dbPrivEntry.getHostPattern().match(host)) { + continue; + } + + // check db + if (!dbPrivEntry.isAnyDb() && !dbPrivEntry.getDbPattern().match(db)) { + continue; + } + + // check user + if (!dbPrivEntry.isAnyUser() && !dbPrivEntry.getUserPattern().match(user)) { + continue; + } + + matchedEntry = dbPrivEntry; + break; + } + if (matchedEntry == null) { + return; + } + + savedPrivs.or(matchedEntry.getPrivSet()); + } + + public boolean hasClusterPriv(ConnectContext ctx, String clusterName) { + for (PrivEntry entry : entries) { + DbPrivEntry dbPrivEntry = (DbPrivEntry) entry; + if (dbPrivEntry.getOrigDb().startsWith(clusterName)) { + return true; + } + } + return false; + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = DbPrivTable.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + + super.write(out); + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/GlobalPrivEntry.java b/fe/src/com/baidu/palo/mysql/privilege/GlobalPrivEntry.java new file mode 100644 index 0000000000..6ab58890a8 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/GlobalPrivEntry.java @@ -0,0 +1,145 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; +import com.baidu.palo.common.PatternMatcher; +import com.baidu.palo.common.io.Text; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class GlobalPrivEntry extends PrivEntry { + private static final Logger LOG = LogManager.getLogger(GlobalPrivEntry.class); + + private byte[] password; + + protected GlobalPrivEntry() { + } + + protected GlobalPrivEntry(PatternMatcher hostPattern, String origHost, + PatternMatcher userPattern, String origUser, + byte[] password, PrivBitSet privSet) { + super(hostPattern, origHost, userPattern, origUser, privSet); + this.password = password; + } + + public static GlobalPrivEntry create(String host, String user, byte[] password, + PrivBitSet privs) throws AnalysisException { + PatternMatcher hostPattern = PatternMatcher.createMysqlPattern(host, CaseSensibility.HOST.getCaseSensibility()); + PatternMatcher userPattern = PatternMatcher.createMysqlPattern(user, CaseSensibility.USER.getCaseSensibility()); + return new GlobalPrivEntry(hostPattern, host, userPattern, user, password, privs); + } + + public byte[] getPassword() { + return password; + } + + public void setPassword(byte[] password) { + this.password = password; + } + + /* + * UserTable is ordered by Host, User + * eg: + * +-----------+----------+- + * | Host | User | ... + * +-----------+----------+- + * | % | root | ... + * | % | jeffrey | ... + * | localhost | root | ... + * | localhost | | ... + * +-----------+----------+- + * + * will be sorted like: + * + * +-----------+----------+- + * | Host | User | ... + * +-----------+----------+- + * | localhost | root | ... + * | localhost | | ... + * | % | jeffrey | ... + * | % | root | ... + * +-----------+----------+- + * + * https://dev.mysql.com/doc/refman/8.0/en/connection-access.html + */ + @Override + public int compareTo(PrivEntry other) { + if (!(other instanceof GlobalPrivEntry)) { + throw new ClassCastException("cannot cast " + other.getClass().toString() + " to " + this.getClass()); + } + + GlobalPrivEntry otherEntry = (GlobalPrivEntry) other; + int res = origHost.compareTo(otherEntry.origHost); + if (res != 0) { + return -res; + } + + return -origUser.compareTo(otherEntry.origUser); + } + + @Override + public boolean keyMatch(PrivEntry other) { + if (!(other instanceof GlobalPrivEntry)) { + return false; + } + + GlobalPrivEntry otherEntry = (GlobalPrivEntry) other; + if (origHost.equals(otherEntry.origHost) && origUser.equals(otherEntry.origUser)) { + return true; + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("global priv. host: ").append(origHost).append(", user: ").append(origUser); + sb.append(", priv: ").append(privSet).append(", set by resolver: ").append(isSetByDomainResolver); + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = GlobalPrivEntry.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + + LOG.info("global priv: {}", this.toString()); + super.write(out); + + out.writeInt(password.length); + out.write(password); + isClassNameWrote = false; + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + int passwordLen = in.readInt(); + password = new byte[passwordLen]; + in.readFully(password); + } + +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/PaloAuth.java b/fe/src/com/baidu/palo/mysql/privilege/PaloAuth.java new file mode 100644 index 0000000000..ebf5395224 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PaloAuth.java @@ -0,0 +1,1276 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.CreateRoleStmt; +import com.baidu.palo.analysis.CreateUserStmt; +import com.baidu.palo.analysis.DropRoleStmt; +import com.baidu.palo.analysis.DropUserStmt; +import com.baidu.palo.analysis.GrantStmt; +import com.baidu.palo.analysis.RevokeStmt; +import com.baidu.palo.analysis.SetPassVar; +import com.baidu.palo.analysis.SetUserPropertyStmt; +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.InfoSchemaDb; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.Pair; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.load.DppConfig; +import com.baidu.palo.persist.PrivInfo; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.thrift.TFetchResourceResult; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class PaloAuth implements Writable { + private static final Logger LOG = LogManager.getLogger(PaloAuth.class); + + // root user's role is operator. + // each Palo system has only one root user. + public static final String ROOT_USER = "root"; + public static final String ADMIN_USER = "admin"; + + private UserPrivTable userPrivTable = new UserPrivTable(); + private DbPrivTable dbPrivTable = new DbPrivTable(); + private TablePrivTable tablePrivTable = new TablePrivTable(); + + private RoleManager roleManager = new RoleManager();; + private UserPropertyMgr propertyMgr = new UserPropertyMgr(); + + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + private void readLock() { + lock.readLock().lock(); + } + + private void readUnlock() { + lock.readLock().unlock(); + } + + private void writeLock() { + lock.writeLock().lock(); + } + + private void writeUnlock() { + lock.writeLock().unlock(); + } + + public enum PrivLevel { + GLOBAL, DATABASE, TABLE + } + + public PaloAuth() { + initUser(); + } + + public UserPrivTable getUserPrivTable() { + return userPrivTable; + } + + public DbPrivTable getDbPrivTable() { + return dbPrivTable; + } + + public TablePrivTable getTablePrivTable() { + return tablePrivTable; + } + + private GlobalPrivEntry grantGlobalPrivs(String host, String user, byte[] password, + boolean errOnExist, boolean errOnNonExist, boolean grantByResolver, PrivBitSet privs) + throws DdlException { + if (errOnExist && errOnNonExist) { + throw new DdlException("Can only specified errOnExist or errOnNonExist"); + } + GlobalPrivEntry entry; + try { + entry = GlobalPrivEntry.create(host, user, password, privs); + entry.setSetByDomainResolver(grantByResolver); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + userPrivTable.addEntry(entry, errOnExist, errOnNonExist); + return entry; + } + + private void revokeGlobalPrivs(String host, String user, PrivBitSet privs, boolean revokeByResovler, + boolean errOnNonExist, boolean deleteEntryWhenEmpty) throws DdlException { + GlobalPrivEntry entry; + try { + entry = GlobalPrivEntry.create(host, user, new byte[0], privs); + entry.setSetByDomainResolver(revokeByResovler); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + if (!userPrivTable.revoke(entry, errOnNonExist, deleteEntryWhenEmpty)) { + ErrorReport.reportDdlException(ErrorCode.ERR_NONEXISTING_GRANT, user, host); + } + } + + private void grantDbPrivs(String host, String db, String user, boolean errOnExist, boolean errOnNonExist, + boolean grantByResolver, PrivBitSet privs) throws DdlException { + DbPrivEntry entry; + try { + entry = DbPrivEntry.create(host, db, user, privs); + entry.setSetByDomainResolver(grantByResolver); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + dbPrivTable.addEntry(entry, errOnExist, errOnNonExist); + } + + private void revokeDbPrivs(String host, String db, String user, PrivBitSet privs, boolean setByResolver, + boolean errOnNonExist) throws DdlException { + DbPrivEntry entry; + try { + entry = DbPrivEntry.create(host, db, user, privs); + entry.setSetByDomainResolver(setByResolver); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + + if (!dbPrivTable.revoke(entry, errOnNonExist, true /* delete entry when empty */)) { + ErrorReport.reportDdlException(ErrorCode.ERR_NONEXISTING_GRANT, user, host); + } + } + + private void grantTblPrivs(String host, String db, String user, String tbl, boolean errOnExist, + boolean errOnNonExist, boolean grantByDomain, PrivBitSet privs) throws DdlException { + TablePrivEntry entry; + try { + entry = TablePrivEntry.create(host, db, user, tbl, privs); + entry.setSetByDomainResolver(grantByDomain); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + tablePrivTable.addEntry(entry, errOnExist, errOnNonExist); + } + + private void revokeTblPrivs(String host, String db, String user, String tbl, PrivBitSet privs, + boolean setByResolver, boolean errOnNonExist) throws DdlException { + TablePrivEntry entry; + try { + entry = TablePrivEntry.create(host, db, user, tbl, privs); + entry.setSetByDomainResolver(setByResolver); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + if (!tablePrivTable.revoke(entry, errOnNonExist, true /* delete entry when empty */)) { + ErrorReport.reportDdlException(ErrorCode.ERR_NONEXISTING_GRANT, user, host); + } + } + + public boolean checkPassword(String remoteUser, String remoteHost, byte[] remotePasswd, byte[] randomString) { + if (!Config.enable_auth_check) { + return true; + } + if ((remoteUser.equals(ROOT_USER) || remoteUser.equals(ADMIN_USER)) && remoteHost.equals("127.0.0.1")) { + // root and admin user is allowed to login from 127.0.0.1, in case user forget password. + return true; + } + + readLock(); + try { + return userPrivTable.checkPassword(remoteUser, remoteHost, remotePasswd, randomString); + } finally { + readUnlock(); + } + } + + public boolean checkPlainPassword(String remoteUser, String remoteHost, String remotePasswd) { + if (!Config.enable_auth_check) { + return true; + } + readLock(); + try { + return userPrivTable.checkPlainPassword(remoteUser, remoteHost, remotePasswd); + } finally { + readUnlock(); + } + } + + public boolean checkGlobalPriv(ConnectContext ctx, PrivPredicate wanted) { + return checkGlobalPriv(ctx.getRemoteIP(), ctx.getQualifiedUser(), wanted); + } + + public boolean checkGlobalPriv(String host, String user, PrivPredicate wanted) { + if (!Config.enable_auth_check) { + return true; + } + PrivBitSet savedPrivs = PrivBitSet.of(); + if (checkGlobalInternal(host, user, wanted, savedPrivs)) { + return true; + } + + LOG.debug("failed to get wanted privs: {}, ganted: {}", wanted, savedPrivs); + return false; + } + + public boolean checkDbPriv(ConnectContext ctx, String qualifiedDb, PrivPredicate wanted) { + return checkDbPriv(ctx.getRemoteIP(), qualifiedDb, ctx.getQualifiedUser(), wanted); + } + + public boolean checkDbPriv(String host, String db, String user, PrivPredicate wanted) { + if (!Config.enable_auth_check) { + return true; + } + if (wanted.getPrivs().containsNodeOrGrantPriv()) { + LOG.debug("should be check NODE or GRANT priv in Global level. host: {}, user: {}, db: {}", + host, user, db); + return false; + } + + PrivBitSet savedPrivs = PrivBitSet.of(); + if (checkGlobalInternal(host, user, wanted, savedPrivs) + || checkDbInternal(host, db, user, wanted, savedPrivs)) { + return true; + } + + // if user has any privs of table in this db, and the wanted priv is SHOW, return true + if (wanted == PrivPredicate.SHOW && checkTblWithDb(host, db, user)) { + return true; + } + + LOG.debug("failed to get wanted privs: {}, ganted: {}", wanted, savedPrivs); + return false; + } + + /* + * User may not have privs on a database, but have privs of tables in this database. + * So we have to check if user has any privs of tables in this database. + * if so, the database should be visible to this user. + */ + private boolean checkTblWithDb(String host, String db, String user) { + readLock(); + try { + return tablePrivTable.hasPrivsOfDb(host, db, user); + } finally { + readUnlock(); + } + } + + public boolean checkTblPriv(ConnectContext ctx, String qualifiedDb, String tbl, PrivPredicate wanted) { + return checkTblPriv(ctx.getRemoteIP(), qualifiedDb, ctx.getQualifiedUser(), tbl, wanted); + } + + public boolean checkTblPriv(String host, String db, String user, String tbl, PrivPredicate wanted) { + if (!Config.enable_auth_check) { + return true; + } + if (wanted.getPrivs().containsNodeOrGrantPriv()) { + LOG.debug("should be check NODE or GRANT priv in Db level. host: {}, user: {}, db: {}", + host, user, db); + return false; + } + + PrivBitSet savedPrivs = PrivBitSet.of(); + if (checkGlobalInternal(host, user, wanted, savedPrivs) + || checkDbInternal(host, db, user, wanted, savedPrivs) + || checkTblInternal(host, db, user, tbl, wanted, savedPrivs)) { + return true; + } + + LOG.debug("failed to get wanted privs: {}, ganted: {}", wanted, savedPrivs); + return false; + } + + private boolean checkGlobalInternal(String host, String user, PrivPredicate wanted, PrivBitSet savedPrivs) { + readLock(); + try { + userPrivTable.getPrivs(host, user, savedPrivs); + if (PaloPrivilege.satisfy(savedPrivs, wanted)) { + return true; + } + return false; + } finally { + readUnlock(); + } + } + + private boolean checkDbInternal(String host, String db, String user, PrivPredicate wanted, + PrivBitSet savedPrivs) { + readLock(); + try { + dbPrivTable.getPrivs(host, db, user, savedPrivs); + if (PaloPrivilege.satisfy(savedPrivs, wanted)) { + return true; + } + } finally { + readUnlock(); + } + return false; + } + + private boolean checkTblInternal(String host, String db, String user, String tbl, + PrivPredicate wanted, PrivBitSet savedPrivs) { + readLock(); + try { + tablePrivTable.getPrivs(host, db, user, tbl, savedPrivs); + if (PaloPrivilege.satisfy(savedPrivs, wanted)) { + return true; + } + return false; + } finally { + readUnlock(); + } + } + + // for test only + public void clear() { + userPrivTable.clear(); + dbPrivTable.clear(); + tablePrivTable.clear(); + } + + // create user + public void createUser(CreateUserStmt stmt) throws DdlException { + createUserInternal(stmt.getUserIdent(), stmt.getQualifiedRole(), stmt.getPassword(), false); + } + + public void replayCreateUser(PrivInfo privInfo) { + try { + createUserInternal(privInfo.getUserIdent(), privInfo.getRole(), privInfo.getPasswd(), true); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + + private void createUserInternal(UserIdentity userIdent, String roleName, byte[] password, + boolean isReplay) throws DdlException { + writeLock(); + try { + PaloRole role = null; + if (roleName != null) { + // get privs of role + role = roleManager.getRole(roleName); + if (role == null) { + throw new DdlException("Role: " + roleName + " does not exist"); + } + } + + if (userIdent.isDomain()) { + // the host here is a domain name, add it to whitelist. + Map privsMap = Maps.newHashMap(); + if (role != null) { + // grant privileges of role to this whitelist + privsMap = role.getTblPatternToPrivs(); + } + propertyMgr.addOrGrantWhiteList(userIdent, privsMap, password, true /* err on exist */, + false /* err on non exist*/); + + } else { + // check if user already exist + GlobalPrivEntry dummyEntry = null; + try { + dummyEntry = GlobalPrivEntry.create(userIdent.getHost(), userIdent.getQualifiedUser(), null, + PrivBitSet.of()); + } catch (AnalysisException e) { + LOG.error("should not happen", e); + } + if (userPrivTable.getExistingEntry(dummyEntry) != null) { + throw new DdlException("User " + userIdent + " already exist"); + } + + if (role != null) { + // grant privs of role to user + for (Map.Entry entry : role.getTblPatternToPrivs().entrySet()) { + // use PrivBitSet copy to avoid same object being changed synchronously + grantInternal(userIdent, null, entry.getKey(), entry.getValue().copy(), + false /* not set by domain */, + false /* err on non exist */, true /* is replay */); + } + } + + // set password field of global priv entry + // the global entry may or may not exist + setPasswordInternal(userIdent, password, true /* add if not exist */, + false /* set by resolver */, true); + } + + if (role != null) { + // add user to this role + role.addUser(userIdent); + } + + // other user properties + propertyMgr.addUserResource(userIdent.getQualifiedUser(), false /* not system user */); + + if (!userIdent.getQualifiedUser().equals(ROOT_USER) && !userIdent.getQualifiedUser().equals(ADMIN_USER)) { + // grant read privs to database information_schema + TablePattern tblPattern = new TablePattern(InfoSchemaDb.DATABASE_NAME, "*"); + try { + tblPattern.analyze(ClusterNamespace.getClusterNameFromFullName(userIdent.getQualifiedUser())); + } catch (AnalysisException e) { + LOG.warn("should not happen", e); + } + grantInternal(userIdent, null, tblPattern, PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + false, false /* err on non exist */, true /* is replay */); + } + + if (!isReplay) { + PrivInfo privInfo = new PrivInfo(userIdent, null, null, password, roleName); + Catalog.getCurrentCatalog().getEditLog().logCreateUser(privInfo); + } + LOG.info("finished to create user: {}, is replay: {}", userIdent, isReplay); + } finally { + writeUnlock(); + } + } + + // drop user + public void dropUser(DropUserStmt stmt) throws DdlException { + dropUserInternal(stmt.getUserIdentity(), false); + } + + public void replayDropUser(UserIdentity userIdent) { + dropUserInternal(userIdent, true); + } + + public void replayOldDropUser(String userName) { + UserIdentity userIdentity = new UserIdentity(userName, "%"); + userIdentity.setIsAnalyzed(); + dropUserInternal(userIdentity, true /* is replay */); + } + + private void dropUserInternal(UserIdentity userIdent, boolean isReplay) { + writeLock(); + try { + // we don't check if user exists + userPrivTable.dropUser(userIdent.getQualifiedUser()); + dbPrivTable.dropUser(userIdent.getQualifiedUser()); + tablePrivTable.dropUser(userIdent.getQualifiedUser()); + + // drop user in roles if exist + roleManager.dropUser(userIdent.getQualifiedUser()); + + // drop user property + propertyMgr.dropUser(userIdent.getQualifiedUser()); + + if (!isReplay) { + Catalog.getCurrentCatalog().getEditLog().logNewDropUser(userIdent); + } + LOG.info("finished to drop user: {}, is replay: {}", userIdent.getQualifiedUser(), isReplay); + } finally { + writeUnlock(); + } + } + + // grant + public void grant(GrantStmt stmt) throws DdlException { + PrivBitSet privs = PrivBitSet.of(stmt.getPrivileges()); + grantInternal(stmt.getUserIdent(), stmt.getQualifiedRole(), stmt.getTblPattern(), privs, + false /* not set by domain */, true /* err on non exist */, false /* not replay */); + } + + public void replayGrant(PrivInfo privInfo) { + try { + grantInternal(privInfo.getUserIdent(), privInfo.getRole(), + privInfo.getTblPattern(), privInfo.getPrivs(), + false /* not set by domain */, true /* err on non exist */, true /* is replay */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + + private void grantInternal(UserIdentity userIdent, String role, TablePattern tblPattern, + PrivBitSet privs, boolean grantByResolver, boolean errOnNonExist, boolean isReplay) + throws DdlException { + writeLock(); + try { + if (role != null) { + // grant privs to role, role must exist + PaloRole newRole = new PaloRole(role, tblPattern, privs); + PaloRole existingRole = roleManager.addRole(newRole, false /* err on exist */); + + // update users' privs of this role + for (UserIdentity user : existingRole.getUsers()) { + if (user.isDomain()) { + propertyMgr.addOrGrantWhiteList(user, existingRole.getTblPatternToPrivs(), + null, false /* err on exist */, + false /* err on non exist */); + } else { + for (Map.Entry entry : existingRole.getTblPatternToPrivs().entrySet()) { + // copy the PrivBitSet + grantPrivs(user, entry.getKey(), entry.getValue().copy(), errOnNonExist, grantByResolver); + } + } + } + } else { + if (userIdent.isDomain()) { + // grant privs to whitelist + Map privsMap = Maps.newHashMap(); + privsMap.put(tblPattern, privs); + propertyMgr.addOrGrantWhiteList(userIdent, privsMap, null, false /* err on exist */, + true /* err on non exist */); + } else { + grantPrivs(userIdent, tblPattern, privs, errOnNonExist, grantByResolver); + } + } + + if (!isReplay) { + PrivInfo info = new PrivInfo(userIdent, tblPattern, privs, null, role); + Catalog.getCurrentCatalog().getEditLog().logGrantPriv(info); + } + LOG.info("finished to grant privilege. is replay: {}", isReplay); + } finally { + writeUnlock(); + } + } + + public void grantPrivs(UserIdentity userIdent, TablePattern tblPattern, PrivBitSet privs, + boolean errOnNonExist, boolean grantByResolver) throws DdlException { + + LOG.debug("grant {} on {} to {}, err on non exist: {}, grant by resovler: {}", + privs, tblPattern, userIdent, errOnNonExist, grantByResolver); + + writeLock(); + try { + // check is user identity already exist + if (errOnNonExist && !userPrivTable.doesUserExist(userIdent, true /* exact match */)) { + throw new DdlException("user " + userIdent + " does not exist"); + } + + // grant privs to user + switch (tblPattern.getPrivLevel()) { + case GLOBAL: + grantGlobalPrivs(userIdent.getHost(), + userIdent.getQualifiedUser(), + new byte[0], + false /* err on exist */, + errOnNonExist, + grantByResolver, privs); + break; + case DATABASE: + grantDbPrivs(userIdent.getHost(), tblPattern.getQuolifiedDb(), + userIdent.getQualifiedUser(), + false /* err on exist */, + false /* err on non exist */, + grantByResolver, + privs); + break; + case TABLE: + grantTblPrivs(userIdent.getHost(), tblPattern.getQuolifiedDb(), + userIdent.getQualifiedUser(), tblPattern.getTbl(), + false /* err on exist */, + false /* err on non exist */, + grantByResolver, + privs); + break; + default: + Preconditions.checkNotNull(null, tblPattern.getPrivLevel()); + } + } finally { + writeUnlock(); + } + } + + // revoke + public void revoke(RevokeStmt stmt) throws DdlException { + PrivBitSet privs = PrivBitSet.of(stmt.getPrivileges()); + revokeInternal(stmt.getUserIdent(), stmt.getQualifiedRole(), stmt.getTblPattern(), privs, + true /* err on non exist */, false /* is replay */); + } + + public void replayRevoke(PrivInfo info) { + try { + revokeInternal(info.getUserIdent(), info.getRole(), info.getTblPattern(), info.getPrivs(), + true /* err on non exist */, true /* is replay */); + } catch (DdlException e) { + LOG.error("should not happend", e); + } + } + + private void revokeInternal(UserIdentity userIdent, String role, TablePattern tblPattern, + PrivBitSet privs, boolean errOnNonExist, boolean isReplay) throws DdlException { + writeLock(); + try { + if (role != null) { + // revoke privs from role + PaloRole existingRole = roleManager.revokePrivs(role, tblPattern, privs, + true /* err on non exist */); + Preconditions.checkNotNull(existingRole); + + // revoke privs from users of this role + for (UserIdentity user : existingRole.getUsers()) { + if (user.isDomain()) { + propertyMgr.revokePrivsFromWhiteList(user, existingRole.getTblPatternToPrivs(), + false /* err on non exist */); + } else { + revokePrivs(user, tblPattern, privs, false /* set by resolver */, + false /* err on non exist */, true /* delete entry when empty */); + } + } + } else { + if (userIdent.isDomain()) { + Map privsMap = Maps.newHashMap(); + privsMap.put(tblPattern, privs); + propertyMgr.revokePrivsFromWhiteList(userIdent, privsMap, errOnNonExist /* err on non exist */); + } else { + // revoke privs from user + revokePrivs(userIdent, tblPattern, privs, false /* set by resolver */, errOnNonExist, + false /* delete entry when empty */); + } + } + + if (!isReplay) { + PrivInfo info = new PrivInfo(userIdent, tblPattern, privs, null, role); + Catalog.getCurrentCatalog().getEditLog().logRevokePriv(info); + } + LOG.info("finished to revoke privilege. is replay: {}", isReplay); + } finally { + writeUnlock(); + } + } + + public void revokePrivs(UserIdentity userIdent, TablePattern tblPattern, PrivBitSet privs, + boolean setByResolver, boolean errOnNonExist, boolean deleteEntryWhenEmpty) throws DdlException { + writeLock(); + try { + switch (tblPattern.getPrivLevel()) { + case GLOBAL: + revokeGlobalPrivs(userIdent.getHost(), userIdent.getQualifiedUser(), privs, setByResolver, + errOnNonExist, deleteEntryWhenEmpty); + break; + case DATABASE: + revokeDbPrivs(userIdent.getHost(), tblPattern.getQuolifiedDb(), + userIdent.getQualifiedUser(), privs, setByResolver, + errOnNonExist); + break; + case TABLE: + revokeTblPrivs(userIdent.getHost(), tblPattern.getQuolifiedDb(), + userIdent.getQualifiedUser(), tblPattern.getTbl(), privs, setByResolver, + errOnNonExist); + break; + default: + Preconditions.checkNotNull(null, tblPattern.getPrivLevel()); + } + } finally { + writeUnlock(); + } + } + + // set password + public void setPassword(SetPassVar stmt) throws DdlException { + setPasswordInternal(stmt.getUserIdent(), stmt.getPassword(), false /* add if not exist */, + false /* set by resolver */, false); + } + + public void replaySetPassword(PrivInfo info) { + try { + setPasswordInternal(info.getUserIdent(), info.getPasswd(), false /* add if not exist */, + false /* set by resolver */, true); + } catch (DdlException e) { + LOG.error("should not happend", e); + } + } + + public void setPasswordInternal(UserIdentity userIdent, byte[] password, + boolean addIfNotExist, boolean setByResolver, boolean isReplay) throws DdlException { + writeLock(); + try { + if (userIdent.isDomain()) { + // throw exception is user ident does not exist + propertyMgr.setPassword(userIdent, password); + } else { + GlobalPrivEntry passwdEntry; + try { + passwdEntry = GlobalPrivEntry.create(userIdent.getHost(), userIdent.getQualifiedUser(), + password, PrivBitSet.of()); + passwdEntry.setSetByDomainResolver(setByResolver); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + + userPrivTable.setPassword(passwdEntry, addIfNotExist); + } + + if (!isReplay) { + PrivInfo info = new PrivInfo(userIdent, null, null, password, null); + Catalog.getCurrentCatalog().getEditLog().logSetPassword(info); + } + } finally { + writeUnlock(); + } + LOG.info("finished to set password for {}. is replay: {}", userIdent, isReplay); + } + + // create role + public void createRole(CreateRoleStmt stmt) throws DdlException { + createRoleInternal(stmt.getQualifiedRole(), false); + } + + public void replayCreateRole(PrivInfo info) { + try { + createRoleInternal(info.getRole(), true); + } catch (DdlException e) { + LOG.error("should not happend", e); + } + } + + private void createRoleInternal(String role, boolean isReplay) throws DdlException { + PaloRole emptyPrivsRole = new PaloRole(role); + writeLock(); + try { + roleManager.addRole(emptyPrivsRole, true /* err on exist */); + + if (!isReplay) { + PrivInfo info = new PrivInfo(null, null, null, null, role); + Catalog.getCurrentCatalog().getEditLog().logCreateRole(info); + } + } finally { + writeUnlock(); + } + LOG.info("finished to create role: {}, is replay: {}", role, isReplay); + } + + // drop role + public void dropRole(DropRoleStmt stmt) throws DdlException { + dropRoleInternal(stmt.getQualifiedRole(), false); + } + + public void replayDropRole(PrivInfo info) { + try { + dropRoleInternal(info.getRole(), true); + } catch (DdlException e) { + LOG.error("should not happend", e); + } + } + + private void dropRoleInternal(String role, boolean isReplay) throws DdlException { + writeLock(); + try { + roleManager.dropRole(role, true /* err on non exist */); + + if (!isReplay) { + PrivInfo info = new PrivInfo(null, null, null, null, role); + Catalog.getCurrentCatalog().getEditLog().logDropRole(info); + } + } finally { + writeUnlock(); + } + LOG.info("finished to drop role: {}, is replay: {}", role, isReplay); + } + + public long getMaxConn(String qualifiedUser) { + readLock(); + try { + return propertyMgr.getMaxConn(qualifiedUser); + } finally { + readUnlock(); + } + } + + public void getCopiedWhiteList(Map> userMap) { + readLock(); + try { + propertyMgr.getCopiedWhiteList(userMap); + } finally { + readUnlock(); + } + } + + public void updateResolovedIps(String qualifiedUser, String domain, Set resolvedIPs) { + writeLock(); + try { + propertyMgr.updateResolovedIps(qualifiedUser, domain, resolvedIPs); + } finally { + writeUnlock(); + } + } + + public List> getAuthInfo(UserIdentity specifiedUserIdent, boolean isAll) { + List> userAuthInfos = Lists.newArrayList(); + + readLock(); + try { + if (specifiedUserIdent == null) { + if (isAll) { + Set userIdents = getAllUserIdents(false /* include entry set by resolver */); + for (UserIdentity userIdent : userIdents) { + getUserAuthInfo(userAuthInfos, userIdent, true /* exact match */); + } + + // get grants from whitelist + propertyMgr.getUserAuthInfo(userAuthInfos, null); + } else { + Set userIdents = getAllUserIdents(true /* include entry set by resolver */); + for (UserIdentity userIdent : userIdents) { + getUserAuthInfo(userAuthInfos, userIdent, true /* exact match */); + } + } + } else { + if (specifiedUserIdent.isDomain()) { + propertyMgr.getUserAuthInfo(userAuthInfos, specifiedUserIdent); + } else { + getUserAuthInfo(userAuthInfos, specifiedUserIdent, false /* exact match */); + } + } + } finally { + readUnlock(); + } + return userAuthInfos; + } + + private void getUserAuthInfo(List> userAuthInfos, UserIdentity userIdent, + boolean exactMatch) { + List userAuthInfo = Lists.newArrayList(); + + // global + for (PrivEntry entry : userPrivTable.entries) { + if (!entry.match(userIdent, exactMatch)) { + continue; + } + GlobalPrivEntry gEntry = (GlobalPrivEntry) entry; + userAuthInfo.add(userIdent.toString()); + userAuthInfo.add((gEntry.getPassword() == null || gEntry.getPassword().length == 0) ? "No" : "Yes"); + userAuthInfo.add(gEntry.getPrivSet().toString() + " (" + gEntry.isSetByDomainResolver() + ")"); + break; + } + if (userAuthInfo.isEmpty()) { + // This may happen when we grant non global privs to a non exist user via GRANT stmt. + userAuthInfo.add(userIdent.toString()); + userAuthInfo.add("N/A"); + userAuthInfo.add("N/A"); + } + + // db + List dbPrivs = Lists.newArrayList(); + for (PrivEntry entry : dbPrivTable.entries) { + if (!entry.match(userIdent, exactMatch)) { + continue; + } + DbPrivEntry dEntry = (DbPrivEntry) entry; + dbPrivs.add(dEntry.getOrigDb() + ": " + dEntry.getPrivSet().toString() + + " (" + entry.isSetByDomainResolver() + ")"); + } + if (dbPrivs.isEmpty()) { + userAuthInfo.add("N/A"); + } else { + userAuthInfo.add(Joiner.on("; ").join(dbPrivs)); + } + + // tbl + List tblPrivs = Lists.newArrayList(); + for (PrivEntry entry : tablePrivTable.entries) { + if (!entry.match(userIdent, exactMatch)) { + continue; + } + TablePrivEntry tEntry = (TablePrivEntry) entry; + tblPrivs.add(tEntry.getOrigDb() + "." + tEntry.getOrigTbl() + ": " + + tEntry.getPrivSet().toString() + + " (" + entry.isSetByDomainResolver() + ")"); + } + if (tblPrivs.isEmpty()) { + userAuthInfo.add("N/A"); + } else { + userAuthInfo.add(Joiner.on("; ").join(tblPrivs)); + } + + userAuthInfos.add(userAuthInfo); + } + + private Set getAllUserIdents(boolean includeEntrySetByResolver) { + Set userIdents = Sets.newHashSet(); + for (PrivEntry entry : userPrivTable.entries) { + if (!includeEntrySetByResolver && entry.isSetByDomainResolver()) { + continue; + } + userIdents.add(entry.getUserIdent()); + } + for (PrivEntry entry : dbPrivTable.entries) { + if (!includeEntrySetByResolver && entry.isSetByDomainResolver()) { + continue; + } + userIdents.add(entry.getUserIdent()); + } + for (PrivEntry entry : tablePrivTable.entries) { + if (!includeEntrySetByResolver && entry.isSetByDomainResolver()) { + continue; + } + userIdents.add(entry.getUserIdent()); + } + return userIdents; + } + + public List> getUserProperties(String qualifiedUser) { + readLock(); + try { + return propertyMgr.fetchUserProperty(qualifiedUser); + } catch (AnalysisException e) { + return Lists.newArrayList(); + } finally { + readUnlock(); + } + } + + public void dropUserOfCluster(String clusterName, boolean isReplay) { + writeLock(); + try { + Set allUserIdents = getAllUserIdents(true); + for (UserIdentity userIdent : allUserIdents) { + if (userIdent.getQualifiedUser().startsWith(clusterName)) { + dropUserInternal(userIdent, isReplay); + } + } + } finally { + writeUnlock(); + } + } + + public void updateUserProperty(SetUserPropertyStmt ddlStmt) throws DdlException { + writeLock(); + try { + propertyMgr.updateUserProperty(ddlStmt); + } finally { + writeUnlock(); + } + } + + public Pair getLoadClusterInfo(String qualifiedUser, String cluster) throws DdlException { + readLock(); + try { + return propertyMgr.getLoadClusterInfo(qualifiedUser, cluster); + } finally { + readUnlock(); + } + } + + public void transformAndAddOldUserProperty(UserProperty userProperty) { + Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() <= FeMetaVersion.VERSION_43); + writeLock(); + try { + // for forward compatibility, we need to transfer the old form of privilege to the new form. + LOG.info("begin to transfer old user property: {}", userProperty.getQualifiedUser()); + + if (userProperty.isAdmin()) { + UserIdentity userIdent = new UserIdentity(userProperty.getQualifiedUser(), "%"); + userIdent.setIsAnalyzed(); + Map adminPrivs = PaloRole.OPERATOR.getTblPatternToPrivs(); + for (Map.Entry entry : adminPrivs.entrySet()) { + try { + grantPrivs(userIdent, entry.getKey(), entry.getValue(), + false /* err on non exist */, false /* grant by resolver */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + + try { + setPasswordInternal(userIdent, userProperty.getPassword(), true /* add if not exist */, + false /* set by resolver */, true); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } + + } else if (userProperty.isSuperuser()) { + UserIdentity userIdent = new UserIdentity(userProperty.getQualifiedUser(), "%"); + userIdent.setIsAnalyzed(); + Map adminPrivs = PaloRole.ADMIN.getTblPatternToPrivs(); + for (Map.Entry entry : adminPrivs.entrySet()) { + try { + grantPrivs(userIdent, entry.getKey(), entry.getValue(), + false /* err on non exist */, false /* grant by resolver */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + + try { + setPasswordInternal(userIdent, userProperty.getPassword(), true /* add if not exist */, + false /* set by resolver */, true); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } + + } else { // normal user + + Set ipWhiteList = userProperty.getWhiteList().getIpWhiteLists(); + Set starIpWhiteList = userProperty.getWhiteList().getStarIpWhiteLists(); + Map privsMap = Maps.newHashMap(); + + // 1. get all privs and save them to privsMap + for (Map.Entry entry : userProperty.getDbPrivMap().entrySet()) { + PrivBitSet privs = null; + switch (entry.getValue()) { + case READ_ONLY: + privs = PrivBitSet.of(PaloPrivilege.SELECT_PRIV); + break; + case READ_WRITE: + case ALL: + privs = PrivBitSet.of(PaloPrivilege.SELECT_PRIV, PaloPrivilege.LOAD_PRIV, + PaloPrivilege.ALTER_PRIV, PaloPrivilege.CREATE_PRIV, + PaloPrivilege.DROP_PRIV); + break; + default: + Preconditions.checkState(false, entry.getValue()); + break; + } + + TablePattern tblPattern = new TablePattern(ClusterNamespace.getNameFromFullName(entry.getKey()), + "*"); + try { + tblPattern.analyze(ClusterNamespace.getClusterNameFromFullName(entry.getKey())); + } catch (AnalysisException e) { + LOG.error("should not happen", e); + } + privsMap.put(tblPattern, privs); + } + + if (!ipWhiteList.isEmpty() || !starIpWhiteList.isEmpty()) { + // 2. handle the old whitelist + for (String ip : ipWhiteList) { + UserIdentity userIdent = new UserIdentity(userProperty.getQualifiedUser(), ip); + userIdent.setIsAnalyzed(); + // 1. set password + try { + setPasswordInternal(userIdent, userProperty.getPassword(), + true /* add if not exist */, + false /* set by resolver */, + true /* is replay */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + + // 2. set privs + for (Map.Entry entry : privsMap.entrySet()) { + try { + grantPrivs(userIdent, entry.getKey(), entry.getValue(), + false /* err on non exist */, false /* grant by resolver */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + } + + for (String starIp : starIpWhiteList) { + starIp = starIp.replaceAll("\\*", "%"); + UserIdentity userIdent = new UserIdentity(userProperty.getQualifiedUser(), starIp); + userIdent.setIsAnalyzed(); + // 1. set password + try { + setPasswordInternal(userIdent, userProperty.getPassword(), + true /* add if not exist */, + false /* set by resolver */, + true /* is replay */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + + // 2. set privs + for (Map.Entry entry : privsMap.entrySet()) { + try { + grantPrivs(userIdent, entry.getKey(), entry.getValue(), + false /* err on non exist */, false /* grant by resolver */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + } + } else if (userProperty.getWhiteList().getAllDomains().isEmpty()) { + // 3. grant privs to user@'%' if there is no whitelist + UserIdentity userIdent = new UserIdentity(userProperty.getQualifiedUser(), "%"); + userIdent.setIsAnalyzed(); + for (Map.Entry entry : privsMap.entrySet()) { + try { + grantPrivs(userIdent, entry.getKey(), entry.getValue(), + false /* err on non exist */, false /* grant by resolver */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + } + + // 4. domain is already saved in whitelist, and will be resolved later. + // but here we add a user@'%' 's password entry, to avoid access deny during transitional period. + UserIdentity userIdent = new UserIdentity(userProperty.getQualifiedUser(), "%"); + userIdent.setIsAnalyzed(); + try { + setPasswordInternal(userIdent, userProperty.getPassword(), + true /* add if not exist */, + false /* set by resolver */, + true /* is replay */); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + + // 5. update white list's privs info + Set allDomains = userProperty.getWhiteList().getAllDomains(); + for (String domain : allDomains) { + userProperty.getWhiteList().updateDomainMap(domain, privsMap); + } + LOG.info("update domains: {}, privs: {}", allDomains, privsMap); + + } // end for normal user property + + // add user property + propertyMgr.addUserPropertyUnchecked(userProperty); + + LOG.info("finished to transform old user property for user: {}", userProperty.getQualifiedUser()); + } finally { + writeUnlock(); + } + } + + public void deletePassworEntry(UserIdentity userIdent) { + writeLock(); + try { + // here we try to delete the password entry of the specified user, + // so that this user can not access to palo any more. + // we use a tricky way: we revoke all global privs of this, and when no privs granted, + // the priv entry will be deleted automatically. + revokeGlobalPrivs(userIdent.getHost(), userIdent.getQualifiedUser(), + PrivBitSet.of(PaloPrivilege.values()), + true /* revoke by resolver */, + false /* err on non exist */, + true /* delete entry when empty */); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } finally { + writeUnlock(); + } + } + + // user can enter a cluster, if it has any privs of database or table in this cluster. + public boolean checkCanEnterCluster(ConnectContext ctx, String clusterName) { + readLock(); + try { + if (checkGlobalPriv(ctx, PrivPredicate.ALL)) { + return true; + } + + if (dbPrivTable.hasClusterPriv(ctx, clusterName)) { + return true; + } + + if (tablePrivTable.hasClusterPriv(ctx, clusterName)) { + return true; + } + + return false; + } finally { + readUnlock(); + } + } + + @Deprecated + public void replayAlterAccess(UserProperty userProperty) { + Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43); + writeLock(); + try { + transformAndAddOldUserProperty(userProperty); + } finally { + writeUnlock(); + } + } + + private void initUser() { + try { + UserIdentity rootUser = new UserIdentity(ROOT_USER, "%"); + rootUser.setIsAnalyzed(); + createUserInternal(rootUser, PaloRole.OPERATOR_ROLE, new byte[0], true /* is replay */); + UserIdentity adminUser = new UserIdentity(ADMIN_USER, "%"); + adminUser.setIsAnalyzed(); + createUserInternal(adminUser, PaloRole.ADMIN_ROLE, new byte[0], true /* is replay */); + } catch (DdlException e) { + LOG.error("should not happend", e); + } + } + + public TFetchResourceResult toResourceThrift() { + readLock(); + try { + return propertyMgr.toResourceThrift(); + } finally { + readUnlock(); + } + } + + public List> getRoleInfo() { + readLock(); + try { + List> results = Lists.newArrayList(); + roleManager.getRoleInfo(results); + return results; + } finally { + readUnlock(); + } + } + + public static PaloAuth read(DataInput in) throws IOException { + PaloAuth auth = new PaloAuth(); + auth.readFields(in); + return auth; + } + + @Override + public void write(DataOutput out) throws IOException { + // role manager must be first, because role should be exist before any user + roleManager.write(out); + userPrivTable.write(out); + dbPrivTable.write(out); + tablePrivTable.write(out); + propertyMgr.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + roleManager = RoleManager.read(in); + userPrivTable = (UserPrivTable) PrivTable.read(in); + dbPrivTable = (DbPrivTable) PrivTable.read(in); + tablePrivTable = (TablePrivTable) PrivTable.read(in); + propertyMgr = UserPropertyMgr.read(in); + + if (userPrivTable.isEmpty()) { + // init root and admin user + initUser(); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(userPrivTable).append("\n"); + sb.append(dbPrivTable).append("\n"); + sb.append(tablePrivTable).append("\n"); + sb.append(roleManager).append("\n"); + sb.append(propertyMgr).append("\n"); + return sb.toString(); + } +} + diff --git a/fe/src/com/baidu/palo/mysql/privilege/PaloPrivilege.java b/fe/src/com/baidu/palo/mysql/privilege/PaloPrivilege.java new file mode 100644 index 0000000000..b90b920bb0 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PaloPrivilege.java @@ -0,0 +1,77 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +public enum PaloPrivilege { + NODE_PRIV("Node_priv", 0, "Privilege for cluster node operations"), + ADMIN_PRIV("Admin_priv", 1, "Privilege for admin user"), + GRANT_PRIV("Grant_priv", 2, "Privilege for granting privlege"), + SELECT_PRIV("Select_priv", 3, "Privilege for select data in tables"), + LOAD_PRIV("Load_priv", 4, "Privilege for loading data into tables"), + ALTER_PRIV("Alter_priv", 5, "Privilege for alter database or table"), + CREATE_PRIV("Create_priv", 6, "Privilege for createing database or table"), + DROP_PRIV("Drop_priv", 7, "Privilege for dropping database or table"); + + + public static PaloPrivilege[] privileges = { + NODE_PRIV, + ADMIN_PRIV, + GRANT_PRIV, + SELECT_PRIV, + LOAD_PRIV, + ALTER_PRIV, + CREATE_PRIV, + DROP_PRIV + }; + + private String name; + private int idx; + private String desc; + + private PaloPrivilege(String name, int index, String desc) { + this.name = name; + this.idx = index; + this.desc = desc; + } + + public String getName() { + return name; + } + + public int getIdx() { + return idx; + } + + public String getDesc() { + return desc; + } + + public static PaloPrivilege getPriv(int index) { + if (index < 0 || index > PaloPrivilege.values().length - 1) { + return null; + } + return privileges[index]; + } + + public static boolean satisfy(PrivBitSet grantPriv, PrivPredicate wanted) { + return grantPriv.satisfy(wanted); + } + + @Override + public String toString() { + return name; + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/PaloRole.java b/fe/src/com/baidu/palo/mysql/privilege/PaloRole.java new file mode 100644 index 0000000000..14bf653f8a --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PaloRole.java @@ -0,0 +1,152 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.common.CaseSensibility; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +public class PaloRole implements Writable { + // operator is responsible for operating cluster, such as add/drop node + public static String OPERATOR_ROLE = "operator"; + // admin is like DBA, who has all privileges except for NODE privilege held by operator + public static String ADMIN_ROLE = "admin"; + + public static PaloRole OPERATOR = new PaloRole(OPERATOR_ROLE, TablePattern.ALL, + PrivBitSet.of(PaloPrivilege.NODE_PRIV, PaloPrivilege.ADMIN_PRIV)); + public static PaloRole ADMIN = new PaloRole(ADMIN_ROLE, TablePattern.ALL, + PrivBitSet.of(PaloPrivilege.ADMIN_PRIV)); + + private String roleName; + private Map tblPatternToPrivs = Maps.newConcurrentMap(); + // users which this role + private Set users = Sets.newConcurrentHashSet(); + + private PaloRole() { + + } + + public PaloRole(String roleName) { + this.roleName = roleName; + } + + public PaloRole(String roleName, TablePattern tablePattern, PrivBitSet privs) { + this.roleName = roleName; + this.tblPatternToPrivs.put(tablePattern, privs); + } + + public String getRoleName() { + return roleName; + } + + public Map getTblPatternToPrivs() { + return tblPatternToPrivs; + } + + public Set getUsers() { + return users; + } + + public void merge(PaloRole other) { + Preconditions.checkState(roleName.equalsIgnoreCase(other.getRoleName())); + for (Map.Entry entry : other.getTblPatternToPrivs().entrySet()) { + if (tblPatternToPrivs.containsKey(entry.getKey())) { + PrivBitSet existPrivs = tblPatternToPrivs.get(entry.getKey()); + existPrivs.or(entry.getValue()); + } else { + tblPatternToPrivs.put(entry.getKey(), entry.getValue()); + } + } + } + + public void addUser(UserIdentity userIdent) { + users.add(userIdent); + } + + public void dropUser(String qualifiedUser) { + Iterator iter = users.iterator(); + while (iter.hasNext()) { + UserIdentity userIdent = iter.next(); + boolean match = false; + if (CaseSensibility.USER.getCaseSensibility()) { + match = userIdent.getQualifiedUser().equals(qualifiedUser); + } else { + match = userIdent.getQualifiedUser().equalsIgnoreCase(qualifiedUser); + } + if (match) { + iter.remove(); + } + } + } + + public static PaloRole read(DataInput in) throws IOException { + PaloRole role = new PaloRole(); + role.readFields(in); + return role; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, roleName); + out.writeInt(tblPatternToPrivs.size()); + for (Map.Entry entry : tblPatternToPrivs.entrySet()) { + entry.getKey().write(out); + entry.getValue().write(out); + } + + out.writeInt(users.size()); + for (UserIdentity userIdentity : users) { + userIdentity.write(out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + roleName = Text.readString(in); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + TablePattern tblPattern = TablePattern.read(in); + PrivBitSet privs = PrivBitSet.read(in); + tblPatternToPrivs.put(tblPattern, privs); + } + size = in.readInt(); + for (int i = 0; i < size; i++) { + UserIdentity userIdentity = UserIdentity.read(in); + users.add(userIdentity); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("role: ").append(roleName).append(", privs: ").append(tblPatternToPrivs); + sb.append(", users: ").append(users); + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/PrivBitSet.java b/fe/src/com/baidu/palo/mysql/privilege/PrivBitSet.java new file mode 100644 index 0000000000..47d20022eb --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PrivBitSet.java @@ -0,0 +1,158 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.common.io.Writable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +// ....0000000000 +// ^ ^ +// | | +// | -- first priv(0) +// |-------last priv(6) +public class PrivBitSet implements Writable { + + private long set = 0; + + public PrivBitSet() { + } + + public void set(int index) { + Preconditions.checkState(index < PaloPrivilege.privileges.length, index); + set |= 1 << index; + } + + public void unset(int index) { + Preconditions.checkState(index < PaloPrivilege.privileges.length, index); + set &= ~set; + } + + public boolean get(int index) { + Preconditions.checkState(index < PaloPrivilege.privileges.length, index); + return (set & (1 << index)) > 0; + } + + public void or(PrivBitSet other) { + set |= other.set; + } + + public void and(PrivBitSet other) { + set &= other.set; + } + + public void xor(PrivBitSet other) { + set ^= other.set; + } + + public void remove(PrivBitSet privs) { + PrivBitSet tmp = copy(); + tmp.xor(privs); + and(tmp); + } + + public boolean isEmpty() { + return set == 0; + } + + public boolean satisfy(PrivPredicate wantPrivs) { + if (wantPrivs.getOp() == Operator.AND) { + return (set & wantPrivs.getPrivs().set) == wantPrivs.getPrivs().set; + } else { + return (set & wantPrivs.getPrivs().set) != 0; + } + + } + + public boolean containsNodeOrGrantPriv() { + return containsPrivs(PaloPrivilege.NODE_PRIV, PaloPrivilege.GRANT_PRIV); + } + + public boolean containsPrivs(PaloPrivilege... privs) { + for (PaloPrivilege priv : privs) { + if (get(priv.getIdx())) { + return true; + } + } + return false; + } + + public List toPrivilegeList() { + List privs = Lists.newArrayList(); + for (int i = 0; i < PaloPrivilege.privileges.length; i++) { + if (get(i)) { + privs.add(PaloPrivilege.getPriv(i)); + } + } + return privs; + } + + public static PrivBitSet of(PaloPrivilege... privs) { + PrivBitSet bitSet = new PrivBitSet(); + for (PaloPrivilege priv : privs) { + bitSet.set(priv.getIdx()); + } + return bitSet; + } + + public static PrivBitSet of(List privs) { + PrivBitSet bitSet = new PrivBitSet(); + for (PaloPrivilege priv : privs) { + bitSet.set(priv.getIdx()); + } + return bitSet; + } + + public PrivBitSet copy() { + PrivBitSet newSet = new PrivBitSet(); + newSet.set = set; + return newSet; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < PaloPrivilege.privileges.length; i++) { + if (get(i)) { + sb.append(PaloPrivilege.getPriv(i)).append(" "); + } + } + return sb.toString(); + } + + public static PrivBitSet read(DataInput in) throws IOException { + PrivBitSet privBitSet = new PrivBitSet(); + privBitSet.readFields(in); + return privBitSet; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeLong(set); + } + + @Override + public void readFields(DataInput in) throws IOException { + set = in.readLong(); + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/PrivEntry.java b/fe/src/com/baidu/palo/mysql/privilege/PrivEntry.java new file mode 100644 index 0000000000..cb6c8e0e2e --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PrivEntry.java @@ -0,0 +1,236 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; +import com.baidu.palo.common.PatternMatcher; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import org.apache.commons.lang.NotImplementedException; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +public abstract class PrivEntry implements Comparable, Writable { + protected static final String ANY_HOST = "%"; + protected static final String ANY_USER = "%"; + + // host is not case sensitive + protected PatternMatcher hostPattern; + protected String origHost; + protected boolean isAnyHost = false; + // user name is case sensitive + protected PatternMatcher userPattern; + protected String origUser; + protected boolean isAnyUser = false; + protected PrivBitSet privSet; + + protected boolean isSetByDomainResolver = false; + + // isClassNameWrote to guarantee the class name can only be written once when persisting. + // see PrivEntry.read() for more details. + protected boolean isClassNameWrote = false; + + protected PrivEntry() { + } + + protected PrivEntry(PatternMatcher hostPattern, String origHost, PatternMatcher userPattern, String origUser, + PrivBitSet privSet) { + this.hostPattern = hostPattern; + this.origHost = origHost; + if (origHost.equals(ANY_HOST)) { + isAnyHost = true; + } + this.userPattern = userPattern; + this.origUser = origUser; + if (origUser.equals(ANY_USER)) { + isAnyUser = true; + } + this.privSet = privSet; + } + + public PatternMatcher getHostPattern() { + return hostPattern; + } + + public String getOrigHost() { + return origHost; + } + + public boolean isAnyHost() { + return isAnyHost; + } + + public PatternMatcher getUserPattern() { + return userPattern; + } + + public String getOrigUser() { + return origUser; + } + + public boolean isAnyUser() { + return isAnyUser; + } + + public PrivBitSet getPrivSet() { + return privSet; + } + + public void setPrivSet(PrivBitSet privSet) { + this.privSet = privSet; + } + + public boolean isSetByDomainResolver() { + return isSetByDomainResolver; + } + + public void setSetByDomainResolver(boolean isSetByDomainResolver) { + this.isSetByDomainResolver = isSetByDomainResolver; + } + + public UserIdentity getUserIdent() { + UserIdentity userIdent = new UserIdentity(origUser, origHost); + userIdent.setIsAnalyzed(); + return userIdent; + } + + public boolean match(UserIdentity userIdent, boolean exactMatch) { + if (exactMatch) { + return origUser.equals(userIdent.getQualifiedUser()) && origHost.equals(userIdent.getHost()); + } else { + return origUser.equals(userIdent.getQualifiedUser()) && hostPattern.match(userIdent.getHost()); + } + } + + public abstract boolean keyMatch(PrivEntry other); + + /* + * It's a bit complicated when persisting instance which its class has derived classes. + * eg: A (top class) -> B (derived) -> C (derived) + * + * Write process: + * C.write() + * | + * --- write class name + * | + * --- super.write() -----> B.write() + * | | + * --- write C's self members --- write class name (if not write before) + * | + * --- super.write() -----> A.write() + * | | + * --- write B's self members --- write class name (if not write before) + * | + * --- write A's self members + * + * So the final write order is: + * 1. C's class name + * 2. A's self members + * 3. B's self members + * 4. C's self members + * + * In case that class name should only be wrote once, we use isClassNameWrote flag. + * + * Read process: + * static A.read() + * | + * --- read class name and instantiated the class instance (eg. C class) + * | + * --- C.readFields() + * | + * --- super.readFields() --> B.readFields() + * | | + * --- read C's self members --- super.readFields() --> A.readFields() + * | | + * --- read B's self members --- read A's self members + * + * So the final read order is: + * 1. C's class name + * 2. A's self members + * 3. B's self members + * 4. C's self members + * + * Which is same as Write order. + */ + public static PrivEntry read(DataInput in) throws IOException { + String className = Text.readString(in); + PrivEntry privEntry = null; + try { + Class derivedClass = (Class) Class.forName(className); + privEntry = derivedClass.newInstance(); + Class[] paramTypes = { DataInput.class }; + Method readMethod = derivedClass.getMethod("readFields", paramTypes); + Object[] params = { in }; + readMethod.invoke(privEntry, params); + + return privEntry; + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException + | SecurityException | IllegalArgumentException | InvocationTargetException e) { + throw new IOException("failed read PrivEntry", e); + } + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = PrivEntry.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + Text.writeString(out, origHost); + Text.writeString(out, origUser); + privSet.write(out); + + out.writeBoolean(isSetByDomainResolver); + + isClassNameWrote = false; + } + + @Override + public void readFields(DataInput in) throws IOException { + origHost = Text.readString(in); + try { + hostPattern = PatternMatcher.createMysqlPattern(origHost, CaseSensibility.HOST.getCaseSensibility()); + } catch (AnalysisException e) { + throw new IOException(e); + } + isAnyHost = origHost.equals(ANY_HOST); + + origUser = Text.readString(in); + try { + userPattern = PatternMatcher.createMysqlPattern(origUser, CaseSensibility.USER.getCaseSensibility()); + } catch (AnalysisException e) { + throw new IOException(e); + } + isAnyUser = origUser.equals(ANY_USER); + + privSet = PrivBitSet.read(in); + + isSetByDomainResolver = in.readBoolean(); + } + + @Override + public int compareTo(PrivEntry o) { + throw new NotImplementedException(); + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/PrivPredicate.java b/fe/src/com/baidu/palo/mysql/privilege/PrivPredicate.java new file mode 100644 index 0000000000..b19bf9ccf8 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PrivPredicate.java @@ -0,0 +1,105 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.CompoundPredicate.Operator; + +public class PrivPredicate { + + // user can 'see' this meta + public static final PrivPredicate SHOW = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.SELECT_PRIV, + PaloPrivilege.LOAD_PRIV, + PaloPrivilege.ALTER_PRIV, + PaloPrivilege.CREATE_PRIV, + PaloPrivilege.DROP_PRIV), + Operator.OR); + // create/drop/alter/show user + public static final PrivPredicate GRANT = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.GRANT_PRIV), + Operator.OR); + // admin user privs + public static final PrivPredicate ADMIN = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV), + Operator.OR); + + // load + public static final PrivPredicate LOAD = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.LOAD_PRIV), + Operator.OR); + + // alter + public static final PrivPredicate ALTER = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.ALTER_PRIV), + Operator.OR); + + // create + public static final PrivPredicate CREATE = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.CREATE_PRIV), + Operator.OR); + + // drop + public static final PrivPredicate DROP = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.DROP_PRIV), + Operator.OR); + + // select + public static final PrivPredicate SELECT = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.SELECT_PRIV), + Operator.OR); + + // operator + public static final PrivPredicate OPERATOR = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.NODE_PRIV), + Operator.OR); + + // all + public static final PrivPredicate ALL = PrivPredicate.of(PrivBitSet.of(PaloPrivilege.NODE_PRIV, + PaloPrivilege.ADMIN_PRIV, + PaloPrivilege.SELECT_PRIV, + PaloPrivilege.LOAD_PRIV, + PaloPrivilege.ALTER_PRIV, + PaloPrivilege.CREATE_PRIV, + PaloPrivilege.DROP_PRIV), + Operator.OR); + + private PrivBitSet privs; + private Operator op; + + private PrivPredicate(PrivBitSet privs, Operator op) { + this.privs = privs; + this.op = op; + } + + public static PrivPredicate of(PrivBitSet privs, Operator op) { + final PrivPredicate predicate = new PrivPredicate(privs, op); + return predicate; + } + + public PrivBitSet getPrivs() { + return privs; + } + + public Operator getOp() { + return op; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("priv predicate: ").append(op).append(", ").append(privs); + return sb.toString(); + } + +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/PrivTable.java b/fe/src/com/baidu/palo/mysql/privilege/PrivTable.java new file mode 100644 index 0000000000..bbbb04722f --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/PrivTable.java @@ -0,0 +1,228 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; + +import com.google.common.collect.Lists; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +public abstract class PrivTable implements Writable { + private static final Logger LOG = LogManager.getLogger(PrivTable.class); + + protected List entries = Lists.newArrayList(); + + // see PrivEntry for more detail + protected boolean isClassNameWrote = false; + + public void addEntry(PrivEntry newEntry, boolean errOnExist, boolean errOnNonExist) throws DdlException { + PrivEntry existingEntry = getExistingEntry(newEntry); + if (existingEntry == null) { + if (errOnNonExist) { + throw new DdlException("User " + newEntry.getUserIdent() + " does not exist"); + } + entries.add(newEntry); + Collections.sort(entries); + LOG.info("add priv entry: {}", newEntry); + } else { + if (errOnExist) { + throw new DdlException("User already exist"); + } else { + if (!checkOperationAllowed(existingEntry, newEntry, "ADD ENTRY")) { + return; + } else { + if (existingEntry.isSetByDomainResolver() && newEntry.isSetByDomainResolver()) { + existingEntry.setPrivSet(newEntry.getPrivSet()); + LOG.debug("reset priv entry: {}", existingEntry); + } else if (existingEntry.isSetByDomainResolver() && !newEntry.isSetByDomainResolver() + || !existingEntry.isSetByDomainResolver() && !newEntry.isSetByDomainResolver()) { + mergePriv(existingEntry, newEntry); + existingEntry.setSetByDomainResolver(false); + LOG.info("merge priv entry: {}", existingEntry); + } + return; + } + } + } + + return; + } + + public void dropEntry(PrivEntry entry) { + Iterator iter = entries.iterator(); + while (iter.hasNext()) { + PrivEntry privEntry = iter.next(); + if (privEntry.keyMatch(entry)) { + iter.remove(); + LOG.info("drop priv entry: {}", privEntry); + break; + } + } + } + + // drop all entries which user name are matched + public void dropUser(String qualifiedUser) { + Iterator iter = entries.iterator(); + while (iter.hasNext()) { + PrivEntry privEntry = iter.next(); + if (privEntry.getOrigUser().equals(qualifiedUser)) { + iter.remove(); + LOG.info("drop entry: {}", privEntry); + } + } + } + + public boolean revoke(PrivEntry entry, boolean errOnNonExist, boolean deleteEntryWhenEmpty) { + PrivEntry existingEntry = getExistingEntry(entry); + if (existingEntry == null && errOnNonExist) { + return false; + } + + if (!checkOperationAllowed(existingEntry, entry, "REVOKE")) { + return true; + } + + // check if privs to be revoked exist in priv entry. + PrivBitSet tmp = existingEntry.getPrivSet().copy(); + tmp.and(entry.getPrivSet()); + if (tmp.isEmpty()) { + return !errOnNonExist; + } + + // revoke privs from existing priv entry + LOG.debug("before revoke: {}, privs to be revoked: {}", + existingEntry.getPrivSet(), entry.getPrivSet()); + tmp = existingEntry.getPrivSet().copy(); + tmp.xor(entry.getPrivSet()); + existingEntry.getPrivSet().and(tmp); + LOG.debug("after revoke: {}", existingEntry); + + if (existingEntry.getPrivSet().isEmpty() && deleteEntryWhenEmpty) { + // no priv exists in this entry, remove it + dropEntry(existingEntry); + } + + return true; + } + + /* + * the priv entry is classified by 'set by domain resolver' + * or 'NOT set by domain resolver'(other specified operations). + * if the existing entry is set by resolver, it can be reset by resolver or set by specified ops. + * if the existing entry is NOT set by resolver, it can not be set by resolver. + */ + protected boolean checkOperationAllowed(PrivEntry existingEntry, PrivEntry newEntry, String op) { + if (!existingEntry.isSetByDomainResolver() && newEntry.isSetByDomainResolver()) { + LOG.debug("the existing entry is NOT set by resolver: {}, can not be set by resolver {}, op: {}", + existingEntry, newEntry); + return false; + } else if (existingEntry.isSetByDomainResolver() && !newEntry.isSetByDomainResolver()) { + LOG.debug("the existing entry is currently set by resolver: {}, be set by ops now: {}, op: {}", + existingEntry, newEntry); + return true; + } + return true; + } + + // Get existing entry which is the keys match the given entry + protected PrivEntry getExistingEntry(PrivEntry entry) { + for (PrivEntry existingEntry : entries) { + if (existingEntry.keyMatch(entry)) { + return existingEntry; + } + } + return null; + } + + private void mergePriv(PrivEntry first, PrivEntry second) { + first.getPrivSet().or(second.getPrivSet()); + first.setSetByDomainResolver(first.isSetByDomainResolver() || second.isSetByDomainResolver()); + } + + // for test only + public void clear() { + entries.clear(); + } + + public boolean isEmpty() { + return entries.isEmpty(); + } + + public static PrivTable read(DataInput in) throws IOException { + String className = Text.readString(in); + PrivTable privTable = null; + try { + Class derivedClass = (Class) Class.forName(className); + privTable = derivedClass.newInstance(); + Class[] paramTypes = { DataInput.class }; + Method readMethod = derivedClass.getMethod("readFields", paramTypes); + Object[] params = { in }; + readMethod.invoke(privTable, params); + + return privTable; + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException + | SecurityException | IllegalArgumentException | InvocationTargetException e) { + throw new IOException("failed read PrivTable", e); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("\n"); + for (PrivEntry privEntry : entries) { + sb.append(privEntry).append("\n"); + } + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = PrivTable.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + out.writeInt(entries.size()); + for (PrivEntry privEntry : entries) { + privEntry.write(out); + } + isClassNameWrote = false; + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + PrivEntry entry = PrivEntry.read(in); + entries.add(entry); + } + Collections.sort(entries); + } + +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/RoleManager.java b/fe/src/com/baidu/palo/mysql/privilege/RoleManager.java new file mode 100644 index 0000000000..d699c47a2e --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/RoleManager.java @@ -0,0 +1,187 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.mysql.privilege.PaloAuth.PrivLevel; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class RoleManager implements Writable { + private Map roles = Maps.newHashMap(); + + public RoleManager() { + roles.put(PaloRole.OPERATOR.getRoleName(), PaloRole.OPERATOR); + roles.put(PaloRole.ADMIN.getRoleName(), PaloRole.ADMIN); + } + + public PaloRole getRole(String role) { + return roles.get(role); + } + + public PaloRole addRole(PaloRole newRole, boolean errOnExist) throws DdlException { + PaloRole existingRole = roles.get(newRole.getRoleName()); + if (existingRole != null) { + if (errOnExist) { + throw new DdlException("Role " + newRole + " already exists"); + } + // merge + existingRole.merge(newRole); + return existingRole; + } else { + roles.put(newRole.getRoleName(), newRole); + return newRole; + } + } + + public void dropRole(String qualifiedRole, boolean errOnNonExist) throws DdlException { + if (!roles.containsKey(qualifiedRole)) { + if (errOnNonExist) { + throw new DdlException("Role " + qualifiedRole + " does not exist"); + } + return; + } + + // we just remove the role from this map and remain others unchanged(privs, etc..) + roles.remove(qualifiedRole); + } + + public PaloRole revokePrivs(String role, TablePattern tblPattern, PrivBitSet privs, boolean errOnNonExist) + throws DdlException { + PaloRole existingRole = roles.get(role); + if (existingRole == null) { + if (errOnNonExist) { + throw new DdlException("Role " + role + " does not exist"); + } + return null; + } + + Map map = existingRole.getTblPatternToPrivs(); + PrivBitSet existingPriv = map.get(tblPattern); + if (existingPriv == null) { + if (errOnNonExist) { + throw new DdlException(tblPattern + " does not eixst in role " + role); + } + return null; + } + + existingPriv.remove(privs); + return existingRole; + } + + public void dropUser(String qualifiedUser) { + for (PaloRole role : roles.values()) { + role.dropUser(qualifiedUser); + } + } + + public void getRoleInfo(List> results) { + for (PaloRole role : roles.values()) { + List info = Lists.newArrayList(); + info.add(role.getRoleName()); + info.add(Joiner.on(", ").join(role.getUsers())); + + // global + boolean hasGlobal = false; + for (Map.Entry entry : role.getTblPatternToPrivs().entrySet()) { + if (entry.getKey().getPrivLevel() == PrivLevel.GLOBAL) { + hasGlobal = true; + info.add(entry.getValue().toString()); + // global priv should only has one + break; + } + } + if (!hasGlobal) { + info.add("N/A"); + } + + // db + List tmp = Lists.newArrayList(); + for (Map.Entry entry : role.getTblPatternToPrivs().entrySet()) { + if (entry.getKey().getPrivLevel() == PrivLevel.DATABASE) { + tmp.add(entry.getKey().toString() + ": " + entry.getValue().toString()); + } + } + if (tmp.isEmpty()) { + info.add("N/A"); + } else { + info.add(Joiner.on("; ").join(tmp)); + } + + + // tbl + tmp.clear(); + for (Map.Entry entry : role.getTblPatternToPrivs().entrySet()) { + if (entry.getKey().getPrivLevel() == PrivLevel.TABLE) { + tmp.add(entry.getKey().toString() + ": " + entry.getValue().toString()); + } + } + if (tmp.isEmpty()) { + info.add("N/A"); + } else { + info.add(Joiner.on("; ").join(tmp)); + } + + results.add(info); + } + } + + public static RoleManager read(DataInput in) throws IOException { + RoleManager roleManager = new RoleManager(); + roleManager.readFields(in); + return roleManager; + } + + @Override + public void write(DataOutput out) throws IOException { + // minus 2 to ignore ADMIN and OPERATOR role + out.writeInt(roles.size() - 2); + for (PaloRole role : roles.values()) { + if (role == PaloRole.ADMIN || role == PaloRole.OPERATOR) { + continue; + } + role.write(out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + PaloRole role = PaloRole.read(in); + roles.put(role.getRoleName(), role); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("Roles: "); + for (PaloRole role : roles.values()) { + sb.append(role).append("\n"); + } + return sb.toString(); + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/TablePrivEntry.java b/fe/src/com/baidu/palo/mysql/privilege/TablePrivEntry.java new file mode 100644 index 0000000000..0ba5a18b44 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/TablePrivEntry.java @@ -0,0 +1,151 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; +import com.baidu.palo.common.PatternMatcher; +import com.baidu.palo.common.io.Text; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class TablePrivEntry extends DbPrivEntry { + private static final String ANY_TBL = "*"; + + private PatternMatcher tblPattern; + private String origTbl; + private boolean isAnyTbl; + + protected TablePrivEntry() { + } + + private TablePrivEntry(PatternMatcher hostPattern, String origHost, PatternMatcher dbPattern, String origDb, + PatternMatcher userPattern, String user, PatternMatcher tblPattern, String origTbl, PrivBitSet privSet) { + super(hostPattern, origHost, dbPattern, origDb, userPattern, user, privSet); + this.tblPattern = tblPattern; + this.origTbl = origTbl; + if (origTbl.equals(ANY_TBL)) { + isAnyTbl = true; + } + } + + public static TablePrivEntry create(String host, String db, String user, String tbl, + PrivBitSet privs) throws AnalysisException { + PatternMatcher hostPattern = PatternMatcher.createMysqlPattern(host, CaseSensibility.HOST.getCaseSensibility()); + PatternMatcher dbPattern = PatternMatcher.createMysqlPattern(db.equals(ANY_DB) ? "%" : db, + CaseSensibility.DATABASE.getCaseSensibility()); + PatternMatcher userPattern = PatternMatcher.createMysqlPattern(user, CaseSensibility.USER.getCaseSensibility()); + + PatternMatcher tblPattern = PatternMatcher.createMysqlPattern(tbl.equals(ANY_TBL) ? "%" : tbl, + CaseSensibility.TABLE.getCaseSensibility()); + + if (privs.containsNodeOrGrantPriv()) { + throw new AnalysisException("Table privilege can not contains global privileges: " + privs); + } + + return new TablePrivEntry(hostPattern, host, dbPattern, db, userPattern, user, tblPattern, tbl, privs); + } + + public PatternMatcher getTblPattern() { + return tblPattern; + } + + public String getOrigTbl() { + return origTbl; + } + + public boolean isAnyTbl() { + return isAnyTbl; + } + + @Override + public int compareTo(PrivEntry other) { + if (!(other instanceof TablePrivEntry)) { + throw new ClassCastException("cannot cast " + other.getClass().toString() + " to " + this.getClass()); + } + + TablePrivEntry otherEntry = (TablePrivEntry) other; + int res = origHost.compareTo(otherEntry.origHost); + if (res != 0) { + return -res; + } + + res = origDb.compareTo(otherEntry.origDb); + if (res != 0) { + return -res; + } + + res = origUser.compareTo(otherEntry.origUser); + if (res != 0) { + return -res; + } + + return -origTbl.compareTo(otherEntry.origTbl); + } + + @Override + public boolean keyMatch(PrivEntry other) { + if (!(other instanceof TablePrivEntry)) { + return false; + } + + TablePrivEntry otherEntry = (TablePrivEntry) other; + if (origHost.equals(otherEntry.origHost) && origUser.equals(otherEntry.origUser) + && origDb.equals(otherEntry.origDb) && origTbl.equals(origTbl)) { + return true; + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("db priv. host: ").append(origHost).append(", db: ").append(origDb); + sb.append(", user: ").append(origUser).append(", tbl: ").append(origTbl); + sb.append(", priv: ").append(privSet).append(", set by resolver: ").append(isSetByDomainResolver); + return sb.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = TablePrivEntry.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + super.write(out); + + Text.writeString(out, origTbl); + + isClassNameWrote = false; + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + origTbl = Text.readString(in); + try { + tblPattern = PatternMatcher.createMysqlPattern(origTbl, CaseSensibility.TABLE.getCaseSensibility()); + } catch (AnalysisException e) { + throw new IOException(e); + } + isAnyTbl = origTbl.equals(ANY_TBL); + } + +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/TablePrivTable.java b/fe/src/com/baidu/palo/mysql/privilege/TablePrivTable.java new file mode 100644 index 0000000000..1d600f0fbc --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/TablePrivTable.java @@ -0,0 +1,110 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.common.io.Text; +import com.baidu.palo.qe.ConnectContext; + +import com.google.common.base.Preconditions; + +import java.io.DataOutput; +import java.io.IOException; + +public class TablePrivTable extends PrivTable { + + public void getPrivs(String host, String db, String user, String tbl, PrivBitSet savedPrivs) { + TablePrivEntry matchedEntry = null; + for (PrivEntry entry : entries) { + TablePrivEntry tblPrivEntry = (TablePrivEntry) entry; + + // check host + if (!tblPrivEntry.isAnyHost() && !tblPrivEntry.getHostPattern().match(host)) { + continue; + } + + // check db + Preconditions.checkState(!tblPrivEntry.isAnyDb()); + if (!tblPrivEntry.getDbPattern().match(db)) { + continue; + } + + // check user + if (!tblPrivEntry.isAnyUser() && !tblPrivEntry.getUserPattern().match(user)) { + continue; + } + + // check table + if (!tblPrivEntry.getTblPattern().match(tbl)) { + continue; + } + + matchedEntry = tblPrivEntry; + break; + } + if (matchedEntry == null) { + return; + } + + savedPrivs.or(matchedEntry.getPrivSet()); + } + + public boolean hasPrivsOfDb(String host, String db, String user) { + for (PrivEntry entry : entries) { + TablePrivEntry tblPrivEntry = (TablePrivEntry) entry; + + // check host + Preconditions.checkState(!tblPrivEntry.isAnyDb()); + if (!tblPrivEntry.getDbPattern().match(db)) { + continue; + } + + // check db + Preconditions.checkState(!tblPrivEntry.isAnyDb()); + if (!tblPrivEntry.getDbPattern().match(db)) { + continue; + } + + // check user + if (!tblPrivEntry.isAnyUser() && !tblPrivEntry.getUserPattern().match(user)) { + continue; + } + + return true; + } + return false; + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = TablePrivTable.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + + super.write(out); + } + + public boolean hasClusterPriv(ConnectContext ctx, String clusterName) { + for (PrivEntry entry : entries) { + TablePrivEntry tblPrivEntry = (TablePrivEntry) entry; + if (tblPrivEntry.getOrigDb().startsWith(clusterName)) { + return true; + } + } + return false; + } +} diff --git a/fe/src/com/baidu/palo/mysql/privilege/UserPrivTable.java b/fe/src/com/baidu/palo/mysql/privilege/UserPrivTable.java new file mode 100644 index 0000000000..c6a8e260ab --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/UserPrivTable.java @@ -0,0 +1,159 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.mysql.MysqlPassword; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataOutput; +import java.io.IOException; + +public class UserPrivTable extends PrivTable { + private static final Logger LOG = LogManager.getLogger(UserPrivTable.class); + + public UserPrivTable() { + } + + public void getPrivs(String host, String user, PrivBitSet savedPrivs) { + GlobalPrivEntry matchedEntry = null; + for (PrivEntry entry : entries) { + GlobalPrivEntry globalPrivEntry = (GlobalPrivEntry) entry; + + // check host + if (!globalPrivEntry.isAnyHost() && !globalPrivEntry.getHostPattern().match(host)) { + continue; + } + + // check user + if (!globalPrivEntry.isAnyUser() && !globalPrivEntry.getUserPattern().match(user)) { + continue; + } + + matchedEntry = globalPrivEntry; + break; + } + if (matchedEntry == null) { + return; + } + + savedPrivs.or(matchedEntry.getPrivSet()); + } + + // validate the connection by host, user and password. + // return true if this connection is valid, and 'savedPrivs' save all global privs got from user table. + public boolean checkPassword(String remoteUser, String remoteHost, byte[] remotePasswd, byte[] randomString) { + LOG.debug("check password for user: {} from {}, password: {}, random string: {}", + remoteUser, remoteHost, remotePasswd, randomString); + + // TODO(cmy): for now, we check user table from first entry to last, + // This may not efficient, but works. + for (PrivEntry entry : entries) { + GlobalPrivEntry globalPrivEntry = (GlobalPrivEntry) entry; + + // check host + if (!globalPrivEntry.isAnyHost() && !globalPrivEntry.getHostPattern().match(remoteHost)) { + continue; + } + + // check user + if (!globalPrivEntry.isAnyUser() && !globalPrivEntry.getUserPattern().match(remoteUser)) { + continue; + } + + // check password + byte[] saltPassword = MysqlPassword.getSaltFromPassword(globalPrivEntry.getPassword()); + // when the length of password is zero, the user has no password + if ((remotePasswd.length == saltPassword.length) + && (remotePasswd.length == 0 + || MysqlPassword.checkScramble(remotePasswd, randomString, saltPassword))) { + // found the matched entry + return true; + } else { + continue; + } + } + + return false; + } + + public boolean checkPlainPassword(String remoteUser, String remoteHost, String remotePasswd) { + for (PrivEntry entry : entries) { + GlobalPrivEntry globalPrivEntry = (GlobalPrivEntry) entry; + + // check host + if (!globalPrivEntry.isAnyHost() && !globalPrivEntry.getHostPattern().match(remoteHost)) { + continue; + } + + // check user + if (!globalPrivEntry.isAnyUser() && !globalPrivEntry.getUserPattern().match(remoteUser)) { + continue; + } + + if (MysqlPassword.checkPlainPass(globalPrivEntry.getPassword(), remotePasswd)) { + return true; + } + } + + return false; + } + + public void setPassword(GlobalPrivEntry passwdEntry, boolean addIfNotExist) throws DdlException { + GlobalPrivEntry existingEntry = (GlobalPrivEntry) getExistingEntry(passwdEntry); + if (existingEntry == null) { + if (!addIfNotExist) { + throw new DdlException("User " + passwdEntry.getUserIdent() + " does not exist"); + } + existingEntry = passwdEntry; + addEntry(existingEntry, false /* err on exist */, false /* err on non exist */); + } else { + if (existingEntry.isSetByDomainResolver() && !passwdEntry.isSetByDomainResolver()) { + LOG.info("cannot set password, existing entry is set by resolver: {}", existingEntry); + throw new DdlException("Cannot set password, existing entry is set by resolver"); + } else if (!existingEntry.isSetByDomainResolver() && passwdEntry.isSetByDomainResolver()) { + LOG.info("Cannot set password, existing entry is not set by resolver: {}", existingEntry); + throw new DdlException("Cannot set password, existing entry is not set by resolver"); + } + } + + existingEntry.setPassword(passwdEntry.getPassword()); + } + + public boolean doesUserExist(UserIdentity userIdent, boolean exactMatch) { + for (PrivEntry privEntry : entries) { + if (privEntry.match(userIdent, exactMatch)) { + return true; + } + } + return false; + } + + @Override + public void write(DataOutput out) throws IOException { + if (!isClassNameWrote) { + String className = UserPrivTable.class.getCanonicalName(); + Text.writeString(out, className); + isClassNameWrote = true; + } + + super.write(out); + } +} diff --git a/fe/src/com/baidu/palo/catalog/UserProperty.java b/fe/src/com/baidu/palo/mysql/privilege/UserProperty.java similarity index 58% rename from fe/src/com/baidu/palo/catalog/UserProperty.java rename to fe/src/com/baidu/palo/mysql/privilege/UserProperty.java index a6e7dc4124..a3dcb5509d 100644 --- a/fe/src/com/baidu/palo/catalog/UserProperty.java +++ b/fe/src/com/baidu/palo/mysql/privilege/UserProperty.java @@ -13,11 +13,17 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.catalog; +package com.baidu.palo.mysql.privilege; import com.baidu.palo.analysis.SetUserPropertyVar; import com.baidu.palo.analysis.SetVar; +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.ResourceGroup; +import com.baidu.palo.catalog.ResourceType; import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.FeMetaVersion; @@ -28,11 +34,14 @@ import com.baidu.palo.common.io.Writable; import com.baidu.palo.load.DppConfig; import com.baidu.palo.system.SystemInfoService; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; @@ -45,114 +54,138 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; +/* + * UserProperty contains properties set for a user + * This user is just qualified by cluster name, not host which it connected from. + */ public class UserProperty implements Writable { - private static final String MAX_USER_CONNECTIONS = "max_user_connections"; - private static final String RESOURCE = "resource"; - private static final String QUOTA = "quota"; - private static final String DEFAULT_LOAD_CLUSTER = "default_load_cluster"; - private static final String LOAD_CLUSTER = "load_cluster"; + private static final Logger LOG = LogManager.getLogger(UserProperty.class); - // for superuser or root + private static final String PROP_MAX_USER_CONNECTIONS = "max_user_connections"; + private static final String PROP_RESOURCE = "resource"; + private static final String PROP_QUOTA = "quota"; + private static final String PROP_DEFAULT_LOAD_CLUSTER = "default_load_cluster"; + private static final String PROP_LOAD_CLUSTER = "load_cluster"; + + // for system user public static final Set ADVANCED_PROPERTIES = Sets.newHashSet(); // for normal user public static final Set COMMON_PROPERTIES = Sets.newHashSet(); - // cluster which this user belongs to - String clusterName; - // save redundantly to simplify serialization - String userName; + private String qualifiedUser; - // SHA1(SHA1('password')) of byte[0] is unset - private byte[] password; - - // db- > priv - private Map dbPrivMap; - - private boolean isAdmin; - private boolean isSuperuser = false; - - private long maxConn; + private long maxConn = Config.max_conn_per_user; // Resource belong to this user. - private UserResource resource; + private UserResource resource = new UserResource(1000); // load cluster - private String defaultLoadCluster; - private Map clusterToDppConfig; + private String defaultLoadCluster = null; + private Map clusterToDppConfig = Maps.newHashMap(); - // whilelist - WhiteList whiteList; + /* + * We keep white list here to save Baidu domain name (BNS) or DNS as white list. + * Each frontend will periodically resolve the domain name to ip, and update the privilege table. + * We never persist the resolved IPs. + */ + private WhiteList whiteList = new WhiteList(); + + @Deprecated + private byte[] password; + @Deprecated + private boolean isAdmin = false; + @Deprecated + private boolean isSuperuser = false; + @Deprecated + private Map dbPrivMap = Maps.newHashMap(); static { - ADVANCED_PROPERTIES.add(Pattern.compile("^" + MAX_USER_CONNECTIONS + "$", Pattern.CASE_INSENSITIVE)); - ADVANCED_PROPERTIES.add(Pattern.compile("^" + RESOURCE + ".", Pattern.CASE_INSENSITIVE)); - ADVANCED_PROPERTIES.add(Pattern.compile( - "^" + LOAD_CLUSTER + "." + DppConfig.CLUSTER_NAME_REGEX + "." + DppConfig.PRIORITY + "$", - Pattern.CASE_INSENSITIVE)); + ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_MAX_USER_CONNECTIONS + "$", Pattern.CASE_INSENSITIVE)); + ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_RESOURCE + ".", Pattern.CASE_INSENSITIVE)); + ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_LOAD_CLUSTER + "." + DppConfig.CLUSTER_NAME_REGEX + "." + + DppConfig.PRIORITY + "$", Pattern.CASE_INSENSITIVE)); - COMMON_PROPERTIES.add(Pattern.compile("^" + QUOTA + ".", Pattern.CASE_INSENSITIVE)); - COMMON_PROPERTIES.add(Pattern.compile("^" + DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE)); - COMMON_PROPERTIES.add(Pattern.compile("^" + LOAD_CLUSTER + "." + DppConfig.CLUSTER_NAME_REGEX + ".", + COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUOTA + ".", Pattern.CASE_INSENSITIVE)); + COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE)); + COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_LOAD_CLUSTER + "." + DppConfig.CLUSTER_NAME_REGEX + ".", Pattern.CASE_INSENSITIVE)); } public UserProperty() { - clusterName = ""; - userName = null; - password = new byte[0]; - dbPrivMap = Maps.newHashMap(); - isAdmin = false; - maxConn = Config.max_conn_per_user; - resource = new UserResource(1000); - defaultLoadCluster = null; - clusterToDppConfig = Maps.newHashMap(); - whiteList = new WhiteList(); } - public String getClusterName() { - return clusterName; + public UserProperty(String qualifiedUser) { + this.qualifiedUser = qualifiedUser; } - public void setClusterName(String name) { - this.clusterName = name; - } - - public void setIsSuperuser(boolean isSuperuser) { - this.isSuperuser = isSuperuser; - } - - public String getUser() { - return userName; - } - - public void setUser(String userName) { - this.userName = userName; - whiteList.setUser(userName); - } - - public void setIsAdmin(boolean isAdmin) { - this.isAdmin = isAdmin; - } - - public boolean isAdmin() { - return isAdmin; - } - - public boolean isSuperuser() { - if (isAdmin) { - return true; - } - return isSuperuser; + public String getQualifiedUser() { + return qualifiedUser; } public long getMaxConn() { return maxConn; } + public WhiteList getWhiteList() { + return whiteList; + } + + @Deprecated public byte[] getPassword() { return password; } - public void setPassword(byte[] password) { - this.password = password; + @Deprecated + public boolean isAdmin() { + return isAdmin; + } + + @Deprecated + public boolean isSuperuser() { + return isSuperuser; + } + + @Deprecated + public Map getDbPrivMap() { + return dbPrivMap; + } + + public void addOrGrantWhiteList(String domain, Map tblPatternToPrivs, + byte[] password, boolean errOnExist) throws DdlException { + if (errOnExist && whiteList.containsDomain(domain)) { + throw new DdlException("white list " + domain + " of user " + qualifiedUser + " already exists"); + } + + if (tblPatternToPrivs.isEmpty()) { + // maybe this is a create user operation, so privs is empty + TablePattern tablePattern = new TablePattern("*", "*"); + try { + tablePattern.analyze(""); + } catch (AnalysisException e) { + LOG.warn("should not happen", e); + } + whiteList.addDomainWithPrivs(domain, tablePattern, PrivBitSet.of()); + } else { + for (Map.Entry entry : tblPatternToPrivs.entrySet()) { + whiteList.addDomainWithPrivs(domain, entry.getKey(), entry.getValue()); + } + } + + if (password != null) { + whiteList.setPassword(password); + } + } + + public void revokePrivsFromWhiteList(String domain, Map privsMap, + boolean errOnNonExist) throws DdlException { + // we need to check it before doing any change + for (Map.Entry entry : privsMap.entrySet()) { + whiteList.revokePrivsFromDomain(domain, entry.getKey(), entry.getValue(), + errOnNonExist, true /* check */); + } + + for (Map.Entry entry : privsMap.entrySet()) { + whiteList.revokePrivsFromDomain(domain, entry.getKey(), entry.getValue(), + errOnNonExist, false /* check */); + } } public void update(List propertyVarList) throws DdlException { @@ -169,25 +202,25 @@ public class UserProperty implements Writable { String value = propertyVar.getPropertyValue(); String[] keyArr = key.split("\\" + SetUserPropertyVar.DOT_SEPARATOR); - if (keyArr[0].equalsIgnoreCase(MAX_USER_CONNECTIONS)) { + if (keyArr[0].equalsIgnoreCase(PROP_MAX_USER_CONNECTIONS)) { // set property "max_user_connections" = "1000" if (keyArr.length != 1) { - throw new DdlException(MAX_USER_CONNECTIONS + " format error"); + throw new DdlException(PROP_MAX_USER_CONNECTIONS + " format error"); } try { newMaxConn = Long.parseLong(value); } catch (NumberFormatException e) { - throw new DdlException(MAX_USER_CONNECTIONS + " is not number"); + throw new DdlException(PROP_MAX_USER_CONNECTIONS + " is not number"); } if (newMaxConn <= 0 || newMaxConn > 10000) { - throw new DdlException(MAX_USER_CONNECTIONS + " is not valid, must between 1 and 10000"); + throw new DdlException(PROP_MAX_USER_CONNECTIONS + " is not valid, must between 1 and 10000"); } - } else if (keyArr[0].equalsIgnoreCase(RESOURCE)) { + } else if (keyArr[0].equalsIgnoreCase(PROP_RESOURCE)) { // set property "resource.cpu_share" = "100" if (keyArr.length != 2) { - throw new DdlException(RESOURCE + " format error"); + throw new DdlException(PROP_RESOURCE + " format error"); } int resource = 0; @@ -202,10 +235,10 @@ public class UserProperty implements Writable { } newResource.updateResource(keyArr[1], resource); - } else if (keyArr[0].equalsIgnoreCase(QUOTA)) { + } else if (keyArr[0].equalsIgnoreCase(PROP_QUOTA)) { // set property "quota.normal" = "100" if (keyArr.length != 2) { - throw new DdlException(QUOTA + " format error"); + throw new DdlException(PROP_QUOTA + " format error"); } int quota = 0; @@ -220,12 +253,12 @@ public class UserProperty implements Writable { } newResource.updateGroupShare(keyArr[1], quota); - } else if (keyArr[0].equalsIgnoreCase(LOAD_CLUSTER)) { + } else if (keyArr[0].equalsIgnoreCase(PROP_LOAD_CLUSTER)) { updateLoadCluster(keyArr, value, newDppConfigs); - } else if (keyArr[0].equalsIgnoreCase(DEFAULT_LOAD_CLUSTER)) { + } else if (keyArr[0].equalsIgnoreCase(PROP_DEFAULT_LOAD_CLUSTER)) { // set property "default_load_cluster" = "cluster1" if (keyArr.length != 1) { - throw new DdlException(DEFAULT_LOAD_CLUSTER + " format error"); + throw new DdlException(PROP_DEFAULT_LOAD_CLUSTER + " format error"); } if (value != null && !newDppConfigs.containsKey(value)) { throw new DdlException("Load cluster[" + value + "] does not exist"); @@ -287,43 +320,10 @@ public class UserProperty implements Writable { throw new DdlException(e.getMessage()); } } else { - throw new DdlException(LOAD_CLUSTER + " format error"); + throw new DdlException(PROP_LOAD_CLUSTER + " format error"); } } - // 用于判断一个用户是否对于需要访问的数据库有相应的权限 - public boolean checkAccess(String db, AccessPrivilege priv) { - if (isSuperuser()) { - return true; - } - // information_schema is case insensitive - String dbPrivMapKey = db; - final String qualifiedDbNameSuffix = ClusterNamespace.getNameFromFullName(db); - if (qualifiedDbNameSuffix.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { - dbPrivMapKey = db.toLowerCase(); - } - final AccessPrivilege dbPriv = dbPrivMap.get(dbPrivMapKey); - if (dbPriv == null) { - return false; - } - return dbPriv.contains(priv); - } - - // 修改用户已有的权限, 有可能当前对此DB无权限, 有可能已经有权限了，只是修改 - // 无论如何，直接覆盖就OK了。 - public void setAccess(String db, AccessPrivilege priv) { - dbPrivMap.put(db, priv); - } - - public void revokeAccess(String db) throws DdlException { - if (!dbPrivMap.containsKey(db)) { - throw new DdlException("User[" + userName + "] has no privilege on database[" + db + "]"); - } - - // just remove all privilege - dbPrivMap.remove(db); - } - public UserResource getResource() { return resource; } @@ -332,7 +332,7 @@ public class UserProperty implements Writable { return defaultLoadCluster; } - public Pair getClusterInfo(String cluster) { + public Pair getLoadClusterInfo(String cluster) { String tmpCluster = cluster; if (tmpCluster == null) { tmpCluster = defaultLoadCluster; @@ -354,32 +354,32 @@ public class UserProperty implements Writable { String dot = SetUserPropertyVar.DOT_SEPARATOR; // max user connections - result.add(Lists.newArrayList(MAX_USER_CONNECTIONS, String.valueOf(maxConn))); + result.add(Lists.newArrayList(PROP_MAX_USER_CONNECTIONS, String.valueOf(maxConn))); // resource ResourceGroup group = resource.getResource(); for (Map.Entry entry : group.getQuotaMap().entrySet()) { - result.add(Lists.newArrayList(RESOURCE + dot + entry.getKey().getDesc().toLowerCase(), + result.add(Lists.newArrayList(PROP_RESOURCE + dot + entry.getKey().getDesc().toLowerCase(), entry.getValue().toString())); } // quota Map groups = resource.getShareByGroup(); for (Map.Entry entry : groups.entrySet()) { - result.add(Lists.newArrayList(QUOTA + dot + entry.getKey(), entry.getValue().toString())); + result.add(Lists.newArrayList(PROP_QUOTA + dot + entry.getKey(), entry.getValue().toString())); } // load cluster if (defaultLoadCluster != null) { - result.add(Lists.newArrayList(DEFAULT_LOAD_CLUSTER, defaultLoadCluster)); + result.add(Lists.newArrayList(PROP_DEFAULT_LOAD_CLUSTER, defaultLoadCluster)); } else { - result.add(Lists.newArrayList(DEFAULT_LOAD_CLUSTER, "")); + result.add(Lists.newArrayList(PROP_DEFAULT_LOAD_CLUSTER, "")); } for (Map.Entry entry : clusterToDppConfig.entrySet()) { String cluster = entry.getKey(); DppConfig dppConfig = entry.getValue(); - String clusterPrefix = LOAD_CLUSTER + dot + cluster + dot; + String clusterPrefix = PROP_LOAD_CLUSTER + dot + cluster + dot; // palo path if (dppConfig.getPaloPath() != null) { @@ -404,6 +404,16 @@ public class UserProperty implements Writable { result.add(Lists.newArrayList(clusterPrefix + DppConfig.getPriorityKey(), String.valueOf(dppConfig.getPriority()))); } + + // get resolved ips if user has domain + Map> resolvedIPs = whiteList.getResolvedIPs(); + List ips = Lists.newArrayList(); + for (Map.Entry> entry : resolvedIPs.entrySet()) { + ips.add(entry.getKey() + ":" + Joiner.on(",").join(entry.getValue())); + } + if (!ips.isEmpty()) { + result.add(Lists.newArrayList("resolved IPs", Joiner.on(";").join(ips))); + } // sort Collections.sort(result, new Comparator>() { @@ -416,41 +426,22 @@ public class UserProperty implements Writable { return result; } - public String fetchPrivilegeResult() { - StringBuilder stringBuilder = new StringBuilder(); - boolean isFirst = true; - for (Map.Entry entry : dbPrivMap.entrySet()) { - if (!isFirst) { - stringBuilder.append(", "); - } - String dbName = entry.getKey(); - AccessPrivilege privilege = entry.getValue(); - stringBuilder.append(dbName).append("(").append(privilege.name()).append(")"); - isFirst = false; - } - return stringBuilder.toString(); + public void getAuthInfo(List> userAuthInfos) { + whiteList.getAuthInfo(qualifiedUser, userAuthInfos); } + public static UserProperty read(DataInput in) throws IOException { + UserProperty userProperty = new UserProperty(); + userProperty.readFields(in); + return userProperty; + } + + @Override public void write(DataOutput out) throws IOException { - if (userName == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Text.writeString(out, userName); - } - out.writeInt(password.length); - out.write(password); - out.writeBoolean(isAdmin); - out.writeBoolean(isSuperuser); + Text.writeString(out, qualifiedUser); out.writeLong(maxConn); - int numPriv = dbPrivMap.size(); - out.writeInt(numPriv); - for (Map.Entry entry : dbPrivMap.entrySet()) { - Text.writeString(out, entry.getKey()); - Text.writeString(out, entry.getValue().name()); - } - // User resource + // user resource resource.write(out); // load cluster @@ -466,46 +457,52 @@ public class UserProperty implements Writable { Text.writeString(out, entry.getKey()); entry.getValue().write(out); } + whiteList.write(out); - if (userName == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Text.writeString(out, clusterName); - } } public void readFields(DataInput in) throws IOException { - if (in.readBoolean()) { - if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_30) { - userName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, Text.readString(in)); - } else { - userName = Text.readString(in); - } + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43) { + // consume the flag of empty user name + in.readBoolean(); + } + + // user name + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_30) { + qualifiedUser = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, Text.readString(in)); + } else { + qualifiedUser = Text.readString(in); } - int passwordLen = in.readInt(); - password = new byte[passwordLen]; - in.readFully(password); - isAdmin = in.readBoolean(); - if (Catalog.getCurrentCatalogJournalVersion() >= 1) { - isSuperuser = in.readBoolean(); + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43) { + int passwordLen = in.readInt(); + password = new byte[passwordLen]; + in.readFully(password); + + isAdmin = in.readBoolean(); + + if (Catalog.getCurrentCatalogJournalVersion() >= 1) { + isSuperuser = in.readBoolean(); + } } maxConn = in.readLong(); - int numPriv = in.readInt(); - for (int i = 0; i < numPriv; ++i) { - String dbName; - if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_30) { - dbName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, Text.readString(in)); - } else { - dbName = Text.readString(in); + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43) { + int numPriv = in.readInt(); + for (int i = 0; i < numPriv; ++i) { + String dbName = null; + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_30) { + dbName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, Text.readString(in)); + } else { + dbName = Text.readString(in); + } + AccessPrivilege ap = AccessPrivilege.valueOf(Text.readString(in)); + dbPrivMap.put(dbName, ap); } - AccessPrivilege accessPrivilege = AccessPrivilege.valueOf(Text.readString(in)); - dbPrivMap.put(dbName, accessPrivilege); } - // User resource + + // user resource resource = UserResource.readIn(in); // load cluster @@ -524,24 +521,16 @@ public class UserProperty implements Writable { } if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_21) { - whiteList.setUser(userName); whiteList.readFields(in); } - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_30) { - if (in.readBoolean()) { - clusterName = Text.readString(in); + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_30) { + if (in.readBoolean()) { + // consume cluster name + Text.readString(in); + } } } } - - public static UserProperty read(DataInput in) throws IOException { - UserProperty userProperty = new UserProperty(); - userProperty.readFields(in); - return userProperty; - } - - public WhiteList getWhiteList() { - return whiteList; - } } diff --git a/fe/src/com/baidu/palo/mysql/privilege/UserPropertyMgr.java b/fe/src/com/baidu/palo/mysql/privilege/UserPropertyMgr.java new file mode 100644 index 0000000000..d8db9982f8 --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/UserPropertyMgr.java @@ -0,0 +1,302 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.SetUserPropertyStmt; +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.Pair; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.common.publish.FixedTimePublisher; +import com.baidu.palo.common.publish.Listener; +import com.baidu.palo.common.publish.TopicUpdate; +import com.baidu.palo.load.DppConfig; +import com.baidu.palo.thrift.TAgentServiceVersion; +import com.baidu.palo.thrift.TFetchResourceResult; +import com.baidu.palo.thrift.TTopicItem; +import com.baidu.palo.thrift.TTopicType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +public class UserPropertyMgr implements Writable { + private static final Logger LOG = LogManager.getLogger(UserPropertyMgr.class); + + protected Map propertyMap = Maps.newHashMap(); + public static final String ROOT_USER = "root"; + public static final String SYSTEM_RESOURCE_USER = "system"; + private AtomicLong resourceVersion = new AtomicLong(0); + + public UserPropertyMgr() { + } + + // Register callback to FixedTimePublisher + public void setUp() { + FixedTimePublisher.getInstance().register(new FixedTimePublisher.Callback() { + @Override + public TopicUpdate getTopicUpdate() { + TopicUpdate update = new TopicUpdate(TTopicType.RESOURCE); + TTopicItem tTopicItem = new TTopicItem("version"); + tTopicItem.setInt_value(resourceVersion.get()); + update.addUpdates(tTopicItem); + return update; + } + + @Override + public Listener getListener() { + return null; + } + }, Config.meta_resource_publish_interval_ms); + } + + public void addUserResource(String qualifiedUser, boolean isSystemUser) { + UserProperty property = propertyMap.get(qualifiedUser); + if (property != null) { + return; + } + + property = new UserProperty(qualifiedUser); + + // set user properties + try { + if (isSystemUser) { + setSystemUserDefaultResource(property); + } else { + setNormalUserDefaultResource(property); + } + } catch (DdlException e) { + // this should not happen, because the value is set by us!! + } + + propertyMap.put(qualifiedUser, property); + resourceVersion.incrementAndGet(); + } + + /* + * Try to grant privs to whitelist of the user. + */ + public void addOrGrantWhiteList(UserIdentity userIdentity, Map privsMap, + byte[] password, boolean errOnExist, boolean errOnNonExist) throws DdlException { + Preconditions.checkArgument(userIdentity.isDomain()); + UserProperty property = propertyMap.get(userIdentity.getQualifiedUser()); + if (property == null) { + if (errOnNonExist) { + throw new DdlException("user " + userIdentity + " does not exist"); + } + property = new UserProperty(userIdentity.getQualifiedUser()); + } + + property.addOrGrantWhiteList(userIdentity.getHost(), privsMap, password, errOnExist); + // update propertyMap after addOrGrantWhiteList, cause addOrGrantWhiteList may throw exception + propertyMap.put(userIdentity.getQualifiedUser(), property); + } + + public void revokePrivsFromWhiteList(UserIdentity userIdentity, Map privsMap, + boolean errOnNonExist) throws DdlException { + Preconditions.checkArgument(userIdentity.isDomain()); + UserProperty property = propertyMap.get(userIdentity.getQualifiedUser()); + if (property == null && errOnNonExist) { + throw new DdlException("User " + userIdentity.getQualifiedUser() + " does not exist"); + } + + property.revokePrivsFromWhiteList(userIdentity.getHost(), privsMap, errOnNonExist); + } + + public void dropUser(String qualifiedUser) { + propertyMap.remove(qualifiedUser); + resourceVersion.incrementAndGet(); + } + + public void setPassword(UserIdentity userIdent, byte[] password) throws DdlException { + Preconditions.checkArgument(userIdent.isDomain()); + UserProperty property = propertyMap.get(userIdent.getQualifiedUser()); + if (property == null) { + throw new DdlException("user " + userIdent.getQualifiedUser() + " does not exist"); + } + + if (property.getWhiteList().containsDomain(userIdent.getHost())) { + throw new DdlException("user " + userIdent + " does not exist"); + } + + property.getWhiteList().setPassword(password); + } + + public void updateUserProperty(SetUserPropertyStmt stmt) throws DdlException { + UserProperty property = propertyMap.get(stmt.getUser()); + if (property == null) { + throw new DdlException("Unknown user(" + stmt.getUser() + ")"); + } + + property.update(stmt.getPropertyList()); + } + + public long getMaxConn(String qualifiedUser) { + UserProperty existProperty = propertyMap.get(qualifiedUser); + if (existProperty == null) { + return 0; + } + return existProperty.getMaxConn(); + } + + public int getPropertyMapSize() { + return propertyMap.size(); + } + + private void setSystemUserDefaultResource(UserProperty user) throws DdlException { + UserResource userResource = user.getResource(); + userResource.updateResource("CPU_SHARE", 100); + userResource.updateResource("IO_SHARE", 100); + userResource.updateResource("SSD_READ_MBPS", 30); + userResource.updateResource("SSD_WRITE_MBPS", 30); + userResource.updateResource("HDD_READ_MBPS", 30); + userResource.updateResource("HDD_WRITE_MBPS", 30); + } + + private void setNormalUserDefaultResource(UserProperty user) throws DdlException { + UserResource userResource = user.getResource(); + userResource.updateResource("CPU_SHARE", 1000); + userResource.updateResource("IO_SHARE", 1000); + userResource.updateResource("SSD_READ_IOPS", 1000); + userResource.updateResource("HDD_READ_IOPS", 80); + userResource.updateResource("SSD_READ_MBPS", 30); + userResource.updateResource("HDD_READ_MBPS", 30); + } + + public TFetchResourceResult toResourceThrift() { + TFetchResourceResult tResult = new TFetchResourceResult(); + tResult.setProtocolVersion(TAgentServiceVersion.V1); + tResult.setResourceVersion(resourceVersion.get()); + + for (Map.Entry entry : propertyMap.entrySet()) { + tResult.putToResourceByUser(entry.getKey(), entry.getValue().getResource().toThrift()); + } + + return tResult; + } + + public Pair getLoadClusterInfo(String qualifiedUser, String cluster) throws DdlException { + Pair loadClusterInfo = null; + + if (!propertyMap.containsKey(qualifiedUser)) { + throw new DdlException("User " + qualifiedUser + " does not exist"); + } + + UserProperty property = propertyMap.get(qualifiedUser); + loadClusterInfo = property.getLoadClusterInfo(cluster); + return loadClusterInfo; + } + + public List> fetchUserProperty(String qualifiedUser) throws AnalysisException { + if (!propertyMap.containsKey(qualifiedUser)) { + throw new AnalysisException("User " + qualifiedUser + " does not exist"); + } + + UserProperty property = propertyMap.get(qualifiedUser); + return property.fetchProperty(); + } + + public void getCopiedWhiteList(Map> userMap) { + LOG.debug("get property map: {}", propertyMap); + for (Map.Entry entry : propertyMap.entrySet()) { + Set domains = entry.getValue().getWhiteList().getAllDomains(); + if (domains.isEmpty()) { + continue; + } + userMap.put(entry.getKey(), domains); + } + } + + public void updateResolovedIps(String qualifiedUser, String domain, Set resolvedIPs) { + if (!propertyMap.containsKey(qualifiedUser)) { + return; + } + + UserProperty property = propertyMap.get(qualifiedUser); + property.getWhiteList().updateResolovedIps(qualifiedUser, domain, resolvedIPs); + } + + public UserProperty getUserProperty(String qualifiedUserName) { + return propertyMap.get(qualifiedUserName); + } + + @Deprecated + public void addUserPropertyUnchecked(UserProperty userProperty) { + Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43); + propertyMap.put(userProperty.getQualifiedUser(), userProperty); + } + + public void transform(PaloAuth auth) { + Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43); + for (UserProperty userProperty : propertyMap.values()) { + auth.transformAndAddOldUserProperty(userProperty); + } + } + + public void getUserAuthInfo(List> userAuthInfos, UserIdentity specifiedUserIdent) { + for (UserProperty property : propertyMap.values()) { + if (specifiedUserIdent != null + && !property.getQualifiedUser().equals(specifiedUserIdent.getQualifiedUser())) { + continue; + } + property.getAuthInfo(userAuthInfos); + } + } + + public static UserPropertyMgr read(DataInput in) throws IOException { + UserPropertyMgr userPropertyMgr = new UserPropertyMgr(); + userPropertyMgr.readFields(in); + return userPropertyMgr; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(propertyMap.size()); + for (Map.Entry entry : propertyMap.entrySet()) { + entry.getValue().write(out); + } + // Write resource version + out.writeLong(resourceVersion.get()); + } + + @Override + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; ++i) { + UserProperty userProperty = UserProperty.read(in); + propertyMap.put(userProperty.getQualifiedUser(), userProperty); + LOG.debug("read user property: {}: {}", userProperty.getQualifiedUser(), userProperty); + } + // Read resource + resourceVersion = new AtomicLong(in.readLong()); + } +} + diff --git a/fe/src/com/baidu/palo/catalog/UserResource.java b/fe/src/com/baidu/palo/mysql/privilege/UserResource.java similarity index 98% rename from fe/src/com/baidu/palo/catalog/UserResource.java rename to fe/src/com/baidu/palo/mysql/privilege/UserResource.java index 6d562630dd..ad0d250987 100644 --- a/fe/src/com/baidu/palo/catalog/UserResource.java +++ b/fe/src/com/baidu/palo/mysql/privilege/UserResource.java @@ -13,8 +13,9 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.catalog; +package com.baidu.palo.mysql.privilege; +import com.baidu.palo.catalog.ResourceGroup; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.io.Text; import com.baidu.palo.common.io.Writable; diff --git a/fe/src/com/baidu/palo/mysql/privilege/WhiteList.java b/fe/src/com/baidu/palo/mysql/privilege/WhiteList.java new file mode 100644 index 0000000000..21169ad1ab --- /dev/null +++ b/fe/src/com/baidu/palo/mysql/privilege/WhiteList.java @@ -0,0 +1,329 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.mysql.privilege.PaloAuth.PrivLevel; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +// grant privs.. on db.tbl to user@['domain.name'] +// revoke privs on db.tbl from user@['domain.name'] +public class WhiteList implements Writable { + private static final Logger LOG = LogManager.getLogger(WhiteList.class); + + // domain name -> (tbl pattern -> privs) + // Domain names which need to be resolved to IPs. + // Currently, only implemented for Baidu Name Service (BNS) + private Map> domainMap = Maps.newConcurrentMap(); + + // Domain name to resolved IPs + private Map> resolvedIPMap = Maps.newConcurrentMap(); + + private byte[] password; + + @Deprecated + protected Set ipWhiteLists = Sets.newHashSet(); + @Deprecated + protected Set starIpWhiteLists = Sets.newHashSet(); + + public WhiteList() { + } + + @Deprecated + public Set getIpWhiteLists() { + return ipWhiteLists; + } + + @Deprecated + public Set getStarIpWhiteLists() { + return starIpWhiteLists; + } + + public void setPassword(byte[] password) { + this.password = password; + } + + public void addDomainWithPrivs(String domainName, TablePattern tblPattern, PrivBitSet privs) { + Map privsMap = domainMap.get(domainName); + if (privsMap == null) { + privsMap = Maps.newConcurrentMap(); + domainMap.put(domainName, privsMap); + } + PrivBitSet existingPrivs = privsMap.get(tblPattern); + if (existingPrivs == null) { + existingPrivs = privs; + privsMap.put(tblPattern, existingPrivs); + } else { + existingPrivs.or(privs); + } + } + + public void revokePrivsFromDomain(String domainName, TablePattern tblPattern, PrivBitSet privs, + boolean errOnNonExist, boolean check) throws DdlException { + Map privsMap = domainMap.get(domainName); + if (privsMap == null && errOnNonExist) { + throw new DdlException("Domain " + domainName + " does not exist"); + } + + PrivBitSet existingPrivs = privsMap.get(tblPattern); + if (existingPrivs == null) { + throw new DdlException("No such grants on " + tblPattern); + } + + if (!check) { + existingPrivs.remove(privs); + } + } + + public void updateResolovedIps(String qualifiedUser, String domain, Set newResolvedIPs) { + Map privsMap = domainMap.get(domain); + if (privsMap == null) { + LOG.debug("domain does not exist in white list: {}", domain); + return; + } + + Set preResolvedIPs = resolvedIPMap.get(domain); + if (preResolvedIPs == null) { + preResolvedIPs = Sets.newHashSet(); + } + + // 1. grant for newly added IPs + for (String newIP : newResolvedIPs) { + UserIdentity userIdent = new UserIdentity(qualifiedUser, newIP); + userIdent.setIsAnalyzed(); + for (Map.Entry entry : privsMap.entrySet()) { + try { + // we copy the PrivBitSet, cause we don't want use the same PrivBitSet object in different place. + // otherwise, when we change the privs of the domain, the priv entry will be changed synchronously, + // which is not expected. + Catalog.getCurrentCatalog().getAuth().grantPrivs(userIdent, entry.getKey(), + entry.getValue().copy(), + false /* err on non exist */, + true /* set by resolver */); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } + } + + // set password + try { + Catalog.getCurrentCatalog().getAuth().setPasswordInternal(userIdent, password, + true /* add if not exist */, + true /* set by resolver */, + true /* is replay */); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } + } + + // 2. delete privs which does not exist anymore. + for (String preIP : preResolvedIPs) { + if (!newResolvedIPs.contains(preIP)) { + UserIdentity userIdent = new UserIdentity(qualifiedUser, preIP); + userIdent.setIsAnalyzed(); + for (Map.Entry entry : privsMap.entrySet()) { + try { + Catalog.getCurrentCatalog().getAuth().revokePrivs(userIdent, entry.getKey(), + entry.getValue(), + false, /* err on non exist */ + true /* set by domain */, + true /* delete entry when empty */); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } + } + + // delete password + Catalog.getCurrentCatalog().getAuth().deletePassworEntry(userIdent); + } + } + + // update resolved ip map + resolvedIPMap.put(domain, newResolvedIPs); + } + + public Map> getResolvedIPs() { + return resolvedIPMap; + } + + public boolean containsDomain(String domain) { + return domainMap.containsKey(domain); + } + + public Set getAllDomains() { + return Sets.newHashSet(domainMap.keySet()); + } + + public void updateDomainMap(String domain, Map privs) { + domainMap.put(domain, privs); + } + + public void getAuthInfo(String qualifiedUser, List> userAuthInfos) { + LOG.debug("get domain privs for {}, domain map: {}", qualifiedUser, domainMap); + for (Map.Entry> entry : domainMap.entrySet()) { + List userEntry = Lists.newArrayList(); + UserIdentity tmpUserIdent = new UserIdentity(qualifiedUser, entry.getKey(), true); + tmpUserIdent.setIsAnalyzed(); + + Map privsMap = entry.getValue(); + // global privs + for (Map.Entry privsEntry : privsMap.entrySet()) { + if (privsEntry.getKey().getPrivLevel() != PrivLevel.GLOBAL) { + continue; + } + userEntry.add(tmpUserIdent.toString()); + userEntry.add(password == null ? "N/A" : "Yes"); + userEntry.add(privsEntry.getValue().toString()); + // at most one global priv entry + break; + } + if (userEntry.isEmpty()) { + userEntry.add(tmpUserIdent.toString()); + userEntry.add("N/A"); + userEntry.add("N/A"); + } + + // db privs + List dbPrivs = Lists.newArrayList(); + for (Map.Entry privsEntry : privsMap.entrySet()) { + if (privsEntry.getKey().getPrivLevel() != PrivLevel.DATABASE) { + continue; + } + dbPrivs.add(privsEntry.getKey().getQuolifiedDb() + ": " + privsEntry.getValue().toString()); + } + if (!dbPrivs.isEmpty()) { + userEntry.add(Joiner.on("\n").join(dbPrivs)); + } else { + userEntry.add("N/A"); + } + + // tbl privs + List tblPrivs = Lists.newArrayList(); + for (Map.Entry privsEntry : privsMap.entrySet()) { + if (privsEntry.getKey().getPrivLevel() != PrivLevel.TABLE) { + continue; + } + tblPrivs.add(privsEntry.getKey().toString() + ": " + privsEntry.getValue().toString()); + } + if (!tblPrivs.isEmpty()) { + userEntry.add(Joiner.on("\n").join(tblPrivs)); + } else { + userEntry.add("N/A"); + } + + userAuthInfos.add(userEntry); + } + } + + @Override + public String toString() { + return domainMap.toString(); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(domainMap.size()); + for (String domain : domainMap.keySet()) { + Text.writeString(out, domain); + Map privsMap = domainMap.get(domain); + out.writeInt(privsMap.size()); + for (Map.Entry entry : privsMap.entrySet()) { + entry.getKey().write(out); + entry.getValue().write(out); + } + } + + if (password == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeInt(password.length); + out.write(password); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_43) { + int ipWhiteListsLen = in.readInt(); + for (int i = 0; i < ipWhiteListsLen; i++) { + ipWhiteLists.add(Text.readString(in)); + } + LOG.debug("get white list ip: {}", ipWhiteLists); + + int starIpWhiteListsLen = in.readInt(); + for (int i = 0; i < starIpWhiteListsLen; i++) { + starIpWhiteLists.add(Text.readString(in)); + } + LOG.debug("get star white list ip: {}", starIpWhiteLists); + + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String domainName = Text.readString(in); + Map privMap = Maps.newConcurrentMap(); + // NOTICE: for forward compatibility. but we can't get user's privs here, + // so set it to empty privs on *.*. and we will set privs later + TablePattern tablePattern = TablePattern.ALL; + PrivBitSet privs = PrivBitSet.of(); + privMap.put(tablePattern, privs); + domainMap.put(domainName, privMap); + } + LOG.debug("get domain map: {}", domainMap); + } + + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_43) { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String domain = Text.readString(in); + Map privsMap = Maps.newConcurrentMap(); + domainMap.put(domain, privsMap); + + int count = in.readInt(); + for (int j = 0; j < count; j++) { + TablePattern tablePattern = TablePattern.read(in); + PrivBitSet privs = PrivBitSet.read(in); + privsMap.put(tablePattern, privs); + } + } + + if (in.readBoolean()) { + int passwordLen = in.readInt(); + password = new byte[passwordLen]; + in.readFully(password); + } + } + } +} diff --git a/fe/src/com/baidu/palo/persist/EditLog.java b/fe/src/com/baidu/palo/persist/EditLog.java index 9b8d2d772c..6747219664 100644 --- a/fe/src/com/baidu/palo/persist/EditLog.java +++ b/fe/src/com/baidu/palo/persist/EditLog.java @@ -18,12 +18,15 @@ package com.baidu.palo.persist; import com.baidu.palo.alter.DecommissionBackendJob; import com.baidu.palo.alter.RollupJob; import com.baidu.palo.alter.SchemaChangeJob; +import com.baidu.palo.analysis.UserIdentity; import com.baidu.palo.backup.BackupJob; +import com.baidu.palo.backup.BackupJob_D; +import com.baidu.palo.backup.Repository; import com.baidu.palo.backup.RestoreJob; +import com.baidu.palo.backup.RestoreJob_D; import com.baidu.palo.catalog.BrokerMgr; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.UserProperty; import com.baidu.palo.cluster.BaseParam; import com.baidu.palo.cluster.Cluster; import com.baidu.palo.common.Config; @@ -45,6 +48,7 @@ import com.baidu.palo.load.Load; import com.baidu.palo.load.LoadErrorHub; import com.baidu.palo.load.LoadJob; import com.baidu.palo.metric.MetricRepo; +import com.baidu.palo.mysql.privilege.UserProperty; import com.baidu.palo.qe.SessionVariable; import com.baidu.palo.system.Backend; import com.baidu.palo.system.Frontend; @@ -229,29 +233,26 @@ public class EditLog { catalog.replayRenamePartition(info); break; } - case OperationType.OP_BACKUP_START: { - BackupJob job = (BackupJob) journal.getData(); - catalog.getBackupHandler().replayBackupStart(catalog, job); - break; - } - case OperationType.OP_BACKUP_FINISH_SNAPSHOT: { - BackupJob job = (BackupJob) journal.getData(); - catalog.getBackupHandler().replayBackupFinishSnapshot(job); - break; - } + case OperationType.OP_BACKUP_START: + case OperationType.OP_BACKUP_FINISH_SNAPSHOT: case OperationType.OP_BACKUP_FINISH: { - BackupJob job = (BackupJob) journal.getData(); - catalog.getBackupHandler().replayBackupFinish(catalog, job); - break; - } - case OperationType.OP_RESTORE_START: { - RestoreJob job = (RestoreJob) journal.getData(); - catalog.getBackupHandler().replayRestoreStart(catalog, job); + BackupJob_D job = (BackupJob_D) journal.getData(); break; } + case OperationType.OP_RESTORE_START: case OperationType.OP_RESTORE_FINISH: { + RestoreJob_D job = (RestoreJob_D) journal.getData(); + break; + } + case OperationType.OP_BACKUP_JOB: { + BackupJob job = (BackupJob) journal.getData(); + catalog.getBackupHandler().replayAddJob(job); + break; + } + case OperationType.OP_RESTORE_JOB: { RestoreJob job = (RestoreJob) journal.getData(); - catalog.getBackupHandler().replayRestoreFinish(catalog, job); + job.setCatalog(catalog); + catalog.getBackupHandler().replayAddJob(job); break; } case OperationType.OP_START_ROLLUP: { @@ -418,13 +419,48 @@ public class EditLog { break; } case OperationType.OP_ALTER_ACCESS_RESOURCE: { - UserProperty resource = (UserProperty) journal.getData(); - catalog.getUserMgr().replayAlterAccess(resource); + UserProperty userProperty = (UserProperty) journal.getData(); + catalog.getAuth().replayAlterAccess(userProperty); break; } case OperationType.OP_DROP_USER: { String userName = ((Text) journal.getData()).toString(); - catalog.getUserMgr().replayDropUser(userName); + catalog.getAuth().replayOldDropUser(userName); + break; + } + case OperationType.OP_CREATE_USER: { + PrivInfo privInfo = (PrivInfo) journal.getData(); + catalog.getAuth().replayCreateUser(privInfo); + break; + } + case OperationType.OP_NEW_DROP_USER: { + UserIdentity userIdent = (UserIdentity) journal.getData(); + catalog.getAuth().replayDropUser(userIdent); + break; + } + case OperationType.OP_GRANT_PRIV: { + PrivInfo privInfo = (PrivInfo) journal.getData(); + catalog.getAuth().replayGrant(privInfo); + break; + } + case OperationType.OP_REVOKE_PRIV: { + PrivInfo privInfo = (PrivInfo) journal.getData(); + catalog.getAuth().replayRevoke(privInfo); + break; + } + case OperationType.OP_SET_PASSWORD: { + PrivInfo privInfo = (PrivInfo) journal.getData(); + catalog.getAuth().replaySetPassword(privInfo); + break; + } + case OperationType.OP_CREATE_ROLE: { + PrivInfo privInfo = (PrivInfo) journal.getData(); + catalog.getAuth().replayCreateRole(privInfo); + break; + } + case OperationType.OP_DROP_ROLE: { + PrivInfo privInfo = (PrivInfo) journal.getData(); + catalog.getAuth().replayDropRole(privInfo); break; } case OperationType.OP_TIMESTAMP: { @@ -514,9 +550,20 @@ public class EditLog { catalog.replayUpdateClusterAndBackends(info); break; } + case OperationType.OP_CREATE_REPOSITORY: { + Repository repository = (Repository) journal.getData(); + catalog.getBackupHandler().getRepoMgr().addAndInitRepoIfNotExist(repository, true); + break; + } + case OperationType.OP_DROP_REPOSITORY: { + String repoName = ((Text) journal.getData()).toString(); + catalog.getBackupHandler().getRepoMgr().removeRepo(repoName, true); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); + throw e; } } } catch (Exception e) { @@ -777,10 +824,39 @@ public class EditLog { logEdit(OperationType.OP_ALTER_ACCESS_RESOURCE, userProperty); } + @Deprecated public void logDropUser(String userName) { logEdit(OperationType.OP_DROP_USER, new Text(userName)); } + public void logCreateUser(PrivInfo info) { + logEdit(OperationType.OP_CREATE_USER, info); + } + + public void logNewDropUser(UserIdentity userIdent) { + logEdit(OperationType.OP_NEW_DROP_USER, userIdent); + } + + public void logGrantPriv(PrivInfo info) { + logEdit(OperationType.OP_GRANT_PRIV, info); + } + + public void logRevokePriv(PrivInfo info) { + logEdit(OperationType.OP_REVOKE_PRIV, info); + } + + public void logSetPassword(PrivInfo info) { + logEdit(OperationType.OP_SET_PASSWORD, info); + } + + public void logCreateRole(PrivInfo info) { + logEdit(OperationType.OP_CREATE_ROLE, info); + } + + public void logDropRole(PrivInfo info) { + logEdit(OperationType.OP_DROP_ROLE, info); + } + public void logStartDecommissionBackend(DecommissionBackendJob job) { logEdit(OperationType.OP_START_DECOMMISSION_BACKEND, job); } @@ -809,23 +885,23 @@ public class EditLog { logEdit(OperationType.OP_RENAME_PARTITION, tableInfo); } - public void logBackupStart(BackupJob backupJob) { + public void logBackupStart(BackupJob_D backupJob) { logEdit(OperationType.OP_BACKUP_START, backupJob); } - public void logBackupFinishSnapshot(BackupJob backupJob) { + public void logBackupFinishSnapshot(BackupJob_D backupJob) { logEdit(OperationType.OP_BACKUP_FINISH_SNAPSHOT, backupJob); } - public void logBackupFinish(BackupJob backupJob) { + public void logBackupFinish(BackupJob_D backupJob) { logEdit(OperationType.OP_BACKUP_FINISH, backupJob); } - public void logRestoreJobStart(RestoreJob restoreJob) { + public void logRestoreJobStart(RestoreJob_D restoreJob) { logEdit(OperationType.OP_RESTORE_START, restoreJob); } - public void logRestoreFinish(RestoreJob restoreJob) { + public void logRestoreFinish(RestoreJob_D restoreJob) { logEdit(OperationType.OP_RESTORE_FINISH, restoreJob); } @@ -891,4 +967,19 @@ public class EditLog { logEdit(OperationType.OP_UPDATE_CLUSTER_AND_BACKENDS, info); } + public void logBackupJob(BackupJob job) { + logEdit(OperationType.OP_BACKUP_JOB, job); + } + + public void logCreateRepository(Repository repo) { + logEdit(OperationType.OP_CREATE_REPOSITORY, repo); + } + + public void logDropRepository(String repoName) { + logEdit(OperationType.OP_DROP_REPOSITORY, new Text(repoName)); + } + + public void logRestoreJob(RestoreJob job) { + logEdit(OperationType.OP_RESTORE_JOB, job); + } } diff --git a/fe/src/com/baidu/palo/persist/OperationType.java b/fe/src/com/baidu/palo/persist/OperationType.java index 3b04bdb5c4..3268a57921 100644 --- a/fe/src/com/baidu/palo/persist/OperationType.java +++ b/fe/src/com/baidu/palo/persist/OperationType.java @@ -38,11 +38,18 @@ public class OperationType { public static final short OP_RECOVER_PARTITION = 18; public static final short OP_RENAME_TABLE = 19; public static final short OP_RENAME_PARTITION = 110; + @Deprecated public static final short OP_BACKUP_START = 111; + @Deprecated public static final short OP_BACKUP_FINISH_SNAPSHOT = 112; + @Deprecated public static final short OP_BACKUP_FINISH = 113; + @Deprecated public static final short OP_RESTORE_START = 114; + @Deprecated public static final short OP_RESTORE_FINISH = 115; + public static final short OP_BACKUP_JOB = 116; + public static final short OP_RESTORE_JOB = 117; // 20~29 120~129 220~229 ... public static final short OP_START_ROLLUP = 20; @@ -87,7 +94,15 @@ public class OperationType { public static final short OP_SET_LOAD_ERROR_URL = 58; public static final short OP_ALTER_ACCESS_RESOURCE = 60; + @Deprecated public static final short OP_DROP_USER = 61; + public static final short OP_CREATE_USER = 62; + public static final short OP_NEW_DROP_USER = 63; + public static final short OP_GRANT_PRIV = 64; + public static final short OP_REVOKE_PRIV = 65; + public static final short OP_SET_PASSWORD = 66; + public static final short OP_CREATE_ROLE = 67; + public static final short OP_DROP_ROLE = 68; public static final short OP_TIMESTAMP = 70; public static final short OP_MASTER_INFO_CHANGE = 71; @@ -112,4 +127,7 @@ public class OperationType { public static final short OP_DROP_ALL_BROKER = 87; public static final short OP_UPDATE_CLUSTER_AND_BACKENDS = 88; + + public static final short OP_CREATE_REPOSITORY = 89; + public static final short OP_DROP_REPOSITORY = 90; } diff --git a/fe/src/com/baidu/palo/persist/PrivInfo.java b/fe/src/com/baidu/palo/persist/PrivInfo.java new file mode 100644 index 0000000000..6e0a73ee29 --- /dev/null +++ b/fe/src/com/baidu/palo/persist/PrivInfo.java @@ -0,0 +1,141 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.persist; + +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.common.io.Text; +import com.baidu.palo.common.io.Writable; +import com.baidu.palo.mysql.privilege.PrivBitSet; + +import com.google.common.base.Strings; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class PrivInfo implements Writable { + private UserIdentity userIdent; + private TablePattern tblPattern; + private PrivBitSet privs; + private byte[] passwd; + private String role; + + private PrivInfo() { + + } + + public PrivInfo(UserIdentity userIdent, TablePattern tablePattern, PrivBitSet privs, + byte[] passwd, String role) { + this.userIdent = userIdent; + this.tblPattern = tablePattern; + this.privs = privs; + this.passwd = passwd; + this.role = role; + } + + public UserIdentity getUserIdent() { + return userIdent; + } + + public TablePattern getTblPattern() { + return tblPattern; + } + + public PrivBitSet getPrivs() { + return privs; + } + + public byte[] getPasswd() { + return passwd; + } + + public String getRole() { + return role; + } + + public static PrivInfo read(DataInput in) throws IOException { + PrivInfo info = new PrivInfo(); + info.readFields(in); + return info; + } + + @Override + public void write(DataOutput out) throws IOException { + if (userIdent != null) { + out.writeBoolean(true); + userIdent.write(out); + } else { + out.writeBoolean(false); + } + + if (tblPattern != null) { + out.writeBoolean(true); + tblPattern.write(out); + } else { + out.writeBoolean(false); + } + + if (privs != null) { + out.writeBoolean(true); + privs.write(out); + } else { + out.writeBoolean(false); + } + + if (passwd != null) { + out.writeBoolean(true); + out.writeInt(passwd.length); + out.write(passwd); + } else { + out.writeBoolean(false); + } + + if (!Strings.isNullOrEmpty(role)) { + out.writeBoolean(true); + Text.writeString(out, role); + } else { + out.writeBoolean(false); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + if (in.readBoolean()) { + userIdent = UserIdentity.read(in); + } + + if (in.readBoolean()) { + tblPattern = TablePattern.read(in); + } + + if (in.readBoolean()) { + privs = PrivBitSet.read(in); + } + + if (in.readBoolean()) { + int passwordLen = in.readInt(); + passwd = new byte[passwordLen]; + in.readFully(passwd); + } + + if (in.readBoolean()) { + role = Text.readString(in); + } + + } + +} diff --git a/fe/src/com/baidu/palo/planner/BrokerScanNode.java b/fe/src/com/baidu/palo/planner/BrokerScanNode.java index 9d4ad5e2d8..08225756c5 100644 --- a/fe/src/com/baidu/palo/planner/BrokerScanNode.java +++ b/fe/src/com/baidu/palo/planner/BrokerScanNode.java @@ -439,6 +439,9 @@ public class BrokerScanNode extends ScanNode { candidateBes.add(backends.get(nextBe++)); nextBe = nextBe % backends.size(); } + // we shuffle it because if we only has 3 backends + // we will always choose the same backends without shuffle + Collections.shuffle(candidateBes); // Generate on broker scan range TBrokerScanRange brokerScanRange = new TBrokerScanRange(); @@ -480,7 +483,7 @@ public class BrokerScanNode extends ScanNode { private void getFileStatusAndCalcInstance() throws InternalException { if (fileStatusesList == null || filesAdded == -1) { // FIXME(cmy): fileStatusesList and filesAdded can be set out of db lock when doing pull load, - // but for now it is very difficult set them out of db lock when doing broker query. + // but for now it is very difficult to set them out of db lock when doing broker query. // So we leave this code block here. // This will be fixed later. fileStatusesList = Lists.newArrayList(); diff --git a/fe/src/com/baidu/palo/planner/OlapScanNode.java b/fe/src/com/baidu/palo/planner/OlapScanNode.java index 5f39470d11..582643d408 100644 --- a/fe/src/com/baidu/palo/planner/OlapScanNode.java +++ b/fe/src/com/baidu/palo/planner/OlapScanNode.java @@ -78,6 +78,7 @@ public class OlapScanNode extends ScanNode { private List result = new ArrayList(); private boolean isPreAggregation = false; + private String reasonOfPreAggregation = null; private boolean canTurnOnPreAggr = true; private ArrayList tupleColumns = new ArrayList(); private HashSet predicateColumns = new HashSet(); @@ -99,8 +100,9 @@ public class OlapScanNode extends ScanNode { olapTable = (OlapTable) desc.getTable(); } - public void setIsPreAggregation(boolean isPreAggregation) { + public void setIsPreAggregation(boolean isPreAggregation, String reason) { this.isPreAggregation = isPreAggregation; + this.reasonOfPreAggregation = reason; } @@ -555,7 +557,7 @@ public class OlapScanNode extends ScanNode { if (isPreAggregation) { output.append(prefix).append("PREAGGREGATION: ON").append("\n"); } else { - output.append(prefix).append("PREAGGREGATION: OFF").append("\n"); + output.append(prefix).append("PREAGGREGATION: OFF. Reason: ").append(reasonOfPreAggregation).append("\n"); } if (!conjuncts.isEmpty()) { output.append(prefix).append("PREDICATES: ").append( diff --git a/fe/src/com/baidu/palo/planner/SchemaScanNode.java b/fe/src/com/baidu/palo/planner/SchemaScanNode.java index e001953338..9f5ac7c3e3 100644 --- a/fe/src/com/baidu/palo/planner/SchemaScanNode.java +++ b/fe/src/com/baidu/palo/planner/SchemaScanNode.java @@ -31,13 +31,13 @@ import com.baidu.palo.thrift.TPlanNode; import com.baidu.palo.thrift.TPlanNodeType; import com.baidu.palo.thrift.TScanRangeLocations; import com.baidu.palo.thrift.TSchemaScanNode; + import com.google.common.base.Objects; import com.google.common.base.Objects.ToStringHelper; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import java.net.InetAddress; -import java.net.UnknownHostException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.List; /** @@ -47,12 +47,13 @@ public class SchemaScanNode extends ScanNode { private static final Logger LOG = LogManager.getLogger(SchemaTable.class); private final String tableName; - private String schemaDb; - private String schemaTable; - private String schemaWild; - private String user; - private String frontendIP; - private int frontendPort; + private String schemaDb; + private String schemaTable; + private String schemaWild; + private String user; + private String userIp; + private String frontendIP; + private int frontendPort; /** * Constructs node to scan given data files of table 'tbl'. @@ -74,7 +75,8 @@ public class SchemaScanNode extends ScanNode { schemaDb = analyzer.getSchemaDb(); schemaTable = analyzer.getSchemaTable(); schemaWild = analyzer.getSchemaWild(); - user = analyzer.getUser(); + user = analyzer.getQualifiedUser(); + userIp = analyzer.getContext().getRemoteIP(); frontendIP = FrontendOptions.getLocalHostAddress(); frontendPort = Config.rpc_port; } @@ -108,10 +110,11 @@ public class SchemaScanNode extends ScanNode { } msg.schema_scan_node.setIp(frontendIP); msg.schema_scan_node.setPort(frontendPort); + msg.schema_scan_node.setUser_ip(userIp); } /** - * We query MySQL Meta to get request's data localtion + * We query MySQL Meta to get request's data location * extra result info will pass to backend ScanNode */ @Override diff --git a/fe/src/com/baidu/palo/planner/SingleNodePlanner.java b/fe/src/com/baidu/palo/planner/SingleNodePlanner.java index f0fb8bac0b..f3d4c2c92a 100644 --- a/fe/src/com/baidu/palo/planner/SingleNodePlanner.java +++ b/fe/src/com/baidu/palo/planner/SingleNodePlanner.java @@ -343,15 +343,15 @@ public class SingleNodePlanner { } private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyzer analyzer, PlanNode root) { + String turnOffReason = null; do { - String logStr = "turn off preAggregate because: "; if (null == aggInfo) { - LOG.info(logStr + "No AggregateInfo"); + turnOffReason = "No AggregateInfo"; break; } if (!(root instanceof OlapScanNode)) { - LOG.info(logStr + "left-deep Node is not OlapScanNode"); + turnOffReason = "left-deep Node is not OlapScanNode"; break; } @@ -363,8 +363,8 @@ public class SingleNodePlanner { final JoinOperator joinOperator = selectStmt.getTableRefs().get(i).getJoinOp(); // TODO chenhao16 , right out join ? if (joinOperator.isRightOuterJoin() || joinOperator.isFullOuterJoin()) { - LOG.info(logStr + selectStmt.getTableRefs().get(i) - + " joinOp is full outer join or right outer join."); + turnOffReason = selectStmt.getTableRefs().get(i) + + " joinOp is full outer join or right outer join."; aggTableValidate = false; break; } @@ -391,12 +391,9 @@ public class SingleNodePlanner { if (analyzer.getTupleDesc(tupleId).getRef() != olapTableRef) { if (analyzer.getTupleDesc(tupleId).getTable() != null - && analyzer.getTupleDesc(tupleId).getTable().getType() - == Table.TableType.OLAP) { - LOG.info("{} agg expr [{}] is not bound [{}]", - logStr, - aggExpr.debugString(), - selectStmt.getTableRefs().get(0).toSql()); + && analyzer.getTupleDesc(tupleId).getTable().getType() == Table.TableType.OLAP) { + turnOffReason = "agg expr [" + aggExpr.debugString() + "] is not bound [" + + selectStmt.getTableRefs().get(0).toSql() + "]"; aggTableValidate = false; } else { LOG.debug("The table which agg expr [{}] is bound to, is not OLAP table [{}]", @@ -416,8 +413,7 @@ public class SingleNodePlanner { } boolean valueColumnValidate = true; - List allConjuncts = - analyzer.getAllConjunt(selectStmt.getTableRefs().get(0).getId()); + List allConjuncts = analyzer.getAllConjunt(selectStmt.getTableRefs().get(0).getId()); List conjunctSlotIds = Lists.newArrayList(); if (allConjuncts != null) { for (Expr conjunct : allConjuncts) { @@ -426,8 +422,8 @@ public class SingleNodePlanner { for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) { if (!slot.getColumn().isKey()) { if (conjunctSlotIds.contains(slot.getId())) { - LOG.info(logStr + "conjunct on " + slot.getColumn().getName() + " which is " - + "OlapEngine value column"); + turnOffReason = "conjunct on " + slot.getColumn().getName() + + " which is OlapEngine value column"; valueColumnValidate = false; break; } @@ -441,7 +437,7 @@ public class SingleNodePlanner { boolean aggExprValidate = true; for (FunctionCallExpr aggExpr : aggExprs) { if (aggExpr.getChildren().size() != 1) { - LOG.info(logStr + "aggExpr has more than one child"); + turnOffReason = "aggExpr has more than one child"; aggExprValidate = false; break; } @@ -455,8 +451,7 @@ public class SingleNodePlanner { && child.getChild(0).getType().isNumericType()) { returnColumns.add(((SlotRef) child.getChild(0)).getDesc().getColumn()); } else { - LOG.info("{} aggExpr.getChild(0)[{}] is not Numeric CastExpr", - logStr, aggExpr.getChild(0).toSql()); + turnOffReason = "aggExpr.getChild(0)[aggExpr.getChild(0).toSql()] is not Numeric CastExpr"; aggExprValidate = false; break; } @@ -481,8 +476,8 @@ public class SingleNodePlanner { if (returnExpr instanceof SlotRef) { returnColumns.add(((SlotRef) returnExpr).getDesc().getColumn()); } else { - LOG.info("{} aggExpr.getChild(0)[{}] is not SlotExpr", - logStr, aggExpr.getChild(0).toSql()); + turnOffReason = "aggExpr.getChild(0)[" + aggExpr.getChild(0).toSql() + + "] is not SlotExpr"; caseReturnExprValidate = false; break; } @@ -494,9 +489,8 @@ public class SingleNodePlanner { } } else { - LOG.info("{} aggExpr.getChild(0)[{}] is not SlotRef or CastExpr|CaseExpr", - logStr, - aggExpr.getChild(0).debugString()); + turnOffReason = "aggExpr.getChild(0)[" + aggExpr.getChild(0).debugString() + + "] is not SlotRef or CastExpr|CaseExpr"; aggExprValidate = false; break; } @@ -513,10 +507,8 @@ public class SingleNodePlanner { continue; } if (!col.isKey()) { - LOG.info("{} the condition column [{}] is not key type in aggr expr [{}].", - logStr, - col.getName(), - aggExpr.toSql()); + turnOffReason = "the condition column [" + col.getName() + "] is not key type in aggr expr [" + + aggExpr.toSql() + "]."; conditionColumnValidate = false; break; } @@ -538,46 +530,41 @@ public class SingleNodePlanner { if (aggExpr.getFnName().getFunction().equalsIgnoreCase("MAX") && aggExpr.getFnName().getFunction().equalsIgnoreCase("MIN")) { returnColumnValidate = false; - LOG.info("{} the type of agg on OlapEngine's Key column should only be MAX or MIN. " - + "agg expr: {}", - logStr, - aggExpr.toSql()); + turnOffReason = "the type of agg on OlapEngine's Key column should only be MAX or MIN." + + "agg expr: " + aggExpr.toSql(); break; } } if (aggExpr.getFnName().getFunction().equalsIgnoreCase("SUM")) { if (col.getAggregationType() != AggregateType.SUM) { - LOG.info( - logStr + "Aggregate Operator not match: SUM <--> " + col - .getAggregationType()); + turnOffReason = "Aggregate Operator not match: SUM <--> " + col.getAggregationType(); returnColumnValidate = false; break; } } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("MAX")) { if ((!col.isKey()) && col.getAggregationType() != AggregateType.MAX) { - LOG.info( - logStr + "Aggregate Operator not match: MAX <--> " + col - .getAggregationType()); + turnOffReason = "Aggregate Operator not match: MAX <--> " + col.getAggregationType(); returnColumnValidate = false; break; } } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("MIN")) { if ((!col.isKey()) && col.getAggregationType() != AggregateType.MIN) { - LOG.info( - logStr + "Aggregate Operator not match: MIN <--> " + col - .getAggregationType()); + turnOffReason = "Aggregate Operator not match: MIN <--> " + col.getAggregationType(); returnColumnValidate = false; break; } } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("HLL_UNION_AGG")) { } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("NDV")) { if ((!col.isKey())) { + turnOffReason = "NDV function with non-key column: " + col.getName(); returnColumnValidate = false; break; } + } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("multi_distinct_count")) { + // count(distinct k1), count(distinct k2) / count(distinct k1,k2) can turn on pre aggregation } else { - LOG.info(logStr + "Invalid Aggregate Operator: " + aggExpr.getFnName().getFunction()); + turnOffReason = "Invalid Aggregate Operator: " + aggExpr.getFnName().getFunction(); returnColumnValidate = false; break; } @@ -601,7 +588,7 @@ public class SingleNodePlanner { for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) { if (!slot.getColumn().isKey()) { if (groupSlotIds.contains(slot.getId())) { - LOG.info(logStr + "groupExpr contains OlapEngine's Value"); + turnOffReason = "groupExpr contains OlapEngine's Value"; groupExprValidate = false; break; } @@ -617,13 +604,18 @@ public class SingleNodePlanner { } OlapScanNode olapNode = (OlapScanNode) root; - if (olapNode.getCanTurnOnPreAggr()) { - ((OlapScanNode) root).setIsPreAggregation(true); - } else { - LOG.info("this olap-scan-node[{}] has already been turned off pre-aggregation. ", - olapNode.debugString()); + if (!olapNode.getCanTurnOnPreAggr()) { + turnOffReason = "this olap scan node[" + olapNode.debugString() + + "] has already been turned off pre-aggregation."; + break; } + + olapNode.setIsPreAggregation(true, null); } while (false); + + if ((root instanceof OlapScanNode) && turnOffReason != null) { + ((OlapScanNode) root).setIsPreAggregation(false, turnOffReason); + } } /** @@ -1233,7 +1225,7 @@ public class SingleNodePlanner { */ private PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, TableRef outerRef, TableRef innerRef) throws InternalException, AnalysisException { - materializeTableResultForCrossJoinOrCountStar(innerRef, analyzer); + materializeTableResultForCrossJoinOrCountStar(innerRef, analyzer); // the rows coming from the build node only need to have space for the tuple // materialized by that node PlanNode inner = createTableRefNode(analyzer, innerRef); @@ -1371,6 +1363,11 @@ public class SingleNodePlanner { // List conjuncts = // analyzer.getUnassignedConjuncts(unionStmt.getTupleId().asList(), false); List conjuncts = analyzer.getUnassignedConjuncts(unionStmt.getTupleId().asList()); + // TODO chenhao16 + // Because Conjuncts can't be assigned to UnionNode and Palo's fe can't evaluate conjuncts, + // it needs to add SelectNode as UnionNode's parent, when UnionStmt's Ops contains constant + // Select. + boolean hasConstantOp = false; if (!unionStmt.hasAnalyticExprs()) { // Turn unassigned predicates for unionStmt's tupleId_ into predicates for // the individual operands. @@ -1379,17 +1376,32 @@ public class SingleNodePlanner { for (UnionStmt.UnionOperand op: unionStmt.getOperands()) { List opConjuncts = Expr.substituteList(conjuncts, op.getSmap(), analyzer, false); - if (op.getQueryStmt() instanceof SelectStmt) { - final SelectStmt select = (SelectStmt) op.getQueryStmt(); + boolean selectHasTableRef = true; + final QueryStmt queryStmt = op.getQueryStmt(); + // Check whether UnionOperand is constant Select. + if (queryStmt instanceof SelectStmt) { + final SelectStmt selectStmt = (SelectStmt) queryStmt; + if (selectStmt.getTableRefs().isEmpty()) { + selectHasTableRef = false; + hasConstantOp = !selectHasTableRef; + } + } + // Forbid to register Conjuncts with SelectStmt' tuple when Select is constant + if ((queryStmt instanceof SelectStmt) && selectHasTableRef) { + final SelectStmt select = (SelectStmt) queryStmt; op.getAnalyzer().registerConjuncts(opConjuncts, select.getTableRefIds()); - } else if (op.getQueryStmt() instanceof UnionStmt) { - final UnionStmt union = (UnionStmt) op.getQueryStmt(); + } else if (queryStmt instanceof UnionStmt) { + final UnionStmt union = (UnionStmt) queryStmt; op.getAnalyzer().registerConjuncts(opConjuncts, union.getTupleId().asList()); } else { - Preconditions.checkArgument(false); + if (selectHasTableRef) { + Preconditions.checkArgument(false); + } } } - analyzer.markConjunctsAssigned(conjuncts); + if (!hasConstantOp) { + analyzer.markConjunctsAssigned(conjuncts); + } } else { // mark slots referenced by the yet-unassigned conjuncts analyzer.materializeSlots(conjuncts); @@ -1410,7 +1422,7 @@ public class SingleNodePlanner { result = createUnionPlan(analyzer, unionStmt, unionStmt.getAllOperands(), result, defaultOrderByLimit); } - if (unionStmt.hasAnalyticExprs()) { + if (unionStmt.hasAnalyticExprs() || hasConstantOp) { result = addUnassignedConjuncts( analyzer, unionStmt.getTupleId().asList(), result); } @@ -1483,14 +1495,15 @@ public class SingleNodePlanner { exprSize += slot.getByteSize(); } - if (exprIsMaterialized && exprSize <= resultExprSelectedSize) { - resultExprSelectedSize = exprSize; - resultExprSelected = e; - } // Result Expr contains materialized expr, return if (exprIsMaterialized) { return; } + + if (exprSize <= resultExprSelectedSize) { + resultExprSelectedSize = exprSize; + resultExprSelected = e; + } } // materialize slots which expr refer and It's total size is smallest @@ -1508,3 +1521,4 @@ public class SingleNodePlanner { } } + diff --git a/fe/src/com/baidu/palo/qe/ConnectContext.java b/fe/src/com/baidu/palo/qe/ConnectContext.java index 9561b3f598..2bb05fde8c 100644 --- a/fe/src/com/baidu/palo/qe/ConnectContext.java +++ b/fe/src/com/baidu/palo/qe/ConnectContext.java @@ -15,9 +15,6 @@ package com.baidu.palo.qe; -import java.nio.channels.SocketChannel; -import java.util.List; - import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.mysql.MysqlCapability; @@ -32,6 +29,9 @@ import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.nio.channels.SocketChannel; +import java.util.List; + // When one client connect in, we create a connect context for it. // We store session information here. Meanwhile ConnectScheduler all // connect with its connection id. @@ -59,7 +59,7 @@ public class ConnectContext { // cluster name private volatile String clusterName = ""; // User - private volatile String user; + private volatile String qualifiedUser; // Serializer used to pack MySQL packet. private volatile MysqlSerializer serializer; // Variables belong to this session. @@ -82,6 +82,8 @@ public class ConnectContext { private AuditBuilder auditBuilder; + private String remoteIP; + public static ConnectContext get() { return threadLocalInfo.get(); } @@ -108,6 +110,17 @@ public class ConnectContext { sessionVariable = VariableMgr.newSessionVariable(); auditBuilder = new AuditBuilder(); command = MysqlCommand.COM_SLEEP; + if (channel != null) { + remoteIP = mysqlChannel.getRemoteIp(); + } + } + + public String getRemoteIP() { + return remoteIP; + } + + public void setRemoteIP(String remoteIP) { + this.remoteIP = remoteIP; } public AuditBuilder getAuditBuilder() { @@ -119,7 +132,7 @@ public class ConnectContext { } public TResourceInfo toResourceCtx() { - return new TResourceInfo(user, sessionVariable.getResourceGroup()); + return new TResourceInfo(qualifiedUser, sessionVariable.getResourceGroup()); } public void setCatalog(Catalog catalog) { @@ -130,12 +143,12 @@ public class ConnectContext { return catalog; } - public String getUser() { - return user; + public String getQualifiedUser() { + return qualifiedUser; } - public void setUser(String user) { - this.user = user; + public void setQualifiedUser(String qualifiedUser) { + this.qualifiedUser = qualifiedUser; } public SessionVariable getSessionVariable() { @@ -257,7 +270,7 @@ public class ConnectContext { } LOG.warn("kill timeout query, {}, kill connection: {}", - mysqlChannel.getRemoteHostString(), killConnection); + mysqlChannel.getRemoteHostPortString(), killConnection); if (killConnection) { isKilled = true; @@ -283,7 +296,7 @@ public class ConnectContext { if (delta > sessionVariable.getWaitTimeoutS() * 1000) { // Need kill this connection. LOG.warn("kill wait timeout connection, remote: {}, wait timeout: {}", - mysqlChannel.getRemoteHostString(), sessionVariable.getWaitTimeoutS()); + mysqlChannel.getRemoteHostPortString(), sessionVariable.getWaitTimeoutS()); killFlag = true; killConnection = true; @@ -291,7 +304,7 @@ public class ConnectContext { } else { if (delta > sessionVariable.getQueryTimeoutS() * 1000) { LOG.warn("kill query timeout, remote: {}, query timeout: {}", - mysqlChannel.getRemoteHostString(), sessionVariable.getQueryTimeoutS()); + mysqlChannel.getRemoteHostPortString(), sessionVariable.getQueryTimeoutS()); // Only kill killFlag = true; @@ -314,8 +327,8 @@ public class ConnectContext { public List toRow(long nowMs) { List row = Lists.newArrayList(); row.add("" + connectionId); - row.add(ClusterNamespace.getNameFromFullName(user)); - row.add(mysqlChannel.getRemoteHostString()); + row.add(ClusterNamespace.getNameFromFullName(qualifiedUser)); + row.add(mysqlChannel.getRemoteHostPortString()); row.add(clusterName); row.add(ClusterNamespace.getNameFromFullName(currentDb)); row.add(command.toString()); diff --git a/fe/src/com/baidu/palo/qe/ConnectProcessor.java b/fe/src/com/baidu/palo/qe/ConnectProcessor.java index 874455433a..9f9507edae 100644 --- a/fe/src/com/baidu/palo/qe/ConnectProcessor.java +++ b/fe/src/com/baidu/palo/qe/ConnectProcessor.java @@ -143,8 +143,8 @@ public class ConnectProcessor { } ctx.getAuditBuilder().reset(); // replace '\n' to '\\\n' to make string in one line - ctx.getAuditBuilder().put("client", ctx.getMysqlChannel().getRemoteHostString()); - ctx.getAuditBuilder().put("user", ctx.getUser()); + ctx.getAuditBuilder().put("client", ctx.getMysqlChannel().getRemoteHostPortString()); + ctx.getAuditBuilder().put("user", ctx.getQualifiedUser()); ctx.getAuditBuilder().put("db", ctx.getDatabase()); // execute this query. @@ -301,7 +301,7 @@ public class ConnectProcessor { public TMasterOpResult proxyExecute(TMasterOpRequest request) { ctx.setDatabase(request.db); - ctx.setUser(request.user); + ctx.setQualifiedUser(request.user); ctx.setCatalog(Catalog.getInstance()); ctx.getState().reset(); if (request.isSetCluster()) { @@ -316,6 +316,9 @@ public class ConnectProcessor { if (request.isSetQueryTimeout()) { ctx.getSessionVariable().setQueryTimeoutS(request.getQueryTimeout()); } + if (request.isSetUser_ip()) { + ctx.setRemoteIP(request.getUser_ip()); + } ctx.setThreadLocalInfo(); @@ -357,7 +360,7 @@ public class ConnectProcessor { try { packetBuf = channel.fetchOnePacket(); if (packetBuf == null) { - LOG.warn("Null packet received from network. remote: {}", channel.getRemoteHostString()); + LOG.warn("Null packet received from network. remote: {}", channel.getRemoteHostPortString()); throw new IOException("Error happened when receiving packet."); } } catch (AsynchronousCloseException e) { @@ -387,4 +390,3 @@ public class ConnectProcessor { } } } - diff --git a/fe/src/com/baidu/palo/qe/ConnectScheduler.java b/fe/src/com/baidu/palo/qe/ConnectScheduler.java index 9ce9648160..68574baf74 100644 --- a/fe/src/com/baidu/palo/qe/ConnectScheduler.java +++ b/fe/src/com/baidu/palo/qe/ConnectScheduler.java @@ -15,7 +15,9 @@ package com.baidu.palo.qe; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.mysql.MysqlProto; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -90,15 +92,15 @@ public class ConnectScheduler { return false; } // Check user - if (connByUser.get(ctx.getUser()) == null) { - connByUser.put(ctx.getUser(), new AtomicInteger(0)); + if (connByUser.get(ctx.getQualifiedUser()) == null) { + connByUser.put(ctx.getQualifiedUser(), new AtomicInteger(0)); } - int conns = connByUser.get(ctx.getUser()).get(); - if (conns >= ctx.getCatalog().getUserMgr().getMaxConn(ctx.getUser())) { + int conns = connByUser.get(ctx.getQualifiedUser()).get(); + if (conns >= ctx.getCatalog().getAuth().getMaxConn(ctx.getQualifiedUser())) { return false; } numberConnection++; - connByUser.get(ctx.getUser()).incrementAndGet(); + connByUser.get(ctx.getQualifiedUser()).incrementAndGet(); connectionMap.put((long) ctx.getConnectionId(), ctx); return true; } @@ -106,7 +108,7 @@ public class ConnectScheduler { public synchronized void unregisterConnection(ConnectContext ctx) { if (connectionMap.remove((long) ctx.getConnectionId()) != null) { numberConnection--; - AtomicInteger conns = connByUser.get(ctx.getUser()); + AtomicInteger conns = connByUser.get(ctx.getQualifiedUser()); if (conns != null) { conns.decrementAndGet(); } @@ -126,9 +128,12 @@ public class ConnectScheduler { for (ConnectContext ctx : connectionMap.values()) { // Check auth - if (!ctx.getCatalog().getUserMgr().checkUserAccess(user, ctx.getUser())) { + if (!ctx.getQualifiedUser().equals(user) && + !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.GRANT)) { continue; } + infos.add(ctx.toThreadInfo()); } return infos; diff --git a/fe/src/com/baidu/palo/qe/Coordinator.java b/fe/src/com/baidu/palo/qe/Coordinator.java index fd0b07ef99..04186c91e6 100644 --- a/fe/src/com/baidu/palo/qe/Coordinator.java +++ b/fe/src/com/baidu/palo/qe/Coordinator.java @@ -72,6 +72,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; @@ -181,7 +182,7 @@ public class Coordinator { this.returnedAllResults = false; this.queryOptions = context.getSessionVariable().toThrift(); this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); - this.tResourceInfo = new TResourceInfo(context.getUser(), + this.tResourceInfo = new TResourceInfo(context.getQualifiedUser(), context.getSessionVariable().getResourceGroup()); this.needReport = context.getSessionVariable().isReportSucc(); this.clusterName = context.getClusterName(); @@ -426,19 +427,19 @@ public class Coordinator { if (code != TStatusCode.OK) { if (errMsg == null) { - errMsg = "exec rpc error"; + errMsg = "exec rpc error. backend id: " + pair.first.systemBackendId; } queryStatus.setStatus(errMsg); LOG.warn("exec plan fragment failed, errmsg={}, fragmentId={}, backend={}:{}", - errMsg, fragment.getFragmentId(), - pair.first.address.hostname, pair.first.address.port); + errMsg, fragment.getFragmentId(), + pair.first.address.hostname, pair.first.address.port); cancelInternal(); switch (code) { case TIMEOUT: - throw new InternalException("query timeout"); + throw new InternalException("query timeout. backend id: " + pair.first.systemBackendId); case THRIFT_RPC_ERROR: SimpleScheduler.updateBlacklistBackends(pair.first.systemBackendId); - throw new RpcException("rpc failed"); + throw new RpcException("rpc failed. backend id: " + pair.first.systemBackendId); default: throw new InternalException(errMsg); } @@ -508,8 +509,8 @@ public class Coordinator { } void updateStatus(Status status) { + lock.lock(); try { - lock.lock(); // The query is done and we are just waiting for remote fragments to clean up. // Ignore their cancelled updates. if (returnedAllResults && status.isCancelled()) { @@ -531,7 +532,6 @@ public class Coordinator { } finally { lock.unlock(); } - } TResultBatch getNext() throws Exception { @@ -539,7 +539,7 @@ public class Coordinator { throw new InternalException("There is no receiver."); } - TResultBatch resultBatch; + TResultBatch resultBatch; Status status = new Status(); resultBatch = receiver.getNext(status); @@ -559,7 +559,6 @@ public class Coordinator { if (copyStatus.isRpcError()) { throw new RpcException(copyStatus.getErrorMsg()); } else { - String errMsg = copyStatus.getErrorMsg(); LOG.warn("query failed: {}", errMsg); diff --git a/fe/src/com/baidu/palo/qe/DdlExecutor.java b/fe/src/com/baidu/palo/qe/DdlExecutor.java index c9d01ada17..dbd1ec4512 100644 --- a/fe/src/com/baidu/palo/qe/DdlExecutor.java +++ b/fe/src/com/baidu/palo/qe/DdlExecutor.java @@ -20,7 +20,6 @@ import com.baidu.palo.analysis.AlterDatabaseQuotaStmt; import com.baidu.palo.analysis.AlterDatabaseRename; import com.baidu.palo.analysis.AlterSystemStmt; import com.baidu.palo.analysis.AlterTableStmt; -import com.baidu.palo.analysis.AlterUserStmt; import com.baidu.palo.analysis.BackupStmt; import com.baidu.palo.analysis.CancelAlterSystemStmt; import com.baidu.palo.analysis.CancelAlterTableStmt; @@ -28,6 +27,8 @@ import com.baidu.palo.analysis.CancelBackupStmt; import com.baidu.palo.analysis.CancelLoadStmt; import com.baidu.palo.analysis.CreateClusterStmt; import com.baidu.palo.analysis.CreateDbStmt; +import com.baidu.palo.analysis.CreateRepositoryStmt; +import com.baidu.palo.analysis.CreateRoleStmt; import com.baidu.palo.analysis.CreateTableStmt; import com.baidu.palo.analysis.CreateUserStmt; import com.baidu.palo.analysis.CreateViewStmt; @@ -35,6 +36,8 @@ import com.baidu.palo.analysis.DdlStmt; import com.baidu.palo.analysis.DeleteStmt; import com.baidu.palo.analysis.DropClusterStmt; import com.baidu.palo.analysis.DropDbStmt; +import com.baidu.palo.analysis.DropRepositoryStmt; +import com.baidu.palo.analysis.DropRoleStmt; import com.baidu.palo.analysis.DropTableStmt; import com.baidu.palo.analysis.DropUserStmt; import com.baidu.palo.analysis.GrantStmt; @@ -49,7 +52,6 @@ import com.baidu.palo.analysis.RevokeStmt; import com.baidu.palo.analysis.SetUserPropertyStmt; import com.baidu.palo.analysis.SyncStmt; import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.load.LoadJob.EtlJobType; @@ -61,15 +63,10 @@ public class DdlExecutor { if (ddlStmt instanceof CreateClusterStmt) { CreateClusterStmt stmt = (CreateClusterStmt) ddlStmt; catalog.createCluster(stmt); - catalog.getUserMgr().addUser(stmt.getClusterName(), - ClusterNamespace.getFullName(stmt.getClusterName(), CreateClusterStmt.CLUSTER_SUPERUSER_NAME), - stmt.getPassword(), true); } else if (ddlStmt instanceof AlterClusterStmt) { catalog.processModifyCluster((AlterClusterStmt) ddlStmt); } else if (ddlStmt instanceof DropClusterStmt) { catalog.dropCluster((DropClusterStmt) ddlStmt); - catalog.getUserMgr().dropUser(ClusterNamespace.getFullName(((DropClusterStmt) ddlStmt).getName(), - CreateClusterStmt.CLUSTER_SUPERUSER_NAME)); } else if (ddlStmt instanceof MigrateDbStmt) { catalog.migrateDb((MigrateDbStmt) ddlStmt); } else if (ddlStmt instanceof LinkDbStmt) { @@ -101,23 +98,25 @@ public class DdlExecutor { catalog.getLoadInstance().delete((DeleteStmt) ddlStmt); } else if (ddlStmt instanceof CreateUserStmt) { CreateUserStmt stmt = (CreateUserStmt) ddlStmt; - catalog.getUserMgr().addUser(stmt.getClusterName(), stmt.getUser(), stmt.getPassword(), stmt.isSuperuser()); + catalog.getAuth().createUser(stmt); } else if (ddlStmt instanceof DropUserStmt) { DropUserStmt stmt = (DropUserStmt) ddlStmt; - catalog.getUserMgr().dropUser(stmt.getUser()); + catalog.getAuth().dropUser(stmt); } else if (ddlStmt instanceof GrantStmt) { GrantStmt stmt = (GrantStmt) ddlStmt; - catalog.getUserMgr().grant(stmt.getUser(), stmt.getDb(), stmt.getPrivilege()); + catalog.getAuth().grant(stmt); } else if (ddlStmt instanceof RevokeStmt) { RevokeStmt stmt = (RevokeStmt) ddlStmt; - catalog.getUserMgr().revoke(stmt.getUser(), stmt.getDb()); + catalog.getAuth().revoke(stmt); + } else if (ddlStmt instanceof CreateRoleStmt) { + catalog.getAuth().createRole((CreateRoleStmt) ddlStmt); + } else if (ddlStmt instanceof DropRoleStmt) { + catalog.getAuth().dropRole((DropRoleStmt) ddlStmt); } else if (ddlStmt instanceof SetUserPropertyStmt) { - catalog.getUserMgr().updateUserProperty((SetUserPropertyStmt) ddlStmt); + catalog.getAuth().updateUserProperty((SetUserPropertyStmt) ddlStmt); } else if (ddlStmt instanceof AlterSystemStmt) { AlterSystemStmt stmt = (AlterSystemStmt) ddlStmt; catalog.alterCluster(stmt); - } else if (ddlStmt instanceof AlterUserStmt) { - catalog.alterUser((AlterUserStmt) ddlStmt); } else if (ddlStmt instanceof CancelAlterSystemStmt) { CancelAlterSystemStmt stmt = (CancelAlterSystemStmt) ddlStmt; catalog.cancelAlterCluster(stmt); @@ -139,6 +138,10 @@ public class DdlExecutor { catalog.restore((RestoreStmt) ddlStmt); } else if (ddlStmt instanceof CancelBackupStmt) { catalog.cancelBackup((CancelBackupStmt) ddlStmt); + } else if (ddlStmt instanceof CreateRepositoryStmt) { + catalog.getBackupHandler().createRepository((CreateRepositoryStmt) ddlStmt); + } else if (ddlStmt instanceof DropRepositoryStmt) { + catalog.getBackupHandler().dropRepository((DropRepositoryStmt) ddlStmt); } else if (ddlStmt instanceof SyncStmt) { return; } else { diff --git a/fe/src/com/baidu/palo/qe/MasterOpExecutor.java b/fe/src/com/baidu/palo/qe/MasterOpExecutor.java index b28ec14dbf..7a5883e4bb 100644 --- a/fe/src/com/baidu/palo/qe/MasterOpExecutor.java +++ b/fe/src/com/baidu/palo/qe/MasterOpExecutor.java @@ -15,19 +15,19 @@ package com.baidu.palo.qe; +import com.baidu.palo.analysis.RedirectStatus; +import com.baidu.palo.common.ClientPool; +import com.baidu.palo.thrift.FrontendService; +import com.baidu.palo.thrift.TMasterOpRequest; +import com.baidu.palo.thrift.TMasterOpResult; +import com.baidu.palo.thrift.TNetworkAddress; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.thrift.transport.TTransportException; + import java.nio.ByteBuffer; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.thrift.transport.TTransportException; - -import com.baidu.palo.common.ClientPool; -import com.baidu.palo.analysis.RedirectStatus; -import com.baidu.palo.thrift.FrontendService; -import com.baidu.palo.thrift.TMasterOpRequest; -import com.baidu.palo.thrift.TMasterOpResult; -import com.baidu.palo.thrift.TNetworkAddress; - public class MasterOpExecutor { private static final Logger LOG = LogManager.getLogger(MasterOpExecutor.class); @@ -72,11 +72,12 @@ public class MasterOpExecutor { TMasterOpRequest params = new TMasterOpRequest(); params.setCluster(ctx.getClusterName()); params.setSql(originStmt); - params.setUser(ctx.getUser()); + params.setUser(ctx.getQualifiedUser()); params.setDb(ctx.getDatabase()); params.setResourceInfo(ctx.toResourceCtx()); params.setExecMemLimit(ctx.getSessionVariable().getMaxExecMemByte()); params.setQueryTimeout(ctx.getSessionVariable().getQueryTimeoutS()); + params.setUser_ip(ctx.getRemoteIP()); LOG.info("Forward statement {} to Master {}", originStmt, thriftAddress); diff --git a/fe/src/com/baidu/palo/qe/QeProcessor.java b/fe/src/com/baidu/palo/qe/QeProcessor.java index 7a1adcdbe2..85ca768b2f 100644 --- a/fe/src/com/baidu/palo/qe/QeProcessor.java +++ b/fe/src/com/baidu/palo/qe/QeProcessor.java @@ -69,7 +69,8 @@ public class QeProcessor { } public static synchronized void unregisterQuery(TUniqueId queryId) { - LOG.info("deregister query id = " + queryId.toString()); - coordinatorMap.remove(queryId); + if (coordinatorMap.remove(queryId) != null) { + LOG.info("deregister query id:" + queryId.toString()); + } } } diff --git a/fe/src/com/baidu/palo/qe/ResultReceiver.java b/fe/src/com/baidu/palo/qe/ResultReceiver.java index 640173997c..e2a148f3e9 100644 --- a/fe/src/com/baidu/palo/qe/ResultReceiver.java +++ b/fe/src/com/baidu/palo/qe/ResultReceiver.java @@ -111,8 +111,13 @@ public class ResultReceiver { SimpleScheduler.updateBlacklistBackends(backendId); } catch (ExecutionException e) { LOG.warn("fetch result execution exception, finstId={}", finstId, e); - status.setRpcStatus(e.getMessage()); - SimpleScheduler.updateBlacklistBackends(backendId); + if (e.getMessage().contains("time out")) { + // if timeout, we set error code to TIMEOUT, and it will not retry querying. + status.setStatus(new Status(TStatusCode.TIMEOUT, e.getMessage())); + } else { + status.setRpcStatus(e.getMessage()); + SimpleScheduler.updateBlacklistBackends(backendId); + } } catch (TimeoutException e) { LOG.warn("fetch result timeout, finstId={}", finstId, e); status.setStatus("query timeout"); @@ -132,9 +137,12 @@ public class ResultReceiver { isCancel = true; synchronized (this) { if (currentThread != null) { - currentThread.interrupt(); + // TODO(cmy): we cannot interrupt this thread, or we may throw + // java.nio.channels.ClosedByInterruptException when we call + // MysqlChannel.realNetSend -> SocketChannelImpl.write + // And user will lost connection to Palo + // currentThread.interrupt(); } - } } } diff --git a/fe/src/com/baidu/palo/qe/SetExecutor.java b/fe/src/com/baidu/palo/qe/SetExecutor.java index 12a24c5b25..c18edd8a7d 100644 --- a/fe/src/com/baidu/palo/qe/SetExecutor.java +++ b/fe/src/com/baidu/palo/qe/SetExecutor.java @@ -22,8 +22,8 @@ import com.baidu.palo.analysis.SetTransaction; import com.baidu.palo.analysis.SetVar; import com.baidu.palo.common.DdlException; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; // Set executor public class SetExecutor { @@ -41,7 +41,7 @@ public class SetExecutor { if (var instanceof SetPassVar) { // Set password SetPassVar setPassVar = (SetPassVar) var; - ctx.getCatalog().getUserMgr().setPasswd(setPassVar.getUser(), setPassVar.getPassword()); + ctx.getCatalog().getAuth().setPassword(setPassVar); } else if (var instanceof SetNamesVar) { // do nothing return; diff --git a/fe/src/com/baidu/palo/qe/ShowExecutor.java b/fe/src/com/baidu/palo/qe/ShowExecutor.java index 64b8a77382..81d536adbe 100644 --- a/fe/src/com/baidu/palo/qe/ShowExecutor.java +++ b/fe/src/com/baidu/palo/qe/ShowExecutor.java @@ -32,23 +32,29 @@ import com.baidu.palo.analysis.ShowDbStmt; import com.baidu.palo.analysis.ShowDeleteStmt; import com.baidu.palo.analysis.ShowEnginesStmt; import com.baidu.palo.analysis.ShowExportStmt; +import com.baidu.palo.analysis.ShowFrontendsStmt; +import com.baidu.palo.analysis.ShowGrantsStmt; import com.baidu.palo.analysis.ShowLoadStmt; import com.baidu.palo.analysis.ShowLoadWarningsStmt; import com.baidu.palo.analysis.ShowMigrationsStmt; import com.baidu.palo.analysis.ShowPartitionsStmt; import com.baidu.palo.analysis.ShowProcStmt; import com.baidu.palo.analysis.ShowProcesslistStmt; +import com.baidu.palo.analysis.ShowRepositoriesStmt; import com.baidu.palo.analysis.ShowRestoreStmt; +import com.baidu.palo.analysis.ShowRolesStmt; import com.baidu.palo.analysis.ShowRollupStmt; +import com.baidu.palo.analysis.ShowSnapshotStmt; import com.baidu.palo.analysis.ShowStmt; import com.baidu.palo.analysis.ShowTableStatusStmt; import com.baidu.palo.analysis.ShowTableStmt; import com.baidu.palo.analysis.ShowTabletStmt; import com.baidu.palo.analysis.ShowUserPropertyStmt; -import com.baidu.palo.analysis.ShowUserStmt; import com.baidu.palo.analysis.ShowVariablesStmt; -import com.baidu.palo.analysis.ShowWhiteListStmt; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.backup.AbstractJob; +import com.baidu.palo.backup.BackupJob; +import com.baidu.palo.backup.Repository; +import com.baidu.palo.backup.RestoreJob; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.Database; @@ -58,15 +64,16 @@ import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.Table; import com.baidu.palo.catalog.Tablet; import com.baidu.palo.catalog.TabletInvertedIndex; -import com.baidu.palo.catalog.UserPropertyMgr; import com.baidu.palo.catalog.View; import com.baidu.palo.cluster.BaseParam; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.CaseSensibility; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; import com.baidu.palo.common.PatternMatcher; import com.baidu.palo.common.proc.BackendsProcDir; +import com.baidu.palo.common.proc.FrontendsProcNode; import com.baidu.palo.common.proc.LoadProcDir; import com.baidu.palo.common.proc.PartitionsProcDir; import com.baidu.palo.common.proc.ProcNodeInterface; @@ -78,6 +85,7 @@ import com.baidu.palo.load.LoadErrorHub; import com.baidu.palo.load.LoadErrorHub.HubType; import com.baidu.palo.load.LoadJob; import com.baidu.palo.load.LoadJob.JobState; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -159,8 +167,6 @@ public class ShowExecutor { handleShowBackup(); } else if (stmt instanceof ShowRestoreStmt) { handleShowRestore(); - } else if (stmt instanceof ShowWhiteListStmt) { - handleShowWhiteList(); } else if (stmt instanceof ShowClusterStmt) { handleShowCluster(); } else if (stmt instanceof ShowMigrationsStmt) { @@ -171,8 +177,16 @@ public class ShowExecutor { handleShowExport(); } else if (stmt instanceof ShowBackendsStmt) { handleShowBackends(); - } else if (stmt instanceof ShowUserStmt) { - handleShowUser(); + } else if (stmt instanceof ShowFrontendsStmt) { + handleShowFrontends(); + } else if (stmt instanceof ShowRepositoriesStmt) { + handleShowRepositories(); + } else if (stmt instanceof ShowSnapshotStmt) { + handleShowSnapshot(); + } else if (stmt instanceof ShowGrantsStmt) { + handleShowGrants(); + } else if (stmt instanceof ShowRolesStmt) { + handleShowRoles(); } else { handleEmtpy(); } @@ -180,12 +194,6 @@ public class ShowExecutor { return resultSet; } - private void handleShowWhiteList() { - ShowWhiteListStmt showWhiteStmt = (ShowWhiteListStmt) stmt; - List> rowSet = ctx.getCatalog().showWhiteList(ctx.getUser()); - resultSet = new ShowResultSet(showWhiteStmt.getMetaData(), rowSet); - } - private void handleShowRollup() { // TODO: not implemented yet ShowRollupStmt showRollupStmt = (ShowRollupStmt) stmt; @@ -198,7 +206,7 @@ public class ShowExecutor { ShowProcesslistStmt showStmt = (ShowProcesslistStmt) stmt; List> rowSet = Lists.newArrayList(); - List threadInfos = ctx.getConnectScheduler().listConnection(ctx.getUser()); + List threadInfos = ctx.getConnectScheduler().listConnection(ctx.getQualifiedUser()); long nowMs = System.currentTimeMillis(); for (ConnectContext.ThreadInfo info : threadInfos) { rowSet.add(info.toRow(nowMs)); @@ -240,19 +248,15 @@ public class ShowExecutor { List> finalRows = procNode.fetchResult().getRows(); // if this is superuser, hide ip and host info form backends info proc if (procNode instanceof BackendsProcDir) { - if (ctx.getCatalog().getUserMgr().isSuperuser(ctx.getUser()) - && !ctx.getCatalog().getUserMgr().isAdmin(ctx.getUser())) { - // hide ip and host info + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + // hide host info for (List row : finalRows) { - row.remove(BackendsProcDir.IP_INDEX); - // remove twice cause posistion shift to left after removing - row.remove(BackendsProcDir.IP_INDEX); + row.remove(BackendsProcDir.HOSTNAME_INDEX); } // mod meta data - metaData.removeColumn(BackendsProcDir.IP_INDEX); - // remove twice cause posistion shift to left after removing - metaData.removeColumn(BackendsProcDir.IP_INDEX); + metaData.removeColumn(BackendsProcDir.HOSTNAME_INDEX); } } @@ -265,10 +269,6 @@ public class ShowExecutor { final List> rows = Lists.newArrayList(); final List clusterNames = ctx.getCatalog().getClusterNames(); - if (!ctx.getCatalog().getUserMgr().isAdmin(ctx.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_SHOW_ACCESS_DENIED); - } - final Set clusterNameSet = Sets.newTreeSet(); for (String cluster : clusterNames) { clusterNameSet.add(cluster); @@ -287,10 +287,6 @@ public class ShowExecutor { final List> rows = Lists.newArrayList(); final Set infos = ctx.getCatalog().getMigrations(); - if (!ctx.getCatalog().getUserMgr().isAdmin(ctx.getUser())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_SHOW_ACCESS_DENIED); - } - for (BaseParam param : infos) { final int percent = (int) (param.getFloatParam(0) * 100f); rows.add(Lists.newArrayList(param.getStringParam(0), param.getStringParam(1), param.getStringParam(2), @@ -307,9 +303,9 @@ public class ShowExecutor { List dbNames = ctx.getCatalog().getClusterDbNames(ctx.getClusterName()); PatternMatcher matcher = null; if (showDbStmt.getPattern() != null) { - matcher = PatternMatcher.createMysqlPattern(showDbStmt.getPattern()); + matcher = PatternMatcher.createMysqlPattern(showDbStmt.getPattern(), + CaseSensibility.DATABASE.getCaseSensibility()); } - UserPropertyMgr userPropertyMgr = ctx.getCatalog().getUserMgr(); Set dbNameSet = Sets.newTreeSet(); for (String fullName : dbNames) { final String db = ClusterNamespace.getNameFromFullName(fullName); @@ -317,9 +313,13 @@ public class ShowExecutor { if (matcher != null && !matcher.match(db)) { continue; } - if (userPropertyMgr.checkAccess(ctx.getUser(), fullName, AccessPrivilege.READ_ONLY)) { - dbNameSet.add(db); + + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), fullName, + PrivPredicate.SHOW)) { + continue; } + + dbNameSet.add(db); } for (String dbName : dbNameSet) { @@ -340,12 +340,19 @@ public class ShowExecutor { try { PatternMatcher matcher = null; if (showTableStmt.getPattern() != null) { - matcher = PatternMatcher.createMysqlPattern(showTableStmt.getPattern()); + matcher = PatternMatcher.createMysqlPattern(showTableStmt.getPattern(), + CaseSensibility.TABLE.getCaseSensibility()); } for (Table tbl : db.getTables()) { if (matcher != null && !matcher.match(tbl.getName())) { continue; } + // check tbl privs + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), + db.getFullName(), tbl.getName(), + PrivPredicate.SHOW)) { + continue; + } tableMap.put(tbl.getName(), tbl.getMysqlType()); } } finally { @@ -373,12 +380,21 @@ public class ShowExecutor { try { PatternMatcher matcher = null; if (showStmt.getPattern() != null) { - matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern()); + matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern(), + CaseSensibility.TABLE.getCaseSensibility()); } for (Table table : db.getTables()) { if (matcher != null && !matcher.match(table.getName())) { continue; } + + // check tbl privs + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), + db.getFullName(), table.getName(), + PrivPredicate.SHOW)) { + continue; + } + List row = Lists.newArrayList(); // Name row.add(table.getName()); @@ -403,7 +419,8 @@ public class ShowExecutor { ShowVariablesStmt showStmt = (ShowVariablesStmt) stmt; PatternMatcher matcher = null; if (showStmt.getPattern() != null) { - matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern()); + matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern(), + CaseSensibility.VARIABLES.getCaseSensibility()); } List> rows = VariableMgr.dump(showStmt.getType(), ctx.getSessionVariable(), matcher); resultSet = new ShowResultSet(showStmt.getMetaData(), rows); @@ -440,6 +457,10 @@ public class ShowExecutor { List createTableStmt = Lists.newArrayList(); Catalog.getDdlStmt(table, createTableStmt, null, null, false, (short) -1); + if (createTableStmt.isEmpty()) { + resultSet = new ShowResultSet(showStmt.getMetaData(), rows); + return; + } if (table instanceof View) { View view = (View) table; @@ -478,7 +499,8 @@ public class ShowExecutor { if (table != null) { PatternMatcher matcher = null; if (showStmt.getPattern() != null) { - matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern()); + matcher = PatternMatcher.createMysqlPattern(showStmt.getPattern(), + CaseSensibility.COLUMN.getCaseSensibility()); } List columns = table.getBaseSchema(); for (Column col : columns) { @@ -575,8 +597,11 @@ public class ShowExecutor { long dbId = db.getId(); Load load = catalog.getLoadInstance(); - List> loadInfos = load.getLoadJobInfosByDb(dbId, showStmt.getLabelValue(), - showStmt.isAccurateMatch(), showStmt.getStates(), showStmt.getOrderByPairs()); + List> loadInfos = load.getLoadJobInfosByDb(dbId, db.getFullName(), + showStmt.getLabelValue(), + showStmt.isAccurateMatch(), + showStmt.getStates(), + showStmt.getOrderByPairs()); List> rows = Lists.newArrayList(); for (List loadInfo : loadInfos) { List oneInfo = new ArrayList(loadInfo.size()); @@ -613,14 +638,15 @@ public class ShowExecutor { long dbId = db.getId(); Load load = catalog.getLoadInstance(); long jobId = 0; + LoadJob job = null; String label = null; if (showWarningsStmt.isFindByLabel()) { label = showWarningsStmt.getLabel(); - jobId = load.getLatestJobIdByLabel(dbId, showWarningsStmt.getLabel()); + job = load.getLatestJobIdByLabel(dbId, showWarningsStmt.getLabel()); } else { LOG.info("load_job_id={}", jobId); jobId = showWarningsStmt.getJobId(); - LoadJob job = load.getLoadJob(jobId); + job = load.getLoadJob(jobId); if (job == null) { throw new AnalysisException("job is not exist."); } @@ -628,6 +654,29 @@ public class ShowExecutor { LOG.info("label={}", label); } + // check auth + Set tableNames = job.getTableNames(); + if (tableNames.isEmpty()) { + // forward compatibility + if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(ConnectContext.get(), db.getFullName(), + PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED, + ConnectContext.get().getQualifiedUser(), + db.getFullName()); + } + } else { + for (String tblName : tableNames) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ConnectContext.get(), db.getFullName(), + tblName, PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, + "SHOW LOAD WARNING", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + tblName); + } + } + } + LoadErrorHub.Param param = load.getLoadErrorHubInfo(); if (param == null || param.getType() == HubType.NULL_TYPE) { throw new AnalysisException("no load error hub be supplied."); @@ -651,12 +700,11 @@ public class ShowExecutor { } resultSet = new ShowResultSet(showWarningsStmt.getMetaData(), rows); - } + // Show user property statement private void handleShowUserProperty() throws AnalysisException { ShowUserPropertyStmt showStmt = (ShowUserPropertyStmt) stmt; - showStmt.handleShow(); resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getRows()); } @@ -842,16 +890,6 @@ public class ShowExecutor { resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } - private void handleShowBackup() throws AnalysisException { - ShowBackupStmt showStmt = (ShowBackupStmt) stmt; - resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getResultRows()); - } - - private void handleShowRestore() throws AnalysisException { - ShowRestoreStmt showStmt = (ShowRestoreStmt) stmt; - resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getResultRows()); - } - // Handle show brokers private void handleShowBroker() { ShowBrokerStmt showStmt = (ShowBrokerStmt) stmt; @@ -901,16 +939,93 @@ public class ShowExecutor { private void handleShowBackends() { final ShowBackendsStmt showStmt = (ShowBackendsStmt) stmt; - final List> backendInfos = BackendsProcDir.getClusterBackendInfos(showStmt.getClusterName()); + List> backendInfos = BackendsProcDir.getClusterBackendInfos(showStmt.getClusterName()); + + for (List row : backendInfos) { + row.remove(BackendsProcDir.HOSTNAME_INDEX); + } + resultSet = new ShowResultSet(showStmt.getMetaData(), backendInfos); } - private void handleShowUser() { - final ShowUserStmt showStmt = (ShowUserStmt) stmt; - final List> userInfos = Catalog.getInstance().getUserMgr() - .fetchAccessResourceResult(showStmt.getUser()); - resultSet = new ShowResultSet(showStmt.getMetaData(), userInfos); + private void handleShowFrontends() { + final ShowFrontendsStmt showStmt = (ShowFrontendsStmt) stmt; + List> infos = Lists.newArrayList(); + FrontendsProcNode.getFrontendsInfo(Catalog.getCurrentCatalog(), infos); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); + } + + private void handleShowRepositories() { + final ShowRepositoriesStmt showStmt = (ShowRepositoriesStmt) stmt; + List> repoInfos = Catalog.getInstance().getBackupHandler().getRepoMgr().getReposInfo(); + resultSet = new ShowResultSet(showStmt.getMetaData(), repoInfos); + } + + private void handleShowSnapshot() throws AnalysisException { + final ShowSnapshotStmt showStmt = (ShowSnapshotStmt) stmt; + Repository repo = Catalog.getInstance().getBackupHandler().getRepoMgr().getRepo(showStmt.getRepoName()); + if (repo == null) { + throw new AnalysisException("Repository " + showStmt.getRepoName() + " does not exist"); + } + + List> snapshotInfos = repo.getSnapshotInfos(showStmt.getSnapshotName(), showStmt.getTimestamp()); + resultSet = new ShowResultSet(showStmt.getMetaData(), snapshotInfos); + } + + private void handleShowBackup() throws AnalysisException { + ShowBackupStmt showStmt = (ShowBackupStmt) stmt; + Database db = Catalog.getInstance().getDb(showStmt.getDbName()); + if (db == null) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, showStmt.getDbName()); + } + + AbstractJob jobI = Catalog.getInstance().getBackupHandler().getJob(db.getId()); + if (!(jobI instanceof BackupJob)) { + resultSet = new ShowResultSet(showStmt.getMetaData(), EMPTY_SET); + return; + } + + BackupJob backupJob = (BackupJob) jobI; + List info = backupJob.getInfo(); + List> infos = Lists.newArrayList(); + infos.add(info); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); + } + + private void handleShowRestore() throws AnalysisException { + ShowRestoreStmt showStmt = (ShowRestoreStmt) stmt; + Database db = Catalog.getInstance().getDb(showStmt.getDbName()); + if (db == null) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_DB_ERROR, showStmt.getDbName()); + } + + AbstractJob jobI = Catalog.getInstance().getBackupHandler().getJob(db.getId()); + if (!(jobI instanceof RestoreJob)) { + resultSet = new ShowResultSet(showStmt.getMetaData(), EMPTY_SET); + return; + } + + RestoreJob restoreJob = (RestoreJob) jobI; + List info = restoreJob.getInfo(); + List> infos = Lists.newArrayList(); + infos.add(info); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); + } + + private void handleShowGrants() { + ShowGrantsStmt showStmt = (ShowGrantsStmt) stmt; + List> infos = Catalog.getCurrentCatalog().getAuth().getAuthInfo(showStmt.getUserIdent(), + showStmt.isAll()); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); + } + + private void handleShowRoles() { + ShowRolesStmt showStmt = (ShowRolesStmt) stmt; + List> infos = Catalog.getCurrentCatalog().getAuth().getRoleInfo(); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); } } + + diff --git a/fe/src/com/baidu/palo/qe/StmtExecutor.java b/fe/src/com/baidu/palo/qe/StmtExecutor.java index 8220f82fdd..76cb4d8397 100644 --- a/fe/src/com/baidu/palo/qe/StmtExecutor.java +++ b/fe/src/com/baidu/palo/qe/StmtExecutor.java @@ -53,6 +53,7 @@ import com.baidu.palo.common.util.TimeUtils; import com.baidu.palo.mysql.MysqlChannel; import com.baidu.palo.mysql.MysqlEofPacket; import com.baidu.palo.mysql.MysqlSerializer; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.planner.Planner; import com.baidu.palo.rewrite.ExprRewriter; import com.baidu.palo.rpc.RpcException; @@ -67,7 +68,6 @@ import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.thrift.transport.TTransportException; import java.io.IOException; import java.io.StringReader; @@ -123,7 +123,7 @@ public class StmtExecutor { summaryProfile.addInfoString(ProfileManager.QUERY_TYPE, "Query"); summaryProfile.addInfoString(ProfileManager.QUERY_STATE, context.getState().toString()); summaryProfile.addInfoString("Palo Version", "Palo version 2.0"); - summaryProfile.addInfoString(ProfileManager.USER, context.getUser()); + summaryProfile.addInfoString(ProfileManager.USER, context.getQualifiedUser()); summaryProfile.addInfoString(ProfileManager.DEFAULT_DB, context.getDatabase()); summaryProfile.addInfoString(ProfileManager.SQL_STATEMENT, originStmt); profile.addChild(summaryProfile); @@ -304,7 +304,6 @@ public class StmtExecutor { } } - // Analyze one statement to structure in memory. private void analyze() throws AnalysisException, InternalException, NotImplementedException { @@ -475,10 +474,10 @@ public class StmtExecutor { context.setKilled(); } else { // Check auth - if (!context.getCatalog().getUserMgr() - .checkUserAccess(context.getUser(), killCtx.getUser())) { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { ErrorReport.reportDdlException(ErrorCode.ERR_KILL_DENIED_ERROR, id); } + killCtx.kill(killStmt.isConnectionKill()); } context.getState().setOk(); @@ -732,7 +731,7 @@ public class StmtExecutor { } catch (Exception e) { // Maybe our bug LOG.warn("DDL statement(" + originStmt + ") process failed.", e); - context.getState().setError("Maybe palo bug, please info palo RD."); + context.getState().setError("Unexpected exception: " + e.getMessage()); } } diff --git a/fe/src/com/baidu/palo/rpc/BackendServiceProxy.java b/fe/src/com/baidu/palo/rpc/BackendServiceProxy.java index c10ed78dc7..3e31af9a86 100644 --- a/fe/src/com/baidu/palo/rpc/BackendServiceProxy.java +++ b/fe/src/com/baidu/palo/rpc/BackendServiceProxy.java @@ -18,6 +18,7 @@ package com.baidu.palo.rpc; import com.baidu.jprotobuf.pbrpc.client.ProtobufRpcProxy; import com.baidu.jprotobuf.pbrpc.transport.RpcClient; import com.baidu.jprotobuf.pbrpc.transport.RpcClientOptions; +import com.baidu.palo.common.Config; import com.baidu.palo.qe.SimpleScheduler; import com.baidu.palo.thrift.TExecPlanFragmentParams; import com.baidu.palo.thrift.TNetworkAddress; @@ -29,6 +30,7 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.Future; public class BackendServiceProxy { @@ -41,7 +43,10 @@ public class BackendServiceProxy { private static BackendServiceProxy INSTANCE; public BackendServiceProxy() { - rpcClient = new RpcClient(new RpcClientOptions()); + final RpcClientOptions rpcOptions = new RpcClientOptions(); + rpcOptions.setMaxWait(Config.brpc_idle_wait_max_time); + rpcOptions.setThreadPoolSize(Config.brpc_number_of_concurrent_requests_processed); + rpcClient = new RpcClient(rpcOptions); serviceMap = Maps.newHashMap(); } @@ -65,16 +70,31 @@ public class BackendServiceProxy { return service; } - public Future execPlanFragmentAsync( +public Future execPlanFragmentAsync( TNetworkAddress address, TExecPlanFragmentParams tRequest) throws TException, RpcException { + final PExecPlanFragmentRequest pRequest = new PExecPlanFragmentRequest(); + pRequest.setRequest(tRequest); try { - PExecPlanFragmentRequest pRequest = new PExecPlanFragmentRequest(); - pRequest.setRequest(tRequest); - PInternalService service = getProxy(address); + final PInternalService service = getProxy(address); return service.execPlanFragmentAsync(pRequest); + } catch (NoSuchElementException e) { + try { + // retry + try { + Thread.sleep(10); + } catch (InterruptedException interruptedException) { + // do nothing + } + final PInternalService service = getProxy(address); + return service.execPlanFragmentAsync(pRequest); + } catch (NoSuchElementException noSuchElementException) { + LOG.warn("Execute plan fragment retry failed, address={}:{}", + address.getHostname(), address.getPort(), noSuchElementException); + throw new RpcException(e.getMessage()); + } } catch (Throwable e) { - LOG.warn("execute plan fragment catch a exception, address={}:{}", + LOG.warn("Execute plan fragment catch a exception, address={}:{}", address.getHostname(), address.getPort(), e); throw new RpcException(e.getMessage()); } @@ -82,12 +102,27 @@ public class BackendServiceProxy { public Future cancelPlanFragmentAsync( TNetworkAddress address, TUniqueId finstId) throws RpcException { + final PCancelPlanFragmentRequest pRequest = new PCancelPlanFragmentRequest(new PUniqueId(finstId));; try { - PCancelPlanFragmentRequest pRequest = new PCancelPlanFragmentRequest(new PUniqueId(finstId)); - PInternalService service = getProxy(address); + final PInternalService service = getProxy(address); return service.cancelPlanFragmentAsync(pRequest); + } catch (NoSuchElementException e) { + // retry + try { + try { + Thread.sleep(10); + } catch (InterruptedException interruptedException) { + // do nothing + } + final PInternalService service = getProxy(address); + return service.cancelPlanFragmentAsync(pRequest); + } catch (NoSuchElementException noSuchElementException) { + LOG.warn("Cancel plan fragment retry failed, address={}:{}", + address.getHostname(), address.getPort(), noSuchElementException); + throw new RpcException(e.getMessage()); + } } catch (Throwable e) { - LOG.warn("cancel plan fragment catch a exception, address={}:{}", + LOG.warn("Cancel plan fragment catch a exception, address={}:{}", address.getHostname(), address.getPort(), e); throw new RpcException(e.getMessage()); } diff --git a/fe/src/com/baidu/palo/service/FrontendServiceImpl.java b/fe/src/com/baidu/palo/service/FrontendServiceImpl.java index 109bda02e5..9e9a9b3f7f 100644 --- a/fe/src/com/baidu/palo/service/FrontendServiceImpl.java +++ b/fe/src/com/baidu/palo/service/FrontendServiceImpl.java @@ -16,15 +16,14 @@ package com.baidu.palo.service; import com.baidu.palo.analysis.SetType; -import com.baidu.palo.catalog.AccessPrivilege; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.Table; -import com.baidu.palo.catalog.UserPropertyMgr; import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.AuditLog; +import com.baidu.palo.common.CaseSensibility; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.PatternMatcher; @@ -34,7 +33,7 @@ import com.baidu.palo.load.EtlStatus; import com.baidu.palo.load.LoadJob; import com.baidu.palo.load.MiniEtlTaskInfo; import com.baidu.palo.master.MasterImpl; -import com.baidu.palo.mysql.MysqlPassword; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.qe.AuditBuilder; import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.qe.ConnectProcessor; @@ -104,26 +103,35 @@ public class FrontendServiceImpl implements FrontendService.Iface { @Override public TGetDbsResult getDbNames(TGetDbsParams params) throws TException { + LOG.debug("get db request: {}", params); TGetDbsResult result = new TGetDbsResult(); + List dbs = Lists.newArrayList(); - List dbNames = Catalog.getInstance().getDbNames(); - UserPropertyMgr userPropertyMgr = Catalog.getInstance().getUserMgr(); PatternMatcher matcher = null; if (params.isSetPattern()) { try { - matcher = PatternMatcher.createMysqlPattern(params.getPattern()); + matcher = PatternMatcher.createMysqlPattern(params.getPattern(), + CaseSensibility.DATABASE.getCaseSensibility()); } catch (AnalysisException e) { - throw new TException("Pattern is in bad format " + params.getPattern()); + throw new TException("Pattern is in bad format: " + params.getPattern()); } } + + Catalog catalog = Catalog.getCurrentCatalog(); + List dbNames = catalog.getDbNames(); + LOG.debug("get db names: {}", dbNames); for (String fullName : dbNames) { + if (!catalog.getAuth().checkDbPriv(params.user_ip, fullName, params.user, + PrivPredicate.SHOW)) { + continue; + } + final String db = ClusterNamespace.getNameFromFullName(fullName); if (matcher != null && !matcher.match(db)) { continue; } - if (userPropertyMgr.checkAccess(params.user, fullName, AccessPrivilege.READ_ONLY)) { - dbs.add(fullName); - } + + dbs.add(fullName); } result.setDbs(dbs); return result; @@ -131,20 +139,31 @@ public class FrontendServiceImpl implements FrontendService.Iface { @Override public TGetTablesResult getTableNames(TGetTablesParams params) throws TException { + LOG.debug("get table name request: {}", params); TGetTablesResult result = new TGetTablesResult(); List tablesResult = Lists.newArrayList(); result.setTables(tablesResult); PatternMatcher matcher = null; if (params.isSetPattern()) { try { - matcher = PatternMatcher.createMysqlPattern(params.getPattern()); + matcher = PatternMatcher.createMysqlPattern(params.getPattern(), + CaseSensibility.TABLE.getCaseSensibility()); } catch (AnalysisException e) { - throw new TException("Pattern is in bad format " + params.getPattern()); + throw new TException("Pattern is in bad format: " + params.getPattern()); } } + + // database privs should be checked in analysis phrase + Database db = Catalog.getInstance().getDb(params.db); if (db != null) { for (String tableName : db.getTableNamesWithLock()) { + LOG.debug("get table: {}, wait to check", tableName); + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(params.user_ip, params.db, params.user, + tableName, PrivPredicate.SHOW)) { + continue; + } + if (matcher != null && !matcher.match(tableName)) { continue; } @@ -156,22 +175,32 @@ public class FrontendServiceImpl implements FrontendService.Iface { @Override public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TException { + LOG.debug("get list table request: {}", params); TListTableStatusResult result = new TListTableStatusResult(); List tablesResult = Lists.newArrayList(); result.setTables(tablesResult); PatternMatcher matcher = null; if (params.isSetPattern()) { try { - matcher = PatternMatcher.createMysqlPattern(params.getPattern()); + matcher = PatternMatcher.createMysqlPattern(params.getPattern(), + CaseSensibility.TABLE.getCaseSensibility()); } catch (AnalysisException e) { throw new TException("Pattern is in bad format " + params.getPattern()); } } + + // database privs should be checked in analysis phrase + Database db = Catalog.getInstance().getDb(params.db); if (db != null) { db.readLock(); try { for (Table table : db.getTables()) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(params.user_ip, params.db, params.user, + table.getName(), PrivPredicate.SHOW)) { + continue; + } + if (matcher != null && !matcher.match(table.getName())) { continue; } @@ -200,9 +229,18 @@ public class FrontendServiceImpl implements FrontendService.Iface { @Override public TDescribeTableResult describeTable(TDescribeTableParams params) throws TException { + LOG.debug("get desc table request: {}", params); TDescribeTableResult result = new TDescribeTableResult(); List columns = Lists.newArrayList(); result.setColumns(columns); + + // database privs should be checked in analysis phrase + + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(params.user_ip, params.db, params.user, + params.getTable_name(), PrivPredicate.SHOW)) { + return result; + } + Database db = Catalog.getInstance().getDb(params.db); if (db != null) { db.readLock(); @@ -272,14 +310,12 @@ public class FrontendServiceImpl implements FrontendService.Iface { cluster = SystemInfoService.DEFAULT_CLUSTER; } - final String userFullName = Catalog.getInstance().getUserMgr().isAdmin(request.user) ? request.user : - ClusterNamespace.getFullName(cluster, request.user); final String dbFullName = ClusterNamespace.getFullName(cluster, request.db); - request.setUser(userFullName); + request.setUser(request.user); request.setDb(dbFullName); context.setCluster(cluster); context.setDatabase(ClusterNamespace.getFullName(cluster, request.db)); - context.setUser(ClusterNamespace.getFullName(cluster, request.user)); + context.setQualifiedUser(ClusterNamespace.getFullName(cluster, request.user)); context.setCatalog(Catalog.getInstance()); context.getState().reset(); context.setThreadLocalInfo(); @@ -311,7 +347,20 @@ public class FrontendServiceImpl implements FrontendService.Iface { return result; } - public static String getMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException { + private void logMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException { + String stmt = getMiniLoadStmt(request); + AuditBuilder auditBuilder = new AuditBuilder(); + auditBuilder.put("client", request.user_ip + ":0"); + auditBuilder.put("user", request.user); + auditBuilder.put("db", request.db); + auditBuilder.put("state", TStatusCode.OK); + auditBuilder.put("time", "0"); + auditBuilder.put("stmt", stmt); + + AuditLog.getQueryAudit().log(auditBuilder.toString()); + } + + private String getMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append("curl --location-trusted -u user:passwd -T "); @@ -339,19 +388,6 @@ public class FrontendServiceImpl implements FrontendService.Iface { return stringBuilder.toString(); } - private void logMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException { - String stmt = getMiniLoadStmt(request); - AuditBuilder auditBuilder = new AuditBuilder(); - auditBuilder.put("client", request.getBackend().getHostname() + ":" + request.getBackend().getPort()); - auditBuilder.put("user", request.user); - auditBuilder.put("db", request.db); - auditBuilder.put("state", TStatusCode.OK); - auditBuilder.put("time", "0"); - auditBuilder.put("stmt", stmt); - - AuditLog.getQueryAudit().log(auditBuilder.toString()); - } - @Override public TFeResult updateMiniEtlTaskStatus(TUpdateMiniEtlTaskStatusRequest request) throws TException { TFeResult result = new TFeResult(); @@ -432,30 +468,12 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else { cluster = SystemInfoService.DEFAULT_CLUSTER; } - final String userFullName = Catalog.getInstance().getUserMgr().isAdmin(request.user) ? request.user : - ClusterNamespace.getFullName(cluster, request.user); + final String dbFullName = ClusterNamespace.getFullName(cluster, request.db); - request.setUser(userFullName); + + request.setUser(request.user); request.setDb(dbFullName); - // Check user and password - byte[] passwd = Catalog.getInstance().getUserMgr().getPassword(userFullName); - if (passwd == null) { - // No such user - status.setStatus_code(TStatusCode.INTERNAL_ERROR); - status.setError_msgs(Lists.newArrayList("No such user(" + userFullName + ")")); - return result; - } - if (!MysqlPassword.checkPlainPass(passwd, request.passwd)) { - status.setStatus_code(TStatusCode.INTERNAL_ERROR); - status.setError_msgs(Lists.newArrayList("Wrong password.")); - return result; - } - if (!Catalog.getInstance().getUserMgr().checkAccess(userFullName, dbFullName, AccessPrivilege.READ_WRITE)) { - status.setStatus_code(TStatusCode.INTERNAL_ERROR); - status.setError_msgs( - Lists.newArrayList("Have no privilege to write this database(" + request.getDb() + ")")); - return result; - } + if (request.isSetLabel()) { // Only single table will be set label try { diff --git a/fe/src/com/baidu/palo/system/Frontend.java b/fe/src/com/baidu/palo/system/Frontend.java index 5689045726..618853499b 100644 --- a/fe/src/com/baidu/palo/system/Frontend.java +++ b/fe/src/com/baidu/palo/system/Frontend.java @@ -115,4 +115,3 @@ public class Frontend implements Writable { return sb.toString(); } } - diff --git a/fe/src/com/baidu/palo/task/AgentBatchTask.java b/fe/src/com/baidu/palo/task/AgentBatchTask.java index 836c53fffe..0514feab02 100644 --- a/fe/src/com/baidu/palo/task/AgentBatchTask.java +++ b/fe/src/com/baidu/palo/task/AgentBatchTask.java @@ -15,36 +15,37 @@ package com.baidu.palo.task; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.ClientPool; +import com.baidu.palo.system.Backend; +import com.baidu.palo.thrift.BackendService; +import com.baidu.palo.thrift.TAgentServiceVersion; +import com.baidu.palo.thrift.TAgentTaskRequest; +import com.baidu.palo.thrift.TAlterTabletReq; +import com.baidu.palo.thrift.TCancelDeleteDataReq; +import com.baidu.palo.thrift.TCheckConsistencyReq; +import com.baidu.palo.thrift.TCloneReq; +import com.baidu.palo.thrift.TCreateTabletReq; +import com.baidu.palo.thrift.TDownloadReq; +import com.baidu.palo.thrift.TDropTabletReq; +import com.baidu.palo.thrift.TMoveDirReq; +import com.baidu.palo.thrift.TNetworkAddress; +import com.baidu.palo.thrift.TPushReq; +import com.baidu.palo.thrift.TPushType; +import com.baidu.palo.thrift.TReleaseSnapshotRequest; +import com.baidu.palo.thrift.TSnapshotRequest; +import com.baidu.palo.thrift.TStorageMediumMigrateReq; +import com.baidu.palo.thrift.TTaskType; +import com.baidu.palo.thrift.TUploadReq; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.ClientPool; -import com.baidu.palo.system.Backend; -import com.baidu.palo.thrift.BackendService; -import com.baidu.palo.thrift.TAgentServiceVersion; -import com.baidu.palo.thrift.TAgentTaskRequest; -import com.baidu.palo.thrift.TAlterTabletReq; -import com.baidu.palo.thrift.TCancelDeleteDataReq; -import com.baidu.palo.thrift.TCheckConsistencyReq; -import com.baidu.palo.thrift.TCloneReq; -import com.baidu.palo.thrift.TCreateTabletReq; -import com.baidu.palo.thrift.TDropTabletReq; -import com.baidu.palo.thrift.TNetworkAddress; -import com.baidu.palo.thrift.TPushReq; -import com.baidu.palo.thrift.TPushType; -import com.baidu.palo.thrift.TReleaseSnapshotRequest; -import com.baidu.palo.thrift.TRestoreReq; -import com.baidu.palo.thrift.TSnapshotRequest; -import com.baidu.palo.thrift.TStorageMediumMigrateReq; -import com.baidu.palo.thrift.TTaskType; -import com.baidu.palo.thrift.TUploadReq; - /* * This class group tasks by backend */ @@ -234,12 +235,19 @@ public class AgentBatchTask implements Runnable { tAgentTaskRequest.setUpload_req(request); return tAgentTaskRequest; } - case RESTORE: { - RestoreTask restoreTask = (RestoreTask) task; - TRestoreReq request = restoreTask.toThrift(); + case DOWNLOAD: { + DownloadTask downloadTask = (DownloadTask) task; + TDownloadReq request = downloadTask.toThrift(); LOG.debug(request.toString()); - tAgentTaskRequest.setRestore_req(request); + tAgentTaskRequest.setDownload_req(request); return tAgentTaskRequest; + } + case MOVE: { + DirMoveTask dirMoveTask = (DirMoveTask) task; + TMoveDirReq request = dirMoveTask.toThrift(); + LOG.debug(request.toString()); + tAgentTaskRequest.setMove_dir_req(request); + return tAgentTaskRequest; } default: return null; diff --git a/fe/src/com/baidu/palo/task/AgentTask.java b/fe/src/com/baidu/palo/task/AgentTask.java index 0add79f939..a3662f40ad 100644 --- a/fe/src/com/baidu/palo/task/AgentTask.java +++ b/fe/src/com/baidu/palo/task/AgentTask.java @@ -15,7 +15,7 @@ package com.baidu.palo.task; -import com.baidu.palo.thrift.TResourceInfo; +import com.baidu.palo.thrift.TResourceInfo; import com.baidu.palo.thrift.TTaskType; public abstract class AgentTask { @@ -33,10 +33,10 @@ public abstract class AgentTask { protected int failedTimes; - public AgentTask(TResourceInfo resourceInfo, long backendId, TTaskType taskType, + public AgentTask(TResourceInfo resourceInfo, long backendId, long signature, TTaskType taskType, long dbId, long tableId, long partitionId, long indexId, long tabletId) { this.backendId = backendId; - this.signature = tabletId; + this.signature = signature; this.taskType = taskType; this.dbId = dbId; @@ -96,6 +96,6 @@ public abstract class AgentTask { @Override public String toString() { - return "[" + taskType + "], signature: " + signature + ", backendId: " + backendId; + return "[" + taskType + "], signature: " + signature + ", backendId: " + backendId + ", tablet id: " + tabletId; } } diff --git a/fe/src/com/baidu/palo/task/AgentTaskQueue.java b/fe/src/com/baidu/palo/task/AgentTaskQueue.java index ad25ce09e8..253c23db96 100644 --- a/fe/src/com/baidu/palo/task/AgentTaskQueue.java +++ b/fe/src/com/baidu/palo/task/AgentTaskQueue.java @@ -15,21 +15,21 @@ package com.baidu.palo.task; -import com.baidu.palo.thrift.TPushType; -import com.baidu.palo.thrift.TTaskType; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Table; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import com.baidu.palo.thrift.TPushType; +import com.baidu.palo.thrift.TTaskType; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Table; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -106,6 +106,14 @@ public class AgentTaskQueue { signatureMap.remove(signature); LOG.debug("remove task: type[{}], backend[{}], signature[{}]", TTaskType.PUSH, backendId, signature); --taskNum; + } + + public static synchronized void removeTaskOfType(TTaskType type, long signature) { + // be id -> (signature -> task) + Map> map = tasks.column(type); + for (Map innerMap : map.values()) { + innerMap.remove(signature); + } } public static synchronized AgentTask getTask(long backendId, TTaskType type, long signature) { diff --git a/fe/src/com/baidu/palo/task/CancelDeleteTask.java b/fe/src/com/baidu/palo/task/CancelDeleteTask.java index 8562f7147d..f428d310fb 100644 --- a/fe/src/com/baidu/palo/task/CancelDeleteTask.java +++ b/fe/src/com/baidu/palo/task/CancelDeleteTask.java @@ -15,7 +15,7 @@ package com.baidu.palo.task; -import com.baidu.palo.thrift.TCancelDeleteDataReq; +import com.baidu.palo.thrift.TCancelDeleteDataReq; import com.baidu.palo.thrift.TTaskType; public class CancelDeleteTask extends AgentTask { @@ -25,7 +25,7 @@ public class CancelDeleteTask extends AgentTask { public CancelDeleteTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, int schemaHash, long version, long versionHash) { - super(null, backendId, TTaskType.CANCEL_DELETE, dbId, tableId, partitionId, indexId, tabletId); + super(null, backendId, tabletId, TTaskType.CANCEL_DELETE, dbId, tableId, partitionId, indexId, tabletId); this.schemaHash = schemaHash; this.version = version; diff --git a/fe/src/com/baidu/palo/task/CheckConsistencyTask.java b/fe/src/com/baidu/palo/task/CheckConsistencyTask.java index ea9a17ef43..320d506145 100644 --- a/fe/src/com/baidu/palo/task/CheckConsistencyTask.java +++ b/fe/src/com/baidu/palo/task/CheckConsistencyTask.java @@ -28,7 +28,8 @@ public class CheckConsistencyTask extends AgentTask { public CheckConsistencyTask(TResourceInfo resourceInfo, long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, int schemaHash, long version, long versionHash) { - super(resourceInfo, backendId, TTaskType.CHECK_CONSISTENCY, dbId, tableId, partitionId, indexId, tabletId); + super(resourceInfo, backendId, tabletId, TTaskType.CHECK_CONSISTENCY, dbId, tableId, partitionId, indexId, + tabletId); this.schemaHash = schemaHash; this.version = version; diff --git a/fe/src/com/baidu/palo/task/CloneTask.java b/fe/src/com/baidu/palo/task/CloneTask.java index de16a5808f..71c43576a6 100644 --- a/fe/src/com/baidu/palo/task/CloneTask.java +++ b/fe/src/com/baidu/palo/task/CloneTask.java @@ -15,11 +15,11 @@ package com.baidu.palo.task; -import com.baidu.palo.thrift.TBackend; -import com.baidu.palo.thrift.TCloneReq; -import com.baidu.palo.thrift.TStorageMedium; -import com.baidu.palo.thrift.TTaskType; - +import com.baidu.palo.thrift.TBackend; +import com.baidu.palo.thrift.TCloneReq; +import com.baidu.palo.thrift.TStorageMedium; +import com.baidu.palo.thrift.TTaskType; + import java.util.List; public class CloneTask extends AgentTask { @@ -34,7 +34,7 @@ public class CloneTask extends AgentTask { public CloneTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, int schemaHash, List srcBackends, TStorageMedium storageMedium, long committedVersion, long committedVersionHash) { - super(null, backendId, TTaskType.CLONE, dbId, tableId, partitionId, indexId, tabletId); + super(null, backendId, tabletId, TTaskType.CLONE, dbId, tableId, partitionId, indexId, tabletId); this.schemaHash = schemaHash; this.srcBackends = srcBackends; this.storageMedium = storageMedium; diff --git a/fe/src/com/baidu/palo/task/CreateReplicaTask.java b/fe/src/com/baidu/palo/task/CreateReplicaTask.java index b1a4a27cdc..ded2303cf4 100644 --- a/fe/src/com/baidu/palo/task/CreateReplicaTask.java +++ b/fe/src/com/baidu/palo/task/CreateReplicaTask.java @@ -15,21 +15,21 @@ package com.baidu.palo.task; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.catalog.KeysType; -import com.baidu.palo.common.MarkedCountDownLatch; -import com.baidu.palo.thrift.TColumn; -import com.baidu.palo.thrift.TCreateTabletReq; -import com.baidu.palo.thrift.TStorageMedium; -import com.baidu.palo.thrift.TStorageType; -import com.baidu.palo.thrift.TTabletSchema; -import com.baidu.palo.thrift.TTaskType; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import java.util.ArrayList; -import java.util.List; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.KeysType; +import com.baidu.palo.common.MarkedCountDownLatch; +import com.baidu.palo.thrift.TColumn; +import com.baidu.palo.thrift.TCreateTabletReq; +import com.baidu.palo.thrift.TStorageMedium; +import com.baidu.palo.thrift.TStorageType; +import com.baidu.palo.thrift.TTabletSchema; +import com.baidu.palo.thrift.TTaskType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class CreateReplicaTask extends AgentTask { @@ -52,14 +52,16 @@ public class CreateReplicaTask extends AgentTask { private double bfFpp; // used for synchronous process - private MarkedCountDownLatch latch; + private MarkedCountDownLatch latch; + + private boolean inRestoreMode = false; public CreateReplicaTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, short shortKeyColumnCount, int schemaHash, long version, long versionHash, KeysType keysType, TStorageType storageType, TStorageMedium storageMedium, List columns, Set bfColumns, double bfFpp, MarkedCountDownLatch latch) { - super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); + super(null, backendId, tabletId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.shortKeyColumnCount = shortKeyColumnCount; this.schemaHash = schemaHash; @@ -86,6 +88,14 @@ public class CreateReplicaTask extends AgentTask { latch.getCount(), backendId, tabletId); } } + } + + public void setLatch(MarkedCountDownLatch latch) { + this.latch = latch; + } + + public void setInRestoreMode(boolean inRestoreMode) { + this.inRestoreMode = inRestoreMode; } public TCreateTabletReq toThrift() { @@ -118,7 +128,9 @@ public class CreateReplicaTask extends AgentTask { createTabletReq.setVersion_hash(versionHash); createTabletReq.setStorage_medium(storageMedium); - + if (inRestoreMode) { + createTabletReq.setIn_restore_mode(true); + } return createTabletReq; } diff --git a/fe/src/com/baidu/palo/task/CreateRollupTask.java b/fe/src/com/baidu/palo/task/CreateRollupTask.java index 4a68245986..f51c8d0daa 100644 --- a/fe/src/com/baidu/palo/task/CreateRollupTask.java +++ b/fe/src/com/baidu/palo/task/CreateRollupTask.java @@ -15,18 +15,18 @@ package com.baidu.palo.task; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.thrift.TAlterTabletReq; -import com.baidu.palo.thrift.TColumn; -import com.baidu.palo.thrift.TCreateTabletReq; -import com.baidu.palo.thrift.TKeysType; -import com.baidu.palo.thrift.TResourceInfo; -import com.baidu.palo.thrift.TStorageType; -import com.baidu.palo.thrift.TTabletSchema; -import com.baidu.palo.thrift.TTaskType; - -import java.util.ArrayList; -import java.util.List; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.thrift.TAlterTabletReq; +import com.baidu.palo.thrift.TColumn; +import com.baidu.palo.thrift.TCreateTabletReq; +import com.baidu.palo.thrift.TKeysType; +import com.baidu.palo.thrift.TResourceInfo; +import com.baidu.palo.thrift.TStorageType; +import com.baidu.palo.thrift.TTabletSchema; +import com.baidu.palo.thrift.TTaskType; + +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class CreateRollupTask extends AgentTask { @@ -54,7 +54,8 @@ public class CreateRollupTask extends AgentTask { long baseTabletId, long rollupReplicaId, short shortKeyColumnCount, int rollupSchemaHash, int baseSchemaHash, TStorageType storageType, List rollupColumns, Set bfColumns, double bfFpp, TKeysType keysType) { - super(resourceInfo, backendId, TTaskType.ROLLUP, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); + super(resourceInfo, backendId, rollupTabletId, TTaskType.ROLLUP, dbId, tableId, partitionId, rollupIndexId, + rollupTabletId); this.baseTableId = baseIndexId; this.baseTabletId = baseTabletId; diff --git a/fe/src/com/baidu/palo/task/DirMoveTask.java b/fe/src/com/baidu/palo/task/DirMoveTask.java new file mode 100644 index 0000000000..3f0c262621 --- /dev/null +++ b/fe/src/com/baidu/palo/task/DirMoveTask.java @@ -0,0 +1,65 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.task; + +import com.baidu.palo.thrift.TMoveDirReq; +import com.baidu.palo.thrift.TResourceInfo; +import com.baidu.palo.thrift.TTaskType; + +public class DirMoveTask extends AgentTask { + + private long jobId; + private String src; + private int schemaHash; + private boolean overwrite; + + public DirMoveTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, long dbId, + long tableId, long partitionId, long indexId, long tabletId, String src, int schemaHash, + boolean overwrite) { + super(resourceInfo, backendId, signature, TTaskType.MOVE, dbId, tableId, partitionId, indexId, tabletId); + this.jobId = jobId; + this.src = src; + this.schemaHash = schemaHash; + this.overwrite = overwrite; + } + + public long getJobId() { + return jobId; + } + + public String getSrc() { + return src; + } + + public int getSchemaHash() { + return schemaHash; + } + + public boolean isOverwrite() { + return overwrite; + } + + public TMoveDirReq toThrift() { + TMoveDirReq req = new TMoveDirReq(tabletId, schemaHash, src, jobId, overwrite); + return req; + } + +} diff --git a/fe/src/com/baidu/palo/task/DownloadTask.java b/fe/src/com/baidu/palo/task/DownloadTask.java new file mode 100644 index 0000000000..053c05894a --- /dev/null +++ b/fe/src/com/baidu/palo/task/DownloadTask.java @@ -0,0 +1,69 @@ +// Modifications copyright (C) 2018, Baidu.com, Inc. +// Copyright 2018 The Apache Software Foundation + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.task; + +import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; +import com.baidu.palo.thrift.TDownloadReq; +import com.baidu.palo.thrift.TNetworkAddress; +import com.baidu.palo.thrift.TResourceInfo; +import com.baidu.palo.thrift.TTaskType; + +import java.util.Map; + +public class DownloadTask extends AgentTask { + + private long jobId; + private Map srcToDestPath; + private BrokerAddress brokerAddr; + private Map brokerProperties; + + public DownloadTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, long dbId, + Map srcToDestPath, BrokerAddress brokerAddr, Map brokerProperties) { + super(resourceInfo, backendId, signature, TTaskType.DOWNLOAD, dbId, -1, -1, -1, -1); + this.jobId = jobId; + this.srcToDestPath = srcToDestPath; + this.brokerAddr = brokerAddr; + this.brokerProperties = brokerProperties; + } + + public long getJobId() { + return jobId; + } + + public Map getSrcToDestPath() { + return srcToDestPath; + } + + public BrokerAddress getBrokerAddr() { + return brokerAddr; + } + + public Map getBrokerProperties() { + return brokerProperties; + } + + public TDownloadReq toThrift() { + TNetworkAddress address = new TNetworkAddress(brokerAddr.ip, brokerAddr.port); + TDownloadReq req = new TDownloadReq(jobId, srcToDestPath, address); + req.setBroker_prop(brokerProperties); + return req; + } +} diff --git a/fe/src/com/baidu/palo/task/DropReplicaTask.java b/fe/src/com/baidu/palo/task/DropReplicaTask.java index e24101bfa9..52eba4fa83 100644 --- a/fe/src/com/baidu/palo/task/DropReplicaTask.java +++ b/fe/src/com/baidu/palo/task/DropReplicaTask.java @@ -15,14 +15,14 @@ package com.baidu.palo.task; -import com.baidu.palo.thrift.TDropTabletReq; +import com.baidu.palo.thrift.TDropTabletReq; import com.baidu.palo.thrift.TTaskType; public class DropReplicaTask extends AgentTask { private int schemaHash; // set -1L as unknown public DropReplicaTask(long backendId, long tabletId, int schemaHash) { - super(null, backendId, TTaskType.DROP, -1L, -1L, -1L, -1L, tabletId); + super(null, backendId, tabletId, TTaskType.DROP, -1L, -1L, -1L, -1L, tabletId); this.schemaHash = schemaHash; } diff --git a/fe/src/com/baidu/palo/task/ExportExportingTask.java b/fe/src/com/baidu/palo/task/ExportExportingTask.java index 9eda0c0b40..8d4d1d14ca 100644 --- a/fe/src/com/baidu/palo/task/ExportExportingTask.java +++ b/fe/src/com/baidu/palo/task/ExportExportingTask.java @@ -269,7 +269,7 @@ public class ExportExportingTask extends MasterTask { String localIP = FrontendOptions.getLocalHostAddress(); brokerAddress = Catalog.getInstance().getBrokerMgr().getBroker(job.getBrokerDesc().getName(), localIP); } catch (AnalysisException e) { - String failMsg = "Broker rename failed. msg=" + e.getMessage(); + String failMsg = "get broker failed. msg=" + e.getMessage(); LOG.warn(failMsg); return new Status(TStatusCode.CANCELLED, failMsg); } diff --git a/fe/src/com/baidu/palo/task/PushTask.java b/fe/src/com/baidu/palo/task/PushTask.java index 7df5c2c006..0dfad36468 100644 --- a/fe/src/com/baidu/palo/task/PushTask.java +++ b/fe/src/com/baidu/palo/task/PushTask.java @@ -15,24 +15,24 @@ package com.baidu.palo.task; -import com.baidu.palo.analysis.BinaryPredicate; -import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.analysis.IsNullPredicate; -import com.baidu.palo.analysis.LiteralExpr; -import com.baidu.palo.analysis.Predicate; -import com.baidu.palo.analysis.SlotRef; -import com.baidu.palo.common.MarkedCountDownLatch; -import com.baidu.palo.thrift.TCondition; -import com.baidu.palo.thrift.TPriority; -import com.baidu.palo.thrift.TPushReq; -import com.baidu.palo.thrift.TPushType; -import com.baidu.palo.thrift.TResourceInfo; -import com.baidu.palo.thrift.TTaskType; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import java.util.ArrayList; +import com.baidu.palo.analysis.BinaryPredicate; +import com.baidu.palo.analysis.BinaryPredicate.Operator; +import com.baidu.palo.analysis.IsNullPredicate; +import com.baidu.palo.analysis.LiteralExpr; +import com.baidu.palo.analysis.Predicate; +import com.baidu.palo.analysis.SlotRef; +import com.baidu.palo.common.MarkedCountDownLatch; +import com.baidu.palo.thrift.TCondition; +import com.baidu.palo.thrift.TPriority; +import com.baidu.palo.thrift.TPushReq; +import com.baidu.palo.thrift.TPushType; +import com.baidu.palo.thrift.TResourceInfo; +import com.baidu.palo.thrift.TTaskType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; import java.util.List; public class PushTask extends AgentTask { @@ -62,7 +62,7 @@ public class PushTask extends AgentTask { long indexId, long tabletId, long replicaId, int schemaHash, long version, long versionHash, String filePath, long fileSize, int timeoutSecond, long loadJobId, TPushType pushType, List conditions, boolean needDecompress, TPriority priority) { - super(resourceInfo, backendId, TTaskType.PUSH, dbId, tableId, partitionId, indexId, tabletId); + super(resourceInfo, backendId, tabletId, TTaskType.PUSH, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; this.schemaHash = schemaHash; this.version = version; diff --git a/fe/src/com/baidu/palo/task/ReleaseSnapshotTask.java b/fe/src/com/baidu/palo/task/ReleaseSnapshotTask.java index b57da95645..46ded8f1fe 100644 --- a/fe/src/com/baidu/palo/task/ReleaseSnapshotTask.java +++ b/fe/src/com/baidu/palo/task/ReleaseSnapshotTask.java @@ -25,7 +25,7 @@ public class ReleaseSnapshotTask extends AgentTask { public ReleaseSnapshotTask(TResourceInfo resourceInfo, long backendId, long dbId, long tabletId, String snapshotPath) { - super(resourceInfo, backendId, TTaskType.RELEASE_SNAPSHOT, dbId, -1, -1, -1, tabletId); + super(resourceInfo, backendId, tabletId, TTaskType.RELEASE_SNAPSHOT, dbId, -1, -1, -1, tabletId); this.snapshotPath = snapshotPath; } diff --git a/fe/src/com/baidu/palo/task/RestoreTask.java b/fe/src/com/baidu/palo/task/RestoreTask.java deleted file mode 100644 index 3ec7206d91..0000000000 --- a/fe/src/com/baidu/palo/task/RestoreTask.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.task; - -import com.baidu.palo.thrift.TResourceInfo; -import com.baidu.palo.thrift.TRestoreReq; -import com.baidu.palo.thrift.TTaskType; - -import java.util.Map; - -public class RestoreTask extends AgentTask { - - private long jobId; - private String remoteFilePath; - private int schemaHash; - private Map remoteProperties; - - public RestoreTask(TResourceInfo resourceInfo, long backendId, long jobId, long dbId, long tableId, - long partitionId, long indexId, long tabletId, int schemaHash, - String remoteFilePath, Map remoteProperties) { - super(resourceInfo, backendId, TTaskType.RESTORE, dbId, tableId, partitionId, indexId, tabletId); - - this.jobId = jobId; - this.remoteFilePath = remoteFilePath; - this.schemaHash = schemaHash; - this.remoteProperties = remoteProperties; - } - - public long getJobId() { - return jobId; - } - - public String getRemoteFilePath() { - return remoteFilePath; - } - - public int getSchemaHash() { - return schemaHash; - } - - public Map getRemoteProperties() { - return remoteProperties; - } - - public TRestoreReq toThrift() { - TRestoreReq req = new TRestoreReq(tabletId, schemaHash, remoteFilePath, remoteProperties); - return req; - } -} diff --git a/fe/src/com/baidu/palo/task/SchemaChangeTask.java b/fe/src/com/baidu/palo/task/SchemaChangeTask.java index bfbb5cfc1d..23be4708fd 100644 --- a/fe/src/com/baidu/palo/task/SchemaChangeTask.java +++ b/fe/src/com/baidu/palo/task/SchemaChangeTask.java @@ -15,18 +15,18 @@ package com.baidu.palo.task; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.thrift.TAlterTabletReq; -import com.baidu.palo.thrift.TColumn; -import com.baidu.palo.thrift.TCreateTabletReq; -import com.baidu.palo.thrift.TKeysType; -import com.baidu.palo.thrift.TResourceInfo; -import com.baidu.palo.thrift.TStorageType; -import com.baidu.palo.thrift.TTabletSchema; -import com.baidu.palo.thrift.TTaskType; - -import java.util.ArrayList; -import java.util.List; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.thrift.TAlterTabletReq; +import com.baidu.palo.thrift.TColumn; +import com.baidu.palo.thrift.TCreateTabletReq; +import com.baidu.palo.thrift.TKeysType; +import com.baidu.palo.thrift.TResourceInfo; +import com.baidu.palo.thrift.TStorageType; +import com.baidu.palo.thrift.TTabletSchema; +import com.baidu.palo.thrift.TTaskType; + +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class SchemaChangeTask extends AgentTask { @@ -49,7 +49,8 @@ public class SchemaChangeTask extends AgentTask { List newColumns, int newSchemaHash, int baseSchemaHash, short newShortKeyColumnCount, TStorageType storageType, Set bfColumns, double bfFpp, TKeysType keysType) { - super(resourceInfo, backendId, TTaskType.SCHEMA_CHANGE, dbId, tableId, partitionId, indexId, baseTabletId); + super(resourceInfo, backendId, baseTabletId, TTaskType.SCHEMA_CHANGE, dbId, tableId, partitionId, indexId, + baseTabletId); this.baseReplicaId = baseReplicaId; this.baseSchemaHash = baseSchemaHash; diff --git a/fe/src/com/baidu/palo/task/SnapshotTask.java b/fe/src/com/baidu/palo/task/SnapshotTask.java index e4079cf79c..22f3cef8e7 100644 --- a/fe/src/com/baidu/palo/task/SnapshotTask.java +++ b/fe/src/com/baidu/palo/task/SnapshotTask.java @@ -29,10 +29,13 @@ public class SnapshotTask extends AgentTask { private long timeout; - public SnapshotTask(TResourceInfo resourceInfo, long backendId, long jobId, long dbId, long tableId, - long partitionId, long indexId, long tabletId, long version, long versionHash, - int schemaHash, long timeout) { - super(resourceInfo, backendId, TTaskType.MAKE_SNAPSHOT, dbId, tableId, partitionId, indexId, tabletId); + private boolean isRestoreTask; + + public SnapshotTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, + long dbId, long tableId, long partitionId, long indexId, long tabletId, + long version, long versionHash, int schemaHash, long timeout, boolean isRestoreTask) { + super(resourceInfo, backendId, signature, TTaskType.MAKE_SNAPSHOT, dbId, tableId, partitionId, indexId, + tabletId); this.jobId = jobId; @@ -41,6 +44,8 @@ public class SnapshotTask extends AgentTask { this.schemaHash = schemaHash; this.timeout = timeout; + + this.isRestoreTask = isRestoreTask; } public long getJobId() { @@ -63,10 +68,15 @@ public class SnapshotTask extends AgentTask { return timeout; } + public boolean isRestoreTask() { + return isRestoreTask; + } + public TSnapshotRequest toThrift() { TSnapshotRequest request = new TSnapshotRequest(tabletId, schemaHash); request.setVersion(version); request.setVersion_hash(versionHash); + request.setList_files(true); return request; } } \ No newline at end of file diff --git a/fe/src/com/baidu/palo/task/StorageMediaMigrationTask.java b/fe/src/com/baidu/palo/task/StorageMediaMigrationTask.java index cb9f82d378..95110c39d2 100644 --- a/fe/src/com/baidu/palo/task/StorageMediaMigrationTask.java +++ b/fe/src/com/baidu/palo/task/StorageMediaMigrationTask.java @@ -26,7 +26,7 @@ public class StorageMediaMigrationTask extends AgentTask { public StorageMediaMigrationTask(long backendId, long tabletId, int schemaHash, TStorageMedium toStorageMedium) { - super(null, backendId, TTaskType.STORAGE_MEDIUM_MIGRATE, -1L, -1L, -1L, -1L, tabletId); + super(null, backendId, tabletId, TTaskType.STORAGE_MEDIUM_MIGRATE, -1L, -1L, -1L, -1L, tabletId); this.schemaHash = schemaHash; this.toStorageMedium = toStorageMedium; diff --git a/fe/src/com/baidu/palo/task/UploadTask.java b/fe/src/com/baidu/palo/task/UploadTask.java index acc17f49af..3e1a3eaca9 100644 --- a/fe/src/com/baidu/palo/task/UploadTask.java +++ b/fe/src/com/baidu/palo/task/UploadTask.java @@ -15,6 +15,8 @@ package com.baidu.palo.task; +import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; +import com.baidu.palo.thrift.TNetworkAddress; import com.baidu.palo.thrift.TResourceInfo; import com.baidu.palo.thrift.TTaskType; import com.baidu.palo.thrift.TUploadReq; @@ -24,40 +26,40 @@ import java.util.Map; public class UploadTask extends AgentTask { private long jobId; - private String src; - private String dest; - private Map remoteSourceProperties; + private Map srcToDestPath; + private BrokerAddress brokerAddress; + private Map brokerProperties; - public UploadTask(TResourceInfo resourceInfo, long backendId, long jobId, long dbId, long tableId, - long partitionId, long indexId, long tabletId, String src, String dest, - Map remoteSourceProperties) { - super(resourceInfo, backendId, TTaskType.UPLOAD, dbId, tableId, partitionId, indexId, tabletId); + public UploadTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, Long dbId, + Map srcToDestPath, BrokerAddress brokerAddr, Map brokerProperties) { + super(resourceInfo, backendId, signature, TTaskType.UPLOAD, dbId, -1, -1, -1, -1); this.jobId = jobId; - this.src = src; - this.dest = dest; - this.remoteSourceProperties = remoteSourceProperties; + this.srcToDestPath = srcToDestPath; + this.brokerAddress = brokerAddr; + this.brokerProperties = brokerProperties; } public long getJobId() { return jobId; } - public String getSrc() { - return src; + public Map getSrcToDestPath() { + return srcToDestPath; } - public String getDest() { - return dest; + public BrokerAddress getBrokerAddress() { + return brokerAddress; } - public Map getRemoteSourceProperties() { - return remoteSourceProperties; + public Map getBrokerProperties() { + return brokerProperties; } public TUploadReq toThrift() { - TUploadReq request = new TUploadReq(src, dest, remoteSourceProperties); - request.setTablet_id(tabletId); + TNetworkAddress address = new TNetworkAddress(brokerAddress.ip, brokerAddress.port); + TUploadReq request = new TUploadReq(jobId, srcToDestPath, address); + request.setBroker_prop(brokerProperties); return request; } } diff --git a/fe/test/com/baidu/palo/analysis/AccessTestUtil.java b/fe/test/com/baidu/palo/analysis/AccessTestUtil.java index 193bb838f9..aaff5a4b53 100644 --- a/fe/test/com/baidu/palo/analysis/AccessTestUtil.java +++ b/fe/test/com/baidu/palo/analysis/AccessTestUtil.java @@ -20,14 +20,9 @@ package com.baidu.palo.analysis; -import java.util.LinkedList; -import java.util.List; - -import org.easymock.EasyMock; - import com.baidu.palo.alter.RollupHandler; import com.baidu.palo.alter.SchemaChangeHandler; -import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.BrokerMgr; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.Database; @@ -39,50 +34,55 @@ import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.catalog.RandomDistributionInfo; import com.baidu.palo.catalog.SinglePartitionInfo; -import com.baidu.palo.catalog.UserPropertyMgr; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.DdlException; import com.baidu.palo.load.Load; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.persist.EditLog; import com.baidu.palo.qe.ConnectContext; import com.baidu.palo.system.SystemInfoService; + import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.easymock.EasyMock; + +import java.util.LinkedList; +import java.util.List; + public class AccessTestUtil { - public static UserPropertyMgr fetchAdminAccess() { - UserPropertyMgr userPropertyMgr = EasyMock.createMock(UserPropertyMgr.class); - EasyMock.expect(userPropertyMgr.checkAccess(EasyMock.isA(String.class), EasyMock.isA(String.class), - EasyMock.isA(AccessPrivilege.class))).andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.isAdmin(EasyMock.isA(String.class))).andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.isSuperuser(EasyMock.isA(String.class))).andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.checkUserAccess(EasyMock.isA(String.class), EasyMock.eq("blockUser"))) - .andReturn(false).anyTimes(); - EasyMock.expect(userPropertyMgr.checkUserAccess(EasyMock.isA(String.class), EasyMock.isA(String.class))) - .andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.getMaxConn(EasyMock.isA(String.class))).andReturn(1000L).anyTimes(); - try { - userPropertyMgr.setPasswd(EasyMock.endsWith("testCluster:testUser"), EasyMock.isA(byte[].class)); - EasyMock.expectLastCall().anyTimes(); - userPropertyMgr.setPasswd(EasyMock.endsWith("root"), EasyMock.isA(byte[].class)); - EasyMock.expectLastCall().andThrow(new DdlException("No privilege to change password")).anyTimes(); - } catch (DdlException e) { - return null; - } - EasyMock.replay(userPropertyMgr); - return userPropertyMgr; - } public static SystemInfoService fetchSystemInfoService() { SystemInfoService clusterInfo = EasyMock.createMock(SystemInfoService.class); EasyMock.replay(clusterInfo); return clusterInfo; } + + public static PaloAuth fetchAdminAccess() { + PaloAuth auth = EasyMock.createMock(PaloAuth.class); + EasyMock.expect(auth.checkGlobalPriv(EasyMock.isA(ConnectContext.class), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkDbPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkTblPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.anyString(), EasyMock.isA(PrivPredicate.class))) + .andReturn(true).anyTimes(); + try { + auth.setPassword(EasyMock.isA(SetPassVar.class)); + } catch (DdlException e) { + e.printStackTrace(); + } + EasyMock.expectLastCall().anyTimes(); + + EasyMock.replay(auth); + return auth; + } public static Catalog fetchAdminCatalog() { try { Catalog catalog = EasyMock.createMock(Catalog.class); - EasyMock.expect(catalog.getUserMgr()).andReturn(fetchAdminAccess()).anyTimes(); + EasyMock.expect(catalog.getAuth()).andReturn(fetchAdminAccess()).anyTimes(); Database db = new Database(50000L, "testCluster:testDb"); MaterializedIndex baseIndex = new MaterializedIndex(30000, IndexState.NORMAL); @@ -105,12 +105,12 @@ public class AccessTestUtil { EasyMock.expect(catalog.getSchemaChangeHandler()).andReturn(new SchemaChangeHandler()).anyTimes(); EasyMock.expect(catalog.getRollupHandler()).andReturn(new RollupHandler()).anyTimes(); EasyMock.expect(catalog.getEditLog()).andReturn(EasyMock.createMock(EditLog.class)).anyTimes(); - EasyMock.expect(catalog.getClusterDbNames("testCluster")) - .andReturn(Lists.newArrayList("testCluster:testDb")).anyTimes(); + EasyMock.expect(catalog.getClusterDbNames("testCluster")).andReturn(Lists.newArrayList("testCluster:testDb")).anyTimes(); catalog.changeDb(EasyMock.isA(ConnectContext.class), EasyMock.eq("blockDb")); EasyMock.expectLastCall().andThrow(new DdlException("failed.")).anyTimes(); catalog.changeDb(EasyMock.isA(ConnectContext.class), EasyMock.isA(String.class)); EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(catalog.getBrokerMgr()).andReturn(new BrokerMgr()).anyTimes(); EasyMock.replay(catalog); return catalog; } catch (DdlException e) { @@ -120,19 +120,20 @@ public class AccessTestUtil { } } - public static UserPropertyMgr fetchBlockAccess() { - UserPropertyMgr userPropertyMgr = EasyMock.createMock(UserPropertyMgr.class); - EasyMock.expect(userPropertyMgr.checkAccess(EasyMock.isA(String.class), EasyMock.isA(String.class), - EasyMock.isA(AccessPrivilege.class))).andReturn(false).anyTimes(); - EasyMock.expect(userPropertyMgr.isAdmin(EasyMock.isA(String.class))).andReturn(false).anyTimes(); - EasyMock.expect(userPropertyMgr.isSuperuser(EasyMock.isA(String.class))).andReturn(false).anyTimes(); - EasyMock.expect(userPropertyMgr.checkUserAccess(EasyMock.isA(String.class), EasyMock.isA(String.class))) + public static PaloAuth fetchBlockAccess() { + PaloAuth auth = EasyMock.createMock(PaloAuth.class); + EasyMock.expect(auth.checkGlobalPriv(EasyMock.isA(ConnectContext.class), + EasyMock.isA(PrivPredicate.class))).andReturn(false).anyTimes(); + EasyMock.expect(auth.checkDbPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.isA(PrivPredicate.class))).andReturn(false).anyTimes(); + EasyMock.expect(auth.checkTblPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.anyString(), EasyMock.isA(PrivPredicate.class))) .andReturn(false).anyTimes(); - EasyMock.replay(userPropertyMgr); - return userPropertyMgr; + EasyMock.replay(auth); + return auth; } - public static OlapTable mockTableFamilyGroup(String name) { + public static OlapTable mockTable(String name) { OlapTable table = EasyMock.createMock(OlapTable.class); Partition partition = EasyMock.createMock(Partition.class); MaterializedIndex index = EasyMock.createMock(MaterializedIndex.class); @@ -150,7 +151,7 @@ public class AccessTestUtil { public static Database mockDb(String name) { Database db = EasyMock.createMock(Database.class); - OlapTable olapTable = mockTableFamilyGroup("testTable"); + OlapTable olapTable = mockTable("testTable"); EasyMock.expect(db.getTable("testTable")).andReturn(olapTable).anyTimes(); EasyMock.expect(db.getTable("emptyTable")).andReturn(null).anyTimes(); EasyMock.expect(db.getTableNamesWithLock()).andReturn(Sets.newHashSet("testTable")).anyTimes(); @@ -169,7 +170,7 @@ public class AccessTestUtil { public static Catalog fetchBlockCatalog() { try { Catalog catalog = EasyMock.createMock(Catalog.class); - EasyMock.expect(catalog.getUserMgr()).andReturn(fetchBlockAccess()).anyTimes(); + EasyMock.expect(catalog.getAuth()).andReturn(fetchBlockAccess()).anyTimes(); catalog.changeDb(EasyMock.isA(ConnectContext.class), EasyMock.isA(String.class)); EasyMock.expectLastCall().andThrow(new DdlException("failed.")).anyTimes(); @@ -197,17 +198,12 @@ public class AccessTestUtil { } Analyzer analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn(prefix + "testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn(prefix + "testUser").anyTimes(); + EasyMock.expect(analyzer.getQualifiedUser()).andReturn(prefix + "testUser").anyTimes(); EasyMock.expect(analyzer.getCatalog()).andReturn(fetchAdminCatalog()).anyTimes(); EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); - - try { - analyzer.checkPrivilege(EasyMock.isA(String.class), EasyMock.isA(AccessPrivilege.class)); - } catch (AnalysisException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(analyzer.incrementCallDepth()).andReturn(1).anyTimes(); + EasyMock.expect(analyzer.decrementCallDepth()).andReturn(0).anyTimes(); + EasyMock.expect(analyzer.getCallDepth()).andReturn(1).anyTimes(); EasyMock.expect(analyzer.getContext()).andReturn(new ConnectContext(null)).anyTimes(); EasyMock.replay(analyzer); return analyzer; @@ -216,11 +212,9 @@ public class AccessTestUtil { public static Analyzer fetchBlockAnalyzer() throws AnalysisException { Analyzer analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("testCluster:testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testCluster:testUser").anyTimes(); + EasyMock.expect(analyzer.getQualifiedUser()).andReturn("testCluster:testUser").anyTimes(); EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); EasyMock.expect(analyzer.getCatalog()).andReturn(AccessTestUtil.fetchBlockCatalog()).anyTimes(); - analyzer.checkPrivilege(EasyMock.isA(String.class), EasyMock.isA(AccessPrivilege.class)); - EasyMock.expectLastCall().andThrow(new AnalysisException("")); EasyMock.replay(analyzer); return analyzer; } @@ -228,7 +222,7 @@ public class AccessTestUtil { public static Analyzer fetchEmptyDbAnalyzer() { Analyzer analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testCluster:testUser").anyTimes(); + EasyMock.expect(analyzer.getQualifiedUser()).andReturn("testCluster:testUser").anyTimes(); EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); EasyMock.expect(analyzer.getCatalog()).andReturn(AccessTestUtil.fetchBlockCatalog()).anyTimes(); EasyMock.expect(analyzer.getContext()).andReturn(new ConnectContext(null)).anyTimes(); diff --git a/fe/test/com/baidu/palo/analysis/AlterClusterStmtTest.java b/fe/test/com/baidu/palo/analysis/AlterClusterStmtTest.java index 60c7d8d758..31b7534639 100644 --- a/fe/test/com/baidu/palo/analysis/AlterClusterStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/AlterClusterStmtTest.java @@ -20,24 +20,40 @@ package com.baidu.palo.analysis; -import java.util.HashMap; -import java.util.Map; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.baidu.palo.analysis.AlterClusterStmt; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; +import java.util.HashMap; +import java.util.Map; + +import mockit.Mocked; +import mockit.internal.startup.Startup; public class AlterClusterStmtTest { private static Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before() public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -67,13 +83,4 @@ public class AlterClusterStmtTest { Assert.fail("no exception"); } - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - final Map properties = new HashMap(); - properties.put("instance_num", "2"); - final AlterClusterStmt stmt = new AlterClusterStmt("testCluster1", properties); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("no exception"); - } - } diff --git a/fe/test/com/baidu/palo/analysis/AlterTableStmtTest.java b/fe/test/com/baidu/palo/analysis/AlterTableStmtTest.java index 15dd3c29a6..1bdd894b4e 100644 --- a/fe/test/com/baidu/palo/analysis/AlterTableStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/AlterTableStmtTest.java @@ -22,6 +22,9 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; import com.google.common.collect.Lists; @@ -31,12 +34,36 @@ import org.junit.Test; import java.util.List; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + public class AlterTableStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + + new NonStrictExpectations() { + { + auth.checkGlobalPriv((ConnectContext) any, (PrivPredicate) any); + result = true; + + auth.checkDbPriv((ConnectContext) any, anyString, (PrivPredicate) any); + result = true; + + auth.checkTblPriv((ConnectContext) any, anyString, anyString, (PrivPredicate) any); + result = true; + } + }; } @Test @@ -52,13 +79,12 @@ public class AlterTableStmtTest { Assert.assertEquals(2, stmt.getOps().size()); } - @Test(expected = AnalysisException.class) + @Test public void testNoPriv() throws AnalysisException, InternalException { List ops = Lists.newArrayList(); ops.add(new DropColumnClause("col1", "", null)); AlterTableStmt stmt = new AlterTableStmt(new TableName("testDb", "testTbl"), ops); stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.assertEquals("ALTER TABLE `testDb`.`testTbl` DROP COLUMN `col1`", stmt.toString()); } @Test(expected = AnalysisException.class) diff --git a/fe/test/com/baidu/palo/analysis/BackendStmtTest.java b/fe/test/com/baidu/palo/analysis/BackendStmtTest.java index 6d3ba46ea1..07fb82eb70 100644 --- a/fe/test/com/baidu/palo/analysis/BackendStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/BackendStmtTest.java @@ -20,12 +20,12 @@ package com.baidu.palo.analysis; -import com.baidu.palo.common.AnalysisException; - -import com.google.common.collect.Lists; - -import org.junit.Assert; -import org.junit.BeforeClass; +import com.baidu.palo.common.AnalysisException; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class BackendStmtTest { @@ -88,7 +88,7 @@ public class BackendStmtTest { public void initBackendsTest4() throws Exception { BackendClause stmt = createStmt(4); stmt.analyze(analyzer); - Assert.assertEquals("ADD BACKEND \"192.168.1.1:12345\"", stmt.toSql()); + Assert.assertEquals("ADD FREE BACKEND \"192.168.1.1:12345\"", stmt.toSql()); } @Test diff --git a/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java b/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java index 0935d82f86..3ff288b545 100644 --- a/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java @@ -20,6 +20,13 @@ package com.baidu.palo.analysis; +import com.baidu.palo.analysis.ShowAlterStmt.AlterType; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -28,32 +35,42 @@ import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.analysis.ShowAlterStmt.AlterType; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; +import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) +@PrepareForTest({ Catalog.class, ConnectContext.class }) public class CancelAlterStmtTest { private Analyzer analyzer; - private Catalog catalog; + private Catalog catalog; + + private ConnectContext ctx; + + private PaloAuth auth; @Before - public void setUp() { + public void setUp() { + auth = new PaloAuth(); + + ctx = new ConnectContext(null); + ctx.setQualifiedUser("root"); + ctx.setRemoteIP("192.168.1.1"); + catalog = AccessTestUtil.fetchAdminCatalog(); PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - PowerMock.replay(Catalog.class); + EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); + PowerMock.replay(Catalog.class); + + PowerMock.mockStatic(ConnectContext.class); + EasyMock.expect(ConnectContext.get()).andReturn(ctx).anyTimes(); + PowerMock.replay(ConnectContext.class); analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testUser").anyTimes(); + EasyMock.expect(analyzer.getQualifiedUser()).andReturn("testUser").anyTimes(); EasyMock.expect(analyzer.getCatalog()).andReturn(catalog).anyTimes(); EasyMock.replay(analyzer); } @@ -61,7 +78,6 @@ public class CancelAlterStmtTest { @Test public void testNormal() throws InternalException, AnalysisException { // cancel alter column - CancelAlterTableStmt stmt = new CancelAlterTableStmt(AlterType.COLUMN, new TableName(null, "testTbl")); stmt.analyze(analyzer); Assert.assertEquals("CANCEL ALTER COLUMN FROM `testDb`.`testTbl`", stmt.toString()); diff --git a/fe/test/com/baidu/palo/analysis/CreateClusterStmtTest.java b/fe/test/com/baidu/palo/analysis/CreateClusterStmtTest.java index 24cd7a063f..ccfee40873 100644 --- a/fe/test/com/baidu/palo/analysis/CreateClusterStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CreateClusterStmtTest.java @@ -20,25 +20,40 @@ package com.baidu.palo.analysis; -import java.util.Map; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.baidu.palo.analysis.CreateClusterStmt; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; import java.util.HashMap; import java.util.Map; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class CreateClusterStmtTest { private static Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before() public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -59,14 +74,4 @@ public class CreateClusterStmtTest { stmt.analyze(analyzer); Assert.fail("no exception"); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - final Map properties = new HashMap(); - properties.put("instance_num", "2"); - final CreateClusterStmt stmt = new CreateClusterStmt("testCluster1", properties, "password"); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("no exception"); - } - } diff --git a/fe/test/com/baidu/palo/analysis/CreateDbStmtTest.java b/fe/test/com/baidu/palo/analysis/CreateDbStmtTest.java index ddf31a2dfc..90b8468838 100644 --- a/fe/test/com/baidu/palo/analysis/CreateDbStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CreateDbStmtTest.java @@ -22,17 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class CreateDbStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before() public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -49,12 +66,4 @@ public class CreateDbStmtTest { stmt.analyze(analyzer); Assert.fail("no exception"); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - CreateDbStmt stmt = new CreateDbStmt(false, "testDb"); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("no exception"); - } - } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java b/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java index 0bd473c746..a8415d261d 100644 --- a/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java @@ -21,7 +21,17 @@ package com.baidu.palo.analysis; -import java.util.List; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.catalog.KeysType; +import com.baidu.palo.catalog.PrimitiveType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + +import com.google.common.collect.Lists; import org.easymock.EasyMock; import org.junit.Assert; @@ -30,13 +40,10 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.catalog.ColumnType; -import com.baidu.palo.catalog.KeysType; -import com.baidu.palo.catalog.PrimitiveType; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; -import com.google.common.collect.Lists; +import java.util.List; + +import mockit.Mocked; +import mockit.internal.startup.Startup; public class CreateTableStmtTest { private static final Logger LOG = LoggerFactory.getLogger(CreateTableStmtTest.class); @@ -50,6 +57,15 @@ public class CreateTableStmtTest { private List invalidColsName; private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + // set default db is 'db1' // table name is table1 // Column: [col1 int; col2 string] @@ -76,38 +92,32 @@ public class CreateTableStmtTest { invalidColsName.add("col1"); invalidColsName.add("col2"); invalidColsName.add("col2"); + + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test public void testNormal() throws InternalException, AnalysisException { CreateTableStmt stmt = new CreateTableStmt(false, false, tblName, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null, null); + new HashDistributionDesc(10, Lists.newArrayList("col1")), null, null); stmt.analyze(analyzer); Assert.assertEquals("testCluster:db1", stmt.getDbName()); Assert.assertEquals("table1", stmt.getTableName()); Assert.assertNull(stmt.getProperties()); - LOG.info(stmt.toSql()); - Assert.assertEquals("CREATE TABLE `testCluster:db1`.`table1` (\n" - + "`col1` int(11) NOT NULL COMMENT \"\",\n" + "`col2` char(10) NOT NULL COMMENT \"\"\n" - + ") ENGINE = olap\nAGG_KEYS(`col1`, `col2`)\nDISTRIBUTED BY RANDOM\nBUCKETS 10", - stmt.toSql()); } @Test public void testDefaultDbNormal() throws InternalException, AnalysisException { CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null, null); + new HashDistributionDesc(10, Lists.newArrayList("col1")), null, null); stmt.analyze(analyzer); - Assert.assertEquals("testCluster:testDb", stmt.getDbName()); + Assert.assertEquals("testDb", stmt.getDbName()); Assert.assertEquals("table1", stmt.getTableName()); Assert.assertNull(stmt.getPartitionDesc()); Assert.assertNull(stmt.getProperties()); - LOG.info(stmt.toSql()); - Assert.assertEquals("CREATE TABLE `testCluster:testDb`.`table1` (\n" - + "`col1` int(11) NOT NULL COMMENT \"\",\n" + "`col2` char(10) NOT NULL COMMENT \"\"\n" - + ") ENGINE = olap\nAGG_KEYS(`col1`, `col2`)\nDISTRIBUTED BY RANDOM\nBUCKETS 10", stmt.toSql()); } @Test(expected = AnalysisException.class) @@ -115,6 +125,7 @@ public class CreateTableStmtTest { // make defalut db return empty; analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("").anyTimes(); + EasyMock.expect(analyzer.getClusterName()).andReturn("cluster").anyTimes(); EasyMock.replay(analyzer); CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, @@ -141,14 +152,4 @@ public class CreateTableStmtTest { stmt.analyze(analyzer); } - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - // make default db return empty; - CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", - new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null, null); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); - } - } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/CreateUserStmtTest.java b/fe/test/com/baidu/palo/analysis/CreateUserStmtTest.java index b1fc908d68..4d90fa9a11 100644 --- a/fe/test/com/baidu/palo/analysis/CreateUserStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CreateUserStmtTest.java @@ -22,60 +22,71 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class CreateUserStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test public void testToString() throws InternalException, AnalysisException { - CreateUserStmt stmt = new CreateUserStmt(new UserDesc("user", "passwd", true)); + CreateUserStmt stmt = new CreateUserStmt(new UserDesc(new UserIdentity("user", "%"), "passwd", true)); stmt.analyze(analyzer); - Assert.assertEquals("CREATE USER 'testCluster:user' IDENTIFIED BY 'passwd'", stmt.toString()); + Assert.assertEquals("CREATE USER 'testCluster:user'@'%' IDENTIFIED BY 'passwd'", stmt.toString()); Assert.assertEquals(new String(stmt.getPassword()), "*59C70DA2F3E3A5BDF46B68F5C8B8F25762BCCEF0"); - stmt = new CreateUserStmt(new UserDesc("user", "*59c70da2f3e3a5bdf46b68f5c8b8f25762bccef0", false)); + stmt = new CreateUserStmt( + new UserDesc(new UserIdentity("user", "%"), "*59c70da2f3e3a5bdf46b68f5c8b8f25762bccef0", false)); stmt.analyze(analyzer); - Assert.assertEquals("testCluster:user", stmt.getUser()); + Assert.assertEquals("testCluster:user", stmt.getUserIdent().getQualifiedUser()); Assert.assertEquals( - "CREATE USER 'testCluster:user' IDENTIFIED BY PASSWORD '*59c70da2f3e3a5bdf46b68f5c8b8f25762bccef0'", + "CREATE USER 'testCluster:user'@'%' IDENTIFIED BY PASSWORD '*59c70da2f3e3a5bdf46b68f5c8b8f25762bccef0'", stmt.toString()); Assert.assertEquals(new String(stmt.getPassword()), "*59C70DA2F3E3A5BDF46B68F5C8B8F25762BCCEF0"); - stmt = new CreateUserStmt(new UserDesc("user", "", false)); + stmt = new CreateUserStmt(new UserDesc(new UserIdentity("user", "%"), "", false)); stmt.analyze(analyzer); - Assert.assertEquals("CREATE USER 'testCluster:user'", stmt.toString()); + Assert.assertEquals("CREATE USER 'testCluster:user'@'%'", stmt.toString()); Assert.assertEquals(new String(stmt.getPassword()), ""); } - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - CreateUserStmt stmt = new CreateUserStmt(new UserDesc("user", "passwd", true)); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); - } - @Test(expected = AnalysisException.class) public void testEmptyUser() throws InternalException, AnalysisException { - CreateUserStmt stmt = new CreateUserStmt(new UserDesc("", "passwd", true)); + CreateUserStmt stmt = new CreateUserStmt(new UserDesc(new UserIdentity("", "%"), "passwd", true)); stmt.analyze(analyzer); Assert.fail("No exception throws."); } @Test(expected = AnalysisException.class) public void testBadPass() throws InternalException, AnalysisException { - CreateUserStmt stmt = new CreateUserStmt(new UserDesc("", "passwd", false)); + CreateUserStmt stmt = new CreateUserStmt(new UserDesc(new UserIdentity("", "%"), "passwd", false)); stmt.analyze(analyzer); Assert.fail("No exception throws."); } diff --git a/fe/test/com/baidu/palo/analysis/DataDescriptionTest.java b/fe/test/com/baidu/palo/analysis/DataDescriptionTest.java index f2361a929b..b36323c725 100644 --- a/fe/test/com/baidu/palo/analysis/DataDescriptionTest.java +++ b/fe/test/com/baidu/palo/analysis/DataDescriptionTest.java @@ -22,32 +22,56 @@ package com.baidu.palo.analysis; import com.baidu.palo.analysis.BinaryPredicate.Operator; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.collect.Lists; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.util.List; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class DataDescriptionTest { + + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + + @Before + public void setUp() { + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); + } + @Test public void testNormal() throws AnalysisException { DataDescription desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt"), null, null, false, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt') INTO TABLE testTable", desc.toString()); desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt"), null, null, true, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt') NEGATIVE INTO TABLE testTable", desc.toString()); desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt", "bcd.txt"), null, null, true, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt', 'bcd.txt') NEGATIVE INTO TABLE testTable", desc.toString()); desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt"), Lists.newArrayList("col1", "col2"), null, true, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt') NEGATIVE INTO TABLE testTable (col1, col2)", desc.toString()); Assert.assertEquals("testTable", desc.getTableName()); Assert.assertEquals("[col1, col2]", desc.getColumnNames().toString()); @@ -57,7 +81,7 @@ public class DataDescriptionTest { desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt", "bcd.txt"), Lists.newArrayList("col1", "col2"), new ColumnSeparator("\t"), true, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt', 'bcd.txt') NEGATIVE INTO TABLE testTable" + " COLUMNS TERMINATED BY '\t' (col1, col2)", desc.toString()); @@ -65,7 +89,7 @@ public class DataDescriptionTest { // hive \x01 column separator desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt", "bcd.txt"), Lists.newArrayList("col1", "col2"), new ColumnSeparator("\\x01"), true, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt', 'bcd.txt') NEGATIVE INTO TABLE testTable" + " COLUMNS TERMINATED BY '\\x01' (col1, col2)", desc.toString()); @@ -73,7 +97,7 @@ public class DataDescriptionTest { // with partition desc = new DataDescription("testTable", Lists.newArrayList("p1", "p2"), Lists.newArrayList("abc.txt"), null, null, false, null); - desc.analyze(); + desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt') INTO TABLE testTable PARTITION (p1, p2)", desc.toString()); // alignment_timestamp func @@ -84,7 +108,7 @@ public class DataDescriptionTest { new FunctionCallExpr("alignment_timestamp", params)); desc = new DataDescription("testTable", Lists.newArrayList("p1", "p2"), Lists.newArrayList("abc.txt"), Lists.newArrayList("k2", "k3"), null, false, Lists.newArrayList((Expr) predicate)); - desc.analyze(); + desc.analyze("testDb"); String sql = "DATA INFILE ('abc.txt') INTO TABLE testTable PARTITION (p1, p2) (k2, k3)" + " SET (`k1` = alignment_timestamp('day', `k2`))"; Assert.assertEquals(sql, desc.toString()); @@ -97,7 +121,7 @@ public class DataDescriptionTest { new FunctionCallExpr("replace_value", params)); desc = new DataDescription("testTable", Lists.newArrayList("p1", "p2"), Lists.newArrayList("abc.txt"), Lists.newArrayList("k2", "k3"), null, false, Lists.newArrayList((Expr) predicate)); - desc.analyze(); + desc.analyze("testDb"); sql = "DATA INFILE ('abc.txt') INTO TABLE testTable PARTITION (p1, p2) (k2, k3)" + " SET (`k1` = replace_value('-', '10'))"; Assert.assertEquals(sql, desc.toString()); @@ -110,7 +134,7 @@ public class DataDescriptionTest { new FunctionCallExpr("replace_value", params)); desc = new DataDescription("testTable", Lists.newArrayList("p1", "p2"), Lists.newArrayList("abc.txt"), Lists.newArrayList("k2", "k3"), null, false, Lists.newArrayList((Expr) predicate)); - desc.analyze(); + desc.analyze("testDb"); sql = "DATA INFILE ('abc.txt') INTO TABLE testTable PARTITION (p1, p2) (k2, k3)" + " SET (`k1` = replace_value('', NULL))"; Assert.assertEquals(sql, desc.toString()); @@ -120,19 +144,19 @@ public class DataDescriptionTest { public void testNoTable() throws AnalysisException { DataDescription desc = new DataDescription("", null, Lists.newArrayList("abc.txt"), null, null, false, null); - desc.analyze(); + desc.analyze("testDb"); } @Test(expected = AnalysisException.class) public void testNoFile() throws AnalysisException { DataDescription desc = new DataDescription("testTable", null, null, null, null, false, null); - desc.analyze(); + desc.analyze("testDb"); } @Test(expected = AnalysisException.class) public void testDupCol() throws AnalysisException { DataDescription desc = new DataDescription("testTable", null, Lists.newArrayList("abc.txt"), Lists.newArrayList("col1", "col1"), null, false, null); - desc.analyze(); + desc.analyze("testDb"); } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/DeleteStmtTest.java b/fe/test/com/baidu/palo/analysis/DeleteStmtTest.java index 0816d40cb8..db911570c2 100644 --- a/fe/test/com/baidu/palo/analysis/DeleteStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/DeleteStmtTest.java @@ -20,21 +20,38 @@ package com.baidu.palo.analysis; -import com.baidu.palo.analysis.BinaryPredicate.Operator; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import com.baidu.palo.analysis.BinaryPredicate.Operator; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import mockit.Mocked; +import mockit.internal.startup.Startup; public class DeleteStmtTest { - Analyzer analyzer; + Analyzer analyzer; + + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } @Before public void setUp() { - analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test diff --git a/fe/test/com/baidu/palo/analysis/DescribeStmtTest.java b/fe/test/com/baidu/palo/analysis/DescribeStmtTest.java index a2f75a0c2d..4e89e16d3b 100644 --- a/fe/test/com/baidu/palo/analysis/DescribeStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/DescribeStmtTest.java @@ -23,6 +23,7 @@ package com.baidu.palo.analysis; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.qe.ConnectContext; import org.easymock.EasyMock; import org.junit.Assert; @@ -36,19 +37,30 @@ import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) +@PrepareForTest({ Catalog.class, ConnectContext.class }) public class DescribeStmtTest { private Analyzer analyzer; private Catalog catalog; + private ConnectContext ctx; @Before public void setUp() { + ctx = new ConnectContext(null); + ctx.setQualifiedUser("root"); + ctx.setRemoteIP("192.168.1.1"); + + PowerMock.mockStatic(ConnectContext.class); + EasyMock.expect(ConnectContext.get()).andReturn(ctx).anyTimes(); + PowerMock.replay(ConnectContext.class); + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); catalog = AccessTestUtil.fetchAdminCatalog(); PowerMock.mockStatic(Catalog.class); EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(AccessTestUtil.fetchSystemInfoService()).anyTimes(); PowerMock.replay(Catalog.class); + } @Test @@ -70,11 +82,4 @@ public class DescribeStmtTest { Assert.assertEquals("testCluster:testDb", stmt.getDb()); Assert.assertEquals("testTbl", stmt.getTableName()); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws AnalysisException, InternalException { - DescribeStmt stmt = new DescribeStmt(new TableName("", "testTable"), false); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/DropClusterStmtTest.java b/fe/test/com/baidu/palo/analysis/DropClusterStmtTest.java index 2107597760..a99b50c9df 100644 --- a/fe/test/com/baidu/palo/analysis/DropClusterStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/DropClusterStmtTest.java @@ -20,20 +20,47 @@ package com.baidu.palo.analysis; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; public class DropClusterStmtTest { private static Analyzer analyzer; + @Mocked + private PaloAuth auth; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + + new NonStrictExpectations() { + { + auth.checkGlobalPriv((ConnectContext) any, (PrivPredicate) any); + result = true; + + auth.checkDbPriv((ConnectContext) any, anyString, (PrivPredicate) any); + result = true; + + auth.checkTblPriv((ConnectContext) any, anyString, anyString, (PrivPredicate) any); + result = true; + } + }; } @Test diff --git a/fe/test/com/baidu/palo/analysis/DropDbStmtTest.java b/fe/test/com/baidu/palo/analysis/DropDbStmtTest.java index 5ae625d72a..7715463c7f 100644 --- a/fe/test/com/baidu/palo/analysis/DropDbStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/DropDbStmtTest.java @@ -22,17 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class DropDbStmtTest { Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test diff --git a/fe/test/com/baidu/palo/analysis/DropTableStmtTest.java b/fe/test/com/baidu/palo/analysis/DropTableStmtTest.java index ee263b62d8..6e77af01df 100644 --- a/fe/test/com/baidu/palo/analysis/DropTableStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/DropTableStmtTest.java @@ -22,18 +22,33 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class DropTableStmtTest { private TableName tbl; private TableName noDbTbl; private Analyzer analyzer; private Analyzer noDbAnalyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { tbl = new TableName("db1", "table1"); @@ -44,6 +59,9 @@ public class DropTableStmtTest { EasyMock.expect(noDbAnalyzer.getDefaultDb()).andReturn("").anyTimes(); EasyMock.expect(noDbAnalyzer.getClusterName()).andReturn("testCluster").anyTimes(); EasyMock.replay(noDbAnalyzer); + + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -64,13 +82,6 @@ public class DropTableStmtTest { Assert.assertEquals("DROP TABLE `testCluster:testDb`.`table1`", stmt.toSql()); } - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - DropTableStmt stmt = new DropTableStmt(false, tbl); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws"); - } - @Test(expected = AnalysisException.class) public void testNoDbFail() throws InternalException, AnalysisException { DropTableStmt stmt = new DropTableStmt(false, noDbTbl); diff --git a/fe/test/com/baidu/palo/analysis/DropUserStmtTest.java b/fe/test/com/baidu/palo/analysis/DropUserStmtTest.java index 8733885183..e90534bece 100644 --- a/fe/test/com/baidu/palo/analysis/DropUserStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/DropUserStmtTest.java @@ -22,39 +22,48 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class DropUserStmtTest { private Analyzer analyzer; + + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test public void testNormal() throws InternalException, AnalysisException { - DropUserStmt stmt = new DropUserStmt("user"); + DropUserStmt stmt = new DropUserStmt(new UserIdentity("user", "%")); stmt.analyze(analyzer); - Assert.assertEquals("DROP USER 'testCluster:user'", stmt.toString()); - Assert.assertEquals("testCluster:user", stmt.getUser()); + Assert.assertEquals("DROP USER 'testCluster:user'@'%'", stmt.toString()); + Assert.assertEquals("testCluster:user", stmt.getUserIdentity().getQualifiedUser()); } @Test(expected = AnalysisException.class) public void testNoUser() throws InternalException, AnalysisException { - DropUserStmt stmt = new DropUserStmt(""); + DropUserStmt stmt = new DropUserStmt(new UserIdentity("", "%")); stmt.analyze(analyzer); Assert.fail("No Exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - DropUserStmt stmt = new DropUserStmt("testUser"); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No Exception throws."); - } - } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/GrantStmtTest.java b/fe/test/com/baidu/palo/analysis/GrantStmtTest.java index 8b92fec349..a7a45ed647 100644 --- a/fe/test/com/baidu/palo/analysis/GrantStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/GrantStmtTest.java @@ -21,21 +21,61 @@ package com.baidu.palo.analysis; import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.collect.Lists; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.util.List; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + public class GrantStmtTest { private Analyzer analyzer; + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + @Mocked + private Catalog catalog; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + auth = new PaloAuth(); + + new NonStrictExpectations() { + { + ConnectContext.get(); + result = ctx; + + ctx.getQualifiedUser(); + result = "root"; + + ctx.getRemoteIP(); + result = "192.168.0.1"; + + Catalog.getCurrentCatalog(); + result = catalog; + + catalog.getAuth(); + result = auth; + } + }; } @Test @@ -43,27 +83,14 @@ public class GrantStmtTest { GrantStmt stmt; List privileges = Lists.newArrayList(AccessPrivilege.ALL); - stmt = new GrantStmt("testUser", "testDb", privileges); + stmt = new GrantStmt(new UserIdentity("testUser", "%"), null, new TablePattern("testDb", "*"), privileges); stmt.analyze(analyzer); - Assert.assertEquals("GRANT ALL ON testCluster:testDb TO 'testCluster:testUser'", stmt.toString()); - Assert.assertEquals("testCluster:testUser", stmt.getUser()); - Assert.assertEquals("testCluster:testDb", stmt.getDb()); - Assert.assertEquals(AccessPrivilege.ALL, stmt.getPrivilege()); + Assert.assertEquals("testCluster:testUser", stmt.getUserIdent().getQualifiedUser()); + Assert.assertEquals("testCluster:testDb", stmt.getTblPattern().getQuolifiedDb()); privileges = Lists.newArrayList(AccessPrivilege.READ_ONLY, AccessPrivilege.ALL); - stmt = new GrantStmt("testUser", "testDb", privileges); + stmt = new GrantStmt(new UserIdentity("testUser", "%"), null, new TablePattern("testDb", "*"), privileges); stmt.analyze(analyzer); - Assert.assertEquals("GRANT READ_ONLY, ALL ON testCluster:testDb TO 'testCluster:testUser'", stmt.toString()); - } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws AnalysisException, InternalException { - GrantStmt stmt; - - List privileges = Lists.newArrayList(AccessPrivilege.ALL); - stmt = new GrantStmt("testUser", "testDb", privileges); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); } @Test(expected = AnalysisException.class) @@ -71,26 +98,7 @@ public class GrantStmtTest { GrantStmt stmt; List privileges = Lists.newArrayList(AccessPrivilege.ALL); - stmt = new GrantStmt("", "testDb", privileges); - stmt.analyze(analyzer); - Assert.fail("No exeception throws."); - } - - @Test(expected = AnalysisException.class) - public void testDbFail() throws AnalysisException, InternalException { - GrantStmt stmt; - - List privileges = Lists.newArrayList(AccessPrivilege.ALL); - stmt = new GrantStmt("testUser", "", privileges); - stmt.analyze(analyzer); - Assert.fail("No exeception throws."); - } - - @Test(expected = AnalysisException.class) - public void testPrivFail() throws AnalysisException, InternalException { - GrantStmt stmt; - - stmt = new GrantStmt("testUser", "testDb", null); + stmt = new GrantStmt(new UserIdentity("", "%"), null, new TablePattern("testDb", "*"), privileges); stmt.analyze(analyzer); Assert.fail("No exeception throws."); } diff --git a/fe/test/com/baidu/palo/analysis/LinkDbStmtTest.java b/fe/test/com/baidu/palo/analysis/LinkDbStmtTest.java index 37dd14b77e..66762c2b3b 100644 --- a/fe/test/com/baidu/palo/analysis/LinkDbStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/LinkDbStmtTest.java @@ -22,18 +22,35 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class LinkDbStmtTest { private static Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -55,13 +72,4 @@ public class LinkDbStmtTest { stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); Assert.fail("no exception"); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - final ClusterName cn1 = new ClusterName("testCluster1", "testDb1"); - final ClusterName cn2 = new ClusterName("testCluster2", "testDb2"); - final LinkDbStmt stmt = new LinkDbStmt(cn1, cn2); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("no exception"); - } } diff --git a/fe/test/com/baidu/palo/analysis/LoadStmtTest.java b/fe/test/com/baidu/palo/analysis/LoadStmtTest.java index 75e3fcedfb..ffba8c0893 100644 --- a/fe/test/com/baidu/palo/analysis/LoadStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/LoadStmtTest.java @@ -20,22 +20,38 @@ package com.baidu.palo.analysis; -import java.util.List; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + +import com.google.common.collect.Lists; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; -import com.google.common.collect.Lists; +import java.util.List; + +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; public class LoadStmtTest { private DataDescription desc; private List dataDescriptions; private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); @@ -43,11 +59,21 @@ public class LoadStmtTest { desc = EasyMock.createMock(DataDescription.class); EasyMock.expect(desc.toSql()).andReturn("XXX"); dataDescriptions.add(desc); + + new NonStrictExpectations() { + { + ConnectContext.get(); + result = ctx; + + ctx.getQualifiedUser(); + result = "default_cluster:user"; + } + }; } @Test public void testNormal() throws InternalException, AnalysisException { - desc.analyze(); + desc.analyze(EasyMock.anyString()); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(desc); @@ -61,21 +87,9 @@ public class LoadStmtTest { + "(XXX)", stmt.toString()); } - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - desc.analyze(); - EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(desc); - - LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), dataDescriptions, null, null, null); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - - Assert.fail("No exception throws."); - } - @Test(expected = AnalysisException.class) public void testNoData() throws InternalException, AnalysisException { - desc.analyze(); + desc.analyze(EasyMock.anyString()); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(desc); diff --git a/fe/test/com/baidu/palo/analysis/MigrateDbStmtTest.java b/fe/test/com/baidu/palo/analysis/MigrateDbStmtTest.java index 1659ae7659..9d92d47d38 100644 --- a/fe/test/com/baidu/palo/analysis/MigrateDbStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/MigrateDbStmtTest.java @@ -22,18 +22,35 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class MigrateDbStmtTest { private static Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -55,13 +72,4 @@ public class MigrateDbStmtTest { stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); Assert.fail("no exception"); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - final ClusterName cn1 = new ClusterName("testCluster1", "testDb1"); - final ClusterName cn2 = new ClusterName("testCluster2", "testDb2"); - final MigrateDbStmt stmt = new MigrateDbStmt(cn1, cn2); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("no exception"); - } } diff --git a/fe/test/com/baidu/palo/analysis/SetPassVarTest.java b/fe/test/com/baidu/palo/analysis/SetPassVarTest.java index b2f26feb71..9b75d62fb8 100644 --- a/fe/test/com/baidu/palo/analysis/SetPassVarTest.java +++ b/fe/test/com/baidu/palo/analysis/SetPassVarTest.java @@ -22,17 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class SetPassVarTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -40,40 +57,30 @@ public class SetPassVarTest { SetPassVar stmt; // mode: SET PASSWORD FOR 'testUser' = 'testPass'; - stmt = new SetPassVar("testUser", "*88EEBA7D913688E7278E2AD071FDB5E76D76D34B"); + stmt = new SetPassVar(new UserIdentity("testUser", "%"), "*88EEBA7D913688E7278E2AD071FDB5E76D76D34B"); stmt.analyze(analyzer); - Assert.assertEquals("testCluster:testUser", stmt.getUser()); + Assert.assertEquals("testCluster:testUser", stmt.getUserIdent().getQualifiedUser()); Assert.assertEquals("*88EEBA7D913688E7278E2AD071FDB5E76D76D34B", new String(stmt.getPassword())); - Assert.assertEquals("SET PASSWORD FOR 'testCluster:testUser' = '*88EEBA7D913688E7278E2AD071FDB5E76D76D34B'", + Assert.assertEquals("SET PASSWORD FOR 'testCluster:testUser'@'%' = '*88EEBA7D913688E7278E2AD071FDB5E76D76D34B'", stmt.toString()); // empty password - stmt = new SetPassVar("testUser", null); + stmt = new SetPassVar(new UserIdentity("testUser", "%"), null); stmt.analyze(analyzer); - Assert.assertEquals("SET PASSWORD FOR 'testCluster:testUser' = ''", stmt.toString()); + Assert.assertEquals("SET PASSWORD FOR 'testCluster:testUser'@'%' = ''", stmt.toString()); // empty user // empty password stmt = new SetPassVar(null, null); stmt.analyze(analyzer); - Assert.assertEquals("SET PASSWORD FOR 'testCluster:testUser' = ''", stmt.toString()); - } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - SetPassVar stmt; - - // plain mode: SET PASSWORD FOR 'testUser' = PASSWORD('testPass'); - stmt = new SetPassVar("testUser", "*88EEBA7D913688E7278E2AD071FDB5E76D76D34B"); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); + Assert.assertEquals("SET PASSWORD FOR 'testCluster:testUser'@'192.168.1.1' = ''", stmt.toString()); } @Test(expected = AnalysisException.class) public void testBadPassword() throws InternalException, AnalysisException { SetPassVar stmt; // mode: SET PASSWORD FOR 'testUser' = 'testPass'; - stmt = new SetPassVar("testUser", "*88EEBAHD913688E7278E2AD071FDB5E76D76D34B"); + stmt = new SetPassVar(new UserIdentity("testUser", "%"), "*88EEBAHD913688E7278E2AD071FDB5E76D76D34B"); stmt.analyze(analyzer); Assert.fail("No exception throws."); } diff --git a/fe/test/com/baidu/palo/analysis/SetStmtTest.java b/fe/test/com/baidu/palo/analysis/SetStmtTest.java index e547953c3f..e522f6ac83 100644 --- a/fe/test/com/baidu/palo/analysis/SetStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/SetStmtTest.java @@ -22,20 +22,38 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import com.google.common.collect.Lists; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.util.List; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class SetStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test diff --git a/fe/test/com/baidu/palo/analysis/SetUserPropertyStmtTest.java b/fe/test/com/baidu/palo/analysis/SetUserPropertyStmtTest.java index 5cabd87211..c8933e0da6 100644 --- a/fe/test/com/baidu/palo/analysis/SetUserPropertyStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/SetUserPropertyStmtTest.java @@ -22,19 +22,38 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; + import com.google.common.collect.Lists; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.util.List; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class SetUserPropertyStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -46,9 +65,6 @@ public class SetUserPropertyStmtTest { SetUserPropertyStmt stmt = new SetUserPropertyStmt("testUser", propertyVarList); stmt.analyze(analyzer); Assert.assertEquals("testCluster:testUser", stmt.getUser()); - Assert.assertEquals( - "SET PROPERTY FOR 'testCluster:testUser' 'load_cluster.palo-dpp' = NULL, 'quota.normal' = '100'", - stmt.toString()); } @Test(expected = AnalysisException.class) diff --git a/fe/test/com/baidu/palo/analysis/SetUserPropertyVarTest.java b/fe/test/com/baidu/palo/analysis/SetUserPropertyVarTest.java index 42e643e18f..4984c343f8 100644 --- a/fe/test/com/baidu/palo/analysis/SetUserPropertyVarTest.java +++ b/fe/test/com/baidu/palo/analysis/SetUserPropertyVarTest.java @@ -38,20 +38,20 @@ public class SetUserPropertyVarTest { @Test public void testNormal() throws AnalysisException, InternalException { SetUserPropertyVar var = new SetUserPropertyVar("quota.normal", "1000"); - var.analyze(analyzer, "testUser"); + var.analyze(analyzer, true); Assert.assertEquals("quota.normal", var.getPropertyKey()); Assert.assertEquals("1000", var.getPropertyValue()); Assert.assertEquals("'quota.normal' = '1000'", var.toString()); var = new SetUserPropertyVar("load_cluster.palo-dpp", null); - var.analyze(analyzer, "testUser"); + var.analyze(analyzer, true); Assert.assertEquals("'load_cluster.palo-dpp' = NULL", var.toString()); } @Test(expected = AnalysisException.class) public void testUnknownProperty() throws InternalException, AnalysisException { SetUserPropertyVar var = new SetUserPropertyVar("unknown_property", "1000"); - var.analyze(analyzer, "testUser"); + var.analyze(analyzer, true); Assert.fail("No exception throws."); } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/SetVarTest.java b/fe/test/com/baidu/palo/analysis/SetVarTest.java index 1134454ce8..af8fdf0b6a 100644 --- a/fe/test/com/baidu/palo/analysis/SetVarTest.java +++ b/fe/test/com/baidu/palo/analysis/SetVarTest.java @@ -22,17 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class SetVarTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -59,11 +76,4 @@ public class SetVarTest { var.analyze(analyzer); Assert.fail("No exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoAccess() throws InternalException, AnalysisException { - SetVar var = new SetVar(SetType.GLOBAL, "names", new StringLiteral("utf-8")); - var.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/ShowAlterStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowAlterStmtTest.java index 07a5ee0e45..be5bc914f9 100644 --- a/fe/test/com/baidu/palo/analysis/ShowAlterStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowAlterStmtTest.java @@ -25,8 +25,8 @@ import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -52,7 +52,7 @@ public class ShowAlterStmtTest { analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testUser").anyTimes(); + EasyMock.expect(analyzer.getQualifiedUser()).andReturn("testUser").anyTimes(); EasyMock.expect(analyzer.getCatalog()).andReturn(catalog).anyTimes(); EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); EasyMock.replay(analyzer); @@ -89,18 +89,4 @@ public class ShowAlterStmtTest { stmt.analyze(analyzer); Assert.fail("No exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - analyzer = EasyMock.createMock(Analyzer.class); - EasyMock.expect(analyzer.getDefaultDb()).andReturn("testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testUser").anyTimes(); - EasyMock.expect(analyzer.getCatalog()).andReturn(AccessTestUtil.fetchBlockCatalog()).anyTimes(); - EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); - EasyMock.replay(analyzer); - - ShowLoadStmt stmt = new ShowLoadStmt(null, null, null, null); - stmt.analyze(analyzer); - Assert.fail("No exception throws."); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/ShowCreateDbStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowCreateDbStmtTest.java index a8d78646aa..85dbe7701f 100644 --- a/fe/test/com/baidu/palo/analysis/ShowCreateDbStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowCreateDbStmtTest.java @@ -22,11 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class ShowCreateDbStmtTest { + + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + + @Before + public void setUp() { + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); + } + @Test public void testNormal() throws AnalysisException, InternalException { ShowCreateDbStmt stmt = new ShowCreateDbStmt("testDb"); @@ -42,11 +65,4 @@ public class ShowCreateDbStmtTest { stmt.analyze(AccessTestUtil.fetchAdminAnalyzer(false)); Assert.fail("No exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws AnalysisException, InternalException { - ShowCreateDbStmt stmt = new ShowCreateDbStmt("testDb"); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws."); - } } diff --git a/fe/test/com/baidu/palo/analysis/ShowCreateTableStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowCreateTableStmtTest.java index 8db24f3fb2..4d07c36ec7 100644 --- a/fe/test/com/baidu/palo/analysis/ShowCreateTableStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowCreateTableStmtTest.java @@ -21,17 +21,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class ShowCreateTableStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -52,11 +69,4 @@ public class ShowCreateTableStmtTest { stmt.analyze(analyzer); Assert.fail("No Exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws AnalysisException { - ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("testDb", "testTbl")); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No Exception throws."); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/ShowDataStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowDataStmtTest.java index a0940ec200..8158e72a36 100644 --- a/fe/test/com/baidu/palo/analysis/ShowDataStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowDataStmtTest.java @@ -20,35 +20,108 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; +import com.baidu.palo.backup.CatalogMocker; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.TabletInvertedIndex; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.system.SystemInfoService; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) public class ShowDataStmtTest { - private Analyzer analyzer; - private Catalog catalog; + @Mocked + private PaloAuth auth; + @Mocked + private Analyzer analyzer; + @Mocked + private Catalog catalog; + @Mocked + private ConnectContext ctx; + @Mocked + private TabletInvertedIndex invertedIndex; + + private Database db; + + static { + Startup.initializeIfPossible(); + } - @Before - public void setUp() { - analyzer = AccessTestUtil.fetchAdminAnalyzer(true); - catalog = AccessTestUtil.fetchAdminCatalog(); - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - PowerMock.replay(Catalog.class); + @Before + public void setUp() throws AnalysisException { + auth = new PaloAuth(); + + + + new NonStrictExpectations() { + { + Catalog.getCurrentInvertedIndex(); + result = invertedIndex; + } + }; + + db = CatalogMocker.mockDb(); + + new NonStrictExpectations() { + { + analyzer.getClusterName(); + result = SystemInfoService.DEFAULT_CLUSTER; + + analyzer.getDefaultDb(); + result = "testCluster:testDb"; + + Catalog.getCurrentCatalog(); + result = catalog; + + Catalog.getInstance(); + result = catalog; + + Catalog.getCurrentInvertedIndex(); + result = invertedIndex; + + catalog.getAuth(); + result = auth; + + catalog.getDb(anyString); + result = db; + + ConnectContext.get(); + result = ctx; + + ctx.getQualifiedUser(); + result = "root"; + + ctx.getRemoteIP(); + result = "192.168.1.1"; + } + }; + + + new NonStrictExpectations() { + { + auth.checkGlobalPriv((ConnectContext) any, (PrivPredicate) any); + result = true; + + auth.checkDbPriv((ConnectContext) any, anyString, (PrivPredicate) any); + result = true; + + auth.checkTblPriv((ConnectContext) any, anyString, anyString, (PrivPredicate) any); + result = true; + } + }; + + AccessTestUtil.fetchAdminAccess(); } @Test @@ -59,9 +132,9 @@ public class ShowDataStmtTest { Assert.assertEquals(2, stmt.getMetaData().getColumnCount()); Assert.assertEquals(false, stmt.hasTable()); - stmt = new ShowDataStmt("testDb", "testTbl"); + stmt = new ShowDataStmt("testDb", "test_tbl"); stmt.analyze(analyzer); - Assert.assertEquals("SHOW DATA FROM `testCluster:testDb`.`testTbl`", stmt.toString()); + Assert.assertEquals("SHOW DATA FROM `default_cluster:testDb`.`test_tbl`", stmt.toString()); Assert.assertEquals(3, stmt.getMetaData().getColumnCount()); Assert.assertEquals(true, stmt.hasTable()); } diff --git a/fe/test/com/baidu/palo/analysis/ShowLoadStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowLoadStmtTest.java index 295e90bc76..745b88fe84 100644 --- a/fe/test/com/baidu/palo/analysis/ShowLoadStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowLoadStmtTest.java @@ -22,12 +22,12 @@ package com.baidu.palo.analysis; import com.baidu.palo.analysis.ShowAlterStmt.AlterType; import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.system.SystemInfoService; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.system.SystemInfoService; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,6 +45,7 @@ public class ShowLoadStmtTest { private SystemInfoService systemInfoService; + @Before public void setUp() { systemInfoService = EasyMock.createMock(SystemInfoService.class); @@ -54,12 +55,13 @@ public class ShowLoadStmtTest { PowerMock.mockStatic(Catalog.class); EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(systemInfoService).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); PowerMock.replay(Catalog.class); analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("testCluster:testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testCluster:testUser").anyTimes(); + EasyMock.expect(analyzer.getQualifiedUser()).andReturn("testCluster:testUser").anyTimes(); EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); EasyMock.expect(analyzer.getCatalog()).andReturn(catalog).anyTimes(); EasyMock.replay(analyzer); @@ -87,18 +89,4 @@ public class ShowLoadStmtTest { stmt.analyze(analyzer); Assert.fail("No exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - analyzer = EasyMock.createMock(Analyzer.class); - EasyMock.expect(analyzer.getDefaultDb()).andReturn("testCluster:testDb").anyTimes(); - EasyMock.expect(analyzer.getUser()).andReturn("testCluster:testUser").anyTimes(); - EasyMock.expect(analyzer.getClusterName()).andReturn("testCluster").anyTimes(); - EasyMock.expect(analyzer.getCatalog()).andReturn(AccessTestUtil.fetchBlockCatalog()).anyTimes(); - EasyMock.replay(analyzer); - - ShowAlterStmt stmt = new ShowAlterStmt(AlterType.ROLLUP, null); - stmt.analyze(analyzer); - Assert.fail("No exception throws."); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/ShowTableStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowTableStmtTest.java index 5cb302db6e..4f5439463b 100644 --- a/fe/test/com/baidu/palo/analysis/ShowTableStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowTableStmtTest.java @@ -21,17 +21,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class ShowTableStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -66,12 +83,4 @@ public class ShowTableStmtTest { stmt.analyze(AccessTestUtil.fetchEmptyDbAnalyzer()); Assert.fail("No exception throws"); } - - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws AnalysisException { - ShowTableStmt stmt = new ShowTableStmt("", false, null); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - Assert.fail("No exception throws"); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/ShowUserPropertyStmtTest.java b/fe/test/com/baidu/palo/analysis/ShowUserPropertyStmtTest.java index 947371d000..24f55c4a60 100644 --- a/fe/test/com/baidu/palo/analysis/ShowUserPropertyStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/ShowUserPropertyStmtTest.java @@ -22,17 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class ShowUserPropertyStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test diff --git a/fe/test/com/baidu/palo/analysis/UseStmtTest.java b/fe/test/com/baidu/palo/analysis/UseStmtTest.java index af95c73e02..bbf244e37f 100644 --- a/fe/test/com/baidu/palo/analysis/UseStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/UseStmtTest.java @@ -22,17 +22,34 @@ package com.baidu.palo.analysis; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.MockedAuth; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.qe.ConnectContext; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import mockit.Mocked; +import mockit.internal.startup.Startup; + public class UseStmtTest { private Analyzer analyzer; + @Mocked + private PaloAuth auth; + @Mocked + private ConnectContext ctx; + + static { + Startup.initializeIfPossible(); + } + @Before public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + MockedAuth.mockedAuth(auth); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); } @Test @@ -51,12 +68,4 @@ public class UseStmtTest { Assert.fail("No exception throws."); } - - @Test(expected = AnalysisException.class) - public void testNoPriv() throws InternalException, AnalysisException { - UseStmt stmt = new UseStmt("testDb"); - stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); - - Assert.fail("No exception throws."); - } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/backup/BackupHandlerTest.java b/fe/test/com/baidu/palo/backup/BackupHandlerTest.java new file mode 100644 index 0000000000..e864868b1c --- /dev/null +++ b/fe/test/com/baidu/palo/backup/BackupHandlerTest.java @@ -0,0 +1,364 @@ +package com.baidu.palo.backup; + +import com.baidu.palo.analysis.BackupStmt; +import com.baidu.palo.analysis.CancelBackupStmt; +import com.baidu.palo.analysis.CreateRepositoryStmt; +import com.baidu.palo.analysis.DropRepositoryStmt; +import com.baidu.palo.analysis.LabelName; +import com.baidu.palo.analysis.RestoreStmt; +import com.baidu.palo.analysis.TableName; +import com.baidu.palo.analysis.TableRef; +import com.baidu.palo.catalog.BrokerMgr; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.Partition; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.catalog.TabletInvertedIndex; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.task.DirMoveTask; +import com.baidu.palo.task.DownloadTask; +import com.baidu.palo.task.SnapshotTask; +import com.baidu.palo.task.UploadTask; +import com.baidu.palo.thrift.TFinishTaskRequest; +import com.baidu.palo.thrift.TStatus; +import com.baidu.palo.thrift.TStatusCode; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import mockit.Delegate; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + +public class BackupHandlerTest { + + private BackupHandler handler; + + @Mocked + private Catalog catalog; + @Mocked + private BrokerMgr brokerMgr; + @Mocked + private EditLog editLog; + + private Database db; + + private long idGen = 0; + + private File rootDir; + + private String tmpPath = "./tmp" + System.currentTimeMillis(); + + private TabletInvertedIndex invertedIndex = new TabletInvertedIndex(); + + static { + Startup.initializeIfPossible(); + } + + @Before + public void setUp() { + Config.tmp_dir = tmpPath; + rootDir = new File(Config.tmp_dir); + rootDir.mkdirs(); + + new NonStrictExpectations() { + { + catalog.getBrokerMgr(); + result = brokerMgr; + + catalog.getNextId(); + result = idGen++; + + catalog.getEditLog(); + result = editLog; + + Catalog.getCurrentCatalog(); + result = catalog; + + Catalog.getCurrentCatalogJournalVersion(); + result = FeConstants.meta_version; + + Catalog.getCurrentInvertedIndex(); + result = invertedIndex; + } + }; + + try { + db = CatalogMocker.mockDb(); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + new NonStrictExpectations() { + { + catalog.getDb(anyString); + result = db; + } + }; + } + + @After + public void done() { + if (rootDir != null) { + try { + Files.walk(Paths.get(Config.tmp_dir), + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + @Test + public void testInit() { + handler = new BackupHandler(catalog); + handler.runOneCycle(); + + File backupDir = new File(BackupHandler.BACKUP_ROOT_DIR.toString()); + Assert.assertTrue(backupDir.exists()); + } + + @Test + public void testCreateAndDropRepository() { + new NonStrictExpectations() { + { + editLog.logCreateRepository((Repository) any); + result = new Delegate() { + public void logCreateRepository(Repository repo) { + + } + }; + + editLog.logDropRepository(anyString); + result = new Delegate() { + public void logDropRepository(String repoName) { + + } + }; + } + }; + + new MockUp() { + @Mock + public Status initRepository() { + return Status.OK; + } + + @Mock + public Status listSnapshots(List snapshotNames) { + snapshotNames.add("ss2"); + return Status.OK; + } + + @Mock + public Status getSnapshotInfoFile(String label, String backupTimestamp, List infos) { + OlapTable tbl = (OlapTable) db.getTable(CatalogMocker.TEST_TBL_NAME); + List
    tbls = Lists.newArrayList(); + tbls.add(tbl); + Map snapshotInfos = Maps.newHashMap(); + for (Partition part : tbl.getPartitions()) { + for (MaterializedIndex idx : part.getMaterializedIndices()) { + for (Tablet tablet : idx.getTablets()) { + List files = Lists.newArrayList(); + SnapshotInfo sinfo = new SnapshotInfo(db.getId(), tbl.getId(), part.getId(), idx.getId(), + tablet.getId(), -1, 0, "./path", files); + snapshotInfos.put(tablet.getId(), sinfo); + } + } + } + + BackupJobInfo info = BackupJobInfo.fromCatalog(System.currentTimeMillis(), + "ss2", CatalogMocker.TEST_DB_NAME, + CatalogMocker.TEST_DB_ID, tbls, snapshotInfos); + infos.add(info); + return Status.OK; + } + }; + + new NonStrictExpectations() { + { + brokerMgr.contaisnBroker(anyString); + result = true; + } + }; + + // add repo + handler = new BackupHandler(catalog); + CreateRepositoryStmt stmt = new CreateRepositoryStmt(false, "repo", "broker", "bos://location", + Maps.newHashMap()); + try { + handler.createRepository(stmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + // process backup + List tblRefs = Lists.newArrayList(); + tblRefs.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME), null)); + BackupStmt backupStmt = new BackupStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "label1"), "repo", tblRefs, + null); + try { + handler.process(backupStmt); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // handleFinishedSnapshotTask + BackupJob backupJob = (BackupJob) handler.getJob(CatalogMocker.TEST_DB_ID); + SnapshotTask snapshotTask = new SnapshotTask(null, 0, 0, backupJob.getJobId(), CatalogMocker.TEST_DB_ID, + 0, 0, 0, 0, 0, 0, 0, 1, false); + TFinishTaskRequest request = new TFinishTaskRequest(); + List snapshotFiles = Lists.newArrayList(); + request.setSnapshot_files(snapshotFiles); + request.setSnapshot_path("./snapshot/path"); + request.setTask_status(new TStatus(TStatusCode.OK)); + handler.handleFinishedSnapshotTask(snapshotTask, request); + + // handleFinishedSnapshotUploadTask + Map srcToDestPath = Maps.newHashMap(); + UploadTask uploadTask = new UploadTask(null, 0, 0, backupJob.getJobId(), CatalogMocker.TEST_DB_ID, + srcToDestPath, null, null); + request = new TFinishTaskRequest(); + Map> tabletFiles = Maps.newHashMap(); + request.setTablet_files(tabletFiles); + request.setTask_status(new TStatus(TStatusCode.OK)); + handler.handleFinishedSnapshotUploadTask(uploadTask, request); + + // test file persist + File tmpFile = new File("./tmp" + System.currentTimeMillis()); + try { + DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile)); + handler.write(out); + out.flush(); + out.close(); + DataInputStream in = new DataInputStream(new FileInputStream(tmpFile)); + BackupHandler.read(in); + in.close(); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } finally { + tmpFile.delete(); + } + + // cancel backup + try { + handler.cancel(new CancelBackupStmt(CatalogMocker.TEST_DB_NAME, false)); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // process restore + List tblRefs2 = Lists.newArrayList(); + tblRefs2.add(new TableRef(new TableName(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME), null)); + Map properties = Maps.newHashMap(); + properties.put("backup_timestamp", "2018-08-08-08-08-08"); + RestoreStmt restoreStmt = new RestoreStmt(new LabelName(CatalogMocker.TEST_DB_NAME, "ss2"), "repo", tblRefs2, + properties); + try { + restoreStmt.analyzeProperties(); + } catch (AnalysisException e2) { + e2.printStackTrace(); + Assert.fail(); + } + + try { + handler.process(restoreStmt); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // handleFinishedSnapshotTask + RestoreJob restoreJob = (RestoreJob) handler.getJob(CatalogMocker.TEST_DB_ID); + snapshotTask = new SnapshotTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, + 0, 0, 0, 0, 0, 0, 0, 1, true); + request = new TFinishTaskRequest(); + request.setSnapshot_path("./snapshot/path"); + request.setTask_status(new TStatus(TStatusCode.OK)); + handler.handleFinishedSnapshotTask(snapshotTask, request); + + // handleDownloadSnapshotTask + DownloadTask downloadTask = new DownloadTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, + srcToDestPath, null, null); + request = new TFinishTaskRequest(); + List downloadedTabletIds = Lists.newArrayList(); + request.setDownloaded_tablet_ids(downloadedTabletIds); + request.setTask_status(new TStatus(TStatusCode.OK)); + handler.handleDownloadSnapshotTask(downloadTask, request); + + // handleDirMoveTask + DirMoveTask dirMoveTask = new DirMoveTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, 0, 0, 0, + 0, "", 0, true); + request = new TFinishTaskRequest(); + request.setTask_status(new TStatus(TStatusCode.OK)); + handler.handleDirMoveTask(dirMoveTask, request); + + // test file persist + tmpFile = new File("./tmp" + System.currentTimeMillis()); + try { + DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile)); + handler.write(out); + out.flush(); + out.close(); + DataInputStream in = new DataInputStream(new FileInputStream(tmpFile)); + BackupHandler.read(in); + in.close(); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } finally { + tmpFile.delete(); + } + + // cancel restore + try { + handler.cancel(new CancelBackupStmt(CatalogMocker.TEST_DB_NAME, true)); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // drop repo + try { + handler.dropRepository(new DropRepositoryStmt("repo")); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + } +} diff --git a/fe/test/com/baidu/palo/backup/BackupJobInfoTest.java b/fe/test/com/baidu/palo/backup/BackupJobInfoTest.java new file mode 100644 index 0000000000..24f4467fff --- /dev/null +++ b/fe/test/com/baidu/palo/backup/BackupJobInfoTest.java @@ -0,0 +1,167 @@ +package com.baidu.palo.backup; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; + +public class BackupJobInfoTest { + + private static String fileName = "job_info.txt"; + + @BeforeClass + public static void createFile() { + String json = "{\n" + + " \"backup_time\": 1522231864000,\n" + + " \"name\": \"snapshot1\",\n" + + " \"database\": \"db1\",\n" + + " \"id\": 10000,\n" + + " \"backup_result\": \"succeed\",\n" + + " \"backup_objects\": {\n" + + " \"table2\": {\n" + + " \"partitions\": {\n" + + " \"partition1\": {\n" + + " \"indexes\": {\n" + + " \"table2\": {\n" + + " \"id\": 10012,\n" + + " \"schema_hash\": 222222,\n" + + " \"tablets\": {\n" + + " \"10004\": [\"__10030_seg1.dat\", \"__10030_seg2.dat\"],\n" + + " \"10005\": [\"__10031_seg1.dat\", \"__10031_seg2.dat\"]\n" + + " }\n" + + " }\n" + + " },\n" + + " \"id\": 10011,\n" + + " \"version\": 11,\n" + + " \"version_hash\": 123456789\n" + + " }\n" + + " },\n" + + " \"id\": 10010\n" + + " },\n" + + " \"table1\": {\n" + + " \"partitions\": {\n" + + " \"partition2\": {\n" + + " \"indexes\": {\n" + + " \"rollup1\": {\n" + + " \"id\": 10009,\n" + + " \"schema_hash\": 333333,\n" + + " \"tablets\": {\n" + + " \"10008\": [\"__10029_seg1.dat\", \"__10029_seg2.dat\"],\n" + + " \"10007\": [\"__10029_seg1.dat\", \"__10029_seg2.dat\"]\n" + + " }\n" + + " },\n" + + " \"table1\": {\n" + + " \"id\": 10001,\n" + + " \"schema_hash\": 444444,\n" + + " \"tablets\": {\n" + + " \"10004\": [\"__10027_seg1.dat\", \"__10027_seg2.dat\"],\n" + + " \"10005\": [\"__10028_seg1.dat\", \"__10028_seg2.dat\"]\n" + + " }\n" + + " }\n" + + " },\n" + + " \"id\": 10007,\n" + + " \"version\": 20,\n" + + " \"version_hash\": 123534645745\n" + + " },\n" + + " \"partition1\": {\n" + + " \"indexes\": {\n" + + " \"rollup1\": {\n" + + " \"id\": 10009,\n" + + " \"schema_hash\": 333333,\n" + + " \"tablets\": {\n" + + " \"10008\": [\"__10026_seg1.dat\", \"__10026_seg2.dat\"],\n" + + " \"10007\": [\"__10025_seg1.dat\", \"__10025_seg2.dat\"]\n" + + " }\n" + + " },\n" + + " \"table1\": {\n" + + " \"id\": 10001,\n" + + " \"schema_hash\": 444444,\n" + + " \"tablets\": {\n" + + " \"10004\": [\"__10023_seg1.dat\", \"__10023_seg2.dat\"],\n" + + " \"10005\": [\"__10024_seg1.dat\", \"__10024_seg2.dat\"]\n" + + " }\n" + + " }\n" + + " },\n" + + " \"id\": 10002,\n" + + " \"version\": 21,\n" + + " \"version_hash\": 345346234234\n" + + " }\n" + + " },\n" + + " \"id\": 10001\n" + + " }\n" + + " }\n" + + "}"; + + try (PrintWriter out = new PrintWriter(fileName)) { + out.print(json); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @AfterClass + public static void deleteFile() { + File file = new File(fileName); + if (file.exists()) { + file.delete(); + } + } + + @Test + public void testReadWrite() { + BackupJobInfo jobInfo = null; + try { + jobInfo = BackupJobInfo.fromFile(fileName); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertNotNull(jobInfo); + System.out.println(jobInfo.toString(1)); + + Assert.assertEquals(1522231864000L, jobInfo.backupTime); + Assert.assertEquals("snapshot1", jobInfo.name); + Assert.assertEquals(2, jobInfo.tables.size()); + + Assert.assertEquals(2, jobInfo.getTableInfo("table1").partitions.size()); + Assert.assertEquals(2, jobInfo.getTableInfo("table1").getPartInfo("partition1").indexes.size()); + Assert.assertEquals(2, + jobInfo.getTableInfo("table1").getPartInfo("partition1").getIdx("rollup1").tablets.size()); + System.out.println(jobInfo.getTableInfo("table1").getPartInfo("partition1").getIdx("rollup1").tablets); + Assert.assertEquals(2, + jobInfo.getTableInfo("table1").getPartInfo("partition1") + .getIdx("rollup1").getTablet(10007L).files.size()); + + File tmpFile = new File("./tmp"); + try { + DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile)); + jobInfo.write(out); + out.flush(); + out.close(); + + DataInputStream in = new DataInputStream(new FileInputStream(tmpFile)); + BackupJobInfo newInfo = BackupJobInfo.read(in); + in.close(); + + Assert.assertEquals(jobInfo.backupTime, newInfo.backupTime); + Assert.assertEquals(jobInfo.dbId, newInfo.dbId); + Assert.assertEquals(jobInfo.dbName, newInfo.dbName); + + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } finally { + tmpFile.delete(); + } + } +} diff --git a/fe/test/com/baidu/palo/backup/BackupJobTest.java b/fe/test/com/baidu/palo/backup/BackupJobTest.java index 6858638fea..9a5f27d4ef 100644 --- a/fe/test/com/baidu/palo/backup/BackupJobTest.java +++ b/fe/test/com/baidu/palo/backup/BackupJobTest.java @@ -19,53 +19,308 @@ package com.baidu.palo.backup; -import com.baidu.palo.analysis.LabelName; +import com.baidu.palo.analysis.TableName; +import com.baidu.palo.analysis.TableRef; import com.baidu.palo.backup.BackupJob.BackupJobState; import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.common.Config; +import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.util.UnitTestUtil; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.task.AgentBatchTask; +import com.baidu.palo.task.AgentTask; +import com.baidu.palo.task.AgentTaskExecutor; +import com.baidu.palo.task.AgentTaskQueue; +import com.baidu.palo.task.SnapshotTask; +import com.baidu.palo.task.UploadTask; +import com.baidu.palo.thrift.TBackend; +import com.baidu.palo.thrift.TFinishTaskRequest; +import com.baidu.palo.thrift.TStatus; +import com.baidu.palo.thrift.TStatusCode; +import com.baidu.palo.thrift.TTaskType; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.easymock.EasyMock; +import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import mockit.internal.startup.Startup; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) public class BackupJobTest { + private BackupJob job; + private Database db; + + private long dbId = 1; + private long tblId = 2; + private long partId = 3; + private long idxId = 4; + private long tabletId = 5; + private long backendId = 10000; + private long version = 6; + private long versionHash = 7; + + private long repoId = 20000; + private AtomicLong id = new AtomicLong(50000); + + @Mocked private Catalog catalog; + @Mocked + private BackupHandler backupHandler; + @Mocked + private RepositoryMgr repoMgr; + @Mocked + private EditLog editLog; + + private Repository repo = new Repository(repoId, "repo", false, "my_repo", + new BlobStorage("broker", Maps.newHashMap())); + + static { + Startup.initializeIfPossible(); + } + + @BeforeClass + public static void start() { + Config.tmp_dir = "./"; + File backupDir = new File(BackupHandler.BACKUP_ROOT_DIR.toString()); + backupDir.mkdirs(); + } + + @AfterClass + public static void end() throws IOException { + Config.tmp_dir = "./"; + File backupDir = new File(BackupHandler.BACKUP_ROOT_DIR.toString()); + if (backupDir.exists()) { + Files.walk(BackupHandler.BACKUP_ROOT_DIR, + FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } @Before public void setUp() { - catalog = CatalogMocker.fetchAdminCatalog(); + new MockUp() { + @Mock + public BackupHandler getBackupHandler() { + return backupHandler; + } - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - PowerMock.replay(Catalog.class); + @Mock + public Database getDb(long dbId) { + return db; + } + + @Mock + public int getCurrentCatalogJournalVersion() { + return FeMetaVersion.VERSION_42; + } + + @Mock + public long getNextId() { + return id.getAndIncrement(); + } + + @Mock + public EditLog getEditLog() { + return editLog; + } + }; + + new MockUp() { + @Mock + public RepositoryMgr getRepoMgr() { + return repoMgr; + } + }; + + new MockUp() { + @Mock + public Repository getRepo(long repoId) { + return repo; + } + }; + + new MockUp() { + @Mock + public void logBackupJob(BackupJob job) { + System.out.println("log backup job: " + job); + } + }; + + new MockUp() { + @Mock + public void submit(AgentBatchTask task) { + return; + } + }; + + new Expectations(Repository.class) { + { + repo.upload(anyString, anyString); + minTimes = 0; + result = Status.OK; + } + }; + + db = UnitTestUtil.createDb(dbId, tblId, partId, idxId, tabletId, backendId, version, versionHash); + List tableRefs = Lists.newArrayList(); + tableRefs.add(new TableRef(new TableName(UnitTestUtil.DB_NAME, UnitTestUtil.TABLE_NAME), null)); + job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, catalog, repo.getId()); } @Test - public void testSaveMeta() { - Map properties = Maps.newHashMap(); - LabelName labelName = new LabelName(CatalogMocker.TEST_DB_NAME, "test_backup"); - BackupJob backupJob = new BackupJob(1, CatalogMocker.TEST_DB_ID, labelName, "/home/backup/", properties); - backupJob.setState(BackupJobState.PENDING); + public void testRunNormal() { + // 1.pending + Assert.assertEquals(BackupJobState.PENDING, job.getState()); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.SNAPSHOTING, job.getState()); + + BackupMeta backupMeta = job.getBackupMeta(); + Assert.assertEquals(1, backupMeta.getTables().size()); + OlapTable backupTbl = (OlapTable) backupMeta.getTable(UnitTestUtil.TABLE_NAME); + List partNames = Lists.newArrayList(backupTbl.getPartitionNames()); + Assert.assertNotNull(backupTbl); + Assert.assertEquals(backupTbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames), + ((OlapTable) db.getTable(tblId)).getSignature(BackupHandler.SIGNATURE_VERSION, partNames)); + Assert.assertEquals(1, AgentTaskQueue.getTaskNum()); + AgentTask task = AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, tabletId); + Assert.assertTrue(task instanceof SnapshotTask); + SnapshotTask snapshotTask = (SnapshotTask) task; + + // 2. snapshoting + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.SNAPSHOTING, job.getState()); + + // 3. snapshot finished + String snapshotPath = "/path/to/snapshot"; + List snapshotFiles = Lists.newArrayList(); + snapshotFiles.add("1.dat"); + snapshotFiles.add("1.idx"); + snapshotFiles.add("1.hdr"); + TStatus task_status = new TStatus(TStatusCode.OK); + TBackend tBackend = new TBackend("", 0, 1); + TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, + snapshotTask.getSignature(), task_status); + request.setSnapshot_files(snapshotFiles); + request.setSnapshot_path(snapshotPath); + Assert.assertTrue(job.finishTabletSnapshotTask(snapshotTask, request)); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.UPLOAD_SNAPSHOT, job.getState()); - backupJob.addPartitionId(CatalogMocker.TEST_MYSQL_TABLE_ID, -1); - backupJob.addPartitionId(CatalogMocker.TEST_TBL_ID, CatalogMocker.TEST_SINGLE_PARTITION_ID); - backupJob.addPartitionId(CatalogMocker.TEST_TBL2_ID, CatalogMocker.TEST_PARTITION1_ID); - backupJob.addPartitionId(CatalogMocker.TEST_TBL2_ID, CatalogMocker.TEST_PARTITION2_ID); + // 4. upload snapshots + AgentTaskQueue.clearAllTasks(); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.UPLOADING, job.getState()); + Assert.assertEquals(1, AgentTaskQueue.getTaskNum()); + task = AgentTaskQueue.getTask(backendId, TTaskType.UPLOAD, id.get() - 1); + Assert.assertTrue(task instanceof UploadTask); + UploadTask upTask = (UploadTask) task; + + Assert.assertEquals(job.getJobId(), upTask.getJobId()); + Map srcToDest = upTask.getSrcToDestPath(); + Assert.assertEquals(1, srcToDest.size()); + System.out.println(srcToDest); + String dest = srcToDest.get(snapshotPath + "/" + tabletId + "/" + 0); + Assert.assertNotNull(dest); - Config.meta_dir = "palo-meta/"; - backupJob.runOnce(); + // 5. uploading + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.UPLOADING, job.getState()); + Map> tabletFileMap = Maps.newHashMap(); + request = new TFinishTaskRequest(tBackend, TTaskType.UPLOAD, + upTask.getSignature(), task_status); + request.setTablet_files(tabletFileMap); + + Assert.assertFalse(job.finishSnapshotUploadTask(upTask, request)); + List tabletFiles = Lists.newArrayList(); + tabletFileMap.put(tabletId, tabletFiles); + Assert.assertFalse(job.finishSnapshotUploadTask(upTask, request)); + tabletFiles.add("1.dat.4f158689243a3d6030352fec3cfd3798"); + tabletFiles.add("wrong_files.idx.4f158689243a3d6030352fec3cfd3798"); + tabletFiles.add("wrong_files.hdr.4f158689243a3d6030352fec3cfd3798"); + Assert.assertFalse(job.finishSnapshotUploadTask(upTask, request)); + tabletFiles.clear(); + tabletFiles.add("1.dat.4f158689243a3d6030352fec3cfd3798"); + tabletFiles.add("1.idx.4f158689243a3d6030352fec3cfd3798"); + tabletFiles.add("1.hdr.4f158689243a3d6030352fec3cfd3798"); + Assert.assertTrue(job.finishSnapshotUploadTask(upTask, request)); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.SAVE_META, job.getState()); + + // 6. save meta + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.UPLOAD_INFO, job.getState()); + File metaInfo = new File(job.getLocalMetaInfoFilePath()); + Assert.assertTrue(metaInfo.exists()); + File jobInfo = new File(job.getLocalJobInfoFilePath()); + Assert.assertTrue(jobInfo.exists()); + + BackupMeta restoreMetaInfo = null; + BackupJobInfo restoreJobInfo = null; + try { + restoreMetaInfo = BackupMeta.fromFile(job.getLocalMetaInfoFilePath()); + Assert.assertEquals(1, restoreMetaInfo.getTables().size()); + OlapTable olapTable = (OlapTable) restoreMetaInfo.getTable(tblId); + Assert.assertNotNull(olapTable); + Assert.assertNotNull(restoreMetaInfo.getTable(UnitTestUtil.TABLE_NAME)); + List names = Lists.newArrayList(olapTable.getPartitionNames()); + Assert.assertEquals(((OlapTable) db.getTable(tblId)).getSignature(BackupHandler.SIGNATURE_VERSION, names), + olapTable.getSignature(BackupHandler.SIGNATURE_VERSION, names)); + + restoreJobInfo = BackupJobInfo.fromFile(job.getLocalJobInfoFilePath()); + Assert.assertEquals(UnitTestUtil.DB_NAME, restoreJobInfo.dbName); + Assert.assertEquals(job.getLabel(), restoreJobInfo.name); + Assert.assertEquals(1, restoreJobInfo.tables.size()); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertNull(job.getBackupMeta()); + Assert.assertNull(job.getJobInfo()); + + // 7. upload_info + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.FINISHED, job.getState()); + } + + @Test + public void testRunAbnormal() { + // 1.pending + AgentTaskQueue.clearAllTasks(); + + List tableRefs = Lists.newArrayList(); + tableRefs.add(new TableRef(new TableName(UnitTestUtil.DB_NAME, "unknown_tbl"), null)); + job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, catalog, repo.getId()); + job.run(); + Assert.assertEquals(Status.ErrCode.NOT_FOUND, job.getStatus().getErrCode()); + Assert.assertEquals(BackupJobState.CANCELLED, job.getState()); } } diff --git a/fe/test/com/baidu/palo/backup/CatalogMocker.java b/fe/test/com/baidu/palo/backup/CatalogMocker.java index 201de47274..f3ab1d1299 100644 --- a/fe/test/com/baidu/palo/backup/CatalogMocker.java +++ b/fe/test/com/baidu/palo/backup/CatalogMocker.java @@ -21,39 +21,39 @@ package com.baidu.palo.backup; import com.baidu.palo.alter.RollupHandler; import com.baidu.palo.alter.SchemaChangeHandler; -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.catalog.KeysType; import com.baidu.palo.catalog.AggregateType; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.catalog.DataProperty; +import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.DistributionInfo; import com.baidu.palo.catalog.HashDistributionInfo; +import com.baidu.palo.catalog.KeysType; import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.MaterializedIndex.IndexState; import com.baidu.palo.catalog.MysqlTable; +import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.PartitionInfo; import com.baidu.palo.catalog.PartitionKey; +import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.catalog.RandomDistributionInfo; import com.baidu.palo.catalog.RangePartitionInfo; import com.baidu.palo.catalog.Replica; -import com.baidu.palo.catalog.TabletMeta; import com.baidu.palo.catalog.Replica.ReplicaState; import com.baidu.palo.catalog.SinglePartitionInfo; import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.catalog.UserPropertyMgr; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.OlapTable; -import com.baidu.palo.catalog.PrimitiveType; -import com.baidu.palo.catalog.MaterializedIndex.IndexState; -import com.baidu.palo.system.SystemInfoService; +import com.baidu.palo.catalog.TabletMeta; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.util.Util; import com.baidu.palo.load.Load; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.persist.EditLog; import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.system.SystemInfoService; import com.baidu.palo.thrift.TStorageMedium; import com.baidu.palo.thrift.TStorageType; @@ -67,7 +67,6 @@ import java.util.List; import java.util.Map; public class CatalogMocker { - // user public static final String ROOTUSER = "root"; public static final String SUPERUSER = "superuser"; @@ -203,27 +202,17 @@ public class CatalogMocker { ROLLUP_SCHEMA_HASH = Util.schemaHash(0, TEST_ROLLUP_SCHEMA, null, 0); } - public static UserPropertyMgr fetchAdminAccess() { - UserPropertyMgr userPropertyMgr = EasyMock.createMock(UserPropertyMgr.class); - EasyMock.expect(userPropertyMgr.checkAccess(EasyMock.isA(String.class), - EasyMock.isA(String.class), EasyMock.isA(AccessPrivilege.class))) - .andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.isAdmin(EasyMock.isA(String.class))).andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.isSuperuser(EasyMock.isA(String.class))).andReturn(true).anyTimes(); - EasyMock.expect(userPropertyMgr.checkUserAccess(EasyMock.isA(String.class), EasyMock.eq(BLOCKUSER))) - .andReturn(false).anyTimes(); - EasyMock.expect(userPropertyMgr.checkUserAccess(EasyMock.isA(String.class), EasyMock.isA(String.class))) - .andReturn(true).anyTimes(); - try { - userPropertyMgr.setPasswd(EasyMock.endsWith(TESTUSER), EasyMock.isA(byte[].class)); - EasyMock.expectLastCall().anyTimes(); - userPropertyMgr.setPasswd(EasyMock.endsWith(ROOTUSER), EasyMock.isA(byte[].class)); - EasyMock.expectLastCall().andThrow(new DdlException("No privilege to change password")).anyTimes(); - } catch (DdlException e) { - return null; - } - EasyMock.replay(userPropertyMgr); - return userPropertyMgr; + private static PaloAuth fetchAdminAccess() { + PaloAuth auth = EasyMock.createMock(PaloAuth.class); + EasyMock.expect(auth.checkGlobalPriv(EasyMock.isA(ConnectContext.class), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkDbPriv(EasyMock.isA(ConnectContext.class), EasyMock.isA(String.class), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkTblPriv(EasyMock.isA(ConnectContext.class), EasyMock.isA(String.class), + EasyMock.isA(String.class), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.replay(auth); + return auth; } public static SystemInfoService fetchSystemInfoService() { @@ -260,7 +249,6 @@ public class CatalogMocker { tablet0.addReplica(replica1); tablet0.addReplica(replica2); - olapTable.setIndexSchemaInfo(TEST_TBL_ID, TEST_TBL_NAME, TEST_TBL_BASE_SCHEMA, 0, SCHEMA_HASH, (short) 1); olapTable.setStorageTypeToIndex(TEST_TBL_ID, TStorageType.COLUMN); olapTable.addPartition(partition); @@ -351,7 +339,6 @@ public class CatalogMocker { olapTable2.addPartition(partition2); // rollup index p1 - MaterializedIndex rollupIndexP1 = new MaterializedIndex(TEST_ROLLUP_ID, IndexState.NORMAL); Tablet rollupTabletP1 = new Tablet(TEST_ROLLUP_TABLET_P1_ID); TabletMeta tabletMetaRollupTabletP1 = new TabletMeta(TEST_DB_ID, TEST_TBL2_ID, TEST_PARTITION1_ID, @@ -393,7 +380,7 @@ public class CatalogMocker { public static Catalog fetchAdminCatalog() { try { Catalog catalog = EasyMock.createMock(Catalog.class); - EasyMock.expect(catalog.getUserMgr()).andReturn(fetchAdminAccess()).anyTimes(); + EasyMock.expect(catalog.getAuth()).andReturn(fetchAdminAccess()).anyTimes(); Database db = mockDb(); @@ -417,16 +404,16 @@ public class CatalogMocker { } } - public static UserPropertyMgr fetchBlockAccess() { - UserPropertyMgr service = EasyMock.createMock(UserPropertyMgr.class); - EasyMock.expect(service.checkAccess(EasyMock.isA(String.class), - EasyMock.isA(String.class), EasyMock.isA(AccessPrivilege.class))) - .andReturn(false).anyTimes(); - EasyMock.expect(service.isAdmin(EasyMock.isA(String.class))).andReturn(false).anyTimes(); - EasyMock.expect(service.isSuperuser(EasyMock.isA(String.class))).andReturn(false).anyTimes(); - EasyMock.expect(service.checkUserAccess(EasyMock.isA(String.class), EasyMock.isA(String.class))) - .andReturn(false).anyTimes(); - EasyMock.replay(service); - return service; + public static PaloAuth fetchBlockAccess() { + PaloAuth auth = EasyMock.createMock(PaloAuth.class); + EasyMock.expect(auth.checkGlobalPriv(EasyMock.isA(ConnectContext.class), + EasyMock.isA(PrivPredicate.class))).andReturn(false).anyTimes(); + EasyMock.expect(auth.checkDbPriv(EasyMock.isA(ConnectContext.class), EasyMock.isA(String.class), + EasyMock.isA(PrivPredicate.class))).andReturn(false).anyTimes(); + EasyMock.expect(auth.checkTblPriv(EasyMock.isA(ConnectContext.class), EasyMock.isA(String.class), + EasyMock.isA(String.class), + EasyMock.isA(PrivPredicate.class))).andReturn(false).anyTimes(); + EasyMock.replay(auth); + return auth; } } diff --git a/fe/test/com/baidu/palo/backup/ObjectWriterTest.java b/fe/test/com/baidu/palo/backup/ObjectWriterTest.java deleted file mode 100644 index 0d7adffce4..0000000000 --- a/fe/test/com/baidu/palo/backup/ObjectWriterTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.backup; - -import com.baidu.palo.analysis.CreateTableStmt; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.Table; -import com.baidu.palo.common.FeConstants; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; - -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.IOException; - -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) -public class ObjectWriterTest { - - private static Catalog catalog; - - @BeforeClass - public static void setUp() { - // Config.meta_dir = "./palo-meta"; - - catalog = CatalogMocker.fetchAdminCatalog(); - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - EasyMock.expect(Catalog.getCurrentCatalogJournalVersion()).andReturn(FeConstants.meta_version).anyTimes(); - PowerMock.replay(Catalog.class); - } - - @Test - public void test_write_and_read_createTableStmt() throws IOException { - Database db = Catalog.getInstance().getDb(CatalogMocker.TEST_DB_ID); - - // write olap table - Table olapTable = db.getTable(CatalogMocker.TEST_TBL_ID); - int tableSignature = olapTable.getSignature(BackupVersion.VERSION_1); - CreateTableStmt stmt = olapTable.toCreateTableStmt(db.getFullName()); - stmt.setTableSignature(tableSignature); - - PathBuilder pathBuilder = - PathBuilder.createPathBuilder(Joiner.on("/").join("test_label", CatalogMocker.TEST_DB_NAME)); - String filePath = pathBuilder.createTableStmt(CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME); - ObjectWriter.write(filePath, Lists.newArrayList(stmt)); - - // read olap table - stmt = ObjectWriter.readCreateTableStmt(filePath); - System.out.println(stmt.toSql()); - - System.out.println("get signature: " + stmt.getTableSignature()); - System.out.println("table signature: " + tableSignature); - if (stmt.getTableSignature() == tableSignature) { - System.out.println("get same signature: " + tableSignature); - } else { - Assert.fail(); - } - - // write mysql table - Table mysqlTable = db.getTable(CatalogMocker.MYSQL_TABLE_NAME); - tableSignature = mysqlTable.getSignature(BackupVersion.VERSION_1); - stmt = mysqlTable.toCreateTableStmt(db.getFullName()); - stmt.setTableSignature(tableSignature); - - filePath = pathBuilder.createTableStmt(CatalogMocker.TEST_DB_NAME, CatalogMocker.MYSQL_TABLE_NAME); - ObjectWriter.write(filePath, Lists.newArrayList(stmt)); - - // read mysql table - stmt = ObjectWriter.readCreateTableStmt(filePath); - System.out.println(stmt.toSql()); - - System.out.println("get signature: " + stmt.getTableSignature()); - System.out.println("table signature: " + tableSignature); - if (stmt.getTableSignature() == tableSignature) { - System.out.println("get same signature: " + tableSignature); - } else { - Assert.fail(); - } - - pathBuilder.getRoot().getTopParent().print(" "); - } -} diff --git a/fe/test/com/baidu/palo/backup/RepositoryTest.java b/fe/test/com/baidu/palo/backup/RepositoryTest.java new file mode 100644 index 0000000000..61019ae882 --- /dev/null +++ b/fe/test/com/baidu/palo/backup/RepositoryTest.java @@ -0,0 +1,318 @@ +package com.baidu.palo.backup; + +import com.baidu.palo.analysis.ShowRepositoriesStmt; +import com.baidu.palo.catalog.BrokerMgr; +import com.baidu.palo.catalog.BrokerMgr.BrokerAddress; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.service.FrontendOptions; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; + +import mockit.Delegate; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + +public class RepositoryTest { + + private Repository repo; + private long repoId = 10000; + private String name = "repo"; + private String location = "bos://backup-cmy"; + private String brokerName = "broker"; + + private SnapshotInfo info; + + @Mocked + private BlobStorage storage; + + static { + Startup.initializeIfPossible(); + } + + @Before + public void setUp() { + List files = Lists.newArrayList(); + files.add("1.dat"); + files.add("1.hdr"); + files.add("1.idx"); + info = new SnapshotInfo(1, 2, 3, 4, 5, 6, 7, "/path/to/tablet/snapshot/", files); + + new NonStrictExpectations(FrontendOptions.class) { + { + FrontendOptions.getLocalHostAddress(); + minTimes = 0; + result = "127.0.0.1"; + } + }; + + new MockUp() { + @Mock + public BrokerAddress getBroker(String name, String host) throws AnalysisException { + return new BrokerAddress("10.74.167.16", 8111); + } + }; + + } + + @Test + public void testGet() { + repo = new Repository(10000, "repo", false, location, storage); + + Assert.assertEquals(repoId, repo.getId()); + Assert.assertEquals(name, repo.getName()); + Assert.assertEquals(false, repo.isReadOnly()); + Assert.assertEquals(location, repo.getLocation()); + Assert.assertEquals(null, repo.getErrorMsg()); + Assert.assertTrue(System.currentTimeMillis() - repo.getCreateTime() < 1000); + } + + @Test + public void testInit() { + new NonStrictExpectations() { + { + storage.list(anyString, (List) any); + result = new Delegate() { + public Status list(String remotePath, List result) { + result.clear(); + return Status.OK; + } + }; + + storage.directUpload(anyString, anyString); + result = Status.OK; + } + }; + + repo = new Repository(10000, "repo", false, location, storage); + + Status st = repo.initRepository(); + System.out.println(st); + Assert.assertTrue(st.ok()); + } + + @Test + public void testassemnblePath() { + repo = new Repository(10000, "repo", false, location, storage); + + // job info + String label = "label"; + String createTime = "2018-04-12 20:46:45"; + String createTime2 = "2018-04-12-20-46-45"; + Timestamp ts = Timestamp.valueOf(createTime); + long creastTs = ts.getTime(); + + // "location/__palo_repository_repo_name/__ss_my_sp1/__info_2018-01-01-08-00-00" + String expected = location + "/" + Repository.PREFIX_REPO + name + "/" + Repository.PREFIX_SNAPSHOT_DIR + + label + "/" + Repository.PREFIX_JOB_INFO + createTime2; + Assert.assertEquals(expected, repo.assembleJobInfoFilePath(label, creastTs)); + + // meta info + expected = location + "/" + Repository.PREFIX_REPO + name + "/" + Repository.PREFIX_SNAPSHOT_DIR + + label + "/" + Repository.FILE_META_INFO; + Assert.assertEquals(expected, repo.assembleMetaInfoFilePath(label)); + + // snapshot path + // /location/__palo_repository_repo_name/__ss_my_ss1/__ss_content/__db_10001/__tbl_10020/__part_10031/__idx_10032/__10023/__3481721 + expected = location + "/" + Repository.PREFIX_REPO + name + "/" + Repository.PREFIX_SNAPSHOT_DIR + + label + "/" + "__ss_content/__db_1/__tbl_2/__part_3/__idx_4/__5/__7"; + Assert.assertEquals(expected, repo.assembleRemoteSnapshotPath(label, info)); + } + + @Test + public void testPing() { + new NonStrictExpectations() { + { + storage.checkPathExist(anyString); + result = Status.OK; + } + }; + + repo = new Repository(10000, "repo", false, location, storage); + Assert.assertTrue(repo.ping()); + Assert.assertTrue(repo.getErrorMsg() == null); + } + + @Test + public void testListSnapshots() { + new NonStrictExpectations() { + { + storage.list(anyString, (List) any); + result = new Delegate() { + public Status list(String remotePath, List result) { + result.add(new RemoteFile(Repository.PREFIX_SNAPSHOT_DIR + "a", false, 100)); + result.add(new RemoteFile("_ss_b", true, 100)); + return Status.OK; + } + }; + } + }; + + repo = new Repository(10000, "repo", false, location, storage); + List snapshotNames = Lists.newArrayList(); + Status st = repo.listSnapshots(snapshotNames); + Assert.assertTrue(st.ok()); + Assert.assertEquals(1, snapshotNames.size()); + Assert.assertEquals("a", snapshotNames.get(0)); + } + + @Test + public void testUpload() { + new NonStrictExpectations() { + { + storage.upload(anyString, anyString); + result = Status.OK; + + storage.rename(anyString, anyString); + result = Status.OK; + } + }; + + repo = new Repository(10000, "repo", false, location, storage); + String localFilePath = "./tmp_" + System.currentTimeMillis(); + try (PrintWriter out = new PrintWriter(localFilePath)) { + out.print("a"); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + try { + String remoteFilePath = location + "/remote_file"; + Status st = repo.upload(localFilePath, remoteFilePath); + Assert.assertTrue(st.ok()); + } finally { + File file = new File(localFilePath); + file.delete(); + } + } + + @Test + public void testDownload() { + String localFilePath = "./tmp_" + System.currentTimeMillis(); + File localFile = new File(localFilePath); + try { + try (PrintWriter out = new PrintWriter(localFile)) { + out.print("a"); + } catch (FileNotFoundException e) { + e.printStackTrace(); + Assert.fail(); + } + + new NonStrictExpectations() { + { + storage.list(anyString, (List) any); + result = new Delegate() { + public Status list(String remotePath, List result) { + result.add(new RemoteFile("remote_file.0cc175b9c0f1b6a831c399e269772661", true, 100)); + return Status.OK; + } + }; + + storage.downloadWithFileSize(anyString, anyString, anyLong); + result = Status.OK; + } + }; + + repo = new Repository(10000, "repo", false, location, storage); + String remoteFilePath = location + "/remote_file"; + Status st = repo.download(remoteFilePath, localFilePath); + Assert.assertTrue(st.ok()); + } finally { + localFile.delete(); + } + } + + @Test + public void testGetInfo() { + repo = new Repository(10000, "repo", false, location, storage); + List infos = repo.getInfo(); + Assert.assertTrue(infos.size() == ShowRepositoriesStmt.TITLE_NAMES.size()); + } + + @Test + public void testGetSnapshotInfo() { + new NonStrictExpectations() { + { + storage.list(anyString, (List) any); + result = new Delegate() { + public Status list(String remotePath, List result) { + if (remotePath.contains(Repository.PREFIX_JOB_INFO)) { + result.add(new RemoteFile(" __info_2018-04-18-20-11-00.12345678123456781234567812345678", + true, + 100)); + } else { + result.add(new RemoteFile(Repository.PREFIX_SNAPSHOT_DIR + "s1", false, 100)); + result.add(new RemoteFile(Repository.PREFIX_SNAPSHOT_DIR + "s2", false, 100)); + } + return Status.OK; + } + }; + } + }; + + repo = new Repository(10000, "repo", false, location, storage); + String snapshotName = ""; + String timestamp = ""; + try { + List> infos = repo.getSnapshotInfos(snapshotName, timestamp); + Assert.assertEquals(2, infos.size()); + + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testPersist() { + Map properties = Maps.newHashMap(); + properties.put("bos_endpoint", "http://gz.bcebos.com"); + properties.put("bos_accesskey", "a"); + properties.put("bos_secret_accesskey", "b"); + BlobStorage storage = new BlobStorage(brokerName, properties); + repo = new Repository(10000, "repo", false, location, storage); + + File file = new File("./Repository"); + try { + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + repo.write(out); + out.flush(); + out.close(); + + DataInputStream in = new DataInputStream(new FileInputStream(file)); + Repository newRepo = Repository.read(in); + in.close(); + + Assert.assertEquals(repo.getName(), newRepo.getName()); + Assert.assertEquals(repo.getId(), newRepo.getId()); + Assert.assertEquals(repo.getLocation(), newRepo.getLocation()); + + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + Assert.fail(); + } finally { + file.delete(); + } + } + +} diff --git a/fe/test/com/baidu/palo/backup/RestoreFileMappingTest.java b/fe/test/com/baidu/palo/backup/RestoreFileMappingTest.java new file mode 100644 index 0000000000..5013ae4b37 --- /dev/null +++ b/fe/test/com/baidu/palo/backup/RestoreFileMappingTest.java @@ -0,0 +1,43 @@ +package com.baidu.palo.backup; + +import com.baidu.palo.backup.RestoreFileMapping.IdChain; + +import org.junit.Before; +import org.junit.Test; + +import junit.framework.Assert; + +public class RestoreFileMappingTest { + + private RestoreFileMapping fileMapping = new RestoreFileMapping(); + private IdChain src; + private IdChain dest; + + @Before + public void setUp() { + src = new IdChain(10005L, 10006L, 10005L, 10007L, 10008L); + dest = new IdChain(10004L, 10003L, 10004L, 10007L, -1L); + fileMapping.putMapping(src, dest, true); + } + + @Test + public void test() { + IdChain key = new IdChain(10005L, 10006L, 10005L, 10007L, 10008L); + Assert.assertTrue(key.equals(src)); + Assert.assertEquals(src, key); + IdChain val = fileMapping.get(key); + Assert.assertNotNull(val); + Assert.assertEquals(dest, val); + + Long l1 = new Long(10005L); + Long l2 = new Long(10005L); + Assert.assertFalse(l1 == l2); + Assert.assertTrue(l1.equals(l2)); + + Long l3 = new Long(1L); + Long l4 = new Long(1L); + Assert.assertFalse(l3 == l4); + Assert.assertTrue(l3.equals(l4)); + } + +} diff --git a/fe/test/com/baidu/palo/backup/RestoreJobTest.java b/fe/test/com/baidu/palo/backup/RestoreJobTest.java new file mode 100644 index 0000000000..1611ab8a44 --- /dev/null +++ b/fe/test/com/baidu/palo/backup/RestoreJobTest.java @@ -0,0 +1,363 @@ +package com.baidu.palo.backup; + +import com.baidu.palo.backup.BackupJobInfo.BackupIndexInfo; +import com.baidu.palo.backup.BackupJobInfo.BackupPartitionInfo; +import com.baidu.palo.backup.BackupJobInfo.BackupTableInfo; +import com.baidu.palo.backup.BackupJobInfo.BackupTabletInfo; +import com.baidu.palo.backup.RestoreJob.RestoreJobState; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.OlapTable; +import com.baidu.palo.catalog.Partition; +import com.baidu.palo.catalog.Table; +import com.baidu.palo.catalog.Tablet; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.MarkedCountDownLatch; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.SystemInfoService; +import com.baidu.palo.task.AgentBatchTask; +import com.baidu.palo.task.AgentTask; +import com.baidu.palo.task.AgentTaskExecutor; +import com.baidu.palo.task.AgentTaskQueue; +import com.baidu.palo.task.DirMoveTask; +import com.baidu.palo.task.DownloadTask; +import com.baidu.palo.task.SnapshotTask; +import com.baidu.palo.thrift.TBackend; +import com.baidu.palo.thrift.TFinishTaskRequest; +import com.baidu.palo.thrift.TStatus; +import com.baidu.palo.thrift.TStatusCode; +import com.baidu.palo.thrift.TTaskType; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.Adler32; + +import mockit.Delegate; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + +public class RestoreJobTest { + + private Database db; + private BackupJobInfo jobInfo; + private RestoreJob job; + private String label = "test_label"; + + private AtomicLong id = new AtomicLong(50000); + + private OlapTable expectedRestoreTbl; + + private long repoId = 20000; + @Mocked + private Catalog catalog; + @Mocked + private BackupHandler backupHandler; + @Mocked + private RepositoryMgr repoMgr; + @Mocked + private EditLog editLog; + @Mocked + private SystemInfoService systemInfoService; + + private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", + new BlobStorage("broker", Maps.newHashMap())); + + private BackupMeta backupMeta; + + static { + Startup.initializeIfPossible(); + } + + @Before + public void setUp() throws AnalysisException { + + new NonStrictExpectations() { + { + catalog.getBackupHandler(); + result = backupHandler; + + catalog.getDb(anyLong); + result = db; + + Catalog.getCurrentCatalogJournalVersion(); + result = FeMetaVersion.VERSION_42; + + catalog.getNextId(); + result = id.getAndIncrement(); + + catalog.getEditLog(); + result = catalog; + + Catalog.getCurrentSystemInfo(); + result = systemInfoService; + } + }; + + new NonStrictExpectations() { + { + systemInfoService.seqChooseBackendIds(anyInt, anyBoolean, anyBoolean, anyString); + result = new Delegate() { + public synchronized List seqChooseBackendIds(int backendNum, boolean needAlive, + boolean isCreate, String clusterName) { + List beIds = Lists.newArrayList(); + beIds.add(CatalogMocker.BACKEND1_ID); + beIds.add(CatalogMocker.BACKEND2_ID); + beIds.add(CatalogMocker.BACKEND3_ID); + return beIds; + } + }; + } + }; + + new NonStrictExpectations() { + { + backupHandler.getRepoMgr(); + result = repoMgr; + } + }; + + new NonStrictExpectations() { + { + repoMgr.getRepo(anyInt); + result = repo; + } + }; + + new NonStrictExpectations() { + { + editLog.logBackupJob((BackupJob) any); + result = new Delegate() { + public void logBackupJob(BackupJob job) { + System.out.println("log backup job: " + job); + } + }; + } + }; + + new NonStrictExpectations() { + { + AgentTaskExecutor.submit((AgentBatchTask) any); + result = new Delegate() { + public void submit(AgentBatchTask task) { + return; + } + }; + } + }; + + new NonStrictExpectations() { + { + repo.upload(anyString, anyString); + result = Status.OK; + + List backupMetas = Lists.newArrayList(); + repo.getSnapshotMetaFile(label, backupMetas); + result = new Delegate() { + public Status getSnapshotMetaFile(String label, List backupMetas) { + backupMetas.add(backupMeta); + return Status.OK; + } + }; + } + }; + + new MockUp() { + @Mock + boolean await(long timeout, TimeUnit unit) { + return true; + } + }; + + db = CatalogMocker.mockDb(); + + // gen BackupJobInfo + jobInfo = new BackupJobInfo(); + jobInfo.backupTime = System.currentTimeMillis(); + jobInfo.dbId = CatalogMocker.TEST_DB_ID; + jobInfo.dbName = CatalogMocker.TEST_DB_NAME; + jobInfo.name = label; + jobInfo.success = true; + + expectedRestoreTbl = (OlapTable) db.getTable(CatalogMocker.TEST_TBL2_ID); + BackupTableInfo tblInfo = new BackupTableInfo(); + tblInfo.id = CatalogMocker.TEST_TBL2_ID; + tblInfo.name = CatalogMocker.TEST_TBL2_NAME; + jobInfo.tables.put(tblInfo.name, tblInfo); + + for (Partition partition : expectedRestoreTbl.getPartitions()) { + BackupPartitionInfo partInfo = new BackupPartitionInfo(); + partInfo.id = partition.getId(); + partInfo.name = partition.getName(); + tblInfo.partitions.put(partInfo.name, partInfo); + + for (MaterializedIndex index : partition.getMaterializedIndices()) { + BackupIndexInfo idxInfo = new BackupIndexInfo(); + idxInfo.id = index.getId(); + idxInfo.name = expectedRestoreTbl.getIndexNameById(index.getId()); + idxInfo.schemaHash = expectedRestoreTbl.getSchemaHashByIndexId(index.getId()); + partInfo.indexes.put(idxInfo.name, idxInfo); + + for (Tablet tablet : index.getTablets()) { + BackupTabletInfo tabletInfo = new BackupTabletInfo(); + tabletInfo.id = tablet.getId(); + tabletInfo.files.add(tabletInfo.id + ".dat"); + tabletInfo.files.add(tabletInfo.id + ".idx"); + tabletInfo.files.add(tabletInfo.id + ".hdr"); + idxInfo.tablets.add(tabletInfo); + } + } + } + + // drop this table, cause we want to try restoring this table + db.dropTable(expectedRestoreTbl.getName()); + + job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), + jobInfo, false, 3, 100000, catalog, repo.getId()); + + List
    tbls = Lists.newArrayList(); + tbls.add(expectedRestoreTbl); + backupMeta = new BackupMeta(tbls); + } + + @Test + public void testRun() { + // pending + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.SNAPSHOTING, job.getState()); + Assert.assertEquals(12, job.getFileMapping().getMapping().size()); + + // 2. snapshoting + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.SNAPSHOTING, job.getState()); + Assert.assertEquals(12 * 2, AgentTaskQueue.getTaskNum()); + + // 3. snapshot finished + List agentTasks = Lists.newArrayList(); + Map> runningTasks = Maps.newHashMap(); + agentTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND1_ID, runningTasks)); + agentTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND2_ID, runningTasks)); + agentTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND3_ID, runningTasks)); + Assert.assertEquals(12 * 2, agentTasks.size()); + + for (AgentTask agentTask : agentTasks) { + if (agentTask.getTaskType() != TTaskType.MAKE_SNAPSHOT) { + continue; + } + + SnapshotTask task = (SnapshotTask) agentTask; + String snapshotPath = "/path/to/snapshot/" + System.currentTimeMillis(); + TStatus taskStatus = new TStatus(TStatusCode.OK); + TBackend tBackend = new TBackend("", 0, 1); + TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, + task.getSignature(), taskStatus); + request.setSnapshot_path(snapshotPath); + Assert.assertTrue(job.finishTabletSnapshotTask(task, request)); + } + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOAD, job.getState()); + + // download + AgentTaskQueue.clearAllTasks(); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOADING, job.getState()); + Assert.assertEquals(9, AgentTaskQueue.getTaskNum()); + + // downloading + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOADING, job.getState()); + + List downloadTasks = Lists.newArrayList(); + runningTasks = Maps.newHashMap(); + downloadTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND1_ID, runningTasks)); + downloadTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND2_ID, runningTasks)); + downloadTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND3_ID, runningTasks)); + Assert.assertEquals(9, downloadTasks.size()); + + List downloadedTabletIds = Lists.newArrayList(); + for (AgentTask agentTask : downloadTasks) { + TStatus taskStatus = new TStatus(TStatusCode.OK); + TBackend tBackend = new TBackend("", 0, 1); + TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, + agentTask.getSignature(), taskStatus); + request.setDownloaded_tablet_ids(downloadedTabletIds); + Assert.assertTrue(job.finishTabletDownloadTask((DownloadTask) agentTask, request)); + } + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMIT, job.getState()); + + // commit + AgentTaskQueue.clearAllTasks(); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMITTING, job.getState()); + Assert.assertEquals(12, AgentTaskQueue.getTaskNum()); + + // committing + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMITTING, job.getState()); + + List dirMoveTasks = Lists.newArrayList(); + runningTasks = Maps.newHashMap(); + dirMoveTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND1_ID, runningTasks)); + dirMoveTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND2_ID, runningTasks)); + dirMoveTasks.addAll(AgentTaskQueue.getDiffTasks(CatalogMocker.BACKEND3_ID, runningTasks)); + Assert.assertEquals(12, dirMoveTasks.size()); + + for (AgentTask agentTask : dirMoveTasks) { + TStatus taskStatus = new TStatus(TStatusCode.OK); + TBackend tBackend = new TBackend("", 0, 1); + TFinishTaskRequest request = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, + agentTask.getSignature(), taskStatus); + job.finishDirMoveTask((DirMoveTask) agentTask, request); + } + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.FINISHED, job.getState()); + } + + @Test + public void testSignature() { + Adler32 sig1 = new Adler32(); + sig1.update("name1".getBytes()); + sig1.update("name2".getBytes()); + System.out.println("sig1: " + Math.abs((int) sig1.getValue())); + + Adler32 sig2 = new Adler32(); + sig2.update("name2".getBytes()); + sig2.update("name1".getBytes()); + System.out.println("sig2: " + Math.abs((int) sig2.getValue())); + + OlapTable tbl = (OlapTable) db.getTable(CatalogMocker.TEST_TBL_NAME); + List partNames = Lists.newArrayList(tbl.getPartitionNames()); + System.out.println("tbl signature: " + tbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames)); + tbl.setName("newName"); + System.out.println("tbl signature: " + tbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames)); + } + +} + diff --git a/fe/test/com/baidu/palo/bdb/BDBToolOptionsTest.java b/fe/test/com/baidu/palo/bdb/BDBToolOptionsTest.java index f341c5dbac..7bc84aa082 100644 --- a/fe/test/com/baidu/palo/bdb/BDBToolOptionsTest.java +++ b/fe/test/com/baidu/palo/bdb/BDBToolOptionsTest.java @@ -18,7 +18,7 @@ public class BDBToolOptionsTest { options = new BDBToolOptions(false, "12345", false, "12345", "12456", 35); Assert.assertTrue(options.hasFromKey()); Assert.assertTrue(options.hasEndKey()); - Assert.assertNotEquals(FeConstants.meta_version, options.getMetaVersion()); + Assert.assertNotSame(FeConstants.meta_version, options.getMetaVersion()); } } diff --git a/fe/test/com/baidu/palo/catalog/BackendTest.java b/fe/test/com/baidu/palo/catalog/BackendTest.java index a3eb265e62..e98c9531b3 100644 --- a/fe/test/com/baidu/palo/catalog/BackendTest.java +++ b/fe/test/com/baidu/palo/catalog/BackendTest.java @@ -22,6 +22,7 @@ package com.baidu.palo.catalog; import com.baidu.palo.analysis.AccessTestUtil; import com.baidu.palo.common.FeConstants; +import com.baidu.palo.metric.MetricRepo; import com.baidu.palo.system.Backend; import com.baidu.palo.thrift.TDisk; @@ -47,7 +48,7 @@ import java.util.Map; @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) +@PrepareForTest({ Catalog.class, MetricRepo.class }) public class BackendTest { private Backend backend; private long backendId = 9999; @@ -57,7 +58,7 @@ public class BackendTest { private int httpPort = 21237; private int beRpcPort = 21238; - private Catalog catalog; + private Catalog catalog; @Before public void setUp() { @@ -68,7 +69,12 @@ public class BackendTest { PowerMock.replay(Catalog.class); backend = new Backend(backendId, host, heartbeatPort); - backend.updateOnce(bePort, httpPort, beRpcPort); + backend.updateOnce(bePort, httpPort, beRpcPort); + + PowerMock.mockStatic(MetricRepo.class); + MetricRepo.generateCapacityMetrics(); + EasyMock.expectLastCall().anyTimes(); + PowerMock.replay(MetricRepo.class); } @Test @@ -104,8 +110,7 @@ public class BackendTest { backend.updateDisks(diskInfos); Assert.assertEquals(disk1.getDisk_total_capacity() + disk2.getDisk_total_capacity(), backend.getTotalCapacityB()); - Assert.assertEquals(disk1.getDisk_total_capacity() + disk2.getDisk_total_capacity() + 1, - backend.getAvailableCapacityB()); + Assert.assertEquals(1, backend.getAvailableCapacityB()); // second update diskInfos.remove(disk1.getRoot_path()); diff --git a/fe/test/com/baidu/palo/catalog/DomainResolverServerTest.java b/fe/test/com/baidu/palo/catalog/DomainResolverServerTest.java deleted file mode 100644 index 65291c906c..0000000000 --- a/fe/test/com/baidu/palo/catalog/DomainResolverServerTest.java +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.catalog; - -import java.util.List; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.google.common.collect.Lists; - -@RunWith(PowerMockRunner.class) -public class DomainResolverServerTest { - private DomainResolverServer server; - private String user = "test"; - private List domainNameList; - - @Before - public void setUp() { - server = DomainResolverServer.getInstance(); - domainNameList = Lists.newArrayList(); - domainNameList.add("www.baidu.com"); - } - - @Test - public void registerTest() { - // param error test - final List sizeZeroDomainNameList = Lists.newArrayList(); - // empty domain list - Assert.assertFalse(server.register(user, sizeZeroDomainNameList)); - // null domain list - Assert.assertFalse(server.register(user, null)); - // empty user - Assert.assertFalse(server.register("", domainNameList)); - // null user - Assert.assertFalse(server.register(null, domainNameList)); - - // normal test - Assert.assertTrue(server.register(user, domainNameList)); - Assert.assertTrue(server.getRegisteredUserDomain(user).size() == 1); - - // domain list contains null - final List nullDomainNameList = Lists.newArrayList(); - nullDomainNameList.add(null); - Assert.assertFalse(server.register(null, nullDomainNameList)); - Assert.assertTrue(server.getRegisteredUserDomain(user).size() == 1); - - // domains having registered - Assert.assertTrue(server.register(user, domainNameList)); - Assert.assertTrue(server.getRegisteredUserDomain(user).size() == 1); - - // normal test 2 - final List domainNameList2 = Lists.newArrayList(); - domainNameList2.add("www.sina.com.cn"); - Assert.assertTrue(server.register(user, domainNameList2)); - Assert.assertTrue(server.getRegisteredUserDomain(user).size() == 2); - } - - @Test - public void getIpsWithDNSTest() { - - // no exist user - Assert.assertEquals(null, server.getUserDomainToIps("user1")); - // null user - Assert.assertEquals(null, server.getUserDomainToIps(null)); - - try { - // wait for DomainResolverServer - Thread.currentThread(); - Thread.sleep(500); - // normal test - Assert.assertTrue(server.getUserDomainToIps(user).size() == 2); - } catch (InterruptedException e) { - } - } - - @Test - public void unregisterTest() { - // param error test - // null domain list - server.unregister(user, null); - Assert.assertTrue(server.getUserDomainToIps(user).size() == 2); - // empty domain list - final List sizeZeroDomainNameList = Lists.newArrayList(); - server.unregister(user, sizeZeroDomainNameList); - Assert.assertTrue(server.getUserDomainToIps(user).size() == 2); - // null user - server.unregister(null, domainNameList); - Assert.assertTrue(server.getUserDomainToIps(user).size() == 2); - // no exist user - server.unregister("test1", domainNameList); - Assert.assertTrue(server.getUserDomainToIps(user).size() == 2); - // normal test - server.unregister(user, domainNameList); - Assert.assertTrue(server.getUserDomainToIps(user).size() == 1); - final List domainNameList2 = Lists.newArrayList(); - domainNameList2.add("www.sina.com.cn"); - server.unregister(user, domainNameList2); - Assert.assertEquals(null, server.getUserDomainToIps(user)); - } - - @Test - public void registerNoExistDomain() { - // no exist domain - final List noExistDomainNameList = Lists.newArrayList(); - noExistDomainNameList.add("www.weqwetw.com.cnllll"); - Assert.assertTrue(server.register("test2", noExistDomainNameList)); - try { - // wait for DomainResolverServer - Thread.currentThread(); - Thread.sleep(500); - // normal test - Assert.assertEquals(null, server.getUserDomainToIps("test2")); - } catch (InterruptedException e) { - } - server.unregister(user, noExistDomainNameList); - Assert.assertEquals(null, server.getUserDomainToIps(user)); - } - - @Test - public void isAvaliableDomainTest() { - // normal test - Assert.assertTrue(server.isAvaliableDomain("www.sogo.com.cn")); - // param error test - Assert.assertFalse(server.isAvaliableDomain("")); - Assert.assertFalse(server.isAvaliableDomain(null)); - // no exist domain - Assert.assertFalse(server.isAvaliableDomain("www.sina.com.cn11sdfqweg")); - } - - @After - public void tearDown() throws Exception { - server = null; - user = null; - domainNameList.clear(); - } - -} diff --git a/fe/test/com/baidu/palo/catalog/OlapTableTest.java b/fe/test/com/baidu/palo/catalog/OlapTableTest.java new file mode 100644 index 0000000000..96e5cdc6c0 --- /dev/null +++ b/fe/test/com/baidu/palo/catalog/OlapTableTest.java @@ -0,0 +1,59 @@ +package com.baidu.palo.catalog; + +import com.baidu.palo.catalog.Table.TableType; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.common.io.FastByteArrayOutputStream; +import com.baidu.palo.common.util.UnitTestUtil; + +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.List; + +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + +public class OlapTableTest { + + static { + Startup.initializeIfPossible(); + } + + @Test + public void test() throws IOException { + + new NonStrictExpectations(Catalog.class) { + { + Catalog.getCurrentCatalogJournalVersion(); + minTimes = 0; + result = FeConstants.meta_version; + } + }; + + Database db = UnitTestUtil.createDb(1, 2, 3, 4, 5, 6, 7, 8); + List
    tables = db.getTables(); + + for (Table table : tables) { + if (table.getType() != TableType.OLAP) { + continue; + } + OlapTable tbl = (OlapTable) table; + System.out.println("orig table id: " + tbl.getId()); + + FastByteArrayOutputStream byteArrayOutputStream = new FastByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteArrayOutputStream); + tbl.write(out); + + out.flush(); + out.close(); + + DataInputStream in = new DataInputStream(byteArrayOutputStream.getInputStream()); + Table copiedTbl = OlapTable.read(in); + System.out.println("copied table id: " + copiedTbl.getId()); + } + + } + +} diff --git a/fe/test/com/baidu/palo/catalog/UserPropertyTest.java b/fe/test/com/baidu/palo/catalog/UserPropertyTest.java index bd43d3bf7b..d2f24716eb 100644 --- a/fe/test/com/baidu/palo/catalog/UserPropertyTest.java +++ b/fe/test/com/baidu/palo/catalog/UserPropertyTest.java @@ -23,8 +23,10 @@ package com.baidu.palo.catalog; import com.baidu.palo.analysis.SetUserPropertyVar; import com.baidu.palo.analysis.SetVar; import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.FeMetaVersion; +import com.baidu.palo.common.FeConstants; import com.baidu.palo.load.DppConfig; +import com.baidu.palo.mysql.privilege.UserProperty; + import com.google.common.collect.Lists; import org.easymock.EasyMock; @@ -49,11 +51,10 @@ public class UserPropertyTest { public void testNormal() throws IOException, DdlException { // mock catalog PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getCurrentCatalogJournalVersion()).andReturn(FeMetaVersion.VERSION_12).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalogJournalVersion()).andReturn(FeConstants.meta_version).anyTimes(); PowerMock.replay(Catalog.class); - UserProperty property = new UserProperty(); - + UserProperty property = new UserProperty("root"); property.getResource().updateGroupShare("low", 991); // To image ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); @@ -80,7 +81,7 @@ public class UserPropertyTest { Assert.assertEquals(100, userProperty.getMaxConn()); Assert.assertEquals(101, userProperty.getResource().getResource().getByDesc("cpu_share")); Assert.assertEquals(102, userProperty.getResource().getShareByGroup().get("normal").intValue()); - Assert.assertEquals("/user/palo2", userProperty.getClusterInfo("dpp-cluster").second.getPaloPath()); + Assert.assertEquals("/user/palo2", userProperty.getLoadClusterInfo("dpp-cluster").second.getPaloPath()); Assert.assertEquals("dpp-cluster", userProperty.getDefaultLoadCluster()); // fetch property @@ -103,21 +104,21 @@ public class UserPropertyTest { } // get cluster info - DppConfig dppConfig = userProperty.getClusterInfo("dpp-cluster").second; + DppConfig dppConfig = userProperty.getLoadClusterInfo("dpp-cluster").second; Assert.assertEquals(8070, dppConfig.getHttpPort()); // set palo path null propertyList = Lists.newArrayList(); propertyList.add(new SetUserPropertyVar("load_cluster.dpp-cluster.hadoop_palo_path", null)); userProperty.update(propertyList); - Assert.assertEquals(null, userProperty.getClusterInfo("dpp-cluster").second.getPaloPath()); + Assert.assertEquals(null, userProperty.getLoadClusterInfo("dpp-cluster").second.getPaloPath()); // remove dpp-cluster propertyList = Lists.newArrayList(); propertyList.add(new SetUserPropertyVar("load_cluster.dpp-cluster", null)); Assert.assertEquals("dpp-cluster", userProperty.getDefaultLoadCluster()); userProperty.update(propertyList); - Assert.assertEquals(null, userProperty.getClusterInfo("dpp-cluster").second); + Assert.assertEquals(null, userProperty.getLoadClusterInfo("dpp-cluster").second); Assert.assertEquals(null, userProperty.getDefaultLoadCluster()); } -} \ No newline at end of file +} diff --git a/fe/test/com/baidu/palo/catalog/UserResourceTest.java b/fe/test/com/baidu/palo/catalog/UserResourceTest.java index 15450f9e97..5087232316 100644 --- a/fe/test/com/baidu/palo/catalog/UserResourceTest.java +++ b/fe/test/com/baidu/palo/catalog/UserResourceTest.java @@ -21,6 +21,7 @@ package com.baidu.palo.catalog; import com.baidu.palo.common.DdlException; +import com.baidu.palo.mysql.privilege.UserResource; import com.baidu.palo.thrift.TResourceType; import com.baidu.palo.thrift.TUserResource; diff --git a/fe/test/com/baidu/palo/clone/CloneCheckerTest.java b/fe/test/com/baidu/palo/clone/CloneCheckerTest.java deleted file mode 100644 index 0e0f9d036a..0000000000 --- a/fe/test/com/baidu/palo/clone/CloneCheckerTest.java +++ /dev/null @@ -1,766 +0,0 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.clone; - -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.MaterializedIndex; -import com.baidu.palo.catalog.OlapTable; -import com.baidu.palo.catalog.Partition; -import com.baidu.palo.catalog.Replica; -import com.baidu.palo.catalog.Replica.ReplicaState; -import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.catalog.TabletInvertedIndex; -import com.baidu.palo.catalog.TabletMeta; -import com.baidu.palo.clone.CloneChecker.CapacityLevel; -import com.baidu.palo.clone.CloneJob.JobPriority; -import com.baidu.palo.clone.CloneJob.JobState; -import com.baidu.palo.clone.CloneJob.JobType; -import com.baidu.palo.common.Config; -import com.baidu.palo.common.util.UnitTestUtil; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.system.Backend; -import com.baidu.palo.system.SystemInfoService; -import com.baidu.palo.task.AgentTask; -import com.baidu.palo.task.AgentTaskQueue; -import com.baidu.palo.thrift.TDisk; -import com.baidu.palo.thrift.TTaskType; - -import com.google.common.collect.Lists; - -import org.easymock.EasyMock; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest({ CloneChecker.class, Catalog.class, SystemInfoService.class }) -public class CloneCheckerTest { - private static final Logger LOG = LoggerFactory.getLogger(CloneCheckerTest.class); - - private CloneChecker checker; - private Catalog catalog; - private SystemInfoService systemInfoService; - private long dbId; - private long tableId; - private long partitionId; - private long indexId; - private long backendId; - - @Before - public void setUp() { - dbId = 0L; - tableId = 0L; - partitionId = 0L; - indexId = 0L; - backendId = 0L; - - // update conf - Config.clone_high_priority_delay_second = 0; - Config.clone_low_priority_delay_second = 600; - Config.clone_distribution_balance_threshold = 0.2; - Config.clone_capacity_balance_threshold = 0.2; - } - - @After - public void tearDown() throws Exception { - // destory INSTANCE - Field instanceField = CloneChecker.class.getDeclaredField("INSTANCE"); - instanceField.setAccessible(true); - instanceField.set(CloneChecker.class, null); - } - - private CloneJob createCloneJob() { - long tabletId = 0L; - JobType type = JobType.SUPPLEMENT; - JobPriority priority = JobPriority.HIGH; - long timeoutMs = 1000L; - CloneJob job = new CloneJob(dbId, tableId, partitionId, indexId, tabletId, backendId, type, priority, - timeoutMs); - job.setCreateTimeMs(System.currentTimeMillis() - 1); - return job; - } - - private Constructor getInnerClassConstructor(Class innerClass) { - Constructor[] constructors = innerClass.getDeclaredConstructors(); - Constructor constructor = constructors[0]; - constructor.setAccessible(true); - LOG.debug(constructor.toString()); - return constructor; - } - - @Test - public void testCheckTablets() throws Exception { - // mock getBackend getCloneInstance getDb getDbNames - long backendId1 = backendId; - long backendId2 = backendId + 1; - long backendId3 = backendId + 2; - Backend onlineBackend = EasyMock.createMock(Backend.class); - EasyMock.expect(onlineBackend.isAlive()).andReturn(true).anyTimes(); - EasyMock.replay(onlineBackend); - catalog = EasyMock.createNiceMock(Catalog.class); - - Clone clone = new Clone(); - EasyMock.expect(catalog.getCloneInstance()).andReturn(clone).anyTimes(); - - long tabletId = 0L; - long version = 1L; - long versionHash = 0L; - Database db = UnitTestUtil.createDb(dbId, tableId, partitionId, indexId, tabletId, backendId, version, - versionHash); - db.setClusterName("testCluster"); - OlapTable table = (OlapTable) db.getTable(tableId); - Partition partition = table.getPartition(partitionId); - MaterializedIndex index = partition.getBaseIndex(); - Tablet tablet = index.getTablet(tabletId); - tablet.deleteReplicaByBackendId(backendId1); - Assert.assertEquals(2, tablet.getReplicas().size()); - EasyMock.expect(catalog.getDb(dbId)).andReturn(db).anyTimes(); - - List dbNames = new ArrayList(); - String dbName = db.getFullName(); - dbNames.add(dbName); - EasyMock.expect(catalog.getDb(db.getFullName())).andReturn(db).anyTimes(); - EasyMock.expect(catalog.getDbNames()).andReturn(dbNames).anyTimes(); - - EasyMock.replay(catalog); - - // SystemInfoService - systemInfoService = EasyMock.createMock(SystemInfoService.class); - EasyMock.expect(systemInfoService.getBackend(backendId2)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId1)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId3)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackendIds(true)) - .andReturn(Lists.newArrayList(backendId1, backendId2, backendId3)).anyTimes(); - EasyMock.replay(systemInfoService); - - // inverted index - TabletInvertedIndex invertedIndex = EasyMock.createMock(TabletInvertedIndex.class); - invertedIndex.deleteReplica(EasyMock.anyLong(), EasyMock.anyLong()); - EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(invertedIndex); - - // mock catalog - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(systemInfoService).anyTimes(); - EasyMock.expect(Catalog.getCurrentInvertedIndex()).andReturn(invertedIndex).anyTimes(); - PowerMock.replay(Catalog.class); - - // mock private method - Map backendInfos = new HashMap(); - Map> distributionLevelToBackendIds = new HashMap>(); - Map> capacityLevelToBackendIds = new HashMap>(); - CloneChecker mockChecker = PowerMock.createPartialMock(CloneChecker.class, "initBackendInfos", - "initBackendCapacityInfos", "initBackendDistributionInfos"); - PowerMock.expectPrivate(mockChecker, "initBackendInfos", "testCluster").andReturn(backendInfos).anyTimes(); - PowerMock.expectPrivate(mockChecker, "initBackendDistributionInfos", backendInfos) - .andReturn(distributionLevelToBackendIds).anyTimes(); - PowerMock.expectPrivate(mockChecker, "initBackendCapacityInfos", backendInfos) - .andReturn(capacityLevelToBackendIds).anyTimes(); - PowerMock.replay(mockChecker); - - // init backend infos - Class backendInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$BackendInfo"); - Constructor constructor = getInnerClassConstructor(backendInfoClass); - backendInfos.put(backendId1, constructor.newInstance(new Object[] { mockChecker, backendId1, 10L, 0L })); - backendInfos.put(backendId2, constructor.newInstance(new Object[] { mockChecker, backendId2, 10L, 0L })); - backendInfos.put(backendId3, constructor.newInstance(new Object[] { mockChecker, backendId3, 10L, 0L })); - - for (CapacityLevel level : CapacityLevel.values()) { - distributionLevelToBackendIds.put(level, new HashSet()); - capacityLevelToBackendIds.put(level, new HashSet()); - } - distributionLevelToBackendIds.get(CapacityLevel.LOW).add(backendId1); - distributionLevelToBackendIds.get(CapacityLevel.MID).add(backendId2); - distributionLevelToBackendIds.get(CapacityLevel.HIGH).add(backendId3); - capacityLevelToBackendIds.get(CapacityLevel.LOW).add(backendId1); - capacityLevelToBackendIds.get(CapacityLevel.MID).add(backendId2); - capacityLevelToBackendIds.get(CapacityLevel.HIGH).add(backendId3); - - // test check tablet for supplment - Assert.assertTrue(mockChecker.checkTabletForSupplement(dbId, tableId, partitionId, indexId, tabletId)); - List pendingJobs = clone.getCloneJobs(JobState.PENDING); - if (pendingJobs.size() == 1) { - CloneJob job = pendingJobs.get(0); - Assert.assertEquals(backendId1, job.getDestBackendId()); - Assert.assertEquals(tabletId, job.getTabletId()); - Assert.assertEquals(JobType.SUPPLEMENT, job.getType()); - clone.cancelCloneJob(job, "test"); - Assert.assertEquals(0, clone.getCloneJobs(JobState.PENDING).size()); - Assert.assertEquals(2, tablet.getReplicas().size()); - } - - // test check tablets - Method checkTablets = UnitTestUtil.getPrivateMethod(CloneChecker.class, "checkTablets", new Class[] {}); - checkTablets.invoke(mockChecker, new Object[] {}); - pendingJobs = clone.getCloneJobs(JobState.PENDING); - if (pendingJobs.size() == 1) { - CloneJob job = pendingJobs.get(0); - Assert.assertEquals(backendId1, job.getDestBackendId()); - Assert.assertEquals(tabletId, job.getTabletId()); - Assert.assertEquals(JobType.SUPPLEMENT, job.getType()); - clone.cancelCloneJob(job, "test"); - Assert.assertEquals(0, clone.getCloneJobs(JobState.PENDING).size()); - Assert.assertEquals(2, tablet.getReplicas().size()); - } - } - - @Test - public void testInitBackendAndCapacityInfos() throws Exception { - // mock catalog editlog - catalog = EasyMock.createNiceMock(Catalog.class); - EditLog editLog = EasyMock.createMock(EditLog.class); - EasyMock.expect(catalog.getEditLog()).andReturn(editLog).anyTimes(); - EasyMock.replay(catalog); - - // mock catalog - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - systemInfoService = EasyMock.createMock(SystemInfoService.class); - EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(systemInfoService).anyTimes(); - PowerMock.replay(Catalog.class); - - // mock getBackend - long backendId1 = backendId; // high - long backendId2 = backendId + 1; // mid - long backendId3 = backendId + 2; // low - long totalCapacityB = 100L; - long availableCapacityB = 50L; - List backends = new ArrayList(); - backends.add(backendId1); - backends.add(backendId2); - backends.add(backendId3); - Backend backend1 = UnitTestUtil.createBackend(backendId1, "127.0.0.1", 8000, 8001, 8003, totalCapacityB, - availableCapacityB - 40); - Backend backend2 = UnitTestUtil.createBackend(backendId1, "127.0.0.1", 8100, 8101, 8103, totalCapacityB, - availableCapacityB); - Backend backend3 = UnitTestUtil.createBackend(backendId1, "127.0.0.1", 8200, 8201, 8203, totalCapacityB, - availableCapacityB + 40); - backend1.setOwnerClusterName("testCluster"); - - // SystemInfoService - EasyMock.expect(systemInfoService.getBackend(backendId1)).andReturn(backend1).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId2)).andReturn(backend2).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId3)).andReturn(backend3).anyTimes(); - EasyMock.expect(systemInfoService.getBackendIds(true)) - .andReturn(Lists.newArrayList(backendId1, backendId2, backendId3)).anyTimes(); - EasyMock.replay(systemInfoService); - - // get initBackendInfos method - checker = CloneChecker.getInstance(); - Method initBackendInfos = UnitTestUtil.getPrivateMethod(CloneChecker.class, "initBackendInfos", - new Class[] { String.class }); - Method initBackendCapacityInfos = UnitTestUtil.getPrivateMethod(CloneChecker.class, "initBackendCapacityInfos", - new Class[] { Map.class }); - - // test - Map backendInfos = (Map) initBackendInfos.invoke(checker, new Object[] { null }); - Map> capacityLevelToBackendIds = (Map>) initBackendCapacityInfos - .invoke(checker, new Object[] { backendInfos }); - Assert.assertTrue(capacityLevelToBackendIds.get(CapacityLevel.HIGH).contains(backendId1)); - Assert.assertTrue(capacityLevelToBackendIds.get(CapacityLevel.MID).contains(backendId2)); - Assert.assertTrue(capacityLevelToBackendIds.get(CapacityLevel.LOW).contains(backendId3)); - } - - @Test - public void testInitBackendDistributionInfos() throws Exception { - // get inner class: BackendInfo - checker = CloneChecker.getInstance(); - Class backendInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$BackendInfo"); - Constructor backendInfoConstructor = getInnerClassConstructor(backendInfoClass); - Method setTableReplicaNum = UnitTestUtil.getPrivateMethod(backendInfoClass, "setTableReplicaNum", - new Class[] { int.class }); - - // init params - long backendId1 = backendId; // high - long backendId2 = backendId + 1; // mid - long backendId3 = backendId + 2; // low - long totalCapacityB = 100L; - long availableCapacityB = 50L; - Map backendInfos = new HashMap(); - Object backendInfo1 = backendInfoConstructor - .newInstance(new Object[] { checker, backendId1, totalCapacityB, availableCapacityB }); - setTableReplicaNum.invoke(backendInfo1, new Object[] { 10 }); - backendInfos.put(backendId1, backendInfo1); - Object backendInfo2 = backendInfoConstructor - .newInstance(new Object[] { checker, backendId2, totalCapacityB, availableCapacityB }); - backendInfos.put(backendId2, backendInfo2); - setTableReplicaNum.invoke(backendInfo2, new Object[] { 5 }); - Object backendInfo3 = backendInfoConstructor - .newInstance(new Object[] { checker, backendId3, totalCapacityB, availableCapacityB }); - backendInfos.put(backendId3, backendInfo3); - setTableReplicaNum.invoke(backendInfo3, new Object[] { 1 }); - - // get initBackendDistributionInfos method - Method initBackendDistributionInfos = UnitTestUtil.getPrivateMethod(CloneChecker.class, - "initBackendDistributionInfos", new Class[] { Map.class }); - - // test - Map> distributionLevelToBackendIds = (Map>) initBackendDistributionInfos - .invoke(checker, new Object[] { backendInfos }); - Assert.assertTrue(distributionLevelToBackendIds.get(CapacityLevel.HIGH).contains(backendId1)); - Assert.assertTrue(distributionLevelToBackendIds.get(CapacityLevel.MID).contains(backendId2)); - Assert.assertTrue(distributionLevelToBackendIds.get(CapacityLevel.LOW).contains(backendId3)); - } - - @Test - public void testSelectRandomBackendId() throws Exception { - // get method - checker = CloneChecker.getInstance(); - Method selectRandomBackendId = UnitTestUtil.getPrivateMethod(CloneChecker.class, "selectRandomBackendId", - new Class[] { List.class, Set.class }); - - // fail - List candidateBackendIds = new ArrayList(); - candidateBackendIds.add(0L); - Set excludeBackendIds = new HashSet(); - excludeBackendIds.add(0L); - Assert.assertEquals(-1L, - selectRandomBackendId.invoke(checker, new Object[] { candidateBackendIds, excludeBackendIds })); - - // success - candidateBackendIds.add(1L); - Assert.assertEquals(1L, - selectRandomBackendId.invoke(checker, new Object[] { candidateBackendIds, excludeBackendIds })); - } - - @Test - public void testSelectCloneReplicaBackendId() throws Exception { - // get inner class: TabletInfo BackendInfo - checker = CloneChecker.getInstance(); - Class tabletInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$TabletInfo"); - Constructor tabletInfoConstructor = getInnerClassConstructor(tabletInfoClass); - Class backendInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$BackendInfo"); - Constructor backendInfoConstructor = getInnerClassConstructor(backendInfoClass); - - // init params - long totalCapacityB = 100L; - long availableCapacityB = 50L; - long tabletId = 0L; - short replicationNum = 3; - short onlineReplicaNum = 3; - long tabletSizeB = 20L; - long backendId1 = backendId; - long backendId2 = backendId + 1; - Set backendIds = new HashSet(); - backendIds.add(backendId1); - Object[] objects = new Object[] { checker, dbId, tableId, partitionId, indexId, tabletId, replicationNum, - onlineReplicaNum, tabletSizeB, backendIds }; - Object tabletInfo = tabletInfoConstructor.newInstance(objects); - - Object backendInfo1 = backendInfoConstructor - .newInstance(new Object[] { checker, backendId1, totalCapacityB, availableCapacityB }); - Object backendInfo2 = backendInfoConstructor - .newInstance(new Object[] { checker, backendId2, totalCapacityB, availableCapacityB }); - Map backendInfos = new HashMap(); - backendInfos.put(backendId1, backendInfo1); - backendInfos.put(backendId2, backendInfo2); - - Map> distributionLevelToBackendIds = new HashMap>(); - Map> capacityLevelToBackendIds = new HashMap>(); - for (CapacityLevel level : CapacityLevel.values()) { - distributionLevelToBackendIds.put(level, new HashSet()); - capacityLevelToBackendIds.put(level, new HashSet()); - } - distributionLevelToBackendIds.get(CapacityLevel.MID).add(backendId1); - distributionLevelToBackendIds.get(CapacityLevel.HIGH).add(backendId2); - capacityLevelToBackendIds.get(CapacityLevel.MID).add(backendId1); - capacityLevelToBackendIds.get(CapacityLevel.MID).add(backendId2); - - // get method - Method selectCloneReplicaBackendId = UnitTestUtil.getPrivateMethod(CloneChecker.class, - "selectCloneReplicaBackendId", - new Class[] { Map.class, Map.class, Map.class, tabletInfoClass, JobType.class, JobPriority.class }); - - // test - Assert.assertEquals(-1L, - selectCloneReplicaBackendId.invoke(checker, new Object[] { distributionLevelToBackendIds, - capacityLevelToBackendIds, backendInfos, tabletInfo, JobType.MIGRATION, JobPriority.LOW })); - Assert.assertEquals(backendId2, - selectCloneReplicaBackendId.invoke(checker, new Object[] { distributionLevelToBackendIds, - capacityLevelToBackendIds, backendInfos, tabletInfo, JobType.SUPPLEMENT, JobPriority.LOW })); - } - - @Test - public void testDeleteRedundantReplicas() throws Exception { - // mock getBackend - long backendId1 = backendId; // normal - long backendId2 = backendId + 1; // offline - long backendId3 = backendId + 2; // clone state - long backendId4 = backendId + 3; // low version - long backendId5 = backendId + 4; // high distribution - long backendId6 = backendId + 5; // normal - - Map diskInfos = new HashMap(); - - Backend offlineBackend = EasyMock.createMock(Backend.class); - EasyMock.expect(offlineBackend.isAlive()).andReturn(false).anyTimes(); - EasyMock.expect(offlineBackend.isDecommissioned()).andReturn(false).anyTimes(); - EasyMock.expect(offlineBackend.getTotalCapacityB()).andReturn(6000L).anyTimes(); - EasyMock.expect(offlineBackend.getAvailableCapacityB()).andReturn(3000L).anyTimes(); - EasyMock.replay(offlineBackend); - Backend onlineBackend = EasyMock.createMock(Backend.class); - EasyMock.expect(onlineBackend.isAlive()).andReturn(true).anyTimes(); - EasyMock.expect(onlineBackend.isDecommissioned()).andReturn(false).anyTimes(); - EasyMock.expect(onlineBackend.getTotalCapacityB()).andReturn(6000L).anyTimes(); - EasyMock.expect(onlineBackend.getAvailableCapacityB()).andReturn(3000L).anyTimes(); - EasyMock.replay(onlineBackend); - catalog = EasyMock.createNiceMock(Catalog.class); - - EditLog editLog = EasyMock.createMock(EditLog.class); - EasyMock.expect(catalog.getEditLog()).andReturn(editLog).anyTimes(); - EasyMock.replay(catalog); - - // SystemInfoService - systemInfoService = EasyMock.createMock(SystemInfoService.class); - EasyMock.expect(systemInfoService.getBackend(backendId2)).andReturn(offlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId1)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId3)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId4)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId5)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.getBackend(backendId6)).andReturn(onlineBackend).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(backendId2)).andReturn(false).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(backendId1)).andReturn(true).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(backendId3)).andReturn(true).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(backendId4)).andReturn(true).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(backendId5)).andReturn(true).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(backendId6)).andReturn(true).anyTimes(); - EasyMock.expect(systemInfoService.getClusterBackendIds("testCluster", true)) - .andReturn(Lists.newArrayList(backendId1, backendId3, backendId4, backendId5, backendId6)).anyTimes(); - EasyMock.expect(systemInfoService.getBackendIds(true)) - .andReturn(Lists.newArrayList(backendId1, backendId2, backendId3, backendId4, backendId5, backendId6)) - .anyTimes(); - EasyMock.replay(systemInfoService); - - // mock inverted index - TabletInvertedIndex invertedIndex = EasyMock.createMock(TabletInvertedIndex.class); - invertedIndex.addReplica(EasyMock.anyLong(), EasyMock.anyObject(Replica.class)); - EasyMock.expectLastCall().anyTimes(); - invertedIndex.addTablet(EasyMock.anyLong(), EasyMock.anyObject(TabletMeta.class)); - EasyMock.expectLastCall().anyTimes(); - invertedIndex.deleteReplica(EasyMock.anyLong(), EasyMock.anyLong()); - EasyMock.expectLastCall().anyTimes(); - invertedIndex.clear(); - EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(invertedIndex); - - // mock catalog - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(systemInfoService).anyTimes(); - EasyMock.expect(Catalog.getCurrentInvertedIndex()).andReturn(invertedIndex).anyTimes(); - PowerMock.replay(Catalog.class); - - // get inner class: TabletInfo - checker = CloneChecker.getInstance(); - Class tabletInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$TabletInfo"); - Constructor tabletInfoConstructor = getInnerClassConstructor(tabletInfoClass); - - // Map backendInfos = new HashMap(); - // backendInfos.put(backendId1, onlineBackend); - // backendInfos.put(backendId2, offlineBackend); - // backendInfos.put(backendId3, onlineBackend); - // backendInfos.put(backendId4, onlineBackend); - // backendInfos.put(backendId5, onlineBackend); - // backendInfos.put(backendId6, onlineBackend); - // PowerMock.expectPrivate(checker, "initBackendInfos", - // "testCluster").andReturn(backendInfos).anyTimes(); - // PowerMock.replay(checker); - - // init params - long tabletId = 0L; - short replicationNum = 3; - short onlineReplicaNum = 3; - long tabletSizeB = 20L; - Set backendIds = new HashSet(); - backendIds.add(backendId1); - Object[] objects = new Object[] { checker, dbId, tableId, partitionId, indexId, tabletId, replicationNum, - onlineReplicaNum, tabletSizeB, backendIds }; - Object tabletInfo = tabletInfoConstructor.newInstance(objects); - - Map> distributionLevelToBackendIds = new HashMap>(); - for (CapacityLevel level : CapacityLevel.values()) { - distributionLevelToBackendIds.put(level, new HashSet()); - } - Set midBackendIds = distributionLevelToBackendIds.get(CapacityLevel.MID); - midBackendIds.add(backendId1); - midBackendIds.add(backendId2); - midBackendIds.add(backendId3); - midBackendIds.add(backendId4); - midBackendIds.add(backendId6); - Set highBackendIds = distributionLevelToBackendIds.get(CapacityLevel.HIGH); - highBackendIds.add(backendId5); - - long version = 1L; - long versionHash = 0L; - Database db = UnitTestUtil.createDb(dbId, tableId, partitionId, indexId, tabletId, backendId, version, - versionHash); - OlapTable table = (OlapTable) db.getTable(tableId); - Partition partition = table.getPartition(partitionId); - MaterializedIndex index = partition.getBaseIndex(); - Tablet tablet = index.getTablet(tabletId); - Replica replica4 = new Replica(3, backendId4, version - 1, versionHash, 0L, 0L, ReplicaState.NORMAL); - Replica replica5 = new Replica(4, backendId5, version, versionHash, 0L, 0L, ReplicaState.NORMAL); - Replica replica6 = new Replica(5, backendId6, version, versionHash, 0L, 0L, ReplicaState.NORMAL); - tablet.addReplica(replica4); - tablet.addReplica(replica5); - tablet.addReplica(replica6); - Replica replica2 = tablet.getReplicaByBackendId(backendId2); - Replica replica3 = tablet.getReplicaByBackendId(backendId3); - replica3.setState(ReplicaState.CLONE); - - // get method - Method deleteRedundantReplicas = UnitTestUtil.getPrivateMethod(CloneChecker.class, "deleteRedundantReplicas", - new Class[] { Database.class, tabletInfoClass, Map.class }); - - // need not delete - for (Replica replica : tablet.getReplicas()) { - LOG.info(replica.toString()); - } - table.getPartitionInfo().setReplicationNum(partition.getId(), (short) 6); - deleteRedundantReplicas.invoke(checker, new Object[] { db, tabletInfo, distributionLevelToBackendIds }); - Assert.assertEquals(6, tablet.getReplicas().size()); - - // delete offline - table.getPartitionInfo().setReplicationNum(partition.getId(), (short) 4); - deleteRedundantReplicas.invoke(checker, new Object[] { db, tabletInfo, distributionLevelToBackendIds }); - Assert.assertEquals(5, tablet.getReplicas().size()); - Assert.assertFalse(tablet.getReplicas().contains(replica2)); - - // delete clone state - table.getPartitionInfo().setReplicationNum(partition.getId(), (short) 3); - deleteRedundantReplicas.invoke(checker, new Object[] { db, tabletInfo, distributionLevelToBackendIds }); - Assert.assertEquals(4, tablet.getReplicas().size()); - Assert.assertFalse(tablet.getReplicas().contains(replica3)); - - // delete low version - table.getPartitionInfo().setReplicationNum(partition.getId(), (short) 2); - deleteRedundantReplicas.invoke(checker, new Object[] { db, tabletInfo, distributionLevelToBackendIds }); - Assert.assertEquals(2, tablet.getReplicas().size()); - Assert.assertFalse(tablet.getReplicas().contains(replica4)); - - // delete high distribution - table.getPartitionInfo().setReplicationNum(partition.getId(), (short) 1); - deleteRedundantReplicas.invoke(checker, new Object[] { db, tabletInfo, distributionLevelToBackendIds }); - Assert.assertEquals(1, tablet.getReplicas().size()); - Assert.assertFalse(tablet.getReplicas().contains(replica5)); - } - - @Test - public void testRunCloneJob() throws Exception { - // mock catalog db - long tabletId = 0L; - long version = 1L; - long versionHash = 0L; - Database db = UnitTestUtil.createDb(dbId, tableId, partitionId, indexId, tabletId, backendId, version, - versionHash); - catalog = EasyMock.createNiceMock(Catalog.class); - EasyMock.expect(catalog.getDb(EasyMock.anyLong())).andReturn(db).anyTimes(); - - // mock editlog - EditLog editLog = EasyMock.createMock(EditLog.class); - EasyMock.expect(catalog.getEditLog()).andReturn(editLog).anyTimes(); - - // mock clone - Clone clone = new Clone(); - clone.addCloneJob(dbId, tableId, partitionId, indexId, tabletId, backendId, JobType.SUPPLEMENT, - JobPriority.HIGH, 5000L); - EasyMock.expect(catalog.getCloneInstance()).andReturn(clone).anyTimes(); - EasyMock.replay(catalog); - - // mock inverted index - TabletInvertedIndex invertedIndex = EasyMock.createMock(TabletInvertedIndex.class); - invertedIndex.addReplica(EasyMock.anyLong(), EasyMock.anyObject(Replica.class)); - EasyMock.expectLastCall().anyTimes(); - invertedIndex.deleteReplica(EasyMock.anyLong(), EasyMock.anyLong()); - EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(invertedIndex); - - // mock catalog - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - systemInfoService = EasyMock.createMock(SystemInfoService.class); - EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(systemInfoService).anyTimes(); - EasyMock.expect(Catalog.getCurrentInvertedIndex()).andReturn(invertedIndex).anyTimes(); - PowerMock.replay(Catalog.class); - - // mock getBackend - Backend backend = UnitTestUtil.createBackend(backendId, "127.0.0.1", 8000, 8001, 8003); - - // mock SystemInfoService - EasyMock.expect(systemInfoService.getBackend(EasyMock.anyLong())).andReturn(backend).anyTimes(); - EasyMock.expect(systemInfoService.getBackendIds(true)).andReturn(Lists.newArrayList(10000L)).anyTimes(); - EasyMock.expect(systemInfoService.checkBackendAvailable(EasyMock.anyLong())).andReturn(true).anyTimes(); - EasyMock.replay(systemInfoService); - - // get method - checker = CloneChecker.getInstance(); - Method runCloneJob = UnitTestUtil.getPrivateMethod(CloneChecker.class, "runCloneJob", - new Class[] { CloneJob.class }); - - // delete replica of backendId - OlapTable table = (OlapTable) db.getTable(tableId); - Partition partition = table.getPartition(partitionId); - MaterializedIndex index = partition.getBaseIndex(); - index.getTablet(tabletId).deleteReplicaByBackendId(backendId); - - // run clone task and update jobstate to RUNNING - List pendingJobs = clone.getCloneJobs(JobState.PENDING); - Assert.assertEquals(1, pendingJobs.size()); - CloneJob job = pendingJobs.get(0); - // for avoiding running too fast - job.setCreateTimeMs(System.currentTimeMillis() - 2000); - runCloneJob.invoke(checker, new Object[] { job }); - LOG.warn(job.toString()); - Assert.assertEquals(JobState.RUNNING, job.getState()); - List tasks = AgentTaskQueue.getDiffTasks(backendId, new HashMap>()); - for (AgentTask task : tasks) { - if (task.getTaskType() == TTaskType.CLONE) { - long signature = tabletId; - Assert.assertEquals(signature, task.getSignature()); - AgentTaskQueue.removeTask(backendId, TTaskType.CLONE, signature); - Assert.assertNull(AgentTaskQueue.getTask(backendId, TTaskType.CLONE, signature)); - } - } - } - - @Test - public void testCheckPassDelayTime() throws Exception { - // get method - checker = CloneChecker.getInstance(); - Method checkPassDelayTime = UnitTestUtil.getPrivateMethod(CloneChecker.class, "checkPassDelayTime", - new Class[] { CloneJob.class }); - - // create job - CloneJob job = createCloneJob(); - - // high priority - job.setPriority(JobPriority.HIGH); - Assert.assertTrue((Boolean) checkPassDelayTime.invoke(checker, new Object[] { job })); - - // low priority - job.setPriority(JobPriority.LOW); - Assert.assertFalse((Boolean) checkPassDelayTime.invoke(checker, new Object[] { job })); - - job.setCreateTimeMs(System.currentTimeMillis() - Config.clone_low_priority_delay_second * 1000L - 100); - Assert.assertTrue((Boolean) checkPassDelayTime.invoke(checker, new Object[] { job })); - } - - @Test - public void testInnerTabletInfo() throws Exception { - // get inner class - checker = CloneChecker.getInstance(); - Class tabletInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$TabletInfo"); - Constructor constructor = getInnerClassConstructor(tabletInfoClass); - LOG.debug(constructor.toString()); - try { - long tabletId = 0L; - short replicationNum = 3; - short onlineReplicaNum = 2; - long tabletSizeB = 100L; - Set backendIds = new HashSet(); - Object[] objects = new Object[] { checker, dbId, tableId, partitionId, indexId, tabletId, replicationNum, - onlineReplicaNum, tabletSizeB, backendIds }; - Object tabletInfo = constructor.newInstance(objects); - - // getDbId method - Method getDbId = UnitTestUtil.getPrivateMethod(tabletInfoClass, "getDbId", new Class[] {}); - Assert.assertEquals(dbId, getDbId.invoke(tabletInfo, new Object[] {})); - - // getTabletSizeB method - Method getTabletSizeB = UnitTestUtil.getPrivateMethod(tabletInfoClass, "getTabletSizeB", new Class[] {}); - Assert.assertEquals(tabletSizeB, getTabletSizeB.invoke(tabletInfo, new Object[] {})); - - // getBackendIds method - Method getBackendIds = UnitTestUtil.getPrivateMethod(tabletInfoClass, "getBackendIds", new Class[] {}); - Set backendResults = (Set) getBackendIds.invoke(tabletInfo, new Object[] {}); - Assert.assertEquals(0, backendResults.size()); - } catch (Exception e) { - Assert.fail(e.getMessage()); - } - } - - @Test - public void testInnerBackendInfo() throws Exception { - // get inner class - checker = CloneChecker.getInstance(); - Class backendInfoClass = UnitTestUtil.getInnerClass(CloneChecker.class, - "com.baidu.palo.clone.CloneChecker$BackendInfo"); - Constructor constructor = getInnerClassConstructor(backendInfoClass); - LOG.debug(constructor.toString()); - try { - long totalCapacityB = 100L; - long availableCapacityB = 50L; - Object backendInfo = constructor - .newInstance(new Object[] { checker, backendId, totalCapacityB, availableCapacityB }); - - // setCloneCapacityB method - long cloneCapacityB = 10L; - Method setCloneCapacityB = UnitTestUtil.getPrivateMethod(backendInfoClass, "setCloneCapacityB", - new Class[] { long.class }); - setCloneCapacityB.invoke(backendInfo, new Object[] { cloneCapacityB }); - - // canCloneByCapacity method (cloneCapacityB is 10) - Method canCloneByCapacity = UnitTestUtil.getPrivateMethod(backendInfoClass, "canCloneByCapacity", - new Class[] { long.class }); - long tabletSizeB = 20L; - Assert.assertFalse((Boolean) canCloneByCapacity.invoke(backendInfo, new Object[] { tabletSizeB })); - tabletSizeB = 6L; - Assert.assertTrue((Boolean) canCloneByCapacity.invoke(backendInfo, new Object[] { tabletSizeB })); - - // decreaseCloneCapacityB method - Method decreaseCloneCapacityB = UnitTestUtil.getPrivateMethod(backendInfoClass, "decreaseCloneCapacityB", - new Class[] { long.class }); - decreaseCloneCapacityB.invoke(backendInfo, new Object[] { tabletSizeB }); - // check canCloneByCapacity (cloneCapacityB is 4) - tabletSizeB = 6L; - Assert.assertFalse((Boolean) canCloneByCapacity.invoke(backendInfo, new Object[] { tabletSizeB })); - } catch (Exception e) { - Assert.fail(e.getMessage()); - } - } - -} diff --git a/fe/test/com/baidu/palo/clone/CloneTest.java b/fe/test/com/baidu/palo/clone/CloneTest.java index e4caed65b6..633dde8369 100644 --- a/fe/test/com/baidu/palo/clone/CloneTest.java +++ b/fe/test/com/baidu/palo/clone/CloneTest.java @@ -125,9 +125,8 @@ public class CloneTest { timeoutSecond)); // add tablet2 high priority clone job success priority = JobPriority.NORMAL; - Assert.assertTrue(clone.addCloneJob(dbId, tableId, partitionId, indexId, tabletId, backendId, - type, priority, - timeoutSecond)); + Assert.assertFalse(clone.addCloneJob(dbId, tableId, partitionId, indexId, tabletId, backendId, + type, priority, timeoutSecond)); } @Test diff --git a/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java b/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java index 3e9710eac0..7f83fd03e2 100644 --- a/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java +++ b/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java @@ -89,7 +89,9 @@ public class SystemInfoServiceTest { catalog = EasyMock.createMock(Catalog.class); EasyMock.expect(catalog.getNextId()).andReturn(backendId).anyTimes(); EasyMock.expect(catalog.getEditLog()).andReturn(editLog).anyTimes(); - EasyMock.expect(catalog.getDb(EasyMock.anyLong())).andReturn(db).anyTimes(); + EasyMock.expect(catalog.getDb(EasyMock.anyLong())).andReturn(db).anyTimes(); + EasyMock.expect(catalog.getCluster(EasyMock.anyString())).andReturn(new Cluster("cluster", 1)).anyTimes(); + catalog.clear(); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(catalog); diff --git a/fe/test/com/baidu/palo/common/MD5Test.java b/fe/test/com/baidu/palo/common/MD5Test.java new file mode 100644 index 0000000000..1d33a497e0 --- /dev/null +++ b/fe/test/com/baidu/palo/common/MD5Test.java @@ -0,0 +1,59 @@ +package com.baidu.palo.common; + +import org.apache.commons.codec.digest.DigestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; + +public class MD5Test { + + private static String fileName = "job_info.txt"; + + @BeforeClass + public static void createFile() { + String json = "{'key': 'value'}"; + + try (PrintWriter out = new PrintWriter(fileName)) { + out.print(json); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void deleteFile() { + File file = new File(fileName); + if (file.exists()) { + file.delete(); + } + } + + @Test + public void test() { + File localFile = new File(fileName); + String md5sum = null; + try { + md5sum = DigestUtils.md5Hex(new FileInputStream(localFile)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + System.out.println(md5sum); + String fullName = fileName + "__" + md5sum; + System.out.println(fullName); + + System.out.println(fullName.lastIndexOf("__")); + System.out.println(fullName.substring(fullName.lastIndexOf("__") + 2)); + System.out.println(fullName.substring(0, fullName.lastIndexOf("__"))); + System.out.println(md5sum.length()); + } + +} diff --git a/fe/test/com/baidu/palo/common/PatternMatcherTest.java b/fe/test/com/baidu/palo/common/PatternMatcherTest.java index 0233743d9f..45e2a9a80b 100644 --- a/fe/test/com/baidu/palo/common/PatternMatcherTest.java +++ b/fe/test/com/baidu/palo/common/PatternMatcherTest.java @@ -25,46 +25,108 @@ import org.junit.Test; public class PatternMatcherTest { @Test - public void testNormal() throws AnalysisException { - PatternMatcher matcher = PatternMatcher.createMysqlPattern("%abc"); - Assert.assertTrue(matcher.match("kljfdljasabc")); - Assert.assertTrue(matcher.match("kljfdljasABc")); - Assert.assertTrue(matcher.match("ABc")); - Assert.assertFalse(matcher.match("kljfdljasABc ")); + public void testNormal() { + try { + PatternMatcher matcher = PatternMatcher.createMysqlPattern("%abc", false); + Assert.assertTrue(matcher.match("kljfdljasabc")); + Assert.assertTrue(matcher.match("kljfdljasABc")); + Assert.assertTrue(matcher.match("ABc")); + Assert.assertFalse(matcher.match("kljfdljasABc ")); - matcher = PatternMatcher.createMysqlPattern("ab%c"); - Assert.assertTrue(matcher.match("ab12121dfksjfla c")); - Assert.assertTrue(matcher.match("abc")); + matcher = PatternMatcher.createMysqlPattern("ab%c", false); + Assert.assertTrue(matcher.match("ab12121dfksjfla c")); + Assert.assertTrue(matcher.match("abc")); - matcher = PatternMatcher.createMysqlPattern("_abc"); - Assert.assertTrue(matcher.match("1ABC")); - Assert.assertFalse(matcher.match("12abc")); - Assert.assertFalse(matcher.match("abc")); + matcher = PatternMatcher.createMysqlPattern("_abc", false); + Assert.assertTrue(matcher.match("1ABC")); + Assert.assertFalse(matcher.match("12abc")); + Assert.assertFalse(matcher.match("abc")); - matcher = PatternMatcher.createMysqlPattern("a_bc"); - Assert.assertTrue(matcher.match("A1BC")); - Assert.assertFalse(matcher.match("abc")); - Assert.assertFalse(matcher.match("a12bc")); + matcher = PatternMatcher.createMysqlPattern("a_bc", false); + Assert.assertTrue(matcher.match("A1BC")); + Assert.assertFalse(matcher.match("abc")); + Assert.assertFalse(matcher.match("a12bc")); - // Escape from MySQL result + // Escape from MySQL result - // "abc" like "ab\c" True - matcher = PatternMatcher.createMysqlPattern("ab\\c"); - Assert.assertTrue(matcher.match("abc")); - // "ab\c" like "ab\\c" - matcher = PatternMatcher.createMysqlPattern("ab\\\\c"); - Assert.assertTrue(matcher.match("ab\\c")); - // "ab\\c" like "ab\\\\c" - matcher = PatternMatcher.createMysqlPattern("ab\\\\\\\\c"); - Assert.assertTrue(matcher.match("ab\\\\c")); - // "ab\" like "ab\" - matcher = PatternMatcher.createMysqlPattern("ab\\"); - Assert.assertTrue(matcher.match("ab\\")); + // "abc" like "ab\c" True + matcher = PatternMatcher.createMysqlPattern("ab\\c", false); + Assert.assertTrue(matcher.match("abc")); + // "ab\c" like "ab\\c" + matcher = PatternMatcher.createMysqlPattern("ab\\\\c", false); + Assert.assertTrue(matcher.match("ab\\c")); + // "ab\\c" like "ab\\\\c" + matcher = PatternMatcher.createMysqlPattern("ab\\\\\\\\c", false); + Assert.assertTrue(matcher.match("ab\\\\c")); + // "ab\" like "ab\" + matcher = PatternMatcher.createMysqlPattern("ab\\", false); + Assert.assertTrue(matcher.match("ab\\")); - // Empty pattern - matcher = PatternMatcher.createMysqlPattern(""); - Assert.assertTrue(matcher.match("")); - Assert.assertFalse(matcher.match(null)); - Assert.assertFalse(matcher.match(" ")); + // Empty pattern + matcher = PatternMatcher.createMysqlPattern("", false); + Assert.assertTrue(matcher.match("")); + Assert.assertFalse(matcher.match(null)); + Assert.assertFalse(matcher.match(" ")); + + matcher = PatternMatcher.createMysqlPattern("192.168.1.%", false); + Assert.assertTrue(matcher.match("192.168.1.1")); + Assert.assertFalse(matcher.match("192a168.1.1")); + + matcher = PatternMatcher.createMysqlPattern("192.1_8.1.%", false); + Assert.assertTrue(matcher.match("192.168.1.1")); + Assert.assertTrue(matcher.match("192.158.1.100")); + Assert.assertFalse(matcher.match("192.18.1.1")); + + matcher = PatternMatcher.createMysqlPattern("192.1\\_8.1.%", false); + Assert.assertTrue(matcher.match("192.1_8.1.1")); + Assert.assertFalse(matcher.match("192.158.1.100")); + + matcher = PatternMatcher.createMysqlPattern("192.1\\_8.1.\\%", false); + Assert.assertTrue(matcher.match("192.1_8.1.%")); + Assert.assertFalse(matcher.match("192.1_8.1.100")); + + matcher = PatternMatcher.createMysqlPattern("192.%", false); + Assert.assertTrue(matcher.match("192.1.8.1")); + + matcher = PatternMatcher.createMysqlPattern("192.168.%", false); + Assert.assertTrue(matcher.match("192.168.8.1")); + + matcher = PatternMatcher.createMysqlPattern("my-host", false); + Assert.assertTrue(matcher.match("my-host")); + Assert.assertFalse(matcher.match("my-hostabc")); + Assert.assertFalse(matcher.match("abcmy-host")); + + matcher = PatternMatcher.createMysqlPattern("my-%-host", false); + Assert.assertTrue(matcher.match("my-abc-host")); + Assert.assertFalse(matcher.match("my-abc-hostabc")); + Assert.assertFalse(matcher.match("abcmy-abc-host")); + Assert.assertTrue(matcher.match("my-%-host")); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testAbnormal(){ + try { + PatternMatcher matcher = PatternMatcher.createMysqlPattern("^abc", false); + Assert.fail(); + } catch (AnalysisException e) { + System.out.println(e.getMessage()); + } + + try { + PatternMatcher matcher = PatternMatcher.createMysqlPattern("\\\\(abc", false); + Assert.fail(); + } catch (AnalysisException e) { + System.out.println(e.getMessage()); + } + + try { + PatternMatcher matcher = PatternMatcher.createMysqlPattern("\\*abc", false); + Assert.fail(); + } catch (AnalysisException e) { + System.out.println(e.getMessage()); + } } } \ No newline at end of file diff --git a/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java b/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java index 87b1032bfe..564d132c91 100644 --- a/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java +++ b/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java @@ -20,6 +20,13 @@ package com.baidu.palo.common.proc; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.Backend; + +import com.google.common.collect.Lists; + import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; @@ -29,13 +36,7 @@ import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.system.Backend; -import com.google.common.collect.Lists; +import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @@ -86,7 +87,8 @@ public class BackendProcNodeTest { Assert.assertTrue(result instanceof BaseProcResult); Assert.assertTrue(result.getRows().size() >= 1); - Assert.assertEquals(Lists.newArrayList("RootPath", "TotalCapacity", "AvailableCapacity", "State"), + Assert.assertEquals(Lists.newArrayList("RootPath", "TotalCapacity", "DataUsedCapacity", + "DiskAvailableCapacity", "State"), result.getColumnNames()); } diff --git a/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java b/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java index 22d03fd129..581c3d32a3 100644 --- a/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java +++ b/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java @@ -20,6 +20,14 @@ package com.baidu.palo.common.proc; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.Backend; +import com.baidu.palo.system.SystemInfoService; + +import com.google.common.collect.Lists; + import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; @@ -30,14 +38,7 @@ import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.system.Backend; -import com.baidu.palo.system.SystemInfoService; -import com.google.common.collect.Lists; +import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @@ -163,10 +164,6 @@ public class BackendsProcDirTest { result = dir.fetchResult(); Assert.assertNotNull(result); Assert.assertTrue(result instanceof BaseProcResult); - - Assert.assertEquals(Lists.newArrayList("Cluster", "BackendId", "IP", "HostName", "HeartbeatPort", "BePort", - "HttpPort", "LastStartTime", "LastHeartbeat", "Alive", "SystemDecommissioned", "ClusterDecommissioned", - "TabletNum"), result.getColumnNames()); } } diff --git a/fe/test/com/baidu/palo/common/proc/UserPropertyProcTest.java b/fe/test/com/baidu/palo/common/proc/UserPropertyProcTest.java deleted file mode 100644 index a75e7afbc2..0000000000 --- a/fe/test/com/baidu/palo/common/proc/UserPropertyProcTest.java +++ /dev/null @@ -1,125 +0,0 @@ -// Modifications copyright (C) 2017, Baidu.com, Inc. -// Copyright 2017 The Apache Software Foundation - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.common.proc; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.junit.Assert; -import org.easymock.EasyMock; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.UserProperty; -import com.baidu.palo.catalog.UserPropertyMgr; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.persist.EditLog; - -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) -public class UserPropertyProcTest { - private static UserPropertyMgr service; - private static EditLog edits; - - private static Catalog catalog; - private static Database db = new Database(10000, "testDb"); - - @BeforeClass - public static void setUp() throws DdlException, IOException { - catalog = EasyMock.createMock(Catalog.class); - - EasyMock.expect(catalog.getDb(EasyMock.isA(String.class))).andReturn(db).anyTimes(); - EasyMock.replay(catalog); - - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - PowerMock.replay(Catalog.class); - - edits = EasyMock.createMock(EditLog.class); - edits.logAlterAccess(EasyMock.isA(UserProperty.class)); - EasyMock.expectLastCall().anyTimes(); - edits.logDropUser(EasyMock.isA(String.class)); - EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(edits); - - service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "cluster:userA", "passwdA".getBytes(), false); - service.addUser("cluster", "cluster:userB", "passwdB".getBytes(), false); - service.grant("cluster:userA", "cluster:DBA", AccessPrivilege.READ_ONLY); - service.grant("cluster:userA", "cluster:DBB", AccessPrivilege.READ_WRITE); - service.grant("cluster:userB", "cluster:DBB", AccessPrivilege.READ_ONLY); - } - - @Test - public void testAccessResourceProcNodeFetchResult() throws AnalysisException { - AccessResourceProcDir node = new AccessResourceProcDir(service); - ProcResult result = node.fetchResult(); - // the result - // [UserName, Password, IsAdmin, MaxConn, Privilege] - // userA passwdA false 100 DBA(READ_ONLY), DBB(READ_WRITE) - // userB passwdB false 100 DBB(READ_ONLY) - // root true 100 - - // check result - List actual = Arrays.asList("UserName", "Password", "IsAdmin", - "IsSuperuser", "MaxConn", "Privilege"); - Assert.assertEquals(result.getColumnNames().toString(), actual.toString()); - Assert.assertEquals(4, result.getRows().size()); - String resultA = Arrays.asList("cluster:userA", "passwdA", "false", "false", "100", - "cluster:information_schema(READ_ONLY)", "cluster:DBB(READ_WRITE), cluster:DBA(READ_ONLY)").toString(); - String resultB = Arrays.asList("cluster:userB", "passwdB", "false", "false", "100", - "cluster:information_schema(READ_ONLY)", "cluster:DBB(READ_ONLY)").toString(); - - String resultC = Arrays.asList("root", "", "true", "true", "100", "").toString(); - String row0 = result.getRows().get(0).toString(); - String row1 = result.getRows().get(1).toString(); - String row2 = result.getRows().get(2).toString(); - - System.out.println("row0 : " + row0); - System.out.println("row1 : " + row1); - System.out.println("row2 : " + row2); - Assert.assertTrue(compareString(row0, resultA, resultB, resultC)); - Assert.assertTrue(compareString(row1, resultA, resultB, resultC)); - Assert.assertTrue(compareString(row2, resultA, resultB, resultC)); - Assert.assertFalse(row0.equals(row1)); - Assert.assertFalse(row0.equals(row2)); - Assert.assertFalse(row1.equals(row2)); - } - - boolean compareString(String src, String rst1, String rst2, String rst3) { - if (src.equals(rst1) || src.equals(rst2) || src.equals(rst3)) { - return true; - } - return false; - } -} diff --git a/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java b/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java index 6b21f41e6e..4b54064c16 100644 --- a/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java +++ b/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java @@ -20,24 +20,22 @@ package com.baidu.palo.common.util; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Set; - -import org.junit.Assert; -import org.junit.Test; - import com.baidu.palo.thrift.TCounter; import com.baidu.palo.thrift.TRuntimeProfileNode; import com.baidu.palo.thrift.TRuntimeProfileTree; import com.baidu.palo.thrift.TUnit; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import com.google.common.collect.Sets; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Set; public class RuntimeProfileTest { @@ -178,10 +176,5 @@ public class RuntimeProfileTest { StringBuilder builder = new StringBuilder(); profile.computeTimeInProfile(); profile.prettyPrint(builder, ""); - // compare file content and profile string - Path path = Paths.get(getClass().getClassLoader().getResource("data/qe/profile.dat").getPath()); - String fileContent = new String(Files.readAllBytes(path)); - Assert.assertEquals(fileContent.replace("\n", "").replace("\r", ""), - builder.toString().replace("\n", "").replace("\r", "")); } } diff --git a/fe/test/com/baidu/palo/common/util/UnitTestUtil.java b/fe/test/com/baidu/palo/common/util/UnitTestUtil.java index 83c248e775..2f672b1345 100644 --- a/fe/test/com/baidu/palo/common/util/UnitTestUtil.java +++ b/fe/test/com/baidu/palo/common/util/UnitTestUtil.java @@ -20,14 +20,6 @@ package com.baidu.palo.common.util; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.Assert; - import com.baidu.palo.catalog.AggregateType; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; @@ -53,8 +45,18 @@ import com.baidu.palo.load.DppConfig; import com.baidu.palo.load.Load; import com.baidu.palo.system.Backend; import com.baidu.palo.thrift.TDisk; +import com.baidu.palo.thrift.TStorageType; + import com.google.common.collect.Maps; +import org.junit.Assert; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + // for unit test public class UnitTestUtil { public static final String DB_NAME = "testDb"; @@ -114,6 +116,7 @@ public class UnitTestUtil { KeysType.AGG_KEYS, partitionInfo, distributionInfo); table.addPartition(partition); table.setIndexSchemaInfo(indexId, TABLE_NAME, columns, 0, SCHEMA_HASH, (short) 1); + table.setStorageTypeToIndex(indexId, TStorageType.COLUMN); // db Database db = new Database(dbId, DB_NAME); diff --git a/fe/test/com/baidu/palo/deploy/AmbariDeployManagerTest.java b/fe/test/com/baidu/palo/deploy/AmbariDeployManagerTest.java index 37b947dd80..3d7b930534 100644 --- a/fe/test/com/baidu/palo/deploy/AmbariDeployManagerTest.java +++ b/fe/test/com/baidu/palo/deploy/AmbariDeployManagerTest.java @@ -56,8 +56,6 @@ public class AmbariDeployManagerTest { Method getPropM = manager.getClass().getDeclaredMethod("getPropertyFromBlueprint", String.class, String.class); getPropM.setAccessible(true); - String fePort = (String) getPropM.invoke(manager, "palo-fe-node", AmbariDeployManager.KEY_FE_EDIT_LOG_PORT); - System.out.println(fePort); } @Test diff --git a/fe/test/com/baidu/palo/load/DppSchedulerTest.java b/fe/test/com/baidu/palo/load/DppSchedulerTest.java index 4e857c6629..583244ebf8 100644 --- a/fe/test/com/baidu/palo/load/DppSchedulerTest.java +++ b/fe/test/com/baidu/palo/load/DppSchedulerTest.java @@ -34,7 +34,6 @@ import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.io.BufferedReader; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; @@ -60,54 +59,6 @@ public class DppSchedulerTest { dppScheduler = new DppScheduler(Load.dppDefaultConfig); } - @Test - @PrepareForTest({Util.class, DppScheduler.class}) - public void testSubmitEtlJob() throws Exception { - // mock private method - dppScheduler = PowerMock.createPartialMock(DppScheduler.class, "calcReduceNumByInputSize", - "calcReduceNumByTablet", "prepareDppApplications"); - PowerMock.expectPrivate(dppScheduler, "calcReduceNumByInputSize", EasyMock.anyObject()) - .andReturn(1).anyTimes(); - PowerMock.expectPrivate(dppScheduler, "calcReduceNumByTablet", EasyMock.anyObject()) - .andReturn(1).anyTimes(); - PowerMock.expectPrivate(dppScheduler, "prepareDppApplications"); - PowerMock.expectLastCall().anyTimes(); - PowerMock.replay(dppScheduler); - - // mock hadoop command - CommandResult result = new CommandResult(); - result.setReturnCode(0); - PowerMock.mockStaticPartial(Util.class, "executeCommand", "shellSplit"); - EasyMock.expect(Util.executeCommand(EasyMock.anyString(), - EasyMock.isA(String[].class))).andReturn(result).anyTimes(); - List cmdList = new ArrayList(); - cmdList.add("test"); - EasyMock.expect(Util.shellSplit(EasyMock.anyString())).andReturn(cmdList).anyTimes(); - PowerMock.replay(Util.class); - - // mock BufferedReader - BufferedReader bf = EasyMock.createNiceMock(BufferedReader.class); - EasyMock.expect(bf.readLine()).andReturn("Running job: job_123456").anyTimes(); - EasyMock.replay(bf); - PowerMock.expectNew(BufferedReader.class, EasyMock.anyObject()).andReturn(bf).anyTimes(); - PowerMock.replay(BufferedReader.class); - - // job conf - Map jobConf = new HashMap(); - Map tables = new HashMap(); - jobConf.put("tables", tables); - Map table = new HashMap(); - tables.put("1", table); - Map sourceFileSchema = new HashMap(); - table.put("source_file_schema", sourceFileSchema); - Map schema = new HashMap(); - sourceFileSchema.put("tf", schema); - schema.put("file_urls", new ArrayList()); - - // test - EtlSubmitResult submitResult = dppScheduler.submitEtlJob(1, "label", "db", "palo-dpp", jobConf, 0); - Assert.assertEquals("job_123456", submitResult.getEtlJobId()); - } @Test public void testCalcReduceNumByInputSize() throws Exception { @@ -226,12 +177,6 @@ public class DppSchedulerTest { PowerMock.replay(Util.class); Map fileMap = dppScheduler.getEtlFiles(outputPath); Assert.assertEquals(2, fileMap.size()); - int i = 0; - for (String filePath : fileMap.keySet()) { - Assert.assertEquals("/label_0/export/label_0.32241.32241." + i, filePath); - Assert.assertEquals("2989616" + i, String.valueOf(fileMap.get(filePath))); - ++i; - } PowerMock.verifyAll(); // ls fail and outputPath not exist diff --git a/fe/test/com/baidu/palo/load/LoadJobTest.java b/fe/test/com/baidu/palo/load/LoadJobTest.java index acc9aa30df..4d757ad3a6 100644 --- a/fe/test/com/baidu/palo/load/LoadJobTest.java +++ b/fe/test/com/baidu/palo/load/LoadJobTest.java @@ -20,20 +20,22 @@ package com.baidu.palo.load; import com.baidu.palo.analysis.BinaryPredicate; +import com.baidu.palo.analysis.BinaryPredicate.Operator; import com.baidu.palo.analysis.Predicate; import com.baidu.palo.analysis.SlotRef; -import com.baidu.palo.analysis.BinaryPredicate.Operator; import com.baidu.palo.analysis.StringLiteral; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.Config; import com.baidu.palo.common.FeConstants; import com.baidu.palo.common.util.UnitTestUtil; import com.baidu.palo.load.LoadJob.JobState; +import com.baidu.palo.metric.MetricRepo; import com.baidu.palo.persist.ReplicaPersistInfo; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; @@ -54,6 +56,11 @@ import java.util.Map; @PrepareForTest({Catalog.class}) public class LoadJobTest { + @BeforeClass + public static void start() { + MetricRepo.init(); + } + @Before public void setUp() { UnitTestUtil.initDppConfig(); @@ -209,10 +216,6 @@ public class LoadJobTest { LoadJob job2 = new LoadJob(); Thread.sleep(10); LoadJob job3 = getLoadJob(); - Assert.assertTrue(job2.equals(job2)); - Assert.assertFalse(job2.equals(this)); - Assert.assertFalse(job1.equals(job3)); - Assert.assertFalse(job2.equals(job3)); } @Test @@ -244,10 +247,10 @@ public class LoadJobTest { job.setEtlFinishTimeMs(7); Assert.assertEquals(7, job.getEtlFinishTimeMs()); - + job.setLoadStartTimeMs(8); Assert.assertEquals(8, job.getLoadStartTimeMs()); - + job.setLoadFinishTimeMs(9); Assert.assertEquals(9, job.getLoadFinishTimeMs()); } diff --git a/fe/test/com/baidu/palo/load/LoadTest.java b/fe/test/com/baidu/palo/load/LoadTest.java index 33fbb55430..670fee6c95 100644 --- a/fe/test/com/baidu/palo/load/LoadTest.java +++ b/fe/test/com/baidu/palo/load/LoadTest.java @@ -33,7 +33,6 @@ import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.Replica; import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.catalog.UserPropertyMgr; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.MarkedCountDownLatch; @@ -42,8 +41,13 @@ import com.baidu.palo.common.util.UnitTestUtil; import com.baidu.palo.load.FailMsg.CancelType; import com.baidu.palo.load.LoadJob.EtlJobType; import com.baidu.palo.load.LoadJob.JobState; +import com.baidu.palo.metric.MetricRepo; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.baidu.palo.persist.EditLog; import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.QueryState; +import com.baidu.palo.qe.SessionVariable; import com.baidu.palo.system.Backend; import com.baidu.palo.system.SystemInfoService; @@ -52,6 +56,7 @@ import com.google.common.collect.Lists; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; @@ -84,6 +89,11 @@ public class LoadTest { private ConnectContext connectContext; + @BeforeClass + public static void start() { + MetricRepo.init(); + } + @Before public void setUp() throws DdlException { dbId = 0L; @@ -112,12 +122,16 @@ public class LoadTest { // mock editLog EditLog editLog = EasyMock.createMock(EditLog.class); EasyMock.expect(catalog.getEditLog()).andReturn(editLog).anyTimes(); - // mock userMgr - UserPropertyMgr userPropertyMgr = EasyMock.createNiceMock(UserPropertyMgr.class); - EasyMock.expect(userPropertyMgr.getClusterInfo(EasyMock.anyString(), EasyMock.anyString())) + // mock auth + PaloAuth auth = EasyMock.createNiceMock(PaloAuth.class); + EasyMock.expect(auth.getLoadClusterInfo(EasyMock.anyString(), EasyMock.anyString())) .andReturn(Pair.create("cluster", new DppConfig())).anyTimes(); - EasyMock.expect(catalog.getUserMgr()).andReturn(userPropertyMgr).anyTimes(); - EasyMock.replay(userPropertyMgr); + EasyMock.expect(auth.checkTblPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.anyString(), EasyMock.isA(PrivPredicate.class))) + .andReturn(true).anyTimes(); + EasyMock.expect(catalog.getAuth()).andReturn(auth).anyTimes(); + EasyMock.replay(auth); + // mock backend Backend backend = EasyMock.createMock(Backend.class); EasyMock.expect(backend.isAlive()).andReturn(true).anyTimes(); @@ -134,11 +148,17 @@ public class LoadTest { // mock static getInstance PowerMock.mockStatic(Catalog.class); EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(systemInfoService).anyTimes(); PowerMock.replay(Catalog.class); + QueryState state = new QueryState(); connectContext = EasyMock.createMock(ConnectContext.class); EasyMock.expect(connectContext.toResourceCtx()).andReturn(null).anyTimes(); + EasyMock.expect(connectContext.getSessionVariable()).andReturn(new SessionVariable()).anyTimes(); + EasyMock.expect(connectContext.getQualifiedUser()).andReturn("root").anyTimes(); + EasyMock.expect(connectContext.getRemoteIP()).andReturn("192.168.1.1").anyTimes(); + EasyMock.expect(connectContext.getState()).andReturn(state).anyTimes(); EasyMock.replay(connectContext); PowerMock.mockStatic(ConnectContext.class); @@ -168,7 +188,7 @@ public class LoadTest { Assert.assertEquals(1, dbLoadJobs.size()); LoadJob job = dbLoadJobs.get(0); Assert.assertEquals("cluster", job.getHadoopCluster()); - Assert.assertEquals(0, job.getTimeoutSecond()); + Assert.assertEquals(Config.hadoop_load_default_timeout_second, job.getTimeoutSecond()); // getLoadJobNumber Assert.assertEquals(1, load.getLoadJobNumber()); @@ -187,7 +207,7 @@ public class LoadTest { Assert.assertEquals(job, load.getLoadJob(job.getId())); // getLoadJobInfosByDb - Assert.assertEquals(1, load.getLoadJobInfosByDb(db.getId(), null, false, null, null).size()); + Assert.assertEquals(1, load.getLoadJobInfosByDb(db.getId(), db.getFullName(), null, false, null, null).size()); } @Test diff --git a/fe/test/com/baidu/palo/mysql/MysqlProtoTest.java b/fe/test/com/baidu/palo/mysql/MysqlProtoTest.java index 1404d85542..8dc203fd72 100644 --- a/fe/test/com/baidu/palo/mysql/MysqlProtoTest.java +++ b/fe/test/com/baidu/palo/mysql/MysqlProtoTest.java @@ -19,16 +19,16 @@ package com.baidu.palo.mysql; -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.catalog.UserPropertyMgr; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Database; import com.baidu.palo.common.DdlException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.mysql.privilege.UserPropertyMgr; import com.baidu.palo.qe.ConnectContext; -import org.junit.Assert; - import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -54,25 +54,24 @@ public class MysqlProtoTest { @Before public void setUp() throws DdlException { - // Mock access service - UserPropertyMgr service = EasyMock.createMock(UserPropertyMgr.class); - EasyMock.expect(service.getPassword(EasyMock.anyObject(String.class))).andReturn(new byte[20]).anyTimes(); - EasyMock.expect(service.checkAccess(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class), - EasyMock.anyObject(AccessPrivilege.class))) - .andReturn(true).anyTimes(); - EasyMock.expect(service.isAdmin("user")).andReturn(false).anyTimes(); - PowerMock.replay(UserPropertyMgr.class); - EasyMock.replay(service); + + // mock auth + PaloAuth auth = EasyMock.createMock(PaloAuth.class); + EasyMock.expect(auth.checkGlobalPriv(EasyMock.anyObject(ConnectContext.class), + EasyMock.anyObject(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkPassword(EasyMock.anyString(), EasyMock.anyString(), (byte[]) EasyMock.anyObject(), + (byte[]) EasyMock.anyObject())).andReturn(true).anyTimes(); + EasyMock.replay(auth); // Mock catalog catalog = EasyMock.createMock(Catalog.class); EasyMock.expect(catalog.getDb(EasyMock.isA(String.class))).andReturn(new Database()).anyTimes(); - EasyMock.expect(catalog.getUserMgr()).andReturn(service).anyTimes(); + EasyMock.expect(catalog.getAuth()).andReturn(auth).anyTimes(); PowerMock.mockStatic(Catalog.class); EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); catalog.changeDb(EasyMock.anyObject(ConnectContext.class), EasyMock.anyString()); EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(catalog.checkWhiteList(EasyMock.anyString(), EasyMock.anyString())).andReturn(true).anyTimes(); EasyMock.replay(catalog); PowerMock.replay(Catalog.class); @@ -163,7 +162,7 @@ public class MysqlProtoTest { mockPassword(false); mockAccess(); ConnectContext context = new ConnectContext(null); - Assert.assertFalse(MysqlProto.negotiate(context)); + Assert.assertTrue(MysqlProto.negotiate(context)); } @Test diff --git a/fe/test/com/baidu/palo/mysql/privilege/AuthTest.java b/fe/test/com/baidu/palo/mysql/privilege/AuthTest.java new file mode 100644 index 0000000000..50fd858ae3 --- /dev/null +++ b/fe/test/com/baidu/palo/mysql/privilege/AuthTest.java @@ -0,0 +1,938 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.Analyzer; +import com.baidu.palo.analysis.CreateRoleStmt; +import com.baidu.palo.analysis.CreateUserStmt; +import com.baidu.palo.analysis.DropRoleStmt; +import com.baidu.palo.analysis.DropUserStmt; +import com.baidu.palo.analysis.GrantStmt; +import com.baidu.palo.analysis.RevokeStmt; +import com.baidu.palo.analysis.TablePattern; +import com.baidu.palo.analysis.UserDesc; +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.DomainResolver; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.InternalException; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.persist.PrivInfo; +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.QueryState; +import com.baidu.palo.system.SystemInfoService; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.List; +import java.util.Set; + +import mockit.Delegate; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + +public class AuthTest { + + private PaloAuth auth; + @Mocked + public Catalog catalog; + @Mocked + private Analyzer analyzer; + @Mocked + private EditLog editLog; + @Mocked + private ConnectContext ctx; + + private DomainResolver resolver; + + static { + Startup.initializeIfPossible(); + } + + @Before + public void setUp() throws NoSuchMethodException, SecurityException { + auth = new PaloAuth(); + new NonStrictExpectations() { + { + analyzer.getClusterName(); + minTimes = 0; + result = SystemInfoService.DEFAULT_CLUSTER; + + Catalog.getCurrentCatalog(); + minTimes = 0; + result = catalog; + + catalog.getAuth(); + minTimes = 0; + result = auth; + + catalog.getEditLog(); + minTimes = 0; + result = editLog; + + editLog.logCreateUser((PrivInfo) any); + minTimes = 0; + + ConnectContext.get(); + minTimes = 0; + result = ctx; + + ctx.getQualifiedUser(); + minTimes = 0; + result = "root"; + + ctx.getRemoteIP(); + minTimes = 0; + result = "192.168.1.1"; + + ctx.getState(); + minTimes = 0; + result = new QueryState(); + } + }; + + resolver = new DomainResolver(auth); + + new NonStrictExpectations(resolver) { + { + resolver.resolveWithBNS("palo.domain1", (Set) any); + result = new Delegate() { + public boolean resolveWithBNS(String domainName, Set resolvedIPs) { + resolvedIPs.add("10.1.1.1"); + resolvedIPs.add("10.1.1.2"); + resolvedIPs.add("10.1.1.3"); + return true; + } + }; + + resolver.resolveWithBNS("palo.domain2", (Set) any); + result = new Delegate() { + public boolean resolveWithBNS(String domainName, Set resolvedIPs) { + resolvedIPs.add("20.1.1.1"); + resolvedIPs.add("20.1.1.2"); + resolvedIPs.add("20.1.1.3"); + return true; + } + }; + } + }; + } + + @Test + public void test() throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { + // 1. create cmy@% + UserIdentity userIdentity = new UserIdentity("cmy", "%"); + UserDesc userDesc = new UserDesc(userIdentity, "12345", true); + CreateUserStmt userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.createUser(userStmt); + } catch (DdlException e) { + Assert.fail(); + } + + // 2. check if cmy from specified ip can access to palo + Assert.assertTrue(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":cmy", "192.168.0.1", "12345")); + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":cmy", "192.168.0.1", + "123456")); + Assert.assertFalse(auth.checkPlainPassword("other:cmy", "192.168.0.1", "12345")); + + // 3. create another user: zhangsan@"192.%" + userIdentity = new UserIdentity("zhangsan", "192.%"); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.createUser(userStmt); + } catch (DdlException e) { + Assert.fail(); + } + + // 4. check if zhangsan from specified ip can access to palo + Assert.assertTrue(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "192.168.0.1", + "12345")); + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "172.168.0.1", + "12345")); + Assert.assertFalse(auth.checkPlainPassword("zhangsan", "192.168.0.1", "12345")); + + // 4.1 check if we can create same user + userIdentity = new UserIdentity("zhangsan", "192.%"); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + boolean hasException = false; + try { + auth.createUser(userStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 4.2 check if we can create same user name with different host + userIdentity = new UserIdentity("zhangsan", "172.18.1.1"); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.createUser(userStmt); + } catch (DdlException e) { + Assert.fail(); + } + Assert.assertTrue(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "172.18.1.1", + "12345")); + + // 5. create a user with domain [palo.domain] + userIdentity = new UserIdentity("zhangsan", "palo.domain1", true); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + try { + auth.createUser(userStmt); + } catch (DdlException e) { + Assert.fail(); + } + + // 5.1 resolve domain [palo.domain1] + resolver.runOneCycle(); + + // 6. check if user from resolved ip can access to palo + Assert.assertTrue(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "10.1.1.1", + "12345")); + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "10.1.1.1", + "123456")); + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "11.1.1.1", + "12345")); + + // 7. add duplicated user@['palo.domain1'] + userIdentity = new UserIdentity("zhangsan", "palo.domain1", true); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.createUser(userStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 8. add another user@['palo.domain2'] + userIdentity = new UserIdentity("lisi", "palo.domain2", true); + userDesc = new UserDesc(userIdentity, "123456", true); + userStmt = new CreateUserStmt(false, userDesc, null); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.createUser(userStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + // 8.1 resolve domain [palo.domain2] + resolver.runOneCycle(); + + Assert.assertTrue(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":lisi", "20.1.1.1", + "123456")); + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":lisi", "10.1.1.1", + "123456")); + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":lisi", "20.1.1.2", + "123455")); + + /* + * Now we have 4 users: + * cmy@'%' + * zhangsan@"192.%" + * zhangsan@['palo.domain1'] + * lisi@['palo.domain2'] + */ + + // 9. grant for cmy@'%' + TablePattern tablePattern = new TablePattern("*", "*"); + List privileges = Lists.newArrayList(AccessPrivilege.CREATE_PRIV, AccessPrivilege.DROP_PRIV); + GrantStmt grantStmt = new GrantStmt(new UserIdentity("cmy", "%"), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + // check auth before grant + Assert.assertFalse(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":cmy", + PrivPredicate.CREATE)); + + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + // 9.1 check auth + Assert.assertTrue(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":cmy", + PrivPredicate.CREATE)); + Assert.assertFalse(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.CREATE)); + + // 10. grant auth for non exist user + tablePattern = new TablePattern("*", "*"); + privileges = Lists.newArrayList(AccessPrivilege.CREATE_PRIV, AccessPrivilege.DROP_PRIV); + grantStmt = new GrantStmt(new UserIdentity("nouser", "%"), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 11. grant auth for user with non exist host + tablePattern = new TablePattern("*", "*"); + privileges = Lists.newArrayList(AccessPrivilege.SELECT_PRIV, AccessPrivilege.DROP_PRIV); + grantStmt = new GrantStmt(new UserIdentity("zhangsan", "%"), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 12. grant db auth to exist user + tablePattern = new TablePattern("db1", "*"); + privileges = Lists.newArrayList(AccessPrivilege.SELECT_PRIV, AccessPrivilege.DROP_PRIV); + grantStmt = new GrantStmt(new UserIdentity("zhangsan", "192.%"), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertFalse(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.1", SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertTrue(auth.checkTblPriv("192.168.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "tbl1", + PrivPredicate.SELECT)); + + // 13. grant tbl auth to exist user + tablePattern = new TablePattern("db2", "tbl2"); + privileges = Lists.newArrayList(AccessPrivilege.ALTER_PRIV, AccessPrivilege.DROP_PRIV); + grantStmt = new GrantStmt(new UserIdentity("zhangsan", "192.%"), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(auth.checkDbPriv("192.168.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db2", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertFalse(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db2", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.1", SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertTrue(auth.checkTblPriv("192.168.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db2", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "tbl2", + PrivPredicate.DROP)); + + // 14. grant db auth to zhangsan@['palo.domain1'] + tablePattern = new TablePattern("db3", "*"); + privileges = Lists.newArrayList(AccessPrivilege.ALTER_PRIV, AccessPrivilege.DROP_PRIV); + grantStmt = new GrantStmt(new UserIdentity("zhangsan", "palo.domain1", true), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertFalse(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.ALTER)); + + resolver.runOneCycle(); + + Assert.assertTrue(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.ALTER)); + Assert.assertFalse(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.ALTER)); + + // 15. grant new auth to exist priv entry (exist ALTER/DROP, add SELECT) + tablePattern = new TablePattern("db3", "*"); + privileges = Lists.newArrayList(AccessPrivilege.SELECT_PRIV); + grantStmt = new GrantStmt(new UserIdentity("zhangsan", "palo.domain1", true), null, tablePattern, privileges); + + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.grant(grantStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertFalse(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + + resolver.runOneCycle(); + + Assert.assertTrue(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + Assert.assertTrue(auth.checkDbPriv("10.1.1.2", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.ALTER)); + Assert.assertTrue(auth.checkDbPriv("10.1.1.3", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.DROP)); + + /* + * for now, we have following auth: + * cmy@'%' + * *.* -> CREATE/DROP + * zhangsan@"192.%" + * db1.* -> SELECT/DROP + * db2.tbl2 -> ALTER/DROP + * zhangsan@['palo.domain1'] + * db3.* -> ALTER/DROP/SELECT + * lisi@['palo.domain2'] + * N/A + */ + + // 16. revoke privs from non exist user + tablePattern = new TablePattern("*", "*"); + privileges = Lists.newArrayList(AccessPrivilege.SELECT_PRIV); + RevokeStmt revokeStmt = new RevokeStmt(new UserIdentity("nouser", "%"), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 17. revoke privs from non exist host + tablePattern = new TablePattern("*", "*"); + privileges = Lists.newArrayList(AccessPrivilege.SELECT_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("cmy", "172.%"), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 18. revoke privs from non exist db + tablePattern = new TablePattern("nodb", "*"); + privileges = Lists.newArrayList(AccessPrivilege.SELECT_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("cmy", "%"), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 19. revoke privs from user @ ip + tablePattern = new TablePattern("*", "*"); + privileges = Lists.newArrayList(AccessPrivilege.CREATE_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("cmy", "%"), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertTrue(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db", + SystemInfoService.DEFAULT_CLUSTER + ":cmy", + PrivPredicate.CREATE)); + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db", + SystemInfoService.DEFAULT_CLUSTER + ":cmy", + PrivPredicate.CREATE)); + Assert.assertTrue(auth.checkDbPriv("172.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db", + SystemInfoService.DEFAULT_CLUSTER + ":cmy", + PrivPredicate.DROP)); + + // 19. revoke tbl privs from user @ ip + tablePattern = new TablePattern("db2", "tbl2"); + privileges = Lists.newArrayList(AccessPrivilege.ALTER_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("zhangsan", "192.%"), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertTrue(auth.checkTblPriv("192.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db2", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "tbl2", + PrivPredicate.ALTER)); + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(auth.checkTblPriv("192.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db2", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "tbl2", + PrivPredicate.ALTER)); + Assert.assertTrue(auth.checkDbPriv("192.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db1", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.SELECT)); + + // 20. revoke privs from non exist user @ domain + tablePattern = new TablePattern("db2", "tbl2"); + privileges = Lists.newArrayList(AccessPrivilege.ALTER_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("zhangsan", "nodomain", true), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 21. revoke privs from non exist db from user @ domain + tablePattern = new TablePattern("nodb", "*"); + privileges = Lists.newArrayList(AccessPrivilege.ALTER_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("zhangsan", "palo.domain1", true), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 22. revoke privs from exist user @ domain + tablePattern = new TablePattern("db3", "*"); + privileges = Lists.newArrayList(AccessPrivilege.DROP_PRIV); + revokeStmt = new RevokeStmt(new UserIdentity("zhangsan", "palo.domain1", true), null, tablePattern, privileges); + + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertTrue(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.DROP)); + + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertTrue(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.DROP)); + resolver.runOneCycle(); + Assert.assertFalse(auth.checkDbPriv("10.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.DROP)); + Assert.assertFalse(auth.checkDbPriv("10.1.1.3", SystemInfoService.DEFAULT_CLUSTER + ":db3", + SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", + PrivPredicate.DROP)); + + /* + * for now, we have following auth: + * cmy@'%' + * *.* -> DROP + * zhangsan@"192.%" + * db1.* -> SELECT/DROP + * db2.tbl2 -> DROP + * zhangsan@['palo.domain1'] + * db3.* -> ALTER/SELECT + * lisi@['palo.domain2'] + * N/A + */ + + // 23. create admin role, which is not allowed + CreateRoleStmt roleStmt = new CreateRoleStmt(PaloRole.ADMIN_ROLE); + hasException = false; + try { + roleStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e1) { + e1.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 23. create operator role, which is not allowed + roleStmt = new CreateRoleStmt(PaloRole.OPERATOR_ROLE); + hasException = false; + try { + roleStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e1) { + e1.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 24. create role + roleStmt = new CreateRoleStmt("rolo1"); + try { + roleStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + try { + auth.createRole(roleStmt); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // 25. grant auth to non exist role, will create this new role + privileges = Lists.newArrayList(AccessPrivilege.DROP_PRIV, AccessPrivilege.SELECT_PRIV); + grantStmt = new GrantStmt(null, "role2", new TablePattern("*", "*"), privileges); + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + try { + auth.grant(grantStmt); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // 26. grant auth to role + privileges = Lists.newArrayList(AccessPrivilege.DROP_PRIV, AccessPrivilege.SELECT_PRIV); + grantStmt = new GrantStmt(null, "role1", new TablePattern("*", "*"), privileges); + try { + grantStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e1) { + e1.printStackTrace(); + Assert.fail(); + } + try { + auth.grant(grantStmt); + } catch (DdlException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // 27. create user and set it as role1 + userIdentity = new UserIdentity("wangwu", "%"); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, "role1"); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertFalse(auth.checkDbPriv("10.17.2.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":wangwu", + PrivPredicate.DROP)); + try { + auth.createUser(userStmt); + } catch (DdlException e) { + Assert.fail(); + } + Assert.assertTrue(auth.checkDbPriv("10.17.2.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":wangwu", + PrivPredicate.DROP)); + + // 28. create user@domain and set it as role1 + userIdentity = new UserIdentity("chenliu", "palo.domain2", true); + userDesc = new UserDesc(userIdentity, "12345", true); + userStmt = new CreateUserStmt(false, userDesc, "role1"); + try { + userStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.createUser(userStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertFalse(auth.checkDbPriv("20.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":chenliu", + PrivPredicate.DROP)); + resolver.runOneCycle(); + Assert.assertTrue(auth.checkDbPriv("20.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":chenliu", + PrivPredicate.DROP)); + + // 29. revoke auth on non exist db from role1 + privileges = Lists.newArrayList(AccessPrivilege.DROP_PRIV); + revokeStmt = new RevokeStmt(null, "role1", new TablePattern("nodb", "*"), privileges); + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + hasException = false; + try { + auth.revoke(revokeStmt); + } catch (DdlException e1) { + e1.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 30. revoke auth from role1 + privileges = Lists.newArrayList(AccessPrivilege.DROP_PRIV); + revokeStmt = new RevokeStmt(null, "role1", new TablePattern("*", "*"), privileges); + try { + revokeStmt.analyze(analyzer); + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.revoke(revokeStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertTrue(auth.checkDbPriv("20.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":chenliu", + PrivPredicate.DROP)); + resolver.runOneCycle(); + Assert.assertFalse(auth.checkDbPriv("20.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":chenliu", + PrivPredicate.DROP)); + + // 31. drop role, privs remain unchanged + DropRoleStmt dropRoleStmt = new DropRoleStmt("role1"); + try { + dropRoleStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.dropRole(dropRoleStmt); + } catch (DdlException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(auth.checkDbPriv("20.1.1.1", SystemInfoService.DEFAULT_CLUSTER + ":db4", + SystemInfoService.DEFAULT_CLUSTER + ":chenliu", + PrivPredicate.DROP)); + + // drop user cmy + DropUserStmt dropUserStmt = new DropUserStmt(new UserIdentity("cmy", "%")); + try { + dropUserStmt.analyze(analyzer); + } catch (AnalysisException | InternalException e) { + e.printStackTrace(); + Assert.fail(); + } + + try { + auth.dropUser(dropUserStmt); + } catch (DdlException e) { + Assert.fail(); + } + + Assert.assertFalse(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":cmy", "192.168.0.1", "12345")); + Assert.assertTrue(auth.checkPlainPassword(SystemInfoService.DEFAULT_CLUSTER + ":zhangsan", "192.168.0.1", + "12345")); + + } + +} diff --git a/fe/test/com/baidu/palo/mysql/privilege/MockedAuth.java b/fe/test/com/baidu/palo/mysql/privilege/MockedAuth.java new file mode 100644 index 0000000000..b3c282e3ea --- /dev/null +++ b/fe/test/com/baidu/palo/mysql/privilege/MockedAuth.java @@ -0,0 +1,57 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.qe.ConnectContext; +import com.baidu.palo.qe.QueryState; + +import mockit.NonStrictExpectations; + +public class MockedAuth { + + public static void mockedAuth(PaloAuth auth) { + new NonStrictExpectations() { + { + auth.checkGlobalPriv((ConnectContext) any, (PrivPredicate) any); + result = true; + + auth.checkDbPriv((ConnectContext) any, anyString, (PrivPredicate) any); + result = true; + + auth.checkTblPriv((ConnectContext) any, anyString, anyString, (PrivPredicate) any); + result = true; + } + }; + } + + public static void mockedConnectContext(ConnectContext ctx, String user, String ip) { + new NonStrictExpectations() { + { + ConnectContext.get(); + result = ctx; + + ctx.getQualifiedUser(); + result = user; + + ctx.getRemoteIP(); + result = ip; + + ctx.getState(); + result = new QueryState(); + } + }; + } +} diff --git a/fe/test/com/baidu/palo/mysql/privilege/PrivTest.java b/fe/test/com/baidu/palo/mysql/privilege/PrivTest.java new file mode 100644 index 0000000000..19c83dcfbb --- /dev/null +++ b/fe/test/com/baidu/palo/mysql/privilege/PrivTest.java @@ -0,0 +1,347 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.CompoundPredicate.Operator; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.FeMetaVersion; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import mockit.Mock; +import mockit.MockUp; + +public class PrivTest { + + private PaloAuth auth; + private byte[] passwd = new byte[] { 'a', 'c' }; + + private Method grantGlobalPrivsM; + private Method grantDbPrivsM; + private Method grantTblPrivsM; + + @Before + public void setUp() { + auth = new PaloAuth(); + + Method[] methods = PaloAuth.class.getDeclaredMethods(); + for (Method method : methods) { + if (method.getName().equals("grantGlobalPrivs")) { + method.setAccessible(true); + grantGlobalPrivsM = method; + } else if (method.getName().equals("grantDbPrivs")) { + method.setAccessible(true); + grantDbPrivsM = method; + } else if (method.getName().equals("grantTblPrivs")) { + method.setAccessible(true); + grantTblPrivsM = method; + } + } + + new MockUp() { + @Mock + public int getCurrentCatalogJournalVersion() { + return FeMetaVersion.VERSION_43; + } + }; + } + + public void grantGlobalPrivs(Object... params) + throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { + grantGlobalPrivsM.invoke(auth, params); + } + + public void grantDbPrivs(Object... params) + throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { + grantDbPrivsM.invoke(auth, params); + } + + public void grantTblPrivs(Object... params) + throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { + grantTblPrivsM.invoke(auth, params); + } + + + @Test + public void testGlobalPriv() + throws DdlException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + // exact match + grantGlobalPrivs("192.168.1.1", "cmy", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.GRANT_PRIV)); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.1", "cmy2", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.NODE_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("192.168.1.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + // fuzzy match + auth.clear(); + grantGlobalPrivs("192.168._.%", "cmy\\_%", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.GRANT_PRIV, + PaloPrivilege.NODE_PRIV)); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("192.168.1.200", "cmy_123", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("192.168.1.200", "cmy_123", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV, + PaloPrivilege.NODE_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("192.168.1.200", "cmy_", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.NODE_PRIV), + Operator.OR))); + + auth.clear(); + grantGlobalPrivs("192.168.%", ".cmy", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.GRANT_PRIV)); + Assert.assertFalse(auth.checkGlobalPriv("192.10.1.1", ".cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("192.168.1.200", ".cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + + // multi priv entries + auth.clear(); + grantGlobalPrivs("%", "cmy", passwd, false, false, false, PrivBitSet.of(PaloPrivilege.GRANT_PRIV)); + grantGlobalPrivs("localhost", "cmy", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.NODE_PRIV)); + grantGlobalPrivs("127.0.0.1", "cmy", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.SELECT_PRIV)); + + Assert.assertTrue(auth.checkGlobalPriv("127.0.0.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("127.0.0.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("localhost", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.NODE_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("localhost", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("192.168.1.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + + // test persist + auth = testPersist(auth); + Assert.assertTrue(auth.checkGlobalPriv("127.0.0.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("127.0.0.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("localhost", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.NODE_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkGlobalPriv("localhost", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkGlobalPriv("192.168.1.1", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.GRANT_PRIV), + Operator.OR))); + } + + @Test + public void testDbPriv() + throws DdlException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + // normal + grantDbPrivs("192.168.1.%", "my\\_database", "cmy", false, false, false, + PrivBitSet.of(PaloPrivilege.SELECT_PRIV, + PaloPrivilege.ALTER_PRIV)); + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", "my_database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", "my_database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.SELECT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkDbPriv("192.168.1.2", "my_database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.LOAD_PRIV), + Operator.AND))); + Assert.assertTrue(auth.checkDbPriv("192.168.1.2", "my_database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.LOAD_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkDbPriv("192.168.1.1", "my_database2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkDbPriv("192.168.2.1", "my_database2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkDbPriv("192.168.1.1", "my_database2", "cmy2", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + // add global priv + auth.clear(); + grantGlobalPrivs("%", "cmy", passwd, false, false, false, PrivBitSet.of(PaloPrivilege.SELECT_PRIV)); + grantDbPrivs("192.168.1.%", "database", "cmy", false, false, false, + PrivBitSet.of(PaloPrivilege.ALTER_PRIV)); + + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", "database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.SELECT_PRIV), + Operator.OR))); + + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", "database2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkDbPriv("192.168.2.1", "database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkDbPriv("192.168.1.1", "database2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + // test persist + auth = testPersist(auth); + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", "database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV, + PaloPrivilege.SELECT_PRIV), + Operator.OR))); + + Assert.assertTrue(auth.checkDbPriv("192.168.1.1", "database2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkDbPriv("192.168.2.1", "database", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkDbPriv("192.168.1.1", "database2", "cmy", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + } + + @Test + public void testTblPriv() + throws DdlException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + // normal + grantTblPrivs("192.%.1.1", "db\\_%", "cmy%", "tbl%", false, false, false, + PrivBitSet.of(PaloPrivilege.SELECT_PRIV, + PaloPrivilege.LOAD_PRIV)); + Assert.assertTrue(auth.checkTblPriv("192.168.1.1", "db_1", "cmy", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.LOAD_PRIV, + PaloPrivilege.SELECT_PRIV), + Operator.OR))); + Assert.assertFalse(auth.checkTblPriv("192.168.1.1", "db_1", "cmy", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + // add db priv + grantDbPrivs("192.%", "db\\_123", "cmy", false, false, false, PrivBitSet.of(PaloPrivilege.ALTER_PRIV)); + Assert.assertTrue(auth.checkTblPriv("192.168.1.1", "db_123", "cmy", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + Assert.assertTrue(auth.checkTblPriv("192.168.1.1", "db_123", "cmy", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.SELECT_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkTblPriv("10.168.1.1", "db_123", "cmy", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.ALTER_PRIV), + Operator.OR))); + + // add global priv + grantGlobalPrivs("192.168.2.1", "cmy\\_admin", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.DROP_PRIV)); + Assert.assertTrue(auth.checkTblPriv("192.168.2.1", "db_123", "cmy_admin", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.DROP_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkTblPriv("192.168.1.1", "db_123", "cmy_admin", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.DROP_PRIV), + Operator.OR))); + + // test persist + auth = testPersist(auth); + Assert.assertTrue(auth.checkTblPriv("192.168.2.1", "db_123", "cmy_admin", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.DROP_PRIV), + Operator.OR))); + + Assert.assertFalse(auth.checkTblPriv("192.168.1.1", "db_123", "cmy_admin", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.DROP_PRIV), + Operator.OR))); + + // add global priv + grantGlobalPrivs("%", "cmy2", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.DROP_PRIV)); + Assert.assertTrue(auth.checkTblPriv("", "db_123", "cmy2", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.DROP_PRIV), + Operator.OR))); + Assert.assertTrue(auth.checkTblPriv(null, "db_123", "cmy2", "tbl", + PrivPredicate.of(PrivBitSet.of(PaloPrivilege.DROP_PRIV), + Operator.OR))); + + } + + private PaloAuth testPersist(PaloAuth auth) { + // 1. Write objects to file + File file = new File("./paloAuth"); + try { + file.createNewFile(); + DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); + + auth.write(dos); + + dos.flush(); + dos.close(); + + // 2. Read objects from file + DataInputStream dis = new DataInputStream(new FileInputStream(file)); + + PaloAuth replayed = PaloAuth.read(dis); + + System.out.println(replayed.toString()); + + return replayed; + + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } finally { + file.delete(); + } + return null; + } + +} diff --git a/fe/test/com/baidu/palo/mysql/privilege/PrivilegeTest.java b/fe/test/com/baidu/palo/mysql/privilege/PrivilegeTest.java new file mode 100644 index 0000000000..658b43b069 --- /dev/null +++ b/fe/test/com/baidu/palo/mysql/privilege/PrivilegeTest.java @@ -0,0 +1,161 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.common.DdlException; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Random; + +public class PrivilegeTest { + + private static int userNum = 250; + private static int dbNum = 250; + private static int tblNum = 3000; + + private static String[] dbs = new String[dbNum]; + private static String[] tbls = new String[tblNum]; + private static String[] users = new String[userNum]; + + private static PaloAuth auth = new PaloAuth(); + private static Random random = new Random(System.currentTimeMillis()); + + private static Method grantGlobalPrivsM; + private static Method grantDbPrivsM; + private static Method grantTblPrivsM; + + static { + Method[] methods = PaloAuth.class.getDeclaredMethods(); + for (Method method : methods) { + if (method.getName().equals("grantGlobalPrivs")) { + method.setAccessible(true); + grantGlobalPrivsM = method; + } else if (method.getName().equals("grantDbPrivs")) { + method.setAccessible(true); + grantDbPrivsM = method; + } else if (method.getName().equals("grantTblPrivs")) { + method.setAccessible(true); + grantTblPrivsM = method; + } + } + } + + private static String getRandomString(String prefix, int length) { + String str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + Random random = new Random(); + StringBuffer sb = new StringBuffer(prefix); + for (int i = 0; i < length; i++) { + int number = random.nextInt(62); + sb.append(str.charAt(number)); + } + return sb.toString(); + } + + private static void genNames() { + for (int i = 0; i < dbNum; i++) { + dbs[i] = getRandomString("db_", 10); + } + + for (int i = 0; i < tblNum; i++) { + tbls[i] = getRandomString("tbl_", 20); + } + + for (int i = 0; i < userNum; i++) { + users[i] = getRandomString("user_", 20); + } + } + + public static void genAuth() throws DdlException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + long start = System.currentTimeMillis(); + + byte[] passwd = new byte[] { 'a', 'b' }; + + genNames(); + + // global privs + Object[] params = { "%", "root", passwd, false, false, false, PrivBitSet.of(PaloPrivilege.NODE_PRIV) }; + grantGlobalPrivsM.invoke(auth, params); + + + Object[] params2 = { "%", "superuser", passwd, false, false, false, + PrivBitSet.of(PaloPrivilege.GRANT_PRIV, PaloPrivilege.SELECT_PRIV, PaloPrivilege.LOAD_PRIV, + PaloPrivilege.ALTER_PRIV, PaloPrivilege.CREATE_PRIV, PaloPrivilege.DROP_PRIV) }; + grantGlobalPrivsM.invoke(auth, params2); + + // db privs + PrivBitSet dbPrivs = PrivBitSet.of(PaloPrivilege.SELECT_PRIV, PaloPrivilege.LOAD_PRIV, + PaloPrivilege.ALTER_PRIV, PaloPrivilege.CREATE_PRIV, + PaloPrivilege.DROP_PRIV); + for (int i = 0; i < dbs.length; i++) { + for (int j = 0; j < 2; j++) { + int idx = Math.abs(random.nextInt()) % userNum; + Object[] params3 = { "%", dbs[i], users[idx], false, false, false, dbPrivs }; + grantDbPrivsM.invoke(auth, params3); + } + } + + // tbl privs + PrivBitSet tblPrivs = PrivBitSet.of(PaloPrivilege.SELECT_PRIV, PaloPrivilege.LOAD_PRIV); + for (int i = 0; i < tbls.length; i++) { + int dbIdx = Math.abs(random.nextInt()) % dbNum; + int userIdx = Math.abs(random.nextInt()) % userNum; + int tblIdx = Math.abs(random.nextInt()) % tblNum; + + Object[] params4 = { "%", dbs[dbIdx], users[userIdx], tbls[tblIdx], false, false, false, tblPrivs }; + grantTblPrivsM.invoke(auth, params4); + } + + // System.out.println("gen auth cost: " + (System.currentTimeMillis() - start)); + } + + private static long randomCheckTablePrivs(PrivPredicate predicate) + throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { + int dbIdx = Math.abs(random.nextInt()) % dbNum; + int userIdx = Math.abs(random.nextInt()) % userNum; + int tblIdx = Math.abs(random.nextInt()) % tblNum; + + long start = System.nanoTime(); + boolean res = auth.checkTblPriv("192.168.1.1", users[userIdx], dbs[dbIdx], tbls[tblIdx], + predicate); + + if (res) { + System.out.println(res); + } + return System.nanoTime() - start; + // System.out.println("check auth cost: " + (System.currentTimeMillis() - start)); + } + + public static void main(String[] args) throws DdlException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + genAuth(); + + // System.out.println(auth.getUserPrivTable()); + // System.out.println(auth.getDbPrivTable()); + // System.out.println(auth.getTablePrivTable()); + + PrivPredicate predicate = PrivPredicate.ADMIN; + int num = 10000; + long cost = 0; + for (int i = 0; i < num; i++) { + cost += randomCheckTablePrivs(predicate); + } + System.out.println("total auth cost: " + cost); + System.out.println("avg auth cost: " + cost / num); + } + +} diff --git a/fe/test/com/baidu/palo/mysql/privilege/UserIdentityTest.java b/fe/test/com/baidu/palo/mysql/privilege/UserIdentityTest.java new file mode 100644 index 0000000000..424d10e8ab --- /dev/null +++ b/fe/test/com/baidu/palo/mysql/privilege/UserIdentityTest.java @@ -0,0 +1,46 @@ +// Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.baidu.palo.mysql.privilege; + +import com.baidu.palo.analysis.UserIdentity; +import com.baidu.palo.system.SystemInfoService; + +import org.junit.Assert; +import org.junit.Test; + + +public class UserIdentityTest { + + @Test + public void test() { + UserIdentity userIdent = new UserIdentity(SystemInfoService.DEFAULT_CLUSTER + ":cmy", "192.%"); + userIdent.setIsAnalyzed(); + + String str = "'" + SystemInfoService.DEFAULT_CLUSTER + ":cmy" + "'@'192.%'"; + Assert.assertEquals(str, userIdent.toString()); + + UserIdentity userIdent2 = UserIdentity.fromString(str); + Assert.assertEquals(userIdent2.toString(), userIdent.toString()); + + String str2 = "'default_cluster:walletdc_write'@['cluster-leida.orp.all']"; + userIdent = UserIdentity.fromString(str2); + Assert.assertNotNull(userIdent); + Assert.assertTrue(userIdent.isDomain()); + userIdent.setIsAnalyzed(); + Assert.assertEquals(str2, userIdent.toString()); + } + +} diff --git a/fe/test/com/baidu/palo/persist/AccessTest.java b/fe/test/com/baidu/palo/persist/AccessTest.java deleted file mode 100644 index c13768f676..0000000000 --- a/fe/test/com/baidu/palo/persist/AccessTest.java +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.baidu.palo.persist; - -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.catalog.UserProperty; -import com.baidu.palo.catalog.UserPropertyMgr; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.FeConstants; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Arrays; - -@RunWith(PowerMockRunner.class) -@PowerMockIgnore("org.apache.log4j.*") -@PrepareForTest(Catalog.class) -public class AccessTest { - private static File file = new File("./AccessTest"); - private static UserProperty userProperty1 = new UserProperty(); - private static UserProperty userProperty2 = new UserProperty(); - private static byte[] passwd = new byte[0]; - private static EditLog edits; - private static Catalog catalog; - - static UserProperty invokeGetAccessResourceFunction(UserPropertyMgr userPropertyMgr, String userName) - throws IllegalAccessException, - IllegalArgumentException, InvocationTargetException, NoSuchFieldException, - SecurityException, NoSuchMethodException { - Method method = userPropertyMgr.getClass().getDeclaredMethod("getAccessResource", String.class); - method.setAccessible(true); - Object object = method.invoke(userPropertyMgr, userName); - return (UserProperty) object; - } - - @BeforeClass - public static void setUpClass() throws IOException { - file.createNewFile(); - passwd = "passwordIsLong".getBytes(); - // ordinary user - userProperty1.setUser("userName"); - userProperty1.setAccess("db1", AccessPrivilege.READ_ONLY); - userProperty1.setAccess("db2", AccessPrivilege.READ_WRITE); - userProperty1.setPassword(passwd); - userProperty1.setIsAdmin(false); - // adminstrator user - userProperty2.setUser("root"); - userProperty2.setPassword(new byte[0]); - userProperty2.setIsAdmin(true); - - edits = EasyMock.createMock(EditLog.class); - edits.logAlterAccess(EasyMock.isA(UserProperty.class)); - EasyMock.expectLastCall().anyTimes(); - edits.logDropUser(EasyMock.isA(String.class)); - EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(edits); - - } - - @Before - public void setUp() { - catalog = EasyMock.createMock(Catalog.class); - - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - EasyMock.expect(Catalog.getCurrentCatalogJournalVersion()).andReturn(FeConstants.meta_version).anyTimes(); - PowerMock.replay(Catalog.class); - } - - @AfterClass - public static void tearDownClass() { - file.delete(); - } - - @Test - public void testAccessResource() throws Exception { - // write ordinary user information to snapshot - DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); - userProperty1.write(dos); - dos.flush(); - dos.close(); - - // read snapshot - UserProperty result = new UserProperty(); - DataInputStream dis = new DataInputStream(new FileInputStream(file)); - result.readFields(dis); - dis.close(); - - // check result - Assert.assertEquals(result.getUser(), "userName"); - Assert.assertTrue(Arrays.equals(result.getPassword(), passwd)); - Assert.assertEquals(result.isAdmin(), false); - Assert.assertEquals(result.getMaxConn(), 100); - Assert.assertTrue(result.checkAccess("db1", AccessPrivilege.READ_ONLY)); - Assert.assertFalse(result.checkAccess("db1", AccessPrivilege.READ_WRITE)); - Assert.assertTrue(result.checkAccess("db2", AccessPrivilege.READ_ONLY)); - Assert.assertTrue(result.checkAccess("db2", AccessPrivilege.READ_WRITE)); - Assert.assertFalse(result.checkAccess("no_exists_db", AccessPrivilege.READ_ONLY)); - Assert.assertFalse(result.checkAccess("no_exists_db", AccessPrivilege.READ_WRITE)); - } - - @Test - public void testAccessService() throws Exception { - file.delete(); - file.createNewFile(); - UserPropertyMgr result = new UserPropertyMgr(); - result.unprotectAlterAccess(userProperty1); - result.unprotectAlterAccess(userProperty2); - - // write snapshot - DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); - result.write(dos); - dos.flush(); - dos.close(); - // read snapshot - DataInputStream dis = new DataInputStream(new FileInputStream(file)); - result.readFields(dis); - dis.close(); - // check root resource - UserProperty rootResource = invokeGetAccessResourceFunction(result, "root"); - Assert.assertEquals(rootResource.getUser(), "root"); - Assert.assertEquals(rootResource.isAdmin(), true); - Assert.assertEquals(rootResource.getMaxConn(), 100); - Assert.assertTrue(Arrays.equals(rootResource.getPassword(), new byte[0])); - Assert.assertTrue(rootResource.checkAccess("db1", AccessPrivilege.READ_WRITE)); - Assert.assertTrue(rootResource.checkAccess("db2", AccessPrivilege.READ_WRITE)); - - UserProperty ordinaryResource = invokeGetAccessResourceFunction(result, "userName"); - Assert.assertEquals(ordinaryResource.getUser(), "userName"); - Assert.assertTrue(Arrays.equals(ordinaryResource.getPassword(), passwd)); - Assert.assertEquals(ordinaryResource.isAdmin(), false); - Assert.assertEquals(ordinaryResource.getMaxConn(), 100); - Assert.assertTrue(ordinaryResource.checkAccess("db1", AccessPrivilege.READ_ONLY)); - Assert.assertFalse(ordinaryResource.checkAccess("db1", AccessPrivilege.READ_WRITE)); - Assert.assertTrue(ordinaryResource.checkAccess("db2", AccessPrivilege.READ_ONLY)); - Assert.assertTrue(ordinaryResource.checkAccess("db2", AccessPrivilege.READ_WRITE)); - } - - // add user - @Test(expected = DdlException.class) - public void testAddUserExceptionUserIsEmpty() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.addUser("", "", new byte[0], false); - Assert.fail("No Exception throws."); - } - - @Test(expected = DdlException.class) - public void testAddUserExceptionUserIsNull() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.addUser(null, null, new byte[0], false); - Assert.fail("No Exception throws."); - } - - @Test - public void testAddUserSuccess() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "user", new byte[0], false); - Assert.assertNotNull(invokeGetAccessResourceFunction(service, "user")); - - Assert.assertTrue(Arrays.equals(invokeGetAccessResourceFunction(service, "user") - .getPassword(), new byte[0])); - } - - @Test(expected = DdlException.class) - public void testAddUserTwoTimes() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "user", "pi".getBytes(), false); - service.addUser("cluster", "user", "pi".getBytes(), false); - Assert.fail("No Exception throws."); - } - - // set Passwd - @Test(expected = DdlException.class) - public void testSetPasswdExceptionUserIsEmpty() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setPasswd("user", new byte[0]); - Assert.fail("No Exception throws."); - } - - @Test - public void testSetPasswdSuccess() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "user", new byte[0], false); - Assert.assertTrue(Arrays.equals(service.getPassword("user"), new byte[0])); - - byte[] newPasswd = "*B6BDA741F59FE8066344FE3E118291C5D7DD12AD".getBytes(); - service.setPasswd("user", newPasswd); - Assert.assertTrue(Arrays.equals(service.getPassword("user"), newPasswd)); - } - - @Test(expected = DdlException.class) - public void testGrandExceptionUserIsEmpty() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.grant("user", "db", AccessPrivilege.READ_ONLY); - Assert.fail("No Exception throws."); - } - - // grant - @Ignore("Not Ready to Run") - @Test(expected = DdlException.class) - public void testGrandExceptionDbIsEmpty() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "user", new byte[0], false); - service.grant("user", "db_not_exists", AccessPrivilege.READ_ONLY); - Assert.fail("No Exception throws."); - } - - @Test - public void testGrandSuccess() throws Exception { - catalog = EasyMock.createMock(Catalog.class); - EasyMock.expect(catalog.getDb("db_exists")).andReturn(new Database()).anyTimes(); - EasyMock.replay(catalog); - PowerMock.mockStatic(Catalog.class); - EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); - PowerMock.replay(Catalog.class); - - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "user", new byte[0], false); - service.grant("user", "db_exists", AccessPrivilege.READ_ONLY); - Assert.assertFalse(service.checkAccess("user", "db_not_exists", AccessPrivilege.READ_ONLY)); - Assert.assertTrue(service.checkAccess("user", "db_exists", AccessPrivilege.READ_ONLY)); - } - - // drop User - @Test(expected = DdlException.class) - public void testDropUserExceptionUserNotExist() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.dropUser("user"); - Assert.fail("No Exception throws."); - } - - @Test - public void testDropUserSuccess() throws Exception { - UserPropertyMgr service = new UserPropertyMgr(); - service.setEditLog(edits); - service.addUser("cluster", "user", new byte[0], false); - Assert.assertNotNull(invokeGetAccessResourceFunction(service, "user")); - service.dropUser("user"); - Assert.assertNull(invokeGetAccessResourceFunction(service, "user")); - } -} diff --git a/fe/test/com/baidu/palo/qe/ConnectContextTest.java b/fe/test/com/baidu/palo/qe/ConnectContextTest.java index cba90dc463..ffe0b714fc 100644 --- a/fe/test/com/baidu/palo/qe/ConnectContextTest.java +++ b/fe/test/com/baidu/palo/qe/ConnectContextTest.java @@ -49,13 +49,14 @@ public class ConnectContextTest { @Before public void setUp() throws Exception { channel = EasyMock.createMock(MysqlChannel.class); - EasyMock.expect(channel.getRemoteHostString()).andReturn("127.0.0.1:12345").anyTimes(); + EasyMock.expect(channel.getRemoteHostPortString()).andReturn("127.0.0.1:12345").anyTimes(); channel.close(); EasyMock.expectLastCall().anyTimes(); executor = EasyMock.createMock(StmtExecutor.class); executor.cancel(); EasyMock.expectLastCall().anyTimes(); PowerMock.expectNew(MysqlChannel.class, EasyMock.isA(SocketChannel.class)).andReturn(channel).anyTimes(); + EasyMock.expect(channel.getRemoteIp()).andReturn("192.168.1.1").anyTimes(); EasyMock.replay(channel); EasyMock.replay(executor); PowerMock.replay(MysqlChannel.class); @@ -92,8 +93,8 @@ public class ConnectContextTest { Assert.assertEquals("testCluster:testDb", ctx.getDatabase()); // User - ctx.setUser("testCluster:testUser"); - Assert.assertEquals("testCluster:testUser", ctx.getUser()); + ctx.setQualifiedUser("testCluster:testUser"); + Assert.assertEquals("testCluster:testUser", ctx.getQualifiedUser()); // Serializer Assert.assertNotNull(ctx.getSerializer()); @@ -119,10 +120,10 @@ public class ConnectContextTest { List row = ctx.toThreadInfo().toRow(1000); Assert.assertEquals(9, row.size()); Assert.assertEquals("101", row.get(0)); - Assert.assertEquals("testCluster:testUser", row.get(1)); + Assert.assertEquals("testUser", row.get(1)); Assert.assertEquals("127.0.0.1:12345", row.get(2)); Assert.assertEquals("testCluster", row.get(3)); - Assert.assertEquals("testCluster:testDb", row.get(4)); + Assert.assertEquals("testDb", row.get(4)); Assert.assertEquals("Ping", row.get(5)); Assert.assertEquals("1", row.get(6)); Assert.assertEquals("", row.get(7)); diff --git a/fe/test/com/baidu/palo/qe/ConnectProcessorTest.java b/fe/test/com/baidu/palo/qe/ConnectProcessorTest.java index 1d6f82d521..9fcc9f7044 100644 --- a/fe/test/com/baidu/palo/qe/ConnectProcessorTest.java +++ b/fe/test/com/baidu/palo/qe/ConnectProcessorTest.java @@ -22,6 +22,7 @@ package com.baidu.palo.qe; import com.baidu.palo.analysis.AccessTestUtil; import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.metric.MetricRepo; import com.baidu.palo.mysql.MysqlChannel; import com.baidu.palo.mysql.MysqlCommand; import com.baidu.palo.mysql.MysqlEofPacket; @@ -29,8 +30,8 @@ import com.baidu.palo.mysql.MysqlErrPacket; import com.baidu.palo.mysql.MysqlOkPacket; import com.baidu.palo.mysql.MysqlSerializer; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -96,6 +97,8 @@ public class ConnectProcessorTest { serializer.writeEofString(""); fieldListPacket = serializer.toByteBuffer(); } + + MetricRepo.init(); } @Before @@ -108,7 +111,7 @@ public class ConnectProcessorTest { // Mock MysqlChannel channel = EasyMock.createNiceMock(MysqlChannel.class); PowerMock.expectNew(MysqlChannel.class, EasyMock.isA(SocketChannel.class)).andReturn(channel).anyTimes(); - EasyMock.expect(channel.getRemoteHostString()).andReturn("127.0.0.1:12345").anyTimes(); + EasyMock.expect(channel.getRemoteHostPortString()).andReturn("127.0.0.1:12345").anyTimes(); PowerMock.replay(MysqlChannel.class); myContext = new ConnectContext(EasyMock.createMock(SocketChannel.class)); } @@ -126,7 +129,7 @@ public class ConnectProcessorTest { channel.sendAndFlush(EasyMock.isA(ByteBuffer.class)); EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(channel.getRemoteHostString()).andReturn("127.0.0.1:12345").anyTimes(); + EasyMock.expect(channel.getRemoteHostPortString()).andReturn("127.0.0.1:12345").anyTimes(); EasyMock.replay(channel); @@ -144,10 +147,12 @@ public class ConnectProcessorTest { EasyMock.expect(context.getCatalog()).andReturn(catalog).anyTimes(); EasyMock.expect(context.getState()).andReturn(myContext.getState()).anyTimes(); EasyMock.expect(context.getAuditBuilder()).andReturn(auditBuilder).anyTimes(); - EasyMock.expect(context.getUser()).andReturn("testCluster:user").anyTimes(); + EasyMock.expect(context.getQualifiedUser()).andReturn("testCluster:user").anyTimes(); EasyMock.expect(context.getClusterName()).andReturn("testCluster").anyTimes(); EasyMock.expect(context.getStartTime()).andReturn(0L).anyTimes(); EasyMock.expect(context.getSerializer()).andDelegateTo(myContext).anyTimes(); + EasyMock.expect(context.getReturnRows()).andReturn(1L).anyTimes(); + EasyMock.expect(context.isKilled()).andReturn(false).anyTimes(); context.setKilled(); EasyMock.expectLastCall().andDelegateTo(myContext).anyTimes(); context.setCommand(EasyMock.anyObject(MysqlCommand.class)); @@ -278,7 +283,6 @@ public class ConnectProcessorTest { processor.processOnce(); Assert.assertEquals(MysqlCommand.COM_QUERY, myContext.getCommand()); - Assert.assertEquals("Maybe palo bug", myContext.getState().getErrorMessage()); Assert.assertTrue(myContext.getState().toResponsePacket() instanceof MysqlErrPacket); } diff --git a/fe/test/com/baidu/palo/qe/ConnectSchedulerTest.java b/fe/test/com/baidu/palo/qe/ConnectSchedulerTest.java index 5f068460d6..c9fbf93acb 100644 --- a/fe/test/com/baidu/palo/qe/ConnectSchedulerTest.java +++ b/fe/test/com/baidu/palo/qe/ConnectSchedulerTest.java @@ -24,8 +24,8 @@ import com.baidu.palo.analysis.AccessTestUtil; import com.baidu.palo.mysql.MysqlChannel; import com.baidu.palo.mysql.MysqlProto; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,7 +38,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.channels.SocketChannel; -import java.util.List; import java.util.concurrent.atomic.AtomicLong; @RunWith(PowerMockRunner.class) @@ -92,6 +91,8 @@ public class ConnectSchedulerTest { succSubmit = new AtomicLong(0); MysqlChannel channel = EasyMock.createMock(MysqlChannel.class); + EasyMock.expect(channel.getRemoteIp()).andReturn("192.168.1.1").anyTimes(); + EasyMock.replay(channel); PowerMock.expectNew(MysqlChannel.class, EasyMock.isA(SocketChannel.class)).andReturn(channel).anyTimes(); PowerMock.replay(MysqlChannel.class); @@ -120,17 +121,10 @@ public class ConnectSchedulerTest { } else { context.setCatalog(AccessTestUtil.fetchAdminCatalog()); } - context.setUser("root"); + context.setQualifiedUser("root"); Assert.assertTrue(scheduler.submit(context)); Assert.assertEquals(i, context.getConnectionId()); } - - Thread.sleep(1000); - Assert.assertNotNull(scheduler.getContext(0)); - List threads = scheduler.listConnection("root"); - Assert.assertEquals(1, threads.size()); - Assert.assertNotNull(scheduler.getContext(0)); - Assert.assertEquals(1, succSubmit.intValue()); } @Test @@ -142,7 +136,7 @@ public class ConnectSchedulerTest { ConnectContext context = new ConnectContext(EasyMock.createMock(SocketChannel.class)); context.setCatalog(AccessTestUtil.fetchAdminCatalog()); - context.setUser("root"); + context.setQualifiedUser("root"); Assert.assertTrue(scheduler.submit(context)); Assert.assertEquals(0, context.getConnectionId()); diff --git a/fe/test/com/baidu/palo/qe/CoordinatorTest.java b/fe/test/com/baidu/palo/qe/CoordinatorTest.java index 01c0695299..ade98a8d36 100644 --- a/fe/test/com/baidu/palo/qe/CoordinatorTest.java +++ b/fe/test/com/baidu/palo/qe/CoordinatorTest.java @@ -35,21 +35,16 @@ import com.baidu.palo.planner.PlanNode; import com.baidu.palo.planner.PlanNodeId; import com.baidu.palo.planner.Planner; import com.baidu.palo.system.Backend; -import com.baidu.palo.thrift.TExecPlanFragmentParams; import com.baidu.palo.thrift.TNetworkAddress; -import com.baidu.palo.thrift.TQueryOptions; import com.baidu.palo.thrift.TScanRange; import com.baidu.palo.thrift.TScanRangeLocation; import com.baidu.palo.thrift.TScanRangeLocations; -import com.baidu.palo.thrift.TUniqueId; import com.google.common.collect.ImmutableMap; -import org.apache.thrift.TException; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PowerMockIgnore; @@ -63,7 +58,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentMap; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"org.apache.log4j.*", "javax.management.*"}) @@ -414,10 +408,6 @@ public class CoordinatorTest extends Coordinator { } } - @Test - public void testNoUsedHosts() { - } - /* public void testNetworkException() throws TException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, diff --git a/fe/test/com/baidu/palo/qe/SetExecutorTest.java b/fe/test/com/baidu/palo/qe/SetExecutorTest.java index 727f7a9456..93a94188b3 100644 --- a/fe/test/com/baidu/palo/qe/SetExecutorTest.java +++ b/fe/test/com/baidu/palo/qe/SetExecutorTest.java @@ -27,32 +27,64 @@ import com.baidu.palo.analysis.SetNamesVar; import com.baidu.palo.analysis.SetPassVar; import com.baidu.palo.analysis.SetStmt; import com.baidu.palo.analysis.SetVar; +import com.baidu.palo.analysis.UserIdentity; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.InternalException; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; import com.google.common.collect.Lists; -import org.junit.Assert; + import org.junit.Before; import org.junit.Test; import java.util.List; +import mockit.Mocked; +import mockit.NonStrictExpectations; +import mockit.internal.startup.Startup; + public class SetExecutorTest { private Analyzer analyzer; private ConnectContext ctx; + @Mocked + private PaloAuth auth; + + static { + Startup.initializeIfPossible(); + } + @Before - public void setUp() { + public void setUp() throws DdlException { analyzer = AccessTestUtil.fetchAdminAnalyzer(false); ctx = new ConnectContext(null); ctx.setCatalog(AccessTestUtil.fetchAdminCatalog()); + ctx.setQualifiedUser("root"); + ctx.setRemoteIP("192.168.1.1"); + + new NonStrictExpectations() { + { + auth.checkGlobalPriv((ConnectContext) any, (PrivPredicate) any); + result = true; + + auth.checkDbPriv((ConnectContext) any, anyString, (PrivPredicate) any); + result = true; + + auth.checkTblPriv((ConnectContext) any, anyString, anyString, (PrivPredicate) any); + result = true; + + auth.setPassword((SetPassVar) any); + minTimes = 0; + } + }; } @Test public void testNormal() throws InternalException, AnalysisException, DdlException { List vars = Lists.newArrayList(); - vars.add(new SetPassVar("testUser", "*88EEBA7D913688E7278E2AD071FDB5E76D76D34B")); + vars.add(new SetPassVar(new UserIdentity("testUser", "%"), "*88EEBA7D913688E7278E2AD071FDB5E76D76D34B")); vars.add(new SetNamesVar("utf8")); vars.add(new SetVar("query_timeout", new IntLiteral(10L))); @@ -63,19 +95,6 @@ public class SetExecutorTest { executor.execute(); } - @Test(expected = DdlException.class) - public void testNoPriv() throws InternalException, AnalysisException, DdlException { - List vars = Lists.newArrayList(); - vars.add(new SetPassVar("root", "*88EEBA7D913688E7278E2AD071FDB5E76D76D34B")); - - SetStmt stmt = new SetStmt(vars); - stmt.analyze(analyzer); - SetExecutor executor = new SetExecutor(ctx, stmt); - - executor.execute(); - Assert.fail("No exception throws"); - } - @Test public void testEmpty() { } diff --git a/fe/test/com/baidu/palo/qe/ShowExecutorTest.java b/fe/test/com/baidu/palo/qe/ShowExecutorTest.java index 3f00d7b3ae..1367606f3a 100644 --- a/fe/test/com/baidu/palo/qe/ShowExecutorTest.java +++ b/fe/test/com/baidu/palo/qe/ShowExecutorTest.java @@ -32,7 +32,6 @@ import com.baidu.palo.analysis.ShowCreateTableStmt; import com.baidu.palo.analysis.ShowDbStmt; import com.baidu.palo.analysis.ShowEnginesStmt; import com.baidu.palo.analysis.ShowProcedureStmt; -import com.baidu.palo.analysis.ShowProcesslistStmt; import com.baidu.palo.analysis.ShowTableStmt; import com.baidu.palo.analysis.ShowVariablesStmt; import com.baidu.palo.analysis.TableName; @@ -46,18 +45,21 @@ import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.catalog.RandomDistributionInfo; import com.baidu.palo.catalog.SinglePartitionInfo; +import com.baidu.palo.catalog.Table; import com.baidu.palo.catalog.Table.TableType; -import com.baidu.palo.system.SystemInfoService; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; import com.baidu.palo.common.PatternMatcher; -import com.baidu.palo.thrift.TStorageType; import com.baidu.palo.mysql.MysqlCommand; +import com.baidu.palo.mysql.privilege.PaloAuth; +import com.baidu.palo.mysql.privilege.PrivPredicate; +import com.baidu.palo.system.SystemInfoService; +import com.baidu.palo.thrift.TStorageType; import com.google.common.collect.Lists; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -72,7 +74,7 @@ import java.util.List; @RunWith(PowerMockRunner.class) @PowerMockIgnore({ "org.apache.log4j.*", "javax.management.*" }) -@PrepareForTest({ ShowExecutor.class, Catalog.class, VariableMgr.class, HelpModule.class }) +@PrepareForTest({ ShowExecutor.class, Catalog.class, VariableMgr.class, HelpModule.class, ConnectContext.class }) public class ShowExecutorTest { private ConnectContext ctx; private Catalog catalog; @@ -82,6 +84,7 @@ public class ShowExecutorTest { ctx = new ConnectContext(null); ctx.setCommand(MysqlCommand.COM_SLEEP); + Column column1 = new Column("col1", PrimitiveType.BIGINT); Column column2 = new Column("col2", PrimitiveType.DOUBLE); column1.setIsKey(true); @@ -121,6 +124,17 @@ public class ShowExecutorTest { EasyMock.expectLastCall().anyTimes(); EasyMock.expect(db.getTable(EasyMock.isA(String.class))).andReturn(table).anyTimes(); EasyMock.replay(db); + + // mock auth + PaloAuth auth = EasyMock.createMock(PaloAuth.class); + EasyMock.expect(auth.checkGlobalPriv(EasyMock.isA(ConnectContext.class), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkDbPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.isA(PrivPredicate.class))).andReturn(true).anyTimes(); + EasyMock.expect(auth.checkTblPriv(EasyMock.isA(ConnectContext.class), EasyMock.anyString(), + EasyMock.anyString(), EasyMock.isA(PrivPredicate.class))) + .andReturn(true).anyTimes(); + EasyMock.replay(auth); // mock catalog. catalog = EasyMock.createMock(Catalog.class); @@ -129,8 +143,20 @@ public class ShowExecutorTest { EasyMock.expect(catalog.getClusterDbNames("testCluster")).andReturn(Lists.newArrayList("testCluster:testDb")) .anyTimes(); EasyMock.expect(catalog.getClusterDbNames("")).andReturn(Lists.newArrayList("")).anyTimes(); + EasyMock.expect(catalog.getAuth()).andReturn(auth).anyTimes(); EasyMock.replay(catalog); - PowerMock.expectNew(Catalog.class).andReturn(catalog).anyTimes(); + + PowerMock.mockStatic(Catalog.class); + EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); + Catalog.getDdlStmt(EasyMock.isA(Table.class), EasyMock.isA(List.class), + EasyMock.isA(List.class), EasyMock.isA(List.class), EasyMock.anyBoolean(), + EasyMock.anyShort()); + EasyMock.expectLastCall().anyTimes(); + Catalog.getDdlStmt(EasyMock.isA(Table.class), EasyMock.isA(List.class), + EasyMock.isNull(List.class), EasyMock.isNull(List.class), EasyMock.anyBoolean(), + EasyMock.anyShort()); + EasyMock.expectLastCall().anyTimes(); PowerMock.replay(Catalog.class); // mock scheduler @@ -140,8 +166,12 @@ public class ShowExecutorTest { EasyMock.replay(scheduler); ctx.setConnectScheduler(scheduler); ctx.setCatalog(AccessTestUtil.fetchAdminCatalog()); - ctx.setUser("testCluster:testUser"); + ctx.setQualifiedUser("testCluster:testUser"); ctx.setCluster("testCluster"); + + PowerMock.mockStatic(ConnectContext.class); + EasyMock.expect(ConnectContext.get()).andReturn(ctx).anyTimes(); + PowerMock.replay(ConnectContext.class); } @Test @@ -169,8 +199,6 @@ public class ShowExecutorTest { ShowExecutor executor = new ShowExecutor(ctx, stmt); ctx.setCatalog(AccessTestUtil.fetchBlockCatalog()); ShowResultSet resultSet = executor.execute(); - - Assert.assertFalse(resultSet.next()); } @Test @@ -211,6 +239,7 @@ public class ShowExecutorTest { Catalog catalog = AccessTestUtil.fetchAdminCatalog(); PowerMock.mockStatic(Catalog.class); EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalog()).andReturn(catalog).anyTimes(); EasyMock.expect(Catalog.getCurrentSystemInfo()).andReturn(clusterInfo).anyTimes(); PowerMock.replay(Catalog.class); @@ -282,7 +311,7 @@ public class ShowExecutorTest { @Test public void testShowCreateDb() throws AnalysisException { ctx.setCatalog(catalog); - ctx.setUser("testCluster:testUser"); + ctx.setQualifiedUser("testCluster:testUser"); ShowCreateDbStmt stmt = new ShowCreateDbStmt("testCluster:testDb"); ShowExecutor executor = new ShowExecutor(ctx, stmt); @@ -297,7 +326,7 @@ public class ShowExecutorTest { @Test(expected = AnalysisException.class) public void testShowCreateNoDb() throws AnalysisException { ctx.setCatalog(catalog); - ctx.setUser("testCluster:testUser"); + ctx.setQualifiedUser("testCluster:testUser"); ShowCreateDbStmt stmt = new ShowCreateDbStmt("testCluster:emptyDb"); ShowExecutor executor = new ShowExecutor(ctx, stmt); @@ -306,33 +335,6 @@ public class ShowExecutorTest { Assert.fail("No exception throws."); } - @Test - public void testShowCreateTable() throws AnalysisException { - ctx.setCatalog(catalog); - ctx.setUser("testCluster:testUser"); - - ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("testCluster:testDb", "testTbl")); - ShowExecutor executor = new ShowExecutor(ctx, stmt); - ShowResultSet resultSet = executor.execute(); - - Assert.assertTrue(resultSet.next()); - Assert.assertEquals("testTbl", resultSet.getString(0)); - - // print to help compare - String result = new String(resultSet.getString(1)); - result = result.replace(' ', '*'); - System.out.println("create table stmt:[" + result + "]"); - - Assert.assertEquals("CREATE TABLE `testTbl` (\n `col1` bigint(20) NOT NULL COMMENT \"\",\n" - + " `col2` double NOT NULL COMMENT \"\"\n" - + ") ENGINE=OLAP\n" - + "AGG_KEYS(`col1`, `col2`)\n" - + "DISTRIBUTED BY RANDOM BUCKETS 10\n" - + "PROPERTIES (\n" - + "\"storage_type\" = \"COLUMN\"\n" - + ");", resultSet.getString(1)); - } - @Test(expected = AnalysisException.class) public void testShowCreateTableEmptyDb() throws AnalysisException { ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("testCluster:emptyDb", "testTable")); @@ -354,7 +356,7 @@ public class ShowExecutorTest { @Test public void testShowColumn() throws AnalysisException { ctx.setCatalog(catalog); - ctx.setUser("testCluster:testUser"); + ctx.setQualifiedUser("testCluster:testUser"); ShowColumnStmt stmt = new ShowColumnStmt(new TableName("testCluster:testDb", "testTbl"), null, null, false); stmt.analyze(AccessTestUtil.fetchAdminAnalyzer(false)); ShowExecutor executor = new ShowExecutor(ctx, stmt); @@ -421,8 +423,6 @@ public class ShowExecutorTest { Assert.assertEquals("Name", resultSet.getMetaData().getColumn(0).getName()); Assert.assertEquals("Location", resultSet.getMetaData().getColumn(1).getName()); Assert.assertEquals("Comment", resultSet.getMetaData().getColumn(2).getName()); - - Assert.assertTrue(resultSet.next()); } @Test @@ -444,16 +444,6 @@ public class ShowExecutorTest { Assert.assertFalse(resultSet.next()); } - @Test - public void testShowProcesslist() throws AnalysisException { - ShowProcesslistStmt stmt = new ShowProcesslistStmt(); - ShowExecutor executor = new ShowExecutor(ctx, stmt); - ShowResultSet resultSet = executor.execute(); - - Assert.assertTrue(resultSet.next()); - Assert.assertFalse(resultSet.next()); - } - @Test public void testHelp() throws AnalysisException, IOException, InternalException { HelpModule module = new HelpModule(); diff --git a/fe/test/com/baidu/palo/qe/StmtExecutorTest.java b/fe/test/com/baidu/palo/qe/StmtExecutorTest.java index 8cb1dbe478..ce8e1f7789 100644 --- a/fe/test/com/baidu/palo/qe/StmtExecutorTest.java +++ b/fe/test/com/baidu/palo/qe/StmtExecutorTest.java @@ -36,19 +36,21 @@ import com.baidu.palo.analysis.UseStmt; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.util.RuntimeProfile; +import com.baidu.palo.metric.MetricRepo; import com.baidu.palo.mysql.MysqlChannel; import com.baidu.palo.mysql.MysqlSerializer; import com.baidu.palo.planner.Planner; +import com.baidu.palo.rewrite.ExprRewriter; +import com.baidu.palo.service.FrontendOptions; import com.baidu.palo.thrift.TQueryOptions; import com.baidu.palo.thrift.TUniqueId; import com.google.common.collect.Lists; -import java_cup.runtime.Symbol; - import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; @@ -58,10 +60,13 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.lang.reflect.Field; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.List; import java.util.SortedMap; +import java_cup.runtime.Symbol; + @RunWith(PowerMockRunner.class) @PowerMockIgnore({"org.apache.log4j.*", "javax.management.*"}) @PrepareForTest({StmtExecutor.class, DdlExecutor.class, Catalog.class}) @@ -70,6 +75,17 @@ public class StmtExecutorTest { private QueryState state; private ConnectScheduler scheduler; + @BeforeClass + public static void start() { + MetricRepo.init(); + try { + FrontendOptions.init(); + } catch (UnknownHostException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + @Before public void setUp() throws IOException { state = new QueryState(); @@ -90,9 +106,11 @@ public class StmtExecutorTest { EasyMock.expect(ctx.getState()).andReturn(state).anyTimes(); EasyMock.expect(ctx.getConnectScheduler()).andReturn(scheduler).anyTimes(); EasyMock.expect(ctx.getConnectionId()).andReturn(1).anyTimes(); - EasyMock.expect(ctx.getUser()).andReturn("testUser").anyTimes(); + EasyMock.expect(ctx.getQualifiedUser()).andReturn("testUser").anyTimes(); ctx.setKilled(); EasyMock.expectLastCall().anyTimes(); + ctx.updateReturnRows(EasyMock.anyInt()); + EasyMock.expectLastCall().anyTimes(); ctx.setQueryId(EasyMock.isA(TUniqueId.class)); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(ctx.queryId()).andReturn(new TUniqueId()).anyTimes(); @@ -114,6 +132,8 @@ public class StmtExecutorTest { queryStmt.getDbs(EasyMock.isA(Analyzer.class), EasyMock.isA(SortedMap.class)); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(queryStmt.getRedirectStatus()).andReturn(RedirectStatus.NO_FORWARD).anyTimes(); + queryStmt.rewriteExprs(EasyMock.isA(ExprRewriter.class)); + EasyMock.expectLastCall().anyTimes(); EasyMock.replay(queryStmt); Symbol symbol = new Symbol(0, queryStmt); @@ -276,7 +296,7 @@ public class StmtExecutorTest { ConnectContext killCtx = EasyMock.createMock(ConnectContext.class); EasyMock.expect(killCtx.getCatalog()).andReturn(AccessTestUtil.fetchAdminCatalog()).anyTimes(); - EasyMock.expect(killCtx.getUser()).andReturn("blockUser").anyTimes(); + EasyMock.expect(killCtx.getQualifiedUser()).andReturn("blockUser").anyTimes(); killCtx.kill(true); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(killCtx); @@ -310,7 +330,7 @@ public class StmtExecutorTest { ConnectContext killCtx = EasyMock.createMock(ConnectContext.class); EasyMock.expect(killCtx.getCatalog()).andReturn(AccessTestUtil.fetchAdminCatalog()).anyTimes(); - EasyMock.expect(killCtx.getUser()).andReturn("killUser").anyTimes(); + EasyMock.expect(killCtx.getQualifiedUser()).andReturn("killUser").anyTimes(); killCtx.kill(true); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(killCtx); @@ -321,7 +341,7 @@ public class StmtExecutorTest { StmtExecutor stmtExecutor = new StmtExecutor(ctx, ""); stmtExecutor.execute(); - Assert.assertEquals(QueryState.MysqlStateType.OK, state.getStateType()); + Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); } @Test diff --git a/fe/test/com/baidu/palo/task/LoadPendingTaskTest.java b/fe/test/com/baidu/palo/task/LoadPendingTaskTest.java index c0b667fa18..7940146c30 100644 --- a/fe/test/com/baidu/palo/task/LoadPendingTaskTest.java +++ b/fe/test/com/baidu/palo/task/LoadPendingTaskTest.java @@ -120,6 +120,7 @@ public class LoadPendingTaskTest { // mock load load = EasyMock.createMock(Load.class); EasyMock.expect(load.updateLoadJobState(job, JobState.ETL)).andReturn(true).times(1); + EasyMock.expect(load.getLoadErrorHubInfo()).andReturn(null).times(1); EasyMock.replay(load); EasyMock.expect(catalog.getLoadInstance()).andReturn(load).times(1); EasyMock.replay(catalog); diff --git a/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/FileSystemManager.java b/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/FileSystemManager.java index 089c417353..6bdc50ce2d 100644 --- a/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/FileSystemManager.java +++ b/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/FileSystemManager.java @@ -25,21 +25,30 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; import com.google.common.base.Strings; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.Base64; +import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.security.MessageDigest; public class FileSystemManager { @@ -49,6 +58,11 @@ public class FileSystemManager { private static final String HDFS_UGI_CONF = "hadoop.job.ugi"; private static final String USER_NAME_KEY = "username"; private static final String PASSWORD_KEY = "password"; + private static final String AUTHENTICATION_SIMPLE = "simple"; + private static final String AUTHENTICATION_KERBEROS = "kerberos"; + private static final String KERBEROS_PRINCIPAL = "kerberos_principal"; + private static final String KERBEROS_KEYTAB = "kerberos_keytab"; + private static final String KERBEROS_KEYTAB_CONTENT = "kerberos_keytab_content"; // arguments for ha hdfs private static final String DFS_NAMESERVICES_KEY = "dfs.nameservices"; @@ -110,10 +124,50 @@ public class FileSystemManager { String password = properties.containsKey(PASSWORD_KEY) ? properties.get(PASSWORD_KEY) : ""; String dfsNameServices = properties.containsKey(DFS_NAMESERVICES_KEY) ? properties.get(DFS_NAMESERVICES_KEY) : ""; + String authentication = AUTHENTICATION_SIMPLE; + if (properties.containsKey(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION)) { + authentication = properties.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION); + if (Strings.isNullOrEmpty(authentication) + || (!authentication.equals(AUTHENTICATION_SIMPLE) + && !authentication.equals(AUTHENTICATION_KERBEROS))) { + logger.warn("invalid authentication:" + authentication); + throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT, + "invalid authentication:" + authentication); + } + } String hdfsUgi = username + "," + password; - FileSystemIdentity fileSystemIdentity = new FileSystemIdentity(host, hdfsUgi); + FileSystemIdentity fileSystemIdentity = null; BrokerFileSystem fileSystem = null; + if (authentication.equals(AUTHENTICATION_SIMPLE)) { + fileSystemIdentity = new FileSystemIdentity(host, hdfsUgi); + } else { + // for kerberos, use host + principal + keytab as filesystemindentity + String kerberosContent = ""; + if (properties.containsKey(KERBEROS_KEYTAB)) { + kerberosContent = properties.get(KERBEROS_KEYTAB); + } else if (properties.containsKey(KERBEROS_KEYTAB_CONTENT)) { + kerberosContent = properties.get(KERBEROS_KEYTAB_CONTENT); + } else { + throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT, + "keytab is required for kerberos authentication"); + } + if (!properties.containsKey(KERBEROS_PRINCIPAL)) { + throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT, + "principal is required for kerberos authentication"); + } else { + kerberosContent = kerberosContent + properties.get(KERBEROS_PRINCIPAL); + } + try { + MessageDigest digest = MessageDigest.getInstance("md5"); + byte[] result = digest.digest(kerberosContent.getBytes()); + String kerberosUgi = new String(result); + fileSystemIdentity = new FileSystemIdentity(host, kerberosUgi); + } catch (NoSuchAlgorithmException e) { + throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT, + e.getMessage()); + } + } cachedFileSystem.putIfAbsent(fileSystemIdentity, new BrokerFileSystem(fileSystemIdentity)); fileSystem = cachedFileSystem.get(fileSystemIdentity); if (fileSystem == null) { @@ -133,7 +187,52 @@ public class FileSystemManager { Configuration conf = new Configuration(); // TODO get this param from properties // conf.set("dfs.replication", "2"); - conf.set(HDFS_UGI_CONF, hdfsUgi); + String tmpFilePath = null; + if (authentication.equals(AUTHENTICATION_SIMPLE)) { + conf.set(HDFS_UGI_CONF, hdfsUgi); + } else if (authentication.equals(AUTHENTICATION_KERBEROS)){ + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + AUTHENTICATION_KERBEROS); + + String principal = properties.get(KERBEROS_PRINCIPAL); + String keytab = ""; + if (properties.containsKey(KERBEROS_KEYTAB)) { + keytab = properties.get(KERBEROS_KEYTAB); + } else if (properties.containsKey(KERBEROS_KEYTAB_CONTENT)) { + // pass kerberos keytab content use base64 encoding + // so decode it and write it to tmp path under /tmp + // because ugi api only accept a local path as argument + String keytab_content = properties.get(KERBEROS_KEYTAB_CONTENT); + byte[] base64decodedBytes = Base64.getDecoder().decode(keytab_content); + long currentTime = System.currentTimeMillis(); + Random random = new Random(currentTime); + int randNumber = random.nextInt(10000); + tmpFilePath = "/tmp/." + Long.toString(currentTime) + "_" + Integer.toString(randNumber); + FileOutputStream fileOutputStream = new FileOutputStream(tmpFilePath); + fileOutputStream.write(base64decodedBytes); + fileOutputStream.close(); + keytab = tmpFilePath; + } else { + throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT, + "keytab is required for kerberos authentication"); + } + UserGroupInformation.setConfiguration(conf); + UserGroupInformation.loginUserFromKeytab(principal, keytab); + if (properties.containsKey(KERBEROS_KEYTAB_CONTENT)) { + try { + File file = new File(tmpFilePath); + if(!file.delete()){ + logger.warn("delete tmp file:" + tmpFilePath + " failed"); + } + } catch (Exception e) { + throw new BrokerException(TBrokerOperationStatusCode.FILE_NOT_FOUND, + e.getMessage()); + } + } + } else { + throw new BrokerException(TBrokerOperationStatusCode.INVALID_ARGUMENT, + "invalid authentication."); + } if (!Strings.isNullOrEmpty(dfsNameServices)) { // ha hdfs arguments final String dfsHaNameNodesKey = DFS_HA_NAMENODES_PREFIX + dfsNameServices; diff --git a/gensrc/parser/sql_parser.y b/gensrc/parser/sql_parser.y index 9af0e57794..e703fd7ef8 100644 --- a/gensrc/parser/sql_parser.y +++ b/gensrc/parser/sql_parser.y @@ -201,14 +201,14 @@ terminal String KW_ADD, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_AND, KW_ANT KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DECIMAL, KW_DECOMMISSION, KW_DEFAULT, KW_DESC, KW_DESCRIBE, KW_DELETE, KW_DISTINCT, KW_DISTINCTPC, KW_DISTINCTPCSA, KW_DISTRIBUTED, KW_BUCKETS, KW_DIV, KW_DOUBLE, KW_DROP, KW_DROPP, KW_DUPLICATE, KW_ELSE, KW_END, KW_ENGINE, KW_ENGINES, KW_ENTER, KW_ERRORS, KW_EVENTS, KW_EXISTS, KW_EXPORT, KW_EXTERNAL, KW_EXTRACT, - KW_FALSE, KW_FOLLOWER, KW_FOLLOWING, KW_FREE, KW_FROM, KW_FIRST, KW_FLOAT, KW_FOR, KW_FULL, KW_FUNCTION, - KW_GLOBAL, KW_GRANT, KW_GROUP, + KW_FALSE, KW_FOLLOWER, KW_FOLLOWING, KW_FREE, KW_FROM, KW_FIRST, KW_FLOAT, KW_FOR, KW_FRONTENDS, KW_FULL, KW_FUNCTION, + KW_GLOBAL, KW_GRANT, KW_GRANTS, KW_GROUP, KW_HASH, KW_HAVING, KW_HELP,KW_HLL, KW_HLL_UNION, KW_IDENTIFIED, KW_IF, KW_IN, KW_INDEX, KW_INDEXES, KW_INFILE, KW_INNER, KW_INSERT, KW_INT, KW_INTERVAL, KW_INTO, KW_IS, KW_ISNULL, KW_ISOLATION, KW_JOIN, KW_KEY, KW_KILL, - KW_LABEL, KW_LARGEINT, KW_LEFT, KW_LESS, KW_LEVEL, KW_LIKE, KW_LIMIT, KW_LINK, KW_LOAD, KW_LOCAL, + KW_LABEL, KW_LARGEINT, KW_LEFT, KW_LESS, KW_LEVEL, KW_LIKE, KW_LIMIT, KW_LINK, KW_LOAD, KW_LOCAL, KW_LOCATION, KW_MAX, KW_MAX_VALUE, KW_MERGE, KW_MIN, KW_MIGRATE, KW_MIGRATIONS, KW_MODIFY, KW_NAME, KW_NAMES, KW_NEGATIVE, KW_NO, KW_NOT, KW_NULL, KW_OBSERVER, KW_OFFSET, KW_ON, KW_ONLY, KW_OPEN, KW_OR, KW_ORDER, KW_OUTER, KW_OVER, @@ -218,13 +218,13 @@ terminal String KW_ADD, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_AND, KW_ANT KW_PROC, KW_PROCEDURE, KW_PROCESSLIST, KW_PROPERTIES, KW_PROPERTY, KW_QUERY, KW_QUOTA, KW_RANDOM, KW_RANGE, KW_READ, KW_RECOVER, KW_REGEXP, KW_RELEASE, KW_RENAME, - KW_REPEATABLE, KW_REPLACE, KW_RESOURCE, KW_RESTORE, KW_REVOKE, - KW_RIGHT, KW_ROLLBACK, KW_ROLLUP, KW_ROW, KW_ROWS, + KW_REPEATABLE, KW_REPOSITORY, KW_REPOSITORIES, KW_REPLACE, KW_RESOURCE, KW_RESTORE, KW_REVOKE, + KW_RIGHT, KW_ROLE, KW_ROLES, KW_ROLLBACK, KW_ROLLUP, KW_ROW, KW_ROWS, KW_SCHEMAS, KW_SELECT, KW_SEMI, KW_SERIALIZABLE, KW_SESSION, KW_SET, KW_SHOW, KW_SMALLINT, KW_SNAPSHOT, KW_SONAME, KW_SPLIT, KW_START, KW_STATUS, KW_STORAGE, KW_STRING, KW_SUM, KW_SUPERUSER, KW_SYNC, KW_SYSTEM, KW_TABLE, KW_TABLES, KW_TABLET, KW_TERMINATED, KW_THAN, KW_THEN, KW_TIMESTAMP, KW_TINYINT, - KW_TO, KW_TRANSACTION, KW_TRIGGERS, KW_TRIM, KW_TRUE, KW_TYPES, + KW_TO, KW_TRANSACTION, KW_TRIGGERS, KW_TRIM, KW_TRUE, KW_TYPE, KW_TYPES, KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNSIGNED, KW_USE, KW_USER, KW_USING, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VIEW, KW_WARNINGS, KW_WHEN, KW_WHITELIST, KW_WHERE, KW_WITH, KW_WORK, KW_WRITE; @@ -260,6 +260,7 @@ nonterminal describe_command, opt_full, opt_inner, opt_outer, from_or_in, keys_o // String nonterminal String user, opt_user; +nonterminal UserIdentity user_identity; // Description of user nonterminal UserDesc grant_user; @@ -309,7 +310,7 @@ nonterminal AnalyticWindow opt_window_clause; nonterminal AnalyticWindow.Type window_type; nonterminal AnalyticWindow.Boundary window_boundary; nonterminal SlotRef column_ref; -nonterminal ArrayList table_ref_list; +nonterminal ArrayList table_ref_list, base_table_ref_list; nonterminal FromClause from_clause; nonterminal TableRef table_ref; nonterminal TableRef base_table_ref; @@ -370,9 +371,12 @@ nonterminal List opt_col_list, opt_dup_keys; nonterminal List opt_partitions; nonterminal List opt_col_mapping_list; nonterminal ColumnSeparator opt_field_term; +nonterminal String opt_user_role; +nonterminal TablePattern tbl_pattern; +nonterminal String ident_or_star; // Boolean -nonterminal Boolean opt_negative, opt_super_user, opt_is_allow_null, opt_is_key; +nonterminal Boolean opt_negative, opt_super_user, opt_is_allow_null, opt_is_key, opt_read_only; nonterminal String opt_from_rollup, opt_to_rollup; nonterminal ColumnPosition opt_col_pos; @@ -539,9 +543,9 @@ alter_stmt ::= {: RESULT = new AlterDatabaseRename(dbName, newDbName); :} - | KW_ALTER KW_USER ident:userName alter_user_clause:clause + | KW_ALTER KW_USER user_identity:userIdent alter_user_clause:clause {: - RESULT = new AlterUserStmt(userName, clause); + RESULT = new AlterUserStmt(userIdent, clause); :} ; @@ -801,9 +805,9 @@ create_stmt ::= RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, engineName, keys, partition, distribution, tblProperties, extProperties); :} /* User */ - | KW_CREATE KW_USER grant_user:user opt_super_user:isSuperuser + | KW_CREATE KW_USER opt_if_not_exists:ifNotExists grant_user:user opt_user_role:userRole {: - RESULT = new CreateUserStmt(user, isSuperuser); + RESULT = new CreateUserStmt(ifNotExists, user, userRole); :} | KW_CREATE KW_VIEW opt_if_not_exists:ifNotExists table_name:viewName opt_col_list:columns KW_AS query_stmt:view_def @@ -815,37 +819,61 @@ create_stmt ::= {: RESULT = new CreateClusterStmt(name, properties, password); :} - ; - -grant_user ::= - user:user + | KW_CREATE opt_read_only:isReadOnly KW_REPOSITORY ident:repoName KW_WITH KW_BROKER ident:brokerName + KW_ON KW_LOCATION STRING_LITERAL:location + opt_properties:properties {: - /* No password */ - RESULT = new UserDesc(user); + RESULT = new CreateRepositoryStmt(isReadOnly, repoName, brokerName, location, properties); :} - | user:user KW_IDENTIFIED KW_BY STRING_LITERAL:password + | KW_CREATE KW_ROLE ident:role {: - /* plain text password */ - RESULT = new UserDesc(user, password, true); - :} - | user:user KW_IDENTIFIED KW_BY KW_PASSWORD STRING_LITERAL:password - {: - /* hashed password */ - RESULT = new UserDesc(user, password, false); + RESULT = new CreateRoleStmt(role); :} ; -opt_super_user ::= - /* Empty */ +opt_read_only ::= {: RESULT = false; :} - | KW_SUPERUSER + | KW_READ KW_ONLY {: RESULT = true; :} ; +grant_user ::= + user_identity:user_id + {: + /* No password */ + RESULT = new UserDesc(user_id); + :} + | user_identity:user_id KW_IDENTIFIED KW_BY STRING_LITERAL:password + {: + /* plain text password */ + RESULT = new UserDesc(user_id, password, true); + :} + | user_identity:user_id KW_IDENTIFIED KW_BY KW_PASSWORD STRING_LITERAL:password + {: + /* hashed password */ + RESULT = new UserDesc(user_id, password, false); + :} + ; + +opt_user_role ::= + /* Empty */ + {: + RESULT = null; + :} + | KW_SUPERUSER /* for forward compatibility*/ + {: + RESULT = "superuser"; + :} + | KW_DEFAULT KW_ROLE STRING_LITERAL:role + {: + RESULT = role; + :} + ; + user ::= ident_or_text:user {: @@ -853,6 +881,21 @@ user ::= :} ; +user_identity ::= + ident_or_text:user + {: + RESULT = new UserIdentity(user, "%", false); + :} + | ident_or_text:user AT ident_or_text:host + {: + RESULT = new UserIdentity(user, host, false); + :} + | ident_or_text:user AT LBRACKET ident_or_text:host RBRACKET + {: + RESULT = new UserIdentity(user, host, true); + :} + ; + column_type_list ::= column_type:type {: @@ -1024,18 +1067,47 @@ opt_cluster ::= // Grant statement grant_stmt ::= - KW_GRANT privilege_list:privs KW_ON ident:dbName KW_TO user:user + KW_GRANT privilege_list:privs KW_ON tbl_pattern:tblPattern KW_TO user_identity:userId {: - RESULT = new GrantStmt(user, dbName, privs); + RESULT = new GrantStmt(userId, null, tblPattern, privs); + :} + | KW_GRANT privilege_list:privs KW_ON tbl_pattern:tblPattern KW_TO KW_ROLE STRING_LITERAL:role + {: + RESULT = new GrantStmt(null, role, tblPattern, privs); + :} + ; + +tbl_pattern ::= + ident_or_star:db + {: + RESULT = new TablePattern(db, "*"); + :} + | ident_or_star:db DOT ident_or_star:tbl + {: + RESULT = new TablePattern(db, tbl); + :} + ; + +ident_or_star ::= + STAR + {: + RESULT = "*"; + :} + | ident:ident + {: + RESULT = ident; :} ; // Revoke statement revoke_stmt ::= - /* for now, simply revoke ALL privilege */ - KW_REVOKE KW_ALL KW_ON ident:dbName KW_FROM user:user + KW_REVOKE privilege_list:privs KW_ON tbl_pattern:tblPattern KW_FROM user_identity:userId {: - RESULT = new RevokeStmt(user, dbName); + RESULT = new RevokeStmt(userId, null, tblPattern, privs); + :} + | KW_REVOKE privilege_list:privs KW_ON tbl_pattern:tblPattern KW_FROM KW_ROLE STRING_LITERAL:role + {: + RESULT = new RevokeStmt(null, role, tblPattern, privs); :} ; @@ -1062,15 +1134,23 @@ drop_stmt ::= RESULT = new DropTableStmt(ifExists, name); :} /* User */ - | KW_DROP KW_USER STRING_LITERAL:user + | KW_DROP KW_USER user_identity:userId {: - RESULT = new DropUserStmt(user); + RESULT = new DropUserStmt(userId); :} /* View */ | KW_DROP KW_VIEW opt_if_exists:ifExists table_name:name {: RESULT = new DropTableStmt(ifExists, name, true); :} + | KW_DROP KW_REPOSITORY ident:repoName + {: + RESULT = new DropRepositoryStmt(repoName); + :} + | KW_DROP KW_ROLE ident:role + {: + RESULT = new DropRoleStmt(role); + :} ; // Recover statement @@ -1659,9 +1739,9 @@ show_param ::= {: RESULT = new ShowUserPropertyStmt(user, parser.wild); :} - | KW_BACKUP opt_db:db opt_wild_where + | KW_BACKUP opt_db:db {: - RESULT = new ShowBackupStmt(db, parser.where); + RESULT = new ShowBackupStmt(db); :} | KW_RESTORE opt_db:db opt_wild_where {: @@ -1675,10 +1755,38 @@ show_param ::= {: RESULT = new ShowBackendsStmt(); :} + | KW_FRONTENDS + {: + RESULT = new ShowFrontendsStmt(); + :} | KW_USER {: RESULT = new ShowUserStmt(); :} + | KW_REPOSITORIES + {: + RESULT = new ShowRepositoriesStmt(); + :} + | KW_SNAPSHOT KW_ON ident:repo opt_wild_where + {: + RESULT = new ShowSnapshotStmt(repo, parser.where); + :} + | KW_ALL KW_GRANTS + {: + RESULT = new ShowGrantsStmt(null, true); + :} + | KW_GRANTS + {: + RESULT = new ShowGrantsStmt(null, false); + :} + | KW_GRANTS KW_FOR user_identity:userIdent + {: + RESULT = new ShowGrantsStmt(userIdent, false); + :} + | KW_ROLES + {: + RESULT = new ShowRolesStmt(); + :} ; keys_or_index ::= @@ -2105,23 +2213,23 @@ insert_source ::= // backup stmt backup_stmt ::= - KW_BACKUP KW_LABEL job_label:label - opt_partition_name_list:backupObjNames - KW_INTO STRING_LITERAL:rootPath + KW_BACKUP KW_SNAPSHOT job_label:label + KW_TO ident:repoName + KW_ON LPAREN base_table_ref_list:tbls RPAREN opt_properties:properties {: - RESULT = new BackupStmt(label, backupObjNames, rootPath, properties); + RESULT = new BackupStmt(label, repoName, tbls, properties); :} ; // Restore statement restore_stmt ::= - KW_RESTORE KW_LABEL job_label:label - opt_partition_name_list:restoreObjNames - KW_FROM STRING_LITERAL:rootPath + KW_RESTORE KW_SNAPSHOT job_label:label + KW_FROM ident:repoName + KW_ON LPAREN base_table_ref_list:tbls RPAREN opt_properties:properties {: - RESULT = new RestoreStmt(label, restoreObjNames, rootPath, properties); + RESULT = new RestoreStmt(label, repoName, tbls, properties); :} ; @@ -2339,9 +2447,9 @@ option_value_no_option_type ::= {: RESULT = new SetPassVar(null, passwd); :} - | KW_PASSWORD KW_FOR STRING_LITERAL:user equal text_or_password:passwd + | KW_PASSWORD KW_FOR user_identity:userId equal text_or_password:passwd {: - RESULT = new SetPassVar(user, passwd); + RESULT = new SetPassVar(userId, passwd); :} ; @@ -2655,6 +2763,20 @@ inline_view_ref ::= :} ; +base_table_ref_list ::= + base_table_ref:tbl + {: + ArrayList list = new ArrayList(); + list.add(tbl); + RESULT = list; + :} + | base_table_ref_list:list COMMA base_table_ref:tbl + {: + list.add(tbl); + RESULT = list; + :} + ; + base_table_ref ::= table_name:name opt_using_partition:parts opt_table_alias:alias {: @@ -3513,6 +3635,8 @@ keyword ::= {: RESULT = id; :} | KW_LOCAL:id {: RESULT = id; :} + | KW_LOCATION:id + {: RESULT = id; :} | KW_MERGE:id {: RESULT = id; :} | KW_MODIFY:id @@ -3557,6 +3681,10 @@ keyword ::= {: RESULT = id; :} | KW_REPEATABLE:id {: RESULT = id; :} + | KW_REPOSITORY:id + {: RESULT = id; :} + | KW_REPOSITORIES:id + {: RESULT = id; :} | KW_RESOURCE:id {: RESULT = id; :} | KW_RESTORE:id @@ -3593,6 +3721,8 @@ keyword ::= {: RESULT = id; :} | KW_TRIGGERS:id {: RESULT = id; :} + | KW_TYPE:id + {: RESULT = id; :} | KW_TYPES:id {: RESULT = id; :} | KW_UNCOMMITTED:id diff --git a/gensrc/parser/sql_scanner.flex b/gensrc/parser/sql_scanner.flex index 8e37293ebc..9a616ca99a 100644 --- a/gensrc/parser/sql_scanner.flex +++ b/gensrc/parser/sql_scanner.flex @@ -154,10 +154,12 @@ import com.baidu.palo.common.util.SqlUtils; keywordMap.put("following", new Integer(SqlParserSymbols.KW_FOLLOWING)); keywordMap.put("for", new Integer(SqlParserSymbols.KW_FOR)); keywordMap.put("from", new Integer(SqlParserSymbols.KW_FROM)); + keywordMap.put("frontends", new Integer(SqlParserSymbols.KW_FRONTENDS)); keywordMap.put("full", new Integer(SqlParserSymbols.KW_FULL)); keywordMap.put("function", new Integer(SqlParserSymbols.KW_FUNCTION)); keywordMap.put("global", new Integer(SqlParserSymbols.KW_GLOBAL)); keywordMap.put("grant", new Integer(SqlParserSymbols.KW_GRANT)); + keywordMap.put("grants", new Integer(SqlParserSymbols.KW_GRANTS)); keywordMap.put("group", new Integer(SqlParserSymbols.KW_GROUP)); keywordMap.put("hash", new Integer(SqlParserSymbols.KW_HASH)); keywordMap.put("having", new Integer(SqlParserSymbols.KW_HAVING)); @@ -190,6 +192,7 @@ import com.baidu.palo.common.util.SqlUtils; keywordMap.put("limit", new Integer(SqlParserSymbols.KW_LIMIT)); keywordMap.put("load", new Integer(SqlParserSymbols.KW_LOAD)); keywordMap.put("local", new Integer(SqlParserSymbols.KW_LOCAL)); + keywordMap.put("location", new Integer(SqlParserSymbols.KW_LOCATION)); keywordMap.put("max", new Integer(SqlParserSymbols.KW_MAX)); keywordMap.put("maxvalue", new Integer(SqlParserSymbols.KW_MAX_VALUE)); keywordMap.put("merge", new Integer(SqlParserSymbols.KW_MERGE)); @@ -235,11 +238,15 @@ import com.baidu.palo.common.util.SqlUtils; keywordMap.put("rename", new Integer(SqlParserSymbols.KW_RENAME)); keywordMap.put("repeatable", new Integer(SqlParserSymbols.KW_REPEATABLE)); keywordMap.put("replace", new Integer(SqlParserSymbols.KW_REPLACE)); + keywordMap.put("repository", new Integer(SqlParserSymbols.KW_REPOSITORY)); + keywordMap.put("repositories", new Integer(SqlParserSymbols.KW_REPOSITORIES)); keywordMap.put("resource", new Integer(SqlParserSymbols.KW_RESOURCE)); keywordMap.put("restore", new Integer(SqlParserSymbols.KW_RESTORE)); keywordMap.put("revoke", new Integer(SqlParserSymbols.KW_REVOKE)); keywordMap.put("right", new Integer(SqlParserSymbols.KW_RIGHT)); keywordMap.put("rlike", new Integer(SqlParserSymbols.KW_REGEXP)); + keywordMap.put("role", new Integer(SqlParserSymbols.KW_ROLE)); + keywordMap.put("roles", new Integer(SqlParserSymbols.KW_ROLES)); keywordMap.put("rollback", new Integer(SqlParserSymbols.KW_ROLLBACK)); keywordMap.put("rollup", new Integer(SqlParserSymbols.KW_ROLLUP)); keywordMap.put("row", new Integer(SqlParserSymbols.KW_ROW)); @@ -275,6 +282,7 @@ import com.baidu.palo.common.util.SqlUtils; keywordMap.put("triggers", new Integer(SqlParserSymbols.KW_TRIGGERS)); keywordMap.put("trim", new Integer(SqlParserSymbols.KW_TRIM)); keywordMap.put("true", new Integer(SqlParserSymbols.KW_TRUE)); + keywordMap.put("type", new Integer(SqlParserSymbols.KW_TYPE)); keywordMap.put("types", new Integer(SqlParserSymbols.KW_TYPES)); keywordMap.put("unbounded", new Integer(SqlParserSymbols.KW_UNBOUNDED)); keywordMap.put("uncommitted", new Integer(SqlParserSymbols.KW_UNCOMMITTED)); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 5ff511ff94..cc64646242 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -87,6 +87,9 @@ message OLAPHeaderMessage { // bloom filter false positive probability optional double bf_fpp = 14; optional KeysType keys_type = 15; + // if true, this tablet will not do compaction, + // and does not create init version + optional bool in_restore_mode = 16 [default = false]; } message OLAPIndexHeaderMessage { diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index db4dcada1c..5506c1e219 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -26,7 +26,7 @@ # contains the build version based on the git hash or svn revision. ############################################################## -build_version="PALO3.3.19-RELEASE" +build_version="3.3-branch" unset LANG unset LC_CTYPE diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 37e7ef42f3..7547ff7516 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -50,6 +50,7 @@ struct TCreateTabletReq { 3: optional Types.TVersion version 4: optional Types.TVersionHash version_hash 5: optional Types.TStorageMedium storage_medium + 6: optional bool in_restore_mode } struct TDropTabletReq { @@ -111,17 +112,17 @@ struct TCheckConsistencyReq { } struct TUploadReq { - 1: required string local_file_path - 2: required string remote_file_path - 3: required map remote_source_properties - 4: optional Types.TTabletId tablet_id + 1: required i64 job_id; + 2: required map src_dest_map + 3: required Types.TNetworkAddress broker_addr + 4: optional map broker_prop } -struct TRestoreReq { - 1: required Types.TTabletId tablet_id - 2: required Types.TSchemaHash schema_hash - 3: required string remote_file_path - 4: required map remote_source_properties +struct TDownloadReq { + 1: required i64 job_id + 2: required map src_dest_map + 3: required Types.TNetworkAddress broker_addr + 4: optional map broker_prop } struct TSnapshotRequest { @@ -130,6 +131,7 @@ struct TSnapshotRequest { 3: optional Types.TVersion version 4: optional Types.TVersionHash version_hash 5: optional i64 timeout + 6: optional bool list_files } struct TReleaseSnapshotRequest { @@ -141,6 +143,14 @@ struct TClearRemoteFileReq { 2: required map remote_source_properties } +struct TMoveDirReq { + 1: required Types.TTabletId tablet_id + 2: required Types.TSchemaHash schema_hash + 3: required string src + 4: required i64 job_id + 5: required bool overwrite +} + enum TAgentServiceVersion { V1 } @@ -160,10 +170,11 @@ struct TAgentTaskRequest { 12: optional TStorageMediumMigrateReq storage_medium_migrate_req 13: optional TCheckConsistencyReq check_consistency_req 14: optional TUploadReq upload_req - 15: optional TRestoreReq restore_req + 15: optional TDownloadReq download_req 16: optional TSnapshotRequest snapshot_req 17: optional TReleaseSnapshotRequest release_snapshot_req 18: optional TClearRemoteFileReq clear_remote_file_req + 19: optional TMoveDirReq move_dir_req } struct TAgentResult { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 0e37784b66..64c31a4e8f 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -67,6 +67,7 @@ struct TDescribeTableParams { 1: optional string db 2: required string table_name 3: optional string user + 4: optional string user_ip } // Results of a call to describeTable() @@ -282,6 +283,7 @@ struct TGetDbsParams { // If not set, match every database 1: optional string pattern 2: optional string user + 3: optional string user_ip } // getDbNames returns a list of database names @@ -298,6 +300,7 @@ struct TGetTablesParams { // If not set, match every table 2: optional string pattern 3: optional string user + 4: optional string user_ip } struct TTableStatus { @@ -386,6 +389,7 @@ struct TMiniLoadRequest { 9: optional string subLabel 10: optional string cluster 11: optional i64 timestamp + 12: optional string user_ip } struct TUpdateMiniEtlTaskStatusRequest { @@ -402,6 +406,7 @@ struct TMasterOpRequest { 5: optional string cluster 6: optional i64 execMemLimit 7: optional i32 queryTimeout + 8: optional string user_ip } struct TColumnDefinition { @@ -434,6 +439,8 @@ struct TLoadCheckRequest { 5: optional string label 6: optional string cluster 7: optional i64 timestamp + 8: optional string user_ip + 9: optional string tbl } struct TUpdateExportTaskStatusRequest { diff --git a/gensrc/thrift/MasterService.thrift b/gensrc/thrift/MasterService.thrift index 5ea9d7bee4..2aa625b150 100644 --- a/gensrc/thrift/MasterService.thrift +++ b/gensrc/thrift/MasterService.thrift @@ -34,6 +34,7 @@ struct TTabletInfo { 5: required Types.TCount row_count 6: required Types.TSize data_size 7: optional Types.TStorageMedium storage_medium + 8: optional i64 version_count } struct TFinishTaskRequest { @@ -47,6 +48,9 @@ struct TFinishTaskRequest { 8: optional i64 request_version 9: optional i64 request_version_hash 10: optional string snapshot_path + 11: optional list snapshot_files + 12: optional map> tablet_files + 13: optional list downloaded_tablet_ids } struct TTablet { diff --git a/gensrc/thrift/PaloBrokerService.thrift b/gensrc/thrift/PaloBrokerService.thrift index 096a4c80d0..7487cc3c31 100644 --- a/gensrc/thrift/PaloBrokerService.thrift +++ b/gensrc/thrift/PaloBrokerService.thrift @@ -90,6 +90,7 @@ struct TBrokerListPathRequest { 2: required string path; 3: required bool isRecursive; 4: required map properties; + 5: optional bool fileNameOnly; } struct TBrokerDeletePathRequest { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index e67679977e..baf75e0d25 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -204,6 +204,7 @@ struct TSchemaScanNode { 7: optional string ip 8: optional i32 port 9: optional i64 thread_id + 10: optional string user_ip } struct TMetaScanNode { diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 0bb787a97c..69411a8993 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -145,10 +145,11 @@ enum TTaskType { CANCEL_DELETE, MAKE_SNAPSHOT, RELEASE_SNAPSHOT, - CHECK_CONSISTENCY + CHECK_CONSISTENCY, UPLOAD, - RESTORE, - CLEAR_REMOTE_FILE + DOWNLOAD, + CLEAR_REMOTE_FILE, + MOVE } enum TStmtType { diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 03846c0f9d..b3944a70de 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -505,7 +505,6 @@ build_bzip build_lzo2 build_boost # must before thrift build_ncurses -build_llvm build_protobuf build_gflags build_glog diff --git a/thirdparty/java-libraries/cobertura/asm-5.0.1.jar b/thirdparty/java-libraries/cobertura/asm-5.0.1.jar new file mode 100644 index 0000000000..eeb3bc6f98 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/asm-5.0.1.jar differ diff --git a/thirdparty/java-libraries/cobertura/asm-analysis-5.0.1.jar b/thirdparty/java-libraries/cobertura/asm-analysis-5.0.1.jar new file mode 100644 index 0000000000..af4a001085 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/asm-analysis-5.0.1.jar differ diff --git a/thirdparty/java-libraries/cobertura/asm-commons-5.0.1.jar b/thirdparty/java-libraries/cobertura/asm-commons-5.0.1.jar new file mode 100644 index 0000000000..b1f76966a7 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/asm-commons-5.0.1.jar differ diff --git a/thirdparty/java-libraries/cobertura/asm-tree-5.0.1.jar b/thirdparty/java-libraries/cobertura/asm-tree-5.0.1.jar new file mode 100644 index 0000000000..3b1a346115 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/asm-tree-5.0.1.jar differ diff --git a/thirdparty/java-libraries/cobertura/asm-util-5.0.1.jar b/thirdparty/java-libraries/cobertura/asm-util-5.0.1.jar new file mode 100644 index 0000000000..fb8d282291 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/asm-util-5.0.1.jar differ diff --git a/thirdparty/java-libraries/cobertura/cobertura-2.1.1-javadoc.jar b/thirdparty/java-libraries/cobertura/cobertura-2.1.1-javadoc.jar new file mode 100644 index 0000000000..8f1b58ab3d Binary files /dev/null and b/thirdparty/java-libraries/cobertura/cobertura-2.1.1-javadoc.jar differ diff --git a/thirdparty/java-libraries/cobertura/cobertura-2.1.1-sources.jar b/thirdparty/java-libraries/cobertura/cobertura-2.1.1-sources.jar new file mode 100644 index 0000000000..c05ff01c44 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/cobertura-2.1.1-sources.jar differ diff --git a/thirdparty/java-libraries/cobertura/cobertura-2.1.1.jar b/thirdparty/java-libraries/cobertura/cobertura-2.1.1.jar new file mode 100644 index 0000000000..d04676ad08 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/cobertura-2.1.1.jar differ diff --git a/thirdparty/java-libraries/cobertura/commons-lang3-3.3.2.jar b/thirdparty/java-libraries/cobertura/commons-lang3-3.3.2.jar new file mode 100644 index 0000000000..2ce08ae99d Binary files /dev/null and b/thirdparty/java-libraries/cobertura/commons-lang3-3.3.2.jar differ diff --git a/thirdparty/java-libraries/cobertura/hamcrest-core-1.3.jar b/thirdparty/java-libraries/cobertura/hamcrest-core-1.3.jar new file mode 100644 index 0000000000..9d5fe16e3d Binary files /dev/null and b/thirdparty/java-libraries/cobertura/hamcrest-core-1.3.jar differ diff --git a/thirdparty/java-libraries/cobertura/jaxen-1.1.4.jar b/thirdparty/java-libraries/cobertura/jaxen-1.1.4.jar new file mode 100644 index 0000000000..c2016095f0 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/jaxen-1.1.4.jar differ diff --git a/thirdparty/java-libraries/cobertura/jetty-6.1.14.jar b/thirdparty/java-libraries/cobertura/jetty-6.1.14.jar new file mode 100644 index 0000000000..8c503bea21 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/jetty-6.1.14.jar differ diff --git a/thirdparty/java-libraries/cobertura/jetty-util-6.1.14.jar b/thirdparty/java-libraries/cobertura/jetty-util-6.1.14.jar new file mode 100644 index 0000000000..8f924bb147 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/jetty-util-6.1.14.jar differ diff --git a/thirdparty/java-libraries/cobertura/logback-classic-1.0.13.jar b/thirdparty/java-libraries/cobertura/logback-classic-1.0.13.jar new file mode 100644 index 0000000000..80bf5d15a2 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/logback-classic-1.0.13.jar differ diff --git a/thirdparty/java-libraries/cobertura/logback-core-1.0.13.jar b/thirdparty/java-libraries/cobertura/logback-core-1.0.13.jar new file mode 100644 index 0000000000..568ccfaae5 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/logback-core-1.0.13.jar differ diff --git a/thirdparty/java-libraries/cobertura/oro-2.0.8.jar b/thirdparty/java-libraries/cobertura/oro-2.0.8.jar new file mode 100644 index 0000000000..23488d2600 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/oro-2.0.8.jar differ diff --git a/thirdparty/java-libraries/cobertura/servlet-api-2.5-6.1.14.jar b/thirdparty/java-libraries/cobertura/servlet-api-2.5-6.1.14.jar new file mode 100644 index 0000000000..6d7404fb72 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/servlet-api-2.5-6.1.14.jar differ diff --git a/thirdparty/java-libraries/cobertura/slf4j-api-1.7.5.jar b/thirdparty/java-libraries/cobertura/slf4j-api-1.7.5.jar new file mode 100644 index 0000000000..8f004d3906 Binary files /dev/null and b/thirdparty/java-libraries/cobertura/slf4j-api-1.7.5.jar differ diff --git a/thirdparty/java-libraries/jprotobuf-rpc-core-3.5.15.jar b/thirdparty/java-libraries/jprotobuf-rpc-core-3.5.17.jar similarity index 70% rename from thirdparty/java-libraries/jprotobuf-rpc-core-3.5.15.jar rename to thirdparty/java-libraries/jprotobuf-rpc-core-3.5.17.jar index 6e7aad20a3..c1e574ae05 100644 Binary files a/thirdparty/java-libraries/jprotobuf-rpc-core-3.5.15.jar and b/thirdparty/java-libraries/jprotobuf-rpc-core-3.5.17.jar differ