database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Fixed: privilege logic error:

Browse Source 
1. No one can set root password expect for root user itself
    2. NODE_PRIV cannot be granted.
    3. ADMIN_PRIV and GRANT_PRIV can only be granted or revoked on *.*
    4. No one can modifly privs of default role 'operator' and 'admin'.
    5. No user can be granted to role 'operator'.
Fixed: the running load limit should not be applied to replay logic. It will cause replay or loading image fail.
Changed: optimize the problem of too many directories under mini load directory.
Fixed: missing password and auth check when handling mini load request in Frontend.
Fixed: DomainResolver should start after Frontends transfer to a certain ROLE, not in Catalog construction methods.
Fixed: a stupid bug that no one can set password for root user... fix it: only root user can set password for root.
Fixed: read null data twice
    When reading data with a null value, in some cases, the same data will be read twice by the storage engine,
    resulting in a wrong result.The reason for this problem is that when splitting,
    and the start key is the minimum value, the data with null is read.
Fixed: add a flag to prevent DomainResovler thread start twice.
Fixed: fixed a mem leak of using ByteBuf when parsing auth info of http request.
Fixed: add a new config 'disable_hadoop_load', default is false, set to true to disable hadoop load.
Changed: add detail error msg of submitting hadoop load job in show load result.
Fixed: Backend process should be crashed if failed to saving header.
Added: exposure backend info to user when encounter error on Backend. for debugging it more convenient.
Fixed: Should remove fd from map when inputstream or outputstream is closed in Broker process.
Fixed: Change all files' LF to unix format.

Internal commit id: merge from dfcd0aca18eed9ff99d188eb3d01c60d419be1b8
This commit is contained in:
morningman
2018-10-01 19:58:41 +08:00
parent ae19fbfc8b
commit 65fe7f65c1
220 changed files with 20004 additions and 20022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/runtime/data_spliter.cpp
View File

@ -21,6 +21,7 @@
#include "exprs/expr.h"
#include "common/object_pool.h"
#include "service/backend_options.h"
#include "runtime/runtime_state.h"
#include "runtime/raw_value.h"
#include "runtime/row_batch.h"
@ -297,7 +298,8 @@ Status DataSpliter::close(RuntimeState* state, Status close_status) {
Status status = iter->finish(state);
if (UNLIKELY(is_ok && !status.ok())) {
LOG(WARNING) << "finish dpp_sink error"
<< " err_msg=" << status.get_error_msg();
<< " err_msg=" << status.get_error_msg()
<< " backend=" << BackendOptions::get_localhost();
is_ok = false;
err_status = status;
}

125
be/src/runtime/export_sink.cpp
View File

@ -121,72 +121,73 @@ Status ExportSink::gen_row_buffer(TupleRow* row, std::stringstream* ss) {
void* item = _output_expr_ctxs[i]->get_value(row);
if (item == nullptr) {
(*ss) << "NULL";
continue;
}
switch (_output_expr_ctxs[i]->root()->type().type) {
case TYPE_BOOLEAN:
case TYPE_TINYINT:
(*ss) << (int)*static_cast<int8_t*>(item);
break;
case TYPE_SMALLINT:
(*ss) << *static_cast<int16_t*>(item);
break;
case TYPE_INT:
(*ss) << *static_cast<int32_t*>(item);
break;
case TYPE_BIGINT:
(*ss) << *static_cast<int64_t*>(item);
break;
case TYPE_LARGEINT:
(*ss) << reinterpret_cast<PackedInt128*>(item)->value;
break;
case TYPE_FLOAT:
(*ss) << *static_cast<float*>(item);
break;
case TYPE_DOUBLE:
(*ss) << *static_cast<double*>(item);
break;
case TYPE_DATE:
case TYPE_DATETIME: {
char buf[64];
const DateTimeValue* time_val = (const DateTimeValue*)(item);
time_val->to_string(buf);
(*ss) << buf;
break;
}
case TYPE_VARCHAR:
case TYPE_CHAR: {
const StringValue* string_val = (const StringValue*)(item);
if (string_val->ptr == NULL) {
if (string_val->len == 0) {
} else {
(*ss) << "NULL";
} else {
switch (_output_expr_ctxs[i]->root()->type().type) {
case TYPE_BOOLEAN:
case TYPE_TINYINT:
(*ss) << (int)*static_cast<int8_t*>(item);
break;
case TYPE_SMALLINT:
(*ss) << *static_cast<int16_t*>(item);
break;
case TYPE_INT:
(*ss) << *static_cast<int32_t*>(item);
break;
case TYPE_BIGINT:
(*ss) << *static_cast<int64_t*>(item);
break;
case TYPE_LARGEINT:
(*ss) << reinterpret_cast<PackedInt128*>(item)->value;
break;
case TYPE_FLOAT:
(*ss) << *static_cast<float*>(item);
break;
case TYPE_DOUBLE:
(*ss) << *static_cast<double*>(item);
break;
case TYPE_DATE:
case TYPE_DATETIME: {
char buf[64];
const DateTimeValue* time_val = (const DateTimeValue*)(item);
time_val->to_string(buf);
(*ss) << buf;
break;
}
} else {
(*ss) << std::string(string_val->ptr, string_val->len);
}
break;
}
case TYPE_DECIMAL: {
const DecimalValue* decimal_val = reinterpret_cast<const DecimalValue*>(item);
std::string decimal_str;
int output_scale = _output_expr_ctxs[i]->root()->output_scale();
case TYPE_VARCHAR:
case TYPE_CHAR: {
const StringValue* string_val = (const StringValue*)(item);
if (output_scale > 0 && output_scale <= 30) {
decimal_str = decimal_val->to_string(output_scale);
} else {
decimal_str = decimal_val->to_string();
if (string_val->ptr == NULL) {
if (string_val->len == 0) {
} else {
(*ss) << "NULL";
}
} else {
(*ss) << std::string(string_val->ptr, string_val->len);
}
break;
}
case TYPE_DECIMAL: {
const DecimalValue* decimal_val = reinterpret_cast<const DecimalValue*>(item);
std::string decimal_str;
int output_scale = _output_expr_ctxs[i]->root()->output_scale();
if (output_scale > 0 && output_scale <= 30) {
decimal_str = decimal_val->to_string(output_scale);
} else {
decimal_str = decimal_val->to_string();
}
(*ss) << decimal_str;
break;
}
default: {
std::stringstream err_ss;
err_ss << "can't export this type. type = " << _output_expr_ctxs[i]->root()->type();
return Status(err_ss.str());
}
}
(*ss) << decimal_str;
break;
}
default: {
std::stringstream err_ss;
err_ss << "can't export this type. type = " << _output_expr_ctxs[i]->root()->type();
return Status(err_ss.str());
}
}
if (i < num_columns - 1) {
(*ss) << _t_export_sink.column_separator;
}

56
be/src/runtime/load_path_mgr.cpp
View File

@ -29,7 +29,10 @@
namespace palo {
LoadPathMgr::LoadPathMgr() : _idx(0) { }
static const uint32_t MAX_SHARD_NUM = 1024;
static const std::string SHARD_PREFIX = "__shard_";
LoadPathMgr::LoadPathMgr() : _idx(0), _next_shard(0) { }
Status LoadPathMgr::init() {
OLAPRootPath::RootPathVec all_available_root_path;
@ -75,14 +78,18 @@ Status LoadPathMgr::allocate_dir(
Status status = Status::OK;
while (retry--) {
{
// add SHARD_PREFIX for compatible purpose
std::lock_guard<std::mutex> l(_lock);
path = _path_vec[_idx] + "/" + db + "/" + label;
std::string shard = SHARD_PREFIX + std::to_string(_next_shard++ % MAX_SHARD_NUM);
path = _path_vec[_idx] + "/" + db + "/" + shard + "/" + label;
_idx = (_idx + 1) % size;
}
status = FileUtils::create_dir(path);
if (LIKELY(status.ok())) {
*prefix = path;
return Status::OK;
} else {
LOG(WARNING) << "create dir failed:" << path << ", error msg:" << status.get_error_msg();
}
}
@ -134,6 +141,19 @@ std::string LoadPathMgr::get_load_error_absolute_path(const std::string& file_na
return path;
}
void LoadPathMgr::process_label_dir(time_t now, const std::string& label_dir) {
if (!is_too_old(now, label_dir)) {
return;
}
LOG(INFO) << "Going to remove load directory. path=" << label_dir;
Status status = FileUtils::remove_all(label_dir);
if (status.ok()) {
LOG(INFO) << "Remove load directory success. path=" << label_dir;
} else {
LOG(WARNING) << "Remove load directory failed. path=" << label_dir;
}
}
void LoadPathMgr::clean_one_path(const std::string& path) {
std::vector<std::string> dbs;
Status status = FileUtils::scan_dir(path, &dbs);
@ -145,24 +165,32 @@ void LoadPathMgr::clean_one_path(const std::string& path) {
time_t now = time(nullptr);
for (auto& db : dbs) {
std::string db_dir = path + "/" + db;
std::vector<std::string> labels;
status = FileUtils::scan_dir(db_dir, &labels);
std::vector<std::string> sub_dirs;
status = FileUtils::scan_dir(db_dir, &sub_dirs);
if (!status.ok()) {
LOG(WARNING) << "scan db of trash dir failed, continue. dir=" << db_dir;
continue;
}
// delete this file
for (auto& label : labels) {
std::string label_dir = db_dir + "/" + label;
if (!is_too_old(now, label_dir)) {
continue;
}
LOG(INFO) << "Going to remove load directory. path=" << label_dir;
status = FileUtils::remove_all(label_dir);
if (status.ok()) {
LOG(INFO) << "Remove load directory success. path=" << label_dir;
for (auto& sub_dir : sub_dirs) {
std::string sub_path = db_dir + "/" + sub_dir;
// for compatible
if (sub_dir.find(SHARD_PREFIX) == 0) {
// sub_dir starts with SHARD_PREFIX
// process shard sub dir
std::vector<std::string> labels;
Status status = FileUtils::scan_dir(sub_path, &labels);
if (!status.ok()) {
LOG(WARNING) << "scan one path to delete directory failed. path=" << path;
continue;
}
for (auto& label : labels) {
std::string label_dir = sub_path + "/" + label;
process_label_dir(now, label_dir);
}
} else {
LOG(WARNING) << "Remove load directory failed. path=" << label_dir;
// process label dir
process_label_dir(now, sub_path);
}
}
}

2
be/src/runtime/load_path_mgr.h
View File

@ -56,6 +56,7 @@ private:
void clean_one_path(const std::string& path);
void clean_error_log();
void clean();
void process_label_dir(time_t now, const std::string& label_dir);
static void* cleaner(void* param);
@ -65,6 +66,7 @@ private:
int _reserved_hours;
pthread_t _cleaner_id;
std::string _error_log_dir;
uint32_t _next_shard;
};
}