database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[CodeFormat] Clang-format cpp sources (#4965)

Browse Source 
Clang-format all c++ source files.
This commit is contained in:
sduzh
2020-11-28 18:36:49 +08:00
committed by GitHub
parent f944bf4d44
commit 6fedf5881b
1331 changed files with 62548 additions and 68514 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
be/src/exec/csv_scan_node.cpp
View File

@ -17,71 +17,57 @@
#include "csv_scan_node.h"
#include <thrift/protocol/TDebugProtocol.h>
#include <string>
#include <vector>
#include <thrift/protocol/TDebugProtocol.h>
#include "exec/text_converter.hpp"
#include "exprs/hll_hash_function.h"
#include "gen_cpp/PlanNodes_types.h"
#include "runtime/runtime_state.h"
#include "runtime/row_batch.h"
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
#include "util/file_utils.h"
#include "util/runtime_profile.h"
#include "util/debug_util.h"
#include "util/hash_util.hpp"
#include "olap/olap_common.h"
#include "olap/utils.h"
#include "runtime/row_batch.h"
#include "runtime/runtime_state.h"
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
#include "util/debug_util.h"
#include "util/file_utils.h"
#include "util/hash_util.hpp"
#include "util/runtime_profile.h"
namespace doris {
class StringRef {
public:
StringRef(char const* const begin, int const size) :
_begin(begin), _size(size) {
}
StringRef(char const* const begin, int const size) : _begin(begin), _size(size) {}
~StringRef() {
// No need to delete _begin, because it only record the index in a std::string.
// The c-string will be released along with the std::string object.
}
int size() const {
return _size;
}
int length() const {
return _size;
}
int size() const { return _size; }
int length() const { return _size; }
char const* c_str() const {
return _begin;
}
char const* begin() const {
return _begin;
}
char const* c_str() const { return _begin; }
char const* begin() const { return _begin; }
char const* end() const { return _begin + _size; }
char const* end() const {
return _begin + _size;
}
private:
char const* _begin;
int _size;
};
void split_line(const std::string& str, char delimiter, std::vector<StringRef>& result) {
enum State {
IN_DELIM = 1,
IN_TOKEN = 0
};
enum State { IN_DELIM = 1, IN_TOKEN = 0 };
// line-begin char and line-end char are considered to be 'delimeter'
State state = IN_DELIM;
char const* p_begin = str.c_str(); // Begin of either a token or a delimiter
char const* p_begin = str.c_str(); // Begin of either a token or a delimiter
for (string::const_iterator it = str.begin(); it != str.end(); ++it) {
State const new_state = (*it == delimiter? IN_DELIM : IN_TOKEN);
State const new_state = (*it == delimiter ? IN_DELIM : IN_TOKEN);
if (new_state != state) {
if (new_state == IN_DELIM) {
result.push_back(StringRef(p_begin, &*it - p_begin));
@ -98,24 +84,21 @@ void split_line(const std::string& str, char delimiter, std::vector<StringRef>&
result.push_back(StringRef(p_begin, (&*str.end() - p_begin) - state));
}
CsvScanNode::CsvScanNode(
ObjectPool* pool,
const TPlanNode& tnode,
const DescriptorTbl& descs) :
ScanNode(pool, tnode, descs),
_tuple_id(tnode.csv_scan_node.tuple_id),
_file_paths(tnode.csv_scan_node.file_paths),
_column_separator(tnode.csv_scan_node.column_separator),
_column_type_map(tnode.csv_scan_node.column_type_mapping),
_column_function_map(tnode.csv_scan_node.column_function_mapping),
_columns(tnode.csv_scan_node.columns),
_unspecified_columns(tnode.csv_scan_node.unspecified_columns),
_default_values(tnode.csv_scan_node.default_values),
_is_init(false),
_tuple_desc(nullptr),
_tuple_pool(nullptr),
_text_converter(nullptr),
_hll_column_num(0) {
CsvScanNode::CsvScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
: ScanNode(pool, tnode, descs),
_tuple_id(tnode.csv_scan_node.tuple_id),
_file_paths(tnode.csv_scan_node.file_paths),
_column_separator(tnode.csv_scan_node.column_separator),
_column_type_map(tnode.csv_scan_node.column_type_mapping),
_column_function_map(tnode.csv_scan_node.column_function_mapping),
_columns(tnode.csv_scan_node.columns),
_unspecified_columns(tnode.csv_scan_node.unspecified_columns),
_default_values(tnode.csv_scan_node.default_values),
_is_init(false),
_tuple_desc(nullptr),
_tuple_pool(nullptr),
_text_converter(nullptr),
_hll_column_num(0) {
// do nothing
LOG(INFO) << "csv scan node: " << apache::thrift::ThriftDebugString(tnode).c_str();
}
@ -178,10 +161,8 @@ Status CsvScanNode::prepare(RuntimeState* state) {
}
// add 'unspecified_columns' which have default values
if (_unspecified_columns.end() != std::find(
_unspecified_columns.begin(),
_unspecified_columns.end(),
column_name)) {
if (_unspecified_columns.end() !=
std::find(_unspecified_columns.begin(), _unspecified_columns.end(), column_name)) {
_column_slot_map[column_name] = slot;
}
}
@ -204,17 +185,17 @@ Status CsvScanNode::prepare(RuntimeState* state) {
}
// new one scanner
_csv_scanner.reset(new(std::nothrow) CsvScanner(_file_paths));
_csv_scanner.reset(new (std::nothrow) CsvScanner(_file_paths));
if (_csv_scanner.get() == nullptr) {
return Status::InternalError("new a csv scanner failed.");
}
_tuple_pool.reset(new(std::nothrow) MemPool(state->instance_mem_tracker().get()));
_tuple_pool.reset(new (std::nothrow) MemPool(state->instance_mem_tracker().get()));
if (_tuple_pool.get() == nullptr) {
return Status::InternalError("new a mem pool failed.");
}
_text_converter.reset(new(std::nothrow) TextConverter('\\'));
_text_converter.reset(new (std::nothrow) TextConverter('\\'));
if (_text_converter.get() == nullptr) {
return Status::InternalError("new a text convertor failed.");
}
@ -322,7 +303,7 @@ Status CsvScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos
state->update_num_rows_load_total(_num_rows_load_total);
state->update_num_rows_load_filtered(_num_rows_load_filtered);
VLOG_ROW << "normal_row_number: " << state->num_rows_load_success()
<< "; error_row_number: " << state->num_rows_load_filtered() << std::endl;
<< "; error_row_number: " << state->num_rows_load_filtered() << std::endl;
row_batch->tuple_data_pool()->acquire_data(_tuple_pool.get(), false);
@ -354,7 +335,7 @@ Status CsvScanNode::close(RuntimeState* state) {
// Summary normal line and error line number info
std::stringstream summary_msg;
summary_msg << "error line: " << _num_rows_load_filtered
<< "; normal line: " << state->num_rows_load_success();
<< "; normal line: " << state->num_rows_load_success();
state->append_error_msg_to_file("", summary_msg.str(), true);
}
@ -375,9 +356,8 @@ Status CsvScanNode::set_scan_ranges(const std::vector<TScanRangeParams>& scan_ra
return Status::OK();
}
void CsvScanNode::fill_fix_length_string(
const char* value, const int value_length, MemPool* pool,
char** new_value_p, const int new_value_length) {
void CsvScanNode::fill_fix_length_string(const char* value, const int value_length, MemPool* pool,
char** new_value_p, const int new_value_length) {
if (new_value_length != 0 && value_length < new_value_length) {
DCHECK(pool != nullptr);
*new_value_p = reinterpret_cast<char*>(pool->allocate(new_value_length));
@ -388,10 +368,10 @@ void CsvScanNode::fill_fix_length_string(
(*new_value_p)[i] = '\0';
}
VLOG_ROW << "Fill fix length string. "
<< "value: [" << std::string(value, value_length) << "]; "
<< "value_length: " << value_length << "; "
<< "*new_value_p: [" << *new_value_p << "]; "
<< "new value length: " << new_value_length << std::endl;
<< "value: [" << std::string(value, value_length) << "]; "
<< "value_length: " << value_length << "; "
<< "*new_value_p: [" << *new_value_p << "]; "
<< "new value length: " << new_value_length << std::endl;
}
}
@ -399,15 +379,14 @@ void CsvScanNode::fill_fix_length_string(
// .123 1.23 123. -1.23
// ATTN: The decimal point and (for negative numbers) the "-" sign are not counted.
// like '.123', it will be regarded as '0.123', but it match decimal(3, 3)
bool CsvScanNode::check_decimal_input(
const char* value, const int value_length,
const int precision, const int scale,
std::stringstream* error_msg) {
bool CsvScanNode::check_decimal_input(const char* value, const int value_length,
const int precision, const int scale,
std::stringstream* error_msg) {
if (value_length > (precision + 2)) {
(*error_msg) << "the length of decimal value is overflow. "
<< "precision in schema: (" << precision << ", " << scale << "); "
<< "value: [" << std::string(value, value_length) << "]; "
<< "str actual length: " << value_length << ";";
<< "precision in schema: (" << precision << ", " << scale << "); "
<< "value: [" << std::string(value, value_length) << "]; "
<< "str actual length: " << value_length << ";";
return false;
}
@ -435,7 +414,7 @@ bool CsvScanNode::check_decimal_input(
int value_int_len = 0;
int value_frac_len = 0;
value_int_len = point_index - begin_index;
value_frac_len = end_index- point_index;
value_frac_len = end_index - point_index;
if (point_index == -1) {
// an int value: like 123
@ -443,18 +422,16 @@ bool CsvScanNode::check_decimal_input(
value_frac_len = 0;
} else {
value_int_len = point_index - begin_index;
value_frac_len = end_index- point_index;
value_frac_len = end_index - point_index;
}
if (value_int_len > (precision - scale)) {
(*error_msg) << "the int part length longer than schema precision ["
<< precision << "]. "
<< "value [" << std::string(value, value_length) << "]. ";
(*error_msg) << "the int part length longer than schema precision [" << precision << "]. "
<< "value [" << std::string(value, value_length) << "]. ";
return false;
} else if (value_frac_len > scale) {
(*error_msg) << "the frac part length longer than schema scale ["
<< scale << "]. "
<< "value [" << std::string(value, value_length) << "]. ";
(*error_msg) << "the frac part length longer than schema scale [" << scale << "]. "
<< "value [" << std::string(value, value_length) << "]. ";
return false;
}
return true;
@ -465,17 +442,15 @@ static bool is_null(const char* value, int value_length) {
}
// Writes a slot in _tuple from an value containing text data.
bool CsvScanNode::check_and_write_text_slot(
const std::string& column_name, const TColumnType& column_type,
const char* value, int value_length,
const SlotDescriptor* slot, RuntimeState* state,
std::stringstream* error_msg) {
bool CsvScanNode::check_and_write_text_slot(const std::string& column_name,
const TColumnType& column_type, const char* value,
int value_length, const SlotDescriptor* slot,
RuntimeState* state, std::stringstream* error_msg) {
if (value_length == 0 && !slot->type().is_string_type()) {
(*error_msg) << "the length of input should not be 0. "
<< "column_name: " << column_name << "; "
<< "type: " << slot->type() << "; "
<< "input_str: [" << std::string(value, value_length) << "].";
<< "column_name: " << column_name << "; "
<< "type: " << slot->type() << "; "
<< "input_str: [" << std::string(value, value_length) << "].";
return false;
}
@ -485,16 +460,16 @@ bool CsvScanNode::check_and_write_text_slot(
return true;
} else {
(*error_msg) << "value cannot be null. column name: " << column_name
<< "; type: " << slot->type() << "; input_str: ["
<< std::string(value, value_length) << "].";
<< "; type: " << slot->type() << "; input_str: ["
<< std::string(value, value_length) << "].";
return false;
}
}
if (!slot->is_nullable() && is_null(value, value_length)) {
(*error_msg) << "value cannot be null. column name: " << column_name
<< "; type: " << slot->type() << "; input_str: ["
<< std::string(value, value_length) << "].";
<< "; type: " << slot->type() << "; input_str: ["
<< std::string(value, value_length) << "].";
return false;
}
@ -506,17 +481,16 @@ bool CsvScanNode::check_and_write_text_slot(
int char_len = column_type.len;
if (slot->type().type != TYPE_HLL && value_length > char_len) {
(*error_msg) << "the length of input is too long than schema. "
<< "column_name: " << column_name << "; "
<< "input_str: [" << std::string(value, value_length) << "] "
<< "type: " << slot->type() << "; "
<< "schema length: " << char_len << "; "
<< "actual length: " << value_length << "; ";
<< "column_name: " << column_name << "; "
<< "input_str: [" << std::string(value, value_length) << "] "
<< "type: " << slot->type() << "; "
<< "schema length: " << char_len << "; "
<< "actual length: " << value_length << "; ";
return false;
}
if (slot->type().type == TYPE_CHAR && value_length < char_len) {
fill_fix_length_string(
value, value_length, _tuple_pool.get(),
&value_to_convert, char_len);
fill_fix_length_string(value, value_length, _tuple_pool.get(), &value_to_convert,
char_len);
value_to_convert_length = char_len;
}
} else if (slot->type().is_decimal_type()) {
@ -528,13 +502,11 @@ bool CsvScanNode::check_and_write_text_slot(
}
}
if (!_text_converter->write_slot(slot, _tuple, value_to_convert, value_to_convert_length,
true, false, _tuple_pool.get())) {
(*error_msg) << "convert csv string to "
<< slot->type() << " failed. "
<< "column_name: " << column_name << "; "
<< "input_str: [" << std::string(value, value_length) << "]; ";
if (!_text_converter->write_slot(slot, _tuple, value_to_convert, value_to_convert_length, true,
false, _tuple_pool.get())) {
(*error_msg) << "convert csv string to " << slot->type() << " failed. "
<< "column_name: " << column_name << "; "
<< "input_str: [" << std::string(value, value_length) << "]; ";
return false;
}
@ -554,14 +526,14 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
if (_hll_column_num == 0 && fields.size() < _columns.size()) {
error_msg << "actual column number is less than schema column number. "
<< "actual number: " << fields.size() << " ,"
<< "schema number: " << _columns.size() << "; ";
<< "actual number: " << fields.size() << " ,"
<< "schema number: " << _columns.size() << "; ";
_runtime_state->append_error_msg_to_file(line, error_msg.str());
return false;
} else if (_hll_column_num == 0 && fields.size() > _columns.size()) {
error_msg << "actual column number is more than schema column number. "
<< "actual number: " << fields.size() << " ,"
<< "schema number: " << _columns.size() << "; ";
<< "actual number: " << fields.size() << " ,"
<< "schema number: " << _columns.size() << "; ";
_runtime_state->append_error_msg_to_file(line, error_msg.str());
return false;
}
@ -583,11 +555,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
const TColumnType& column_type = _column_type_vec[i];
bool flag = check_and_write_text_slot(
column_name, column_type,
fields[i].c_str(),
fields[i].length(),
slot, state, &error_msg);
bool flag = check_and_write_text_slot(column_name, column_type, fields[i].c_str(),
fields[i].length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
@ -611,11 +580,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
const TColumnType& column_type = _unspecified_colomn_type_vec[i];
bool flag = check_and_write_text_slot(
column_name, column_type,
_default_values[i].c_str(),
_default_values[i].length(),
slot, state, &error_msg);
bool flag = check_and_write_text_slot(column_name, column_type, _default_values[i].c_str(),
_default_values[i].length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
@ -624,8 +590,7 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
for (std::map<std::string, TMiniLoadEtlFunction>::iterator iter = _column_function_map.begin();
iter != _column_function_map.end();
iter++) {
iter != _column_function_map.end(); iter++) {
TMiniLoadEtlFunction& function = iter->second;
const std::string& column_name = iter->first;
const SlotDescriptor* slot = _column_slot_map[column_name];
@ -634,11 +599,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
const char* src = fields[function.param_column_index].c_str();
int src_column_len = fields[function.param_column_index].length();
hll_hash(src, src_column_len, &column_string);
bool flag = check_and_write_text_slot(
column_name, column_type,
column_string.c_str(),
column_string.length(),
slot, state, &error_msg);
bool flag = check_and_write_text_slot(column_name, column_type, column_string.c_str(),
column_string.length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
return false;
@ -649,9 +611,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
}
bool CsvScanNode::check_hll_function(TMiniLoadEtlFunction& function) {
if (function.function_name.empty()
|| function.function_name != "hll_hash"
|| function.param_column_index < 0) {
if (function.function_name.empty() || function.function_name != "hll_hash" ||
function.param_column_index < 0) {
return false;
}
return true;
@ -676,4 +637,3 @@ void CsvScanNode::hll_hash(const char* src, int len, std::string* result) {
}
} // end namespace doris