database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](struct-type) refactor and clean unused code for struct type (#17257)

Browse Source 
remove unused code for struct type
This commit is contained in:
xy720
2023-03-01 15:49:31 +08:00
committed by GitHub
parent 0732eb54bc
commit 48ef61780d
4 changed files with 3 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
be/src/vec/columns/column_struct.cpp
View File

@ -178,24 +178,6 @@ void ColumnStruct::update_hash_with_value(size_t n, SipHash& hash) const {
}
}
// void ColumnStruct::update_weak_hash32(WeakHash32 & hash) const {
// auto s = size();
// if (hash.get_data().size() != s) {
// throw Exception("Size of WeakHash32 does not match size of column: column size is " + std::to_string(s) +
// ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR);
// }
// for (const auto & column : columns) {
// column->update_weak_hash32(hash);
// }
// }
// void ColumnStruct::update_hash_fast(SipHash & hash) const {
// for (const auto & column : columns) {
// column->update_hash_fast(hash);
// }
// }
void ColumnStruct::insert_indices_from(const IColumn& src, const int* indices_begin,
const int* indices_end) {
const ColumnStruct& src_concrete = assert_cast<const ColumnStruct&>(src);
@ -204,32 +186,6 @@ void ColumnStruct::insert_indices_from(const IColumn& src, const int* indices_be
}
}
// const char * ColumnStruct::skip_serialized_in_arena(const char * pos) const {
// for (const auto & column : columns) {
// pos = column->skip_serialized_in_arena(pos);
// }
// return pos;
// }
// void ColumnStruct::expand(const Filter & mask, bool inverted)
// {
// for (auto & column : columns) {
// column->expand(mask, inverted);
// }
// }
// ColumnPtr ColumnStruct::index(const IColumn & indexes, size_t limit) const
// {
// const size_t tuple_size = columns.size();
// Columns new_columns(tuple_size);
// for (size_t i = 0; i < tuple_size; ++i) {
// new_columns[i] = columns[i]->index(indexes, limit);
// }
// return ColumnStruct::create(new_columns);
// }
void ColumnStruct::insert_range_from(const IColumn& src, size_t start, size_t length) {
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i) {
@ -315,158 +271,6 @@ MutableColumns ColumnStruct::scatter(ColumnIndex num_columns, const Selector& se
return res;
}
// int ColumnStruct::compare_at_impl(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint,
// const Collator* collator) const {
// const size_t tuple_size = columns.size();
// for (size_t i = 0; i < tuple_size; ++i) {
// int res = 0;
// if (collator && columns[i]->is_collation_supported()) {
// res = columns[i]->compare_at_with_collation(
// n, m, *assert_cast<const ColumnStruct&>(rhs).columns[i], nan_direction_hint,
// *collator);
// } else {
// res = columns[i]->compare_at(n, m, *assert_cast<const ColumnStruct&>(rhs).columns[i],
// nan_direction_hint);
// }
// if (res) {
// return res;
// }
// }
// return 0;
// }
// int ColumnStruct::compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const {
// return compare_at_impl(n, m, rhs, nan_direction_hint);
// }
// void ColumnStruct::compare_column(const IColumn& rhs, size_t rhs_row_num,
// PaddedPODArray<UInt64>* row_indexes,
// PaddedPODArray<Int8>& compare_results, int direction,
// int nan_direction_hint) const {
// return do_compare_column<ColumnStruct>(assert_cast<const ColumnStruct&>(rhs), rhs_row_num,
// row_indexes, compare_results, direction,
// nan_direction_hint);
// }
// int ColumnStruct::compare_at_with_collation(size_t n, size_t m, const IColumn& rhs,
// int nan_direction_hint,
// const Collator& collator) const {
// return compare_at_impl(n, m, rhs, nan_direction_hint, &collator);
// }
// bool ColumnStruct::has_equal_values() const {
// return has_equal_values_impl<ColumnStruct>();
// }
// template <bool positive>
// struct ColumnStruct::Less {
// TupleColumns columns;
// int nan_direction_hint;
// const Collator* collator;
// Less(const TupleColumns& columns_, int nan_direction_hint_, const Collator* collator_ = nullptr)
// : columns(columns_), nan_direction_hint(nan_direction_hint_), collator(collator_) {}
// bool operator()(size_t a, size_t b) const {
// for (const auto& column : columns) {
// int res;
// if (collator && column->isCollationSupported()) {
// res = column->compareAtWithCollation(a, b, *column, nan_direction_hint, *collator);
// } else {
// res = column->compareAt(a, b, *column, nan_direction_hint);
// }
// if (res < 0) {
// return positive;
// } else if (res > 0) {
// return !positive;
// }
// }
// return false;
// }
// };
// void ColumnStruct::get_permutation_impl(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, Permutation& res,
// const Collator* collator) const {
// size_t rows = size();
// res.resize(rows);
// for (size_t i = 0; i < rows; ++i) {
// res[i] = i;
// }
// if (limit >= rows) {
// limit = 0;
// }
// EqualRange ranges;
// ranges.emplace_back(0, rows);
// update_permutation_impl(direction, stability, limit, nan_direction_hint, res, ranges, collator);
// }
// void ColumnStruct::update_permutation_impl(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability,
// size_t limit, int nan_direction_hint,
// IColumn::Permutation& res, EqualRanges& equal_ranges,
// const Collator* collator) const {
// if (equal_ranges.empty()) {
// return;
// }
// for (const auto& column : columns) {
// while (!equal_ranges.empty() && limit && limit <= equal_ranges.back().first) {
// equal_ranges.pop_back();
// }
// if (collator && column->isCollationSupported()) {
// column->update_permutation_with_collation(*collator, direction, stability, limit,
// nan_direction_hint, res, equal_ranges);
// } else {
// column->update_permutation(direction, stability, limit, nan_direction_hint, res,
// equal_ranges);
// }
// if (equal_ranges.empty()) {
// break;
// }
// }
// }
// void ColumnStruct::get_permutation(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, Permutation& res) const {
// get_permutation_impl(direction, stability, limit, nan_direction_hint, res, nullptr);
// }
// void ColumnStruct::update_permutation(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, IColumn::Permutation& res,
// EqualRanges& equal_ranges) const {
// update_permutation_impl(direction, stability, limit, nan_direction_hint, res, equal_ranges);
// }
// void ColumnStruct::get_permutation_with_collation(const Collator& collator,
// IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability,
// size_t limit, int nan_direction_hint,
// Permutation& res) const {
// get_permutation_impl(direction, stability, limit, nan_direction_hint, res, &collator);
// }
// void ColumnStruct::update_permutation_with_collation(const Collator& collator,
// IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability,
// size_t limit, int nan_direction_hint,
// Permutation& res,
// EqualRanges& equal_ranges) const {
// update_permutation_impl(direction, stability, limit, nan_direction_hint, res, equal_ranges,
// &collator);
// }
// void ColumnStruct::gather(ColumnGathererStream& gatherer) {
// gatherer.gather(*this);
// }
void ColumnStruct::reserve(size_t n) {
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i) {
@ -482,21 +286,6 @@ size_t ColumnStruct::byte_size() const {
return res;
}
// size_t ColumnStruct::byte_size_at(size_t n) const {
// size_t res = 0;
// for (const auto& column : columns) {
// res += column->byte_size_at(n);
// }
// return res;
// }
// void ColumnStruct::ensure_ownership() {
// const size_t tuple_size = columns.size();
// for (size_t i = 0; i < tuple_size; ++i) {
// get_column(i).ensure_ownership();
// }
// }
size_t ColumnStruct::allocated_bytes() const {
size_t res = 0;
for (const auto& column : columns) {
@ -549,59 +338,4 @@ bool ColumnStruct::structure_equals(const IColumn& rhs) const {
}
}
// void ColumnStruct::for_each_subcolumn_recursively(ColumnCallback callback) {
// for (auto& column : columns) {
// callback(column);
// column->for_each_subcolumn_recursively(callback);
// }
// }
// bool ColumnStruct::is_collation_supported() const {
// for (const auto& column : columns) {
// if (column->is_collation_supported()) {
// return true;
// }
// }
// return false;
// }
// ColumnPtr ColumnStruct::compress() const {
// size_t byte_size = 0;
// Columns compressed;
// compressed.reserve(columns.size());
// for (const auto& column : columns) {
// auto compressed_column = column->compress();
// byte_size += compressed_column->byteSize();
// compressed.emplace_back(std::move(compressed_column));
// }
// return ColumnCompressed::create(size(), byte_size,
// [compressed = std::move(compressed)]() mutable {
// for (auto& column : compressed) {
// column = column->decompress();
// }
// return ColumnStruct::create(compressed);
// });
// }
// double ColumnStruct::get_ratio_of_default_rows(double sample_ratio) const {
// return get_ratio_of_default_rows_impl<ColumnStruct>(sample_ratio);
// }
// void ColumnStruct::get_indices_of_nondefault_rows(Offsets& indices, size_t from,
// size_t limit) const {
// return get_indices_of_nondefault_rows_impl<ColumnStruct>(indices, from, limit);
// }
// void ColumnStruct::finalize() {
// for (auto& column : columns) {
// column->finalize();
// }
// }
// bool ColumnStruct::is_finalized() const {
// return std::all_of(columns.begin(), columns.end(),
// [](const auto& column) { return column->is_finalized(); });
// }
} // namespace doris::vectorized

96
be/src/vec/columns/column_struct.h
View File

@ -18,38 +18,6 @@
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnTuple.h
// and modified by Doris
/********************************************************************************
// doris/core/be/src/vec/core/field.h
class Field;
using FieldVector = std::vector<Field>;
/// Array and Tuple use the same storage type -- FieldVector, but we declare
/// distinct types for them, so that the caller can choose whether it wants to
/// construct a Field of Array or a Tuple type. An alternative approach would be
/// to construct both of these types from FieldVector, and have the caller
/// specify the desired Field type explicitly.
#define DEFINE_FIELD_VECTOR(X) \
struct X : public FieldVector { \
using FieldVector::FieldVector; \
}
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
#undef DEFINE_FIELD_VECTOR
// defination of some pointer
using WrappedPtr = chameleon_ptr<Derived>;
using Ptr = immutable_ptr<Derived>;
using ColumnPtr = IColumn::Ptr;
using Columns = std::vector<ColumnPtr>;
using MutablePtr = mutable_ptr<Derived>;
using MutableColumnPtr = IColumn::MutablePtr;
using MutableColumns = std::vector<MutableColumnPtr>;
****************************************************************************/
#pragma once
#include "vec/columns/column.h"
@ -64,10 +32,6 @@ using MutableColumns = std::vector<MutableColumnPtr>;
namespace doris::vectorized {
/** Column, that is just group of few another columns.
*
* For constant Tuples, see ColumnConst.
* Mixed constant/non-constant columns is prohibited in tuple
* for implementation simplicity.
*/
class ColumnStruct final : public COWHelper<IColumn, ColumnStruct> {
private:
@ -123,10 +87,6 @@ public:
const char* deserialize_and_insert_from_arena(const char* pos) override;
void update_hash_with_value(size_t n, SipHash& hash) const override;
// const char * skip_serialized_in_arena(const char * pos) const override;
// void update_weak_hash32(WeakHash32 & hash) const override;
// void update_hash_fast(SipHash & hash) const override;
void insert_indices_from(const IColumn& src, const int* indices_begin,
const int* indices_end) override;
@ -153,60 +113,18 @@ public:
MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override;
// ColumnPtr index(const IColumn & indexes, size_t limit) const override;
// void expand(const Filter & mask, bool inverted) override;
// void gather(ColumnGathererStream & gatherer_stream) override;
// bool has_equal_values() const override;
// void compare_column(const IColumn& rhs, size_t rhs_row_num, PaddedPODArray<UInt64>* row_indexes,
// PaddedPODArray<Int8>& compare_results, int direction,
// int nan_direction_hint) const override;
// int compare_at_with_collation(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint,
// const Collator& collator) const override;
[[noreturn]] int compare_at(size_t n, size_t m, const IColumn& rhs_,
int nan_direction_hint) const override {
LOG(FATAL) << "compare_at not implemented";
}
void get_extremes(Field& min, Field& max) const override;
// void get_permutation(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, IColumn::Permutation& res) const override;
// void update_permutation(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, IColumn::Permutation& res,
// EqualRanges& equal_ranges) const override;
// void get_permutation_with_collation(const Collator& collator,
// IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint,
// IColumn::Permutation& res) const override;
// void update_permutation_with_collation(const Collator& collator,
// IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability,
// size_t limit, int nan_direction_hint,
// IColumn::Permutation& res,
// EqualRanges& equal_ranges) const override;
void reserve(size_t n) override;
size_t byte_size() const override;
// size_t byte_size_at(size_t n) const override;
// void ensure_ownership() override;
size_t allocated_bytes() const override;
void protect() override;
void for_each_subcolumn(ColumnCallback callback) override;
bool structure_equals(const IColumn& rhs) const override;
// void for_each_subcolumn_recursively(ColumnCallback callback) override;
// bool is_collation_supported() const override;
// ColumnPtr compress() const override;
// double get_ratio_of_default_rows(double sample_ratio) const override;
// void get_indices_of_nondefault_rows(Offsets & indices, size_t from, size_t limit) const override;
// void finalize() override;
// bool is_finalized() const override;
size_t tuple_size() const { return columns.size(); }
const IColumn& get_column(size_t idx) const { return *columns[idx]; }
@ -223,20 +141,6 @@ public:
col->clear();
}
}
private:
int compare_at_impl(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const;
// void get_permutation_impl(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, Permutation& res,
// const Collator* collator) const;
// void update_permutation_impl(IColumn::PermutationSortDirection direction,
// IColumn::PermutationSortStability stability, size_t limit,
// int nan_direction_hint, IColumn::Permutation& res,
// EqualRanges& equal_ranges,
// const Collator* collator = nullptr) const;
};
} // namespace doris::vectorized

124
be/src/vec/data_types/data_type_struct.cpp
View File

@ -301,7 +301,6 @@ static void add_element_safe(const DataTypes& elems, IColumn& column, F&& impl)
// Check that all columns now have the same size.
size_t new_size = column.size();
// for (auto i : collections::range(0, elems.size())) {
for (auto i = 0; i < elems.size(); i++) {
const auto& element_column = extract_element_column(column, i);
if (element_column.size() != new_size) {
@ -311,7 +310,6 @@ static void add_element_safe(const DataTypes& elems, IColumn& column, F&& impl)
}
}
} catch (...) {
// for (const auto& i : collections::range(0, elems.size())) {
for (auto i = 0; i < elems.size(); i++) {
auto& element_column = extract_element_column(column, i);
@ -333,42 +331,12 @@ MutableColumnPtr DataTypeStruct::create_column() const {
return ColumnStruct::create(std::move(tuple_columns));
}
// MutableColumnPtr DataTypeStruct::create_column(const ISerialization& serialization) const {
// /// If we read subcolumn of nested Tuple, it may be wrapped to SerializationNamed
// /// several times to allow to reconstruct the substream path name.
// /// Here we don't need substream path name, so we drop first several wrapper serializations.
// const auto* current_serialization = &serialization;
// while (const auto* serialization_named =
// typeid_cast<const SerializationNamed*>(current_serialization))
// current_serialization = serialization_named->get_nested().get();
// const auto* serialization_tuple = typeid_cast<const SerializationTuple*>(current_serialization);
// if (!serialization_tuple)
// throw Exception(ErrorCodes::LOGICAL_ERROR,
// "Unexpected serialization to create column of type Tuple");
// const auto& element_serializations = serialization_tuple->getElementsSerializations();
// size_t size = elems.size();
// assert(element_serializations.size() == size);
// MutableColumns tuple_columns(size);
// for (size_t i = 0; i < size; ++i) {
// tuple_columns[i] = elems[i]->create_column(*element_serializations[i]->get_nested());
// }
// return ColumnStruct::create(std::move(tuple_columns));
// }
Field DataTypeStruct::get_default() const {
return Tuple();
//return Tuple(collections::map<Tuple>(
// elems, [](const DataTypePtr& elem) { return elem->get_default(); }));
}
void DataTypeStruct::insert_default_into(IColumn& column) const {
add_element_safe(elems, column, [&] {
// for (const auto& i : collections::range(0, elems.size()))
for (auto i = 0; i < elems.size(); i++) {
elems[i]->insert_default_into(extract_element_column(column, i));
}
@ -501,96 +469,4 @@ size_t DataTypeStruct::get_size_of_value_in_memory() const {
return res;
}
// bool DataTypeStruct::has_dynamic_subcolumns() const {
// return std::any_of(elems.begin(), elems.end(),
// [](auto&& elem) { return elem->has_dynamic_subcolumns(); });
// }
// SerializationPtr DataTypeStruct::do_get_default_serialization() const {
// SerializationTuple::ElementSerializations serializations(elems.size());
// for (size_t i = 0; i < elems.size(); ++i) {
// String elem_name = have_explicit_names ? names[i] : toString(i + 1);
// auto serialization = elems[i]->get_default_serialization();
// serializations[i] = std::make_shared<SerializationNamed>(serialization, elem_name);
// }
// return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
// }
// SerializationPtr DataTypeStruct::get_serialization(const SerializationInfo& info) const {
// SerializationTuple::ElementSerializations serializations(elems.size());
// const auto& info_tuple = assert_cast<const SerializationInfoTuple&>(info);
// for (size_t i = 0; i < elems.size(); ++i) {
// String elem_name = have_explicit_names ? names[i] : toString(i + 1);
// auto serialization = elems[i]->get_serialization(*info_tuple.get_element_info(i));
// serializations[i] = std::make_shared<SerializationNamed>(serialization, elem_name);
// }
// return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
// }
// MutableSerializationInfoPtr DataTypeStruct::create_serialization_info(
// const SerializationInfo::Settings& settings) const {
// MutableSerializationInfos infos;
// infos.reserve(elems.size());
// for (const auto& elem : elems) {
// infos.push_back(elem->create_serializationInfo(settings));
// }
// return std::make_shared<SerializationInfoTuple>(std::move(infos), names, settings);
// }
// SerializationInfoPtr DataTypeStruct::get_serialization_info(const IColumn& column) const {
// if (const auto* column_const = check_and_get_column<ColumnConst>(&column)) {
// return get_serialization_info(column_const->get_data_column());
// }
// MutableSerializationInfos infos;
// infos.reserve(elems.size());
// const auto& column_tuple = assert_cast<const ColumnStruct&>(column);
// assert(elems.size() == column_tuple.get_columns().size());
// for (size_t i = 0; i < elems.size(); ++i) {
// auto element_info = elems[i]->get_serialization_info(column_tuple.getColumn(i));
// infos.push_back(const_pointer_cast<SerializationInfo>(element_info));
// }
// return std::make_shared<SerializationInfoTuple>(std::move(infos), names,
// SerializationInfo::Settings {});
// }
// static DataTypePtr create(const ASTPtr& arguments) {
// if (!arguments || arguments->children.empty())
// throw Exception("Struct cannot be empty", ErrorCodes::EMPTY_DATA_PASSED);
// DataTypes nested_types;
// nested_types.reserve(arguments->children.size());
// Strings names;
// names.reserve(arguments->children.size());
// for (const ASTPtr& child : arguments->children) {
// if (const auto* name_and_type_pair = child->as<ASTNameTypePair>()) {
// nested_types.emplace_back(DataTypeFactory::instance().get(name_and_type_pair->type));
// names.emplace_back(name_and_type_pair->name);
// } else
// nested_types.emplace_back(DataTypeFactory::instance().get(child));
// }
// if (names.empty())
// return std::make_shared<DataTypeStruct>(nested_types);
// else if (names.size() != nested_types.size())
// throw Exception("Names are specified not for all elements of Struct type",
// ErrorCodes::BAD_ARGUMENTS);
// else
// return std::make_shared<DataTypeStruct>(nested_types, names);
// }
// void registerDataTypeStruct(DataTypeFactory& factory) {
// factory.registerDataType("Struct", create);
// }
} // namespace doris::vectorized

18
be/src/vec/data_types/data_type_struct.h
View File

@ -40,20 +40,16 @@ namespace doris::vectorized {
* If an element is unnamed, it will have automatically assigned name like '1', '2', '3' corresponding to its position.
* Manually assigned names must not begin with digit. Names must be unique.
*
* All tuples with same size and types of elements are equivalent for expressions, regardless to names of elements.
* All structs with same size and types of elements are equivalent for expressions, regardless to names of elements.
*/
class DataTypeStruct final : public IDataType {
private:
// using DataTypePtr = std::shared_ptr<const IDataType>;
// using DataTypes = std::vector<DataTypePtr>;
// using Strings = std::vector<std::string>;
DataTypes elems;
Strings names;
bool have_explicit_names;
public:
// static constexpr bool is_parametric = true;
static constexpr bool is_parametric = true;
explicit DataTypeStruct(const DataTypes& elems);
DataTypeStruct(const DataTypes& elems, const Strings& names);
@ -66,7 +62,6 @@ public:
bool supports_sparse_serialization() const { return true; }
MutableColumnPtr create_column() const override;
// MutableColumnPtr create_column(const ISerialization& serialization) const override;
Field get_default() const override;
void insert_default_into(IColumn& column) const override;
@ -78,7 +73,6 @@ public:
bool is_comparable() const override;
bool text_can_contain_only_valid_utf8() const override;
bool have_maximum_size_of_value() const override;
bool has_dynamic_subcolumns() const;
size_t get_maximum_size_of_value_in_memory() const override;
size_t get_size_of_value_in_memory() const override;
@ -99,13 +93,7 @@ public:
Status from_string(ReadBuffer& rb, IColumn* column) const override;
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
// bool is_parametric() const { return true; }
// SerializationPtr do_get_default_serialization() const override;
// SerializationPtr get_serialization(const SerializationInfo& info) const override;
// MutableSerializationInfoPtr create_serialization_info(
// const SerializationInfo::Settings& settings) const override;
// SerializationInfoPtr get_serialization_info(const IColumn& column) const override;
// bool have_explicit_names() const { return have_explicit_names; }
bool get_have_explicit_names() const { return have_explicit_names; }
};
} // namespace doris::vectorized