database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](delete-pred) enable delete sub predicate v2 for compaction (#35859) (#35895)

Browse Source 
## Proposed changes

This PR enable `delete sub predicate v2` for compaction, and legacy
version of delete predicate will be processed in the original way.
This commit is contained in:
Siyang Tang
2024-06-05 12:05:21 +08:00
committed by GitHub
parent 630fd06ccf
commit fdd87fe008
3 changed files with 45 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
be/src/olap/delete_handler.cpp
View File

@ -21,12 +21,9 @@
#include <gen_cpp/olap_file.pb.h>
#include <thrift/protocol/TDebugProtocol.h>
#include <algorithm>
#include <boost/regex.hpp>
#include <limits>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include "common/config.h"
@ -49,6 +46,47 @@ using ::google::protobuf::RepeatedPtrField;
namespace doris {
using namespace ErrorCode;
// construct sub condition from TCondition
std::string construct_sub_predicate(const TCondition& condition) {
string op = condition.condition_op;
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
string condition_str;
if ("IS" == op) {
// ATTN: tricky! Surround IS with spaces to make it "special"
condition_str = condition.column_name + " IS " + condition.condition_values[0];
} else { // multi-elements IN expr has been processed with InPredicatePB
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
condition_str = condition.column_name + op + "'" + condition.condition_values[0] + "'";
}
return condition_str;
}
// make operators from FE adaptive to BE
std::string trans_op(const std::string& opt) {
std::string op = string(opt);
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
if ("IS" != op) {
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
}
return op;
}
Status DeleteHandler::generate_delete_predicate(const TabletSchema& schema,
const std::vector<TCondition>& conditions,
DeletePredicatePB* del_pred) {
@ -126,45 +164,6 @@ void DeleteHandler::convert_to_sub_pred_v2(DeletePredicatePB* delete_pred,
}
}
std::string DeleteHandler::construct_sub_predicate(const TCondition& condition) {
string op = condition.condition_op;
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
string condition_str;
if ("IS" == op) {
// ATTN: tricky! Surround IS with spaces to make it "special"
condition_str = condition.column_name + " IS " + condition.condition_values[0];
} else { // multi-elements IN expr has been processed with InPredicatePB
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
condition_str = condition.column_name + op + "'" + condition.condition_values[0] + "'";
}
return condition_str;
}
std::string DeleteHandler::trans_op(const std::string& opt) {
std::string op = string(opt);
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
if ("IS" != op) {
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
}
return op;
}
bool DeleteHandler::is_condition_value_valid(const TabletColumn& column,
const std::string& condition_op,
const string& value_str) {
@ -395,7 +394,7 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema,
const auto& delete_condition = delete_pred->delete_predicate();
DeleteConditions temp;
temp.filter_version = delete_pred->version().first;
if (with_sub_pred_v2) {
if (with_sub_pred_v2 && !delete_condition.sub_predicates_v2().empty()) {
RETURN_IF_ERROR(_parse_column_pred(tablet_schema, delete_pred_related_schema,
delete_condition.sub_predicates_v2(), &temp));
} else {

14
be/src/olap/delete_handler.h
View File

@ -18,12 +18,9 @@
#pragma once
#include <butil/macros.h>
#include <stdint.h>
#include <memory>
#include <cstdint>
#include <string>
#include <unordered_map>
#include <vector>
#include "common/factory_creator.h"
#include "common/status.h"
@ -88,12 +85,6 @@ private:
const std::string& condition_op,
const std::string& value_str);
// construct sub condition from TCondition
static std::string construct_sub_predicate(const TCondition& condition);
// make operators from FE adaptive to BE
[[nodiscard]] static std::string trans_op(const string& op);
// extract 'column_name', 'op' and 'operands' to condition
static Status parse_condition(const DeleteSubPredicatePB& sub_cond, TCondition* condition);
@ -107,7 +98,8 @@ public:
// input:
// * schema: tablet's schema, the delete conditions and data rows are in this schema
// * version: maximum version
// * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is based on PB, v1 is based on condition string)
// * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is based on PB and use column uid to specify a column,
// v1 is based on condition string, and relies on regex for parse)
// return:
// * Status::Error<DELETE_INVALID_PARAMETERS>(): input parameters are not valid
// * Status::Error<MEM_ALLOC_FAILED>(): alloc memory failed

6
be/src/olap/tablet_reader.cpp
View File

@ -643,13 +643,7 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
((read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
config::enable_delete_when_cumu_compaction)) ||
read_params.reader_type == ReaderType::READER_CHECKSUM);
if (_filter_delete) {
// note(tsy): for compaction, keep delete sub pred v1 temporarily
return _delete_handler.init(_tablet_schema, read_params.delete_predicates,
read_params.version.second, false);
}
auto* runtime_state = read_params.runtime_state;
// note(tsy): for query, use session var to enable delete sub pred v2, for schema change, use v2 directly
bool enable_sub_pred_v2 =
runtime_state == nullptr ? true : runtime_state->enable_delete_sub_pred_v2();
return _delete_handler.init(_tablet_schema, read_params.delete_predicates,