251 lines
8.8 KiB
Thrift
251 lines
8.8 KiB
Thrift
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
namespace java org.apache.doris.thrift
|
|
namespace cpp doris
|
|
|
|
include "Exprs.thrift"
|
|
include "Opcodes.thrift"
|
|
include "Status.thrift"
|
|
include "Types.thrift"
|
|
|
|
// A result set column descriptor.
|
|
// this definition id different from column desc in palo, the column desc in palo only support scalar type, does not support map, array
|
|
// so that should convert palo column desc into ExtColumnDesc
|
|
struct TExtColumnDesc {
|
|
// The column name as given in the Create .. statement. Always set.
|
|
1: optional string name
|
|
// The column type. Always set.
|
|
2: optional Types.TTypeDesc type
|
|
}
|
|
|
|
// Metadata used to describe the schema (column names, types, comments)
|
|
// of result sets.
|
|
struct TExtTableSchema {
|
|
// List of columns. Always set.
|
|
1: optional list<TExtColumnDesc> cols
|
|
}
|
|
|
|
struct TExtLiteral {
|
|
1: required Exprs.TExprNodeType node_type
|
|
2: optional Exprs.TBoolLiteral bool_literal
|
|
3: optional Exprs.TDateLiteral date_literal
|
|
4: optional Exprs.TFloatLiteral float_literal
|
|
5: optional Exprs.TIntLiteral int_literal
|
|
6: optional Exprs.TStringLiteral string_literal
|
|
7: optional Exprs.TDecimalLiteral decimal_literal
|
|
8: optional Exprs.TLargeIntLiteral large_int_literal
|
|
}
|
|
|
|
// Binary predicates that can be pushed to the external data source and
|
|
// are of the form <col> <op> <val>. Sources can choose to accept or reject
|
|
// predicates via the return value of prepare(), see TPrepareResult.
|
|
// The column and the value are guaranteed to be type compatible in Impala,
|
|
// but they are not necessarily the same type, so the data source
|
|
// implementation may need to do an implicit cast.
|
|
// > < = != >= <=
|
|
struct TExtBinaryPredicate {
|
|
// Column on which the predicate is applied. Always set.
|
|
1: optional TExtColumnDesc col
|
|
// Comparison operator. Always set.
|
|
2: optional Opcodes.TExprOpcode op
|
|
// Value on the right side of the binary predicate. Always set.
|
|
3: optional TExtLiteral value
|
|
}
|
|
|
|
struct TExtInPredicate {
|
|
1: optional bool is_not_in
|
|
// Column on which the predicate is applied. Always set.
|
|
2: optional TExtColumnDesc col
|
|
// Value on the right side of the binary predicate. Always set.
|
|
3: optional list<TExtLiteral> values
|
|
}
|
|
|
|
struct TExtLikePredicate {
|
|
1: optional TExtColumnDesc col
|
|
2: optional TExtLiteral value
|
|
}
|
|
|
|
struct TExtIsNullPredicate {
|
|
1: optional bool is_not_null
|
|
2: optional TExtColumnDesc col
|
|
}
|
|
|
|
struct TExtFunction {
|
|
1: optional string func_name
|
|
// input parameter column descs
|
|
2: optional list<TExtColumnDesc> cols
|
|
// input parameter column literals
|
|
3: optional list<TExtLiteral> values
|
|
}
|
|
|
|
// a union of all predicates
|
|
struct TExtPredicate {
|
|
1: required Exprs.TExprNodeType node_type
|
|
2: optional TExtBinaryPredicate binary_predicate
|
|
3: optional TExtInPredicate in_predicate
|
|
4: optional TExtLikePredicate like_predicate
|
|
5: optional TExtIsNullPredicate is_null_predicate
|
|
6: optional TExtFunction ext_function
|
|
}
|
|
|
|
// A union over all possible return types for a column of data
|
|
// Currently only used by ExternalDataSource types
|
|
//
|
|
struct TExtColumnData {
|
|
// One element in the list for every row in the column indicating if there is
|
|
// a value in the vals list or a null.
|
|
1: required list<bool> is_null;
|
|
|
|
// Only one is set, only non-null values are set. this indicates one column data for a row batch
|
|
2: optional list<bool> bool_vals;
|
|
3: optional list<byte> byte_vals;
|
|
4: optional list<i16> short_vals;
|
|
5: optional list<i32> int_vals;
|
|
6: optional list<i64> long_vals;
|
|
7: optional list<double> double_vals;
|
|
8: optional list<string> string_vals;
|
|
9: optional list<binary> binary_vals;
|
|
}
|
|
|
|
// Serialized batch of rows returned by getNext().
|
|
// one row batch contains mult rows, and the result is arranged in column style
|
|
struct TExtRowBatch {
|
|
// Each TColumnData contains the data for an entire column. Always set.
|
|
1: optional list<TExtColumnData> cols
|
|
|
|
// The number of rows returned. For count(*) queries, there may not be
|
|
// any materialized columns so cols will be an empty list and this value
|
|
// will indicate how many rows are returned. When there are materialized
|
|
// columns, this number should be the same as the size of each
|
|
// TColumnData.is_null list.
|
|
2: optional i64 num_rows
|
|
}
|
|
|
|
// Parameters to prepare().
|
|
struct TExtPrepareParams {
|
|
// The name of the table. Always set.
|
|
1: optional string table_name
|
|
|
|
// A string specified for the table that is passed to the external data source.
|
|
// Always set, may be an empty string.
|
|
2: optional string init_string
|
|
|
|
// A list of conjunctive (AND) clauses, each of which contains a list of
|
|
// disjunctive (OR) binary predicates. Always set, may be an empty list.
|
|
3: optional list<list<TExtPredicate>> predicates
|
|
}
|
|
|
|
// Returned by prepare().
|
|
struct TExtPrepareResult {
|
|
1: required Status.TStatus status
|
|
|
|
// Estimate of the total number of rows returned when applying the predicates indicated
|
|
// by accepted_conjuncts. Not set if the data source does not support providing
|
|
// this statistic.
|
|
2: optional i64 num_rows_estimate
|
|
|
|
// Accepted conjuncts. References the 'predicates' parameter in the prepare()
|
|
// call. It contains the 0-based indices of the top-level list elements (the
|
|
// AND elements) that the library will be able to apply during the scan. Those
|
|
// elements that aren’t referenced in accepted_conjuncts will be evaluated by
|
|
// Impala itself.
|
|
3: optional list<i32> accepted_conjuncts
|
|
}
|
|
|
|
// Parameters to open().
|
|
struct TExtOpenParams {
|
|
// A unique identifier for the query. Always set.
|
|
1: optional Types.TUniqueId query_id
|
|
|
|
// The name of the table. Always set.
|
|
2: optional string table_name
|
|
|
|
// A string specified for the table that is passed to the external data source.
|
|
// Always set, may be an empty string.
|
|
3: optional map<string,string> properties
|
|
|
|
// The authenticated user name. Always set.
|
|
4: optional string authenticated_user_name
|
|
|
|
// The schema of the rows that the scan needs to return. Always set.
|
|
5: optional TExtTableSchema row_schema
|
|
|
|
// The expected size of the row batches it returns in the subsequent getNext() calls.
|
|
// Always set.
|
|
6: optional i32 batch_size
|
|
|
|
7: optional list<list<TExtPredicate>> predicates
|
|
|
|
// The query limit, if specified.
|
|
8: optional i64 limit
|
|
}
|
|
|
|
// Returned by open().
|
|
struct TExtOpenResult {
|
|
1: required Status.TStatus status
|
|
|
|
// An opaque handle used in subsequent getNext()/close() calls. Required.
|
|
2: optional string scan_handle
|
|
3: optional list<i32> accepted_conjuncts
|
|
}
|
|
|
|
// Parameters to getNext()
|
|
struct TExtGetNextParams {
|
|
// The opaque handle returned by the previous open() call. Always set.
|
|
1: optional string scan_handle // es search context id
|
|
2: optional i64 offset // es should check the offset to prevent duplicate rpc calls
|
|
}
|
|
|
|
// Returned by getNext().
|
|
struct TExtGetNextResult {
|
|
1: required Status.TStatus status
|
|
|
|
// If true, reached the end of the result stream; subsequent calls to
|
|
// getNext() won’t return any more results. Required.
|
|
2: optional bool eos
|
|
|
|
// A batch of rows to return, if any exist. The number of rows in the batch
|
|
// should be less than or equal to the batch_size specified in TOpenParams.
|
|
3: optional TExtRowBatch rows
|
|
}
|
|
|
|
// Parameters to close()
|
|
struct TExtCloseParams {
|
|
// The opaque handle returned by the previous open() call. Always set.
|
|
1: optional string scan_handle
|
|
}
|
|
|
|
// Returned by close().
|
|
struct TExtCloseResult {
|
|
1: required Status.TStatus status
|
|
}
|
|
|
|
// 这个data source可以认为是palo统一的外部data source的入口
|
|
service TExtDataSourceService {
|
|
// 1. palo be call this api to send index, type, shard id to es
|
|
// 2. es will open a search context and prepare data, will return a context id
|
|
TExtOpenResult open(1: TExtOpenParams params);
|
|
// 1. palo be will send a search context id to es
|
|
// 2. es will find the search context and find a batch rows and send to palo
|
|
// 3. palo will run the remaining predicates when receving data
|
|
// 4. es should check the offset when receive the request
|
|
TExtGetNextResult getNext(1: TExtGetNextParams params);
|
|
// 1. es will release the context when receiving the data
|
|
TExtCloseResult close(1: TExtCloseParams params);
|
|
}
|