database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

(feature)[DOE]Support array for Doris on ES (#16941)

Browse Source 
* (feature)[DOE]Support array for Doris on ES
This commit is contained in:
qiye
2023-02-23 19:31:18 +08:00
committed by GitHub
parent 48fd528a2b
commit 92ecd16573
15 changed files with 703 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
be/src/exec/es/es_scroll_parser.cpp
View File

@ -30,7 +30,9 @@
#include "runtime/mem_pool.h"
#include "runtime/memory/mem_tracker.h"
#include "util/string_parser.hpp"
#include "vec/columns/column_array.h"
#include "vec/common/string_ref.h"
#include "vec/core/field.h"
#include "vec/runtime/vdatetime_value.h"
namespace doris {
@ -169,6 +171,48 @@ static Status get_int_value(const rapidjson::Value& col, PrimitiveType type, voi
return Status::OK();
}
template <typename T, typename RT>
static RT get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool is_date_str) {
vectorized::DateV2Value<T> dt_slot;
if ((is_date_str &&
!dt_slot.from_date_str(static_cast<const std::string>(col.GetString()).c_str(),
col.GetStringLength())) ||
(!is_date_str && !dt_slot.from_unixtime(col.GetInt64() / 1000, "+08:00"))) {
RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
}
return binary_cast<doris::vectorized::DateV2Value<T>, RT>(
*reinterpret_cast<vectorized::DateV2Value<T>*>(&dt_slot));
}
template <typename T, typename RT>
static RT get_date_int(const rapidjson::Value& sub_col, PrimitiveType sub_type,
bool pure_doc_value) {
// this would happend just only when `enable_docvalue_scan = false`, and field has timestamp format date from _source
if (sub_col.IsNumber()) {
// ES process date/datetime field would use millisecond timestamp for index or docvalue
// processing date type field, if a number is encountered, Doris On ES will force it to be processed according to ms
// Doris On ES needs to be consistent with ES, so just divided by 1000 because the unit for from_unixtime is seconds
return get_date_value_int<T, RT>(sub_col, sub_type, false);
} else if (sub_col.IsArray() && pure_doc_value) {
// this would happened just only when `enable_docvalue_scan = true`
// ES add default format for all field after ES 6.4, if we not provided format for `date` field ES would impose
// a standard date-format for date field as `2020-06-16T00:00:00.000Z`
// At present, we just process this string format date. After some PR were merged into Doris, we would impose `epoch_mills` for
// date field's docvalue
if (sub_col[0].IsString()) {
return get_date_value_int<T, RT>(sub_col, sub_type, true);
}
// ES would return millisecond timestamp for date field, divided by 1000 because the unit for from_unixtime is seconds
return get_date_value_int<T, RT>(sub_col, sub_type, false);
} else {
// this would happened just only when `enable_docvalue_scan = false`, and field has string format date from _source
RETURN_ERROR_IF_COL_IS_ARRAY(sub_col, sub_type);
RETURN_ERROR_IF_COL_IS_NOT_STRING(sub_col, sub_type);
return get_date_value_int<T, RT>(sub_col, sub_type, true);
}
}
template <typename T>
static Status get_float_value(const rapidjson::Value& col, PrimitiveType type, void* slot,
bool pure_doc_value) {
@ -519,7 +563,134 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc,
}
break;
}
case TYPE_ARRAY: {
vectorized::Array array;
const auto& sub_type = tuple_desc->slots()[i]->type().children[0].type;
for (auto& sub_col : col.GetArray()) {
switch (sub_type) {
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING: {
std::string val;
if (pure_doc_value) {
if (!sub_col[0].IsString()) {
val = json_value_to_string(sub_col[0]);
} else {
val = sub_col[0].GetString();
}
} else {
RETURN_ERROR_IF_COL_IS_ARRAY(sub_col, type);
if (!sub_col.IsString()) {
val = json_value_to_string(sub_col);
} else {
val = sub_col.GetString();
}
}
array.push_back(val);
break;
}
case TYPE_TINYINT: {
int8_t val;
RETURN_IF_ERROR(get_int_value<int8_t>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_SMALLINT: {
int16_t val;
RETURN_IF_ERROR(
get_int_value<int16_t>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_INT: {
int32 val;
RETURN_IF_ERROR(get_int_value<int32>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_BIGINT: {
int64_t val;
RETURN_IF_ERROR(
get_int_value<int64_t>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_LARGEINT: {
__int128 val;
RETURN_IF_ERROR(
get_int_value<__int128>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_FLOAT: {
float val;
RETURN_IF_ERROR(
get_float_value<float>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_DOUBLE: {
double val {};
RETURN_IF_ERROR(
get_float_value<double>(sub_col, sub_type, &val, pure_doc_value));
array.push_back(val);
break;
}
case TYPE_BOOLEAN: {
if (sub_col.IsBool()) {
array.push_back(sub_col.GetBool());
break;
}
if (sub_col.IsNumber()) {
array.push_back(sub_col.GetInt());
break;
}
bool is_nested_str = false;
if (pure_doc_value && sub_col.IsArray() && sub_col[0].IsBool()) {
array.push_back(sub_col[0].GetBool());
break;
} else if (pure_doc_value && sub_col.IsArray() && sub_col[0].IsString()) {
is_nested_str = true;
} else if (pure_doc_value && sub_col.IsArray()) {
return Status::InternalError(ERROR_INVALID_COL_DATA, "BOOLEAN");
}
const rapidjson::Value& str_col = is_nested_str ? sub_col[0] : sub_col;
const std::string& val = str_col.GetString();
size_t val_size = str_col.GetStringLength();
StringParser::ParseResult result;
bool b = StringParser::string_to_bool(val.c_str(), val_size, &result);
RETURN_ERROR_IF_PARSING_FAILED(result, str_col, type);
array.push_back(b);
break;
}
// date/datetime v2 is the default type for catalog table,
// see https://github.com/apache/doris/pull/16304
// No need to support date and datetime types.
case TYPE_DATEV2: {
array.push_back(get_date_int<vectorized::DateV2ValueType, uint32_t>(
sub_col, sub_type, pure_doc_value));
break;
}
case TYPE_DATETIMEV2: {
array.push_back(get_date_int<vectorized::DateTimeV2ValueType, uint64_t>(
sub_col, sub_type, pure_doc_value));
break;
}
default: {
LOG(ERROR) << "Do not support Array type: " << sub_type;
break;
}
}
}
col_ptr->insert(array);
break;
}
default: {
LOG(ERROR) << "Unsupported data type: " << type_to_string(type);
DCHECK(false);
break;
}
@ -578,7 +749,7 @@ Status ScrollParser::fill_date_col(vectorized::IColumn* col_ptr, const rapidjson
return Status::OK();
} else {
return Status::InternalError("Unsupported datetime type.");
return Status::InternalError("Unsupported datetime type: " + type_to_string(type));
}
}

21
docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json
View File

@ -2,5 +2,24 @@
"test1": "string1",
"test2": "text#1",
"test3": 3.14,
"test4": "2022-08-08"
"test4": "2022-08-08",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
"c_integer": [32768, 32769, -32769, -32770],
"c_long": [-1, 0, 1, 2],
"c_unsigned_long": [0, 1, 2, 3],
"c_float": [1.0, 1.1, 1.2, 1.3],
"c_half_float": [1, 2, 3, 4],
"c_double": [1, 2, 3, 4],
"c_scaled_float": [1, 2, 3, 4],
"c_date": ["2020-01-01", "2020-01-02"],
"c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
"c_keyword": ["a", "b", "c"],
"c_text": ["d", "e", "f"],
"c_ip": ["192.168.0.1", "127.0.0.1"],
"c_person": [
{"name": "Andy", "age": 18},
{"name": "Tim", "age": 28}
]
}

21
docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json
View File

@ -2,5 +2,24 @@
"test1": "string2",
"test2": "text2",
"test3": 4,
"test4": "2022-09-08"
"test4": "2022-08-08",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
"c_integer": [32768, 32769, -32769, -32770],
"c_long": [-1, 0, 1, 2],
"c_unsigned_long": [0, 1, 2, 3],
"c_float": [1.0, 1.1, 1.2, 1.3],
"c_half_float": [1, 2, 3, 4],
"c_double": [1, 2, 3, 4],
"c_scaled_float": [1, 2, 3, 4],
"c_date": ["2020-01-01", "2020-01-02"],
"c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
"c_keyword": ["a", "b", "c"],
"c_text": ["d", "e", "f"],
"c_ip": ["192.168.0.1", "127.0.0.1"],
"c_person": [
{"name": "Andy", "age": 18},
{"name": "Tim", "age": 28}
]
}

21
docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json
View File

@ -2,5 +2,24 @@
"test1": "string3",
"test2": "text3_4*5",
"test3": 5.0,
"test4": "2022-08-08"
"test4": "2022-08-08",
"c_bool": [true, false, true, true],
"c_byte": [1, -2, -3, 4],
"c_short": [128, 129, -129, -130],
"c_integer": [32768, 32769, -32769, -32770],
"c_long": [-1, 0, 1, 2],
"c_unsigned_long": [0, 1, 2, 3],
"c_float": [1.0, 1.1, 1.2, 1.3],
"c_half_float": [1, 2, 3, 4],
"c_double": [1, 2, 3, 4],
"c_scaled_float": [1, 2, 3, 4],
"c_date": ["2020-01-01", "2020-01-02"],
"c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
"c_keyword": ["a", "b", "c"],
"c_text": ["d", "e", "f"],
"c_ip": ["192.168.0.1", "127.0.0.1"],
"c_person": [
{"name": "Andy", "age": 18},
{"name": "Tim", "age": 28}
]
}

9
docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
View File

@ -29,6 +29,9 @@ curl "http://${ES_6_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -
curl "http://${ES_6_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_6_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_6_HOST}:9200/test2_20220808/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
# put _meta for array
curl "http://${ES_6_HOST}:9200/test1/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
curl "http://${ES_6_HOST}:9200/test2_20220808/doc/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
# es7
# create index test1
@ -42,6 +45,9 @@ curl "http://${ES_7_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json"
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
# put _meta for array
curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
# es8
# create index test1
@ -55,3 +61,6 @@ curl "http://${ES_8_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json"
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
# put _meta for array
curl "http://${ES_8_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
curl "http://${ES_8_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"

24
docker/thirdparties/docker-compose/elasticsearch/scripts/index/array_meta.json Normal file
View File

@ -0,0 +1,24 @@
{
"_meta": {
"doris":{
"array_fields":[
"c_bool",
"c_byte",
"c_short",
"c_integer",
"c_long",
"c_unsigned_long",
"c_float",
"c_half_float",
"c_double",
"c_scaled_float",
"c_date",
"c_datetime",
"c_keyword",
"c_text",
"c_ip",
"c_person"
]
}
}
}

58
docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
View File

@ -23,6 +23,64 @@
},
"test4": {
"type": "date"
},
"c_bool": {
"type": "boolean"
},
"c_byte": {
"type": "byte"
},
"c_short": {
"type": "short"
},
"c_integer": {
"type": "integer"
},
"c_long": {
"type": "long"
},
"c_unsigned_long": {
"type": "unsigned_long"
},
"c_float": {
"type": "float"
},
"c_half_float": {
"type": "half_float"
},
"c_double": {
"type": "double"
},
"c_scaled_float": {
"type": "scaled_float",
"scaling_factor": 0.01
},
"c_date": {
"type": "date",
"format": "yyyy-MM-dd"
},
"c_datetime": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"c_keyword": {
"type": "keyword"
},
"c_text": {
"type": "text"
},
"c_ip": {
"type": "ip"
},
"c_person": {
"properties": {
"name": {
"type": "keyword"
},
"age": {
"type": "integer"
}
}
}
}
}

58
docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json
View File

@ -26,6 +26,64 @@
},
"test4": {
"type": "date"
},
"c_bool": {
"type": "boolean"
},
"c_byte": {
"type": "byte"
},
"c_short": {
"type": "short"
},
"c_integer": {
"type": "integer"
},
"c_long": {
"type": "long"
},
"c_unsigned_long": {
"type": "unsigned_long"
},
"c_float": {
"type": "float"
},
"c_half_float": {
"type": "half_float"
},
"c_double": {
"type": "double"
},
"c_scaled_float": {
"type": "scaled_float",
"scaling_factor": 0.01
},
"c_date": {
"type": "date",
"format": "yyyy-MM-dd"
},
"c_datetime": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"c_keyword": {
"type": "keyword"
},
"c_text": {
"type": "text"
},
"c_ip": {
"type": "ip"
},
"c_person": {
"properties": {
"name": {
"type": "keyword"
},
"age": {
"type": "integer"
}
}
}
}
}

58
docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json
View File

@ -22,6 +22,64 @@
},
"test4": {
"type": "date"
},
"c_bool": {
"type": "boolean"
},
"c_byte": {
"type": "byte"
},
"c_short": {
"type": "short"
},
"c_integer": {
"type": "integer"
},
"c_long": {
"type": "long"
},
"c_unsigned_long": {
"type": "unsigned_long"
},
"c_float": {
"type": "float"
},
"c_half_float": {
"type": "half_float"
},
"c_double": {
"type": "double"
},
"c_scaled_float": {
"type": "scaled_float",
"scaling_factor": 0.01
},
"c_date": {
"type": "date",
"format": "yyyy-MM-dd"
},
"c_datetime": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"c_keyword": {
"type": "keyword"
},
"c_text": {
"type": "text"
},
"c_ip": {
"type": "ip"
},
"c_person": {
"properties": {
"name": {
"type": "keyword"
},
"age": {
"type": "integer"
}
}
}
}
}

58
docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json
View File

@ -25,6 +25,64 @@
},
"test4": {
"type": "date"
},
"c_bool": {
"type": "boolean"
},
"c_byte": {
"type": "byte"
},
"c_short": {
"type": "short"
},
"c_integer": {
"type": "integer"
},
"c_long": {
"type": "long"
},
"c_unsigned_long": {
"type": "unsigned_long"
},
"c_float": {
"type": "float"
},
"c_half_float": {
"type": "half_float"
},
"c_double": {
"type": "double"
},
"c_scaled_float": {
"type": "scaled_float",
"scaling_factor": 0.01
},
"c_date": {
"type": "date",
"format": "yyyy-MM-dd"
},
"c_datetime": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"c_keyword": {
"type": "keyword"
},
"c_text": {
"type": "text"
},
"c_ip": {
"type": "ip"
},
"c_person": {
"properties": {
"name": {
"type": "keyword"
},
"age": {
"type": "integer"
}
}
}
}
}

64
docs/en/docs/lakehouse/multi-catalog/es.md
View File

@ -89,6 +89,70 @@ After switching to the ES Catalog, you will be in the `dafault_db` so you don't
| object | string | |
| other | unsupported | |
### Array Type
Elasticsearch does not have an explicit array type, but one of its fields can contain
[0 or more values](https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html).
To indicate that a field is an array type, a specific `doris` structural annotation can be added to the
[_meta](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-meta-field.html) section of the index mapping.
For Elasticsearch 6.x and before release, please refer [_meta](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/mapping-meta-field.html).
For example, suppose there is an index `doc` containing the following data structure.
```json
{
"array_int_field": [1, 2, 3, 4],
"array_string_field": ["doris", "is", "the", "best"],
"id_field": "id-xxx-xxx",
"timestamp_field": "2022-11-12T12:08:56Z",
"array_object_field": [
{
"name": "xxx",
"age": 18
}
]
}
```
The array fields of this structure can be defined by using the following command to add the field property definition
to the `_meta.doris` property of the target index mapping.
```bash
# ES 7.x and above
curl -X PUT "localhost:9200/doc/_mapping?pretty" -H 'Content-Type:application/json' -d '
{
"_meta": {
"doris":{
"array_fields":[
"array_int_field",
"array_string_field",
"array_object_field"
]
}
}
}'
# ES 6.x and before
curl -X PUT "localhost:9200/doc/_mapping?pretty" -H 'Content-Type: application/json' -d '
{
"_doc": {
"_meta": {
"doris":{
"array_fields":[
"array_int_field",
"array_string_field",
"array_object_field"
]
}
}
}
}
'
```
`array_fields`：Used to indicate a field that is an array type.
## Best Practice
### Predicate Pushdown

59
docs/zh-CN/docs/lakehouse/multi-catalog/es.md
View File

@ -89,6 +89,65 @@ CREATE CATALOG es PROPERTIES (
| object |string | |
|other| unsupported ||
### Array 类型
Elasticsearch 没有明确的数组类型，但是它的某个字段可以含有[0个或多个值](https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html)。
为了表示一个字段是数组类型，可以在索引映射的[_meta](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-meta-field.html)部分添加特定的`doris`结构注释。
对于 Elasticsearch 6.x 及之前版本，请参考[_meta](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/mapping-meta-field.html)。
举例说明，假设有一个索引`doc`包含以下的数据结构：
```json
{
"array_int_field": [1, 2, 3, 4],
"array_string_field": ["doris", "is", "the", "best"],
"id_field": "id-xxx-xxx",
"timestamp_field": "2022-11-12T12:08:56Z",
"array_object_field": [
{
"name": "xxx",
"age": 18
}
]
}
```
该结构的数组字段可以通过使用以下命令将字段属性定义添加到目标索引映射的`_meta.doris`属性来定义。
```bash
# ES 7.x and above
curl -X PUT "localhost:9200/doc/_mapping?pretty" -H 'Content-Type:application/json' -d '
{
"_meta": {
"doris":{
"array_fields":[
"array_int_field",
"array_string_field",
"array_object_field"
]
}
}
}'
# ES 6.x and before
curl -X PUT "localhost:9200/doc/_mapping?pretty" -H 'Content-Type: application/json' -d '
{
"_doc": {
"_meta": {
"doris":{
"array_fields":[
"array_int_field",
"array_string_field",
"array_object_field"
]
}
}
}
}
```
`array_fields`：用来表示是数组类型的字段。
## 最佳实践
### 过滤条件下推

34
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java vendored
View File

@ -118,15 +118,29 @@ public class EsUtil {
**/
public static List<String> getArrayFields(String indexMapping) {
JSONObject mappings = getMapping(indexMapping);
JSONObject meta;
if (!mappings.containsKey("_meta")) {
return new ArrayList<>();
// For ES 6.x and 7.x
String firstType = (String) mappings.keySet().iterator().next();
if (!"properties".equals(firstType)) {
// If type is not passed in takes the first type.
JSONObject firstData = (JSONObject) mappings.get(firstType);
if (!firstData.containsKey("_meta")) {
return new ArrayList<>();
} else {
meta = (JSONObject) firstData.get("_meta");
}
} else {
return new ArrayList<>();
}
} else {
meta = (JSONObject) mappings.get("_meta");
}
JSONObject meta = (JSONObject) mappings.get("_meta");
if (!meta.containsKey("doris")) {
return new ArrayList<>();
}
JSONObject dorisMeta = (JSONObject) meta.get("doris");
return (List<String>) dorisMeta.get("array_field");
return (List<String>) dorisMeta.get("array_fields");
}
private static JSONObject getMapping(String indexMapping) {
@ -145,13 +159,13 @@ public class EsUtil {
// 3. Equal 6.8.x and before user not passed
if (mappingType == null) {
// remove dynamic templates, for ES 7.x and 8.x
checkDynamicTemplates(mappings);
checkNonPropertiesFields(mappings);
String firstType = (String) mappings.keySet().iterator().next();
if (!"properties".equals(firstType)) {
// If type is not passed in takes the first type.
JSONObject firstData = (JSONObject) mappings.get(firstType);
// check for ES 6.x and before
checkDynamicTemplates(firstData);
checkNonPropertiesFields(firstData);
return firstData;
}
// Equal 7.x and after
@ -160,7 +174,7 @@ public class EsUtil {
if (mappings.containsKey(mappingType)) {
JSONObject jsonData = (JSONObject) mappings.get(mappingType);
// check for ES 6.x and before
checkDynamicTemplates(jsonData);
checkNonPropertiesFields(jsonData);
return jsonData;
}
// Compatible type error
@ -169,12 +183,16 @@ public class EsUtil {
}
/**
* Remove `dynamic_templates` and check explicit mapping
* Check non properties fields
*
* @param mappings
*/
private static void checkDynamicTemplates(JSONObject mappings) {
private static void checkNonPropertiesFields(JSONObject mappings) {
// remove `_meta` field
mappings.remove("_meta");
// remove `dynamic_templates` field
mappings.remove("dynamic_templates");
// check explicit mapping
if (mappings.isEmpty()) {
throw new DorisEsException("Do not support index without explicit mapping.");
}

55
regression-test/data/es_p0/test_es_query.out
View File

@ -1,21 +1,60 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql62 --
2022-08-08 text#1 3.14 string1
['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql63 --
2022-08-08 text#1 3.14 string1
2022-08-08 text3_4*5 5.0 string3
['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string3 [1, 2, 3, 4] 2022-08-08 text3_4*5 [2020-01-01, 2020-01-02] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql64 --
2022-09-08 text2 4.0 string2
['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql65 --
true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18}
-- !sql66 --
true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18}
-- !sql72 --
2022-08-08 text#1 3.14 string1
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql73 --
2022-08-08 text#1 3.14 string1
2022-08-08 text3_4*5 5.0 string3
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string3 [1, 2, 3, 4] 2022-08-08 text3_4*5 [2020-01-01, 2020-01-02] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql74 --
2022-09-08 text2 4.0 string2
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql75 --
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
-- !sql76 --
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
-- !sql81 --
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql82 --
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string3 [1, 2, 3, 4] 2022-08-08 text3_4*5 [2020-01-01, 2020-01-02] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770]
-- !sql83 --
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
-- !sql84 --
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}
true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18}

14
regression-test/suites/es_p0/test_es_query.groovy
View File

@ -63,15 +63,21 @@ suite("test_es_query", "p0") {
order_qt_sql62 """select * from test1 where test2='text#1'"""
order_qt_sql63 """select * from test2_20220808 where test4='2022-08-08'"""
order_qt_sql64 """select * from test2_20220808 where substring(test2, 2) = 'ext2'"""
order_qt_sql65 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1"""
order_qt_sql66 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2_20220808"""
sql """switch es7"""
// order_qt_sql71 """show tables"""
order_qt_sql72 """select * from test1 where test2='text#1'"""
order_qt_sql73 """select * from test2_20220808 where test4='2022-08-08'"""
order_qt_sql74 """select * from test2_20220808 where substring(test2, 2) = 'ext2'"""
// es8 has some problem, need fix
// sql """switch es8"""
// order_qt_sql1 """select * from test1 where test2='text'"""
// order_qt_sql2 """select * from test2_20220808 where test4='2022-08-08'"""
order_qt_sql75 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1"""
order_qt_sql76 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2"""
sql """switch es8"""
order_qt_sql81 """select * from test1 where test2='text#1'"""
order_qt_sql82 """select * from test2_20220808 where test4='2022-08-08'"""
order_qt_sql83 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1"""
order_qt_sql84 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2"""
sql """drop catalog if exists es6;"""
sql """drop catalog if exists es7;"""