database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](es-catalog)fix error when querying the index ,elasticsearch version 8.9.1 (#24839)

Browse Source 
Issue Number: close #24833
This commit is contained in:
Guangdong Liu
2023-10-08 10:19:45 +08:00
committed by GitHub
parent f66708db0e
commit fddef8b473
6 changed files with 52 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json Executable file
View File

@ -0,0 +1,4 @@
{
"field1": "value1",
"field2": "value2"
}

9
docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
View File

@ -39,6 +39,8 @@ curl "http://${ES_6_HOST}:9200/test2_20220808/doc/_mapping" -H "Content-Type:app
curl "http://${ES_7_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json"
# create index test2_20220808
curl "http://${ES_7_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json'
# create index test3_20231005
curl "http://${ES_7_HOST}:9200/test3_20231005" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json'
# put data for tese1
curl "http://${ES_7_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_7_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
@ -49,6 +51,8 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:applicati
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
# put data for test3_20231005
curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
# put _meta for array
curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
@ -59,6 +63,9 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:applica
curl "http://${ES_8_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json"
# create index test2_20220808
curl "http://${ES_8_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json'
# create index test3_20231005
curl "http://${ES_8_HOST}:9200/test3_20231005" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json'
# put data for tese1
curl "http://${ES_8_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_8_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
@ -69,6 +76,8 @@ curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:applicati
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
# put data for test3_20231005
curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
# put _meta for array
curl "http://${ES_8_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"

22
docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json Normal file
View File

@ -0,0 +1,22 @@
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"dynamic": "true",
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates": [
],
"date_detection": true,
"numeric_detection": false,
"properties": {
"field1": { "type": "text" },
"field2": { "type": "keyword" }
}
}
}

14
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java vendored
View File

@ -39,6 +39,9 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.stream.StreamSupport;
/**
* Util for ES, some static method.
@ -110,15 +113,18 @@ public class EsUtil {
// remove dynamic templates, for ES 7.x and 8.x
checkNonPropertiesFields(mappings, arrayFields);
String firstType = mappings.fieldNames().next();
if (!"properties".equals(firstType)) {
// If type is not passed in takes the first type.
//The first parameter may not be properties, so we need to first determine whether it is 7.x or above.
if (StreamSupport.stream(Spliterators
.spliteratorUnknownSize(mappings.fieldNames(), Spliterator.ORDERED), false)
.anyMatch(s -> s.contains("properties"))) {
// Equal 7.x and after
return mappings;
} else {
ObjectNode firstData = (ObjectNode) mappings.get(firstType);
// check for ES 6.x and before
checkNonPropertiesFields(firstData, arrayFields);
return firstData;
}
// Equal 7.x and after
return mappings;
} else {
if (mappings.has(mappingType)) {
ObjectNode jsonData = (ObjectNode) mappings.get(mappingType);

5
regression-test/data/external_table_p0/es/test_es_query.out
View File

@ -96,6 +96,9 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
-- !sql710 --
value1 value2
-- !sql81 --
[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345
@ -132,3 +135,5 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
-- !sql88 --
value1 value2

3
regression-test/suites/external_table_p0/es/test_es_query.groovy
View File

@ -158,6 +158,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql77 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
order_qt_sql78 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1"""
order_qt_sql79 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2"""
order_qt_sql710 """select * from test3_20231005"""
sql """switch es8"""
order_qt_sql81 """select * from test1 where test2='text#1'"""
order_qt_sql82 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'"""
@ -166,6 +167,6 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql85 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
order_qt_sql86 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1"""
order_qt_sql87 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2"""
order_qt_sql88 """select * from test3_20231005"""
}
}