database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](DOE)Fix es p0 case error (#17502)

Browse Source 
Fix es array parse error, introduced by #16806
This commit is contained in:
qiye
2023-03-08 08:06:30 +08:00
committed by GitHub
parent 69c62b6c6c
commit a767472c56
3 changed files with 30 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java vendored
View File

@ -101,20 +101,20 @@ public class EsUtil {
}
@VisibleForTesting
public static ObjectNode getRootSchema(ObjectNode mappings, String mappingType) {
public static ObjectNode getRootSchema(ObjectNode mappings, String mappingType, List<String> arrayFields) {
// Type is null in the following three cases
// 1. Equal 6.8.x and after
// 2. Multi-catalog auto infer
// 3. Equal 6.8.x and before user not passed
if (mappingType == null) {
// remove dynamic templates, for ES 7.x and 8.x
checkNonPropertiesFields(mappings);
checkNonPropertiesFields(mappings, arrayFields);
String firstType = mappings.fieldNames().next();
if (!"properties".equals(firstType)) {
// If type is not passed in takes the first type.
ObjectNode firstData = (ObjectNode) mappings.get(firstType);
// check for ES 6.x and before
checkNonPropertiesFields(firstData);
checkNonPropertiesFields(firstData, arrayFields);
return firstData;
}
// Equal 7.x and after
@ -123,11 +123,11 @@ public class EsUtil {
if (mappings.has(mappingType)) {
ObjectNode jsonData = (ObjectNode) mappings.get(mappingType);
// check for ES 6.x and before
checkNonPropertiesFields(jsonData);
checkNonPropertiesFields(jsonData, arrayFields);
return jsonData;
}
// Compatible type error
return getRootSchema(mappings, null);
return getRootSchema(mappings, null, arrayFields);
}
}
@ -136,9 +136,21 @@ public class EsUtil {
*
* @param mappings
*/
private static void checkNonPropertiesFields(ObjectNode mappings) {
// remove `_meta` field
mappings.remove("_meta");
private static void checkNonPropertiesFields(ObjectNode mappings, List<String> arrayFields) {
// remove `_meta` field and parse array_fields
JsonNode metaNode = mappings.remove("_meta");
if (metaNode != null) {
JsonNode dorisMeta = metaNode.get("doris");
if (dorisMeta != null) {
JsonNode arrayNode = dorisMeta.get("array_fields");
if (arrayNode != null) {
Iterator<JsonNode> iterator = arrayNode.iterator();
while (iterator.hasNext()) {
arrayFields.add(iterator.next().asText());
}
}
}
}
// remove `dynamic_templates` field
mappings.remove("dynamic_templates");
// check explicit mapping
@ -152,7 +164,7 @@ public class EsUtil {
**/
public static ObjectNode getMappingProps(String sourceIndex, String indexMapping, String mappingType) {
ObjectNode mappings = getMapping(indexMapping);
ObjectNode rootSchema = getRootSchema(mappings, mappingType);
ObjectNode rootSchema = getRootSchema(mappings, mappingType, new ArrayList<>());
ObjectNode properties = (ObjectNode) rootSchema.get("properties");
if (properties == null) {
throw new DorisEsException(
@ -169,13 +181,15 @@ public class EsUtil {
boolean mappingEsId) {
String mapping = client.getMapping(indexName);
ObjectNode mappings = getMapping(mapping);
ObjectNode rootSchema = getRootSchema(mappings, mappingType);
return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId);
// Get array_fields while removing _meta property.
List<String> arrayFields = new ArrayList<>();
ObjectNode rootSchema = getRootSchema(mappings, mappingType, arrayFields);
return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId, arrayFields);
}
@VisibleForTesting
public static List<Column> genColumnsFromEs(String indexName, String mappingType, ObjectNode rootSchema,
boolean mappingEsId) {
boolean mappingEsId, List<String> arrayFields) {
List<Column> columns = new ArrayList<>();
if (mappingEsId) {
Column column = new Column();
@ -191,14 +205,6 @@ public class EsUtil {
throw new DorisEsException(
"index[" + indexName + "] type[" + mappingType + "] mapping not found for the ES Cluster");
}
List<String> arrayFields = new ArrayList<>();
JsonNode meta = mappingProps.get("_meta");
if (meta != null) {
JsonNode dorisMeta = meta.get("doris");
if (dorisMeta != null) {
arrayFields = dorisMeta.findValuesAsText("array_fields");
}
}
Iterator<String> iterator = mappingProps.fieldNames();
while (iterator.hasNext()) {
String fieldName = iterator.next();

4
fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java vendored
View File

@ -220,8 +220,8 @@ public class EsUtilTest extends EsTestCase {
@Test
public void testDateType() throws IOException, URISyntaxException {
ObjectNode testDateFormat = EsUtil.getRootSchema(
EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null);
List<Column> parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false);
EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null, new ArrayList<>());
List<Column> parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false, new ArrayList<>());
Assertions.assertEquals(8, parseColumns.size());
for (Column column : parseColumns) {
String name = column.getName();