database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](DOE): Fix query _id error and es properties error (#15792)

Browse Source 
Fix query _id error
_id not exist mapping, but be can query it, we need skip check it exist mapping.
This commit is contained in:
Stalary
2023-01-11 17:00:59 +08:00
committed by GitHub
parent 18a3b75626
commit d4e4e18b47
4 changed files with 39 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
View File

@ -61,6 +61,10 @@ public class EsResource extends Resource {
public static final String HTTP_SSL_ENABLED = "http_ssl_enabled";
public static final String QUERY_DSL = "query_dsl";
public static final String DOC_VALUE_SCAN_DEFAULT_VALUE = "true";
public static final String KEYWORD_SNIFF_DEFAULT_VALUE = "true";
public static final String HTTP_SSL_ENABLED_DEFAULT_VALUE = "false";
public static final String NODES_DISCOVERY_DEFAULT_VALUE = "true";
@SerializedName(value = "properties")
private Map<String, String> properties;

36
fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
View File

@ -50,9 +50,9 @@ import java.util.Set;
@Getter
@Setter
public class EsTable extends Table {
private static final Logger LOG = LogManager.getLogger(EsTable.class);
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new HashSet<>(Arrays.asList("text"));
private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:
// It is possible to notice that retrieving an high number of fields leads
// to a sensible worsening of performance if DocValues are used.
@ -80,16 +80,16 @@ public class EsTable extends Table {
private EsTablePartitions esTablePartitions;
// Whether to enable docvalues scan optimization for fetching fields more fast, default to true
private boolean enableDocValueScan = true;
private boolean enableDocValueScan = Boolean.parseBoolean(EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE);
// Whether to enable sniffing keyword for filtering more reasonable, default to true
private boolean enableKeywordSniff = true;
private boolean enableKeywordSniff = Boolean.parseBoolean(EsResource.KEYWORD_SNIFF_DEFAULT_VALUE);
// if the number of fields which value extracted from `doc_value` exceeding this max limitation
// would downgrade to extract value from `stored_fields`
private int maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
private boolean nodesDiscovery = true;
private boolean nodesDiscovery = Boolean.parseBoolean(EsResource.NODES_DISCOVERY_DEFAULT_VALUE);
private boolean httpSslEnabled = false;
private boolean httpSslEnabled = Boolean.parseBoolean(EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE);
// tableContext is used for being convenient to persist some configuration parameters uniformly
private Map<String, String> tableContext = new HashMap<>();
@ -260,12 +260,10 @@ public class EsTable extends Table {
indexName = tableContext.get("indexName");
mappingType = tableContext.get("mappingType");
enableDocValueScan = Boolean.parseBoolean(tableContext.get("enableDocValueScan"));
if (tableContext.containsKey("enableKeywordSniff")) {
enableKeywordSniff = Boolean.parseBoolean(tableContext.get("enableKeywordSniff"));
} else {
enableKeywordSniff = true;
}
enableDocValueScan = Boolean.parseBoolean(tableContext.getOrDefault("enableDocValueScan",
EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
enableKeywordSniff = Boolean.parseBoolean(tableContext.getOrDefault("enableKeywordSniff",
EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
if (tableContext.containsKey("maxDocValueFields")) {
try {
maxDocValueFields = Integer.parseInt(tableContext.get("maxDocValueFields"));
@ -273,16 +271,10 @@ public class EsTable extends Table {
maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
}
}
if (tableContext.containsKey(EsResource.NODES_DISCOVERY)) {
nodesDiscovery = Boolean.parseBoolean(tableContext.get(EsResource.NODES_DISCOVERY));
} else {
nodesDiscovery = true;
}
if (tableContext.containsKey(EsResource.HTTP_SSL_ENABLED)) {
httpSslEnabled = Boolean.parseBoolean(tableContext.get(EsResource.HTTP_SSL_ENABLED));
} else {
httpSslEnabled = false;
}
nodesDiscovery = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.NODES_DISCOVERY,
EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
httpSslEnabled = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED,
EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
PartitionType partType = PartitionType.valueOf(Text.readString(in));
if (partType == PartitionType.UNPARTITIONED) {
partitionInfo = SinglePartitionInfo.read(in);

28
fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.datasource;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EsResource;
@ -41,10 +40,9 @@ import java.util.Map;
*/
@Getter
public class EsExternalCatalog extends ExternalCatalog {
private static final Logger LOG = LogManager.getLogger(EsExternalCatalog.class);
public static final String DEFAULT_DB = "default_db";
private static final Logger LOG = LogManager.getLogger(EsExternalCatalog.class);
private EsRestClient esRestClient;
/**
@ -66,27 +64,15 @@ public class EsExternalCatalog extends ExternalCatalog {
if (properties.containsKey("ssl")) {
properties.put(EsResource.HTTP_SSL_ENABLED, properties.remove("ssl"));
}
if (!properties.containsKey(EsResource.HTTP_SSL_ENABLED)) {
properties.put(EsResource.HTTP_SSL_ENABLED, String.valueOf(false));
}
if (properties.containsKey("username")) {
properties.put(EsResource.USER, properties.remove("username"));
}
if (properties.containsKey("doc_value_scan")) {
properties.put(EsResource.DOC_VALUE_SCAN, properties.remove("doc_value_scan"));
}
if (!properties.containsKey(EsResource.DOC_VALUE_SCAN)) {
properties.put(EsResource.DOC_VALUE_SCAN, "true");
}
if (properties.containsKey("keyword_sniff")) {
properties.put(EsResource.KEYWORD_SNIFF, properties.remove("keyword_sniff"));
}
if (!properties.containsKey(EsResource.KEYWORD_SNIFF)) {
properties.put(EsResource.KEYWORD_SNIFF, "true");
}
if (!properties.containsKey(EsResource.NODES_DISCOVERY)) {
properties.put(EsResource.NODES_DISCOVERY, "true");
}
return properties;
}
@ -104,19 +90,23 @@ public class EsExternalCatalog extends ExternalCatalog {
}
public boolean enableDocValueScan() {
return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.DOC_VALUE_SCAN, "true"));
return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.DOC_VALUE_SCAN,
EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
}
public boolean enableKeywordSniff() {
return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF, "true"));
return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF,
EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
}
public boolean enableSsl() {
return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED, "false"));
return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED,
EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
}
public boolean enableNodesDiscovery() {
return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.NODES_DISCOVERY, "true"));
return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.NODES_DISCOVERY,
EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
}
@Override

22
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java vendored
View File

@ -58,15 +58,17 @@ public class MappingPhase implements SearchPhase {
JSONObject properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type());
for (Column col : searchContext.columns()) {
String colName = col.getName();
// if column exists in Doris Table but no found in ES's mapping, we choose to ignore this situation?
if (!properties.containsKey(colName)) {
throw new DorisEsException(
"index[" + searchContext.sourceIndex() + "] type[" + indexMapping + "] mapping not found column"
+ colName + " for the ES Cluster");
// _id not exist mapping, but be can query it.
if (!"_id".equals(colName)) {
if (!properties.containsKey(colName)) {
throw new DorisEsException(
"index[" + searchContext.sourceIndex() + "] mapping[" + indexMapping + "] not found "
+ "column " + colName + " for the ES Cluster");
}
JSONObject fieldObject = (JSONObject) properties.get(colName);
resolveKeywordFields(searchContext, fieldObject, colName);
resolveDocValuesFields(searchContext, fieldObject, colName);
}
JSONObject fieldObject = (JSONObject) properties.get(colName);
resolveKeywordFields(searchContext, fieldObject, colName);
resolveDocValuesFields(searchContext, fieldObject, colName);
}
}
@ -80,9 +82,9 @@ public class MappingPhase implements SearchPhase {
JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
if (fieldsObject != null) {
for (Object key : fieldsObject.keySet()) {
JSONObject innerTypeObject = (JSONObject) fieldsObject.get((String) key);
JSONObject innerTypeObject = (JSONObject) fieldsObject.get(key);
// just for text type
if ("keyword".equals((String) innerTypeObject.get("type"))) {
if ("keyword".equals(innerTypeObject.get("type"))) {
searchContext.fetchFieldsContext().put(colName, colName + "." + key);
}
}