database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](es-catalog) add include_hidden_indexin order to get the hidden index. (#24826)

Browse Source
This commit is contained in:
Guangdong Liu
2023-10-08 14:35:08 +08:00
committed by GitHub
parent 934e9d5617
commit 541f48a754
11 changed files with 316 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
View File

@ -64,6 +64,7 @@ public class EsResource extends Resource {
public static final String LIKE_PUSH_DOWN = "like_push_down";
public static final String QUERY_DSL = "query_dsl";
public static final String INCLUDE_HIDDEN_INDEX = "include_hidden_index";
public static final String DOC_VALUE_SCAN_DEFAULT_VALUE = "true";
public static final String KEYWORD_SNIFF_DEFAULT_VALUE = "true";
public static final String HTTP_SSL_ENABLED_DEFAULT_VALUE = "false";
@ -71,6 +72,8 @@ public class EsResource extends Resource {
public static final String MAPPING_ES_ID_DEFAULT_VALUE = "false";
public static final String LIKE_PUSH_DOWN_DEFAULT_VALUE = "true";
public static final String INCLUDE_HIDDEN_INDEX_DEFAULT_VALUE = "false";
@SerializedName(value = "properties")
private Map<String, String> properties;
@ -134,6 +137,9 @@ public class EsResource extends Resource {
if (properties.containsKey(EsResource.LIKE_PUSH_DOWN)) {
EsUtil.getBoolean(properties, EsResource.LIKE_PUSH_DOWN);
}
if (properties.containsKey(EsResource.INCLUDE_HIDDEN_INDEX)) {
EsUtil.getBoolean(properties, EsResource.INCLUDE_HIDDEN_INDEX);
}
}
public static void fillUrlsWithSchema(String[] urls, boolean isSslEnabled) {

14
fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
View File

@ -37,7 +37,7 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -50,7 +50,7 @@ import java.util.Set;
@Getter
@Setter
public class EsTable extends Table {
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new HashSet<>(Arrays.asList("text"));
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new HashSet<>(Collections.singletonList("text"));
private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:
@ -96,6 +96,9 @@ public class EsTable extends Table {
// Whether pushdown like expr, like will trans to wildcard query, consumes too many es cpu resources
private boolean likePushDown = Boolean.parseBoolean(EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE);
// Whether to include hidden index, default to false
private boolean includeHiddenIndex = Boolean.parseBoolean(EsResource.INCLUDE_HIDDEN_INDEX_DEFAULT_VALUE);
// tableContext is used for being convenient to persist some configuration parameters uniformly
private Map<String, String> tableContext = new HashMap<>();
@ -202,6 +205,10 @@ public class EsTable extends Table {
// parse httpSslEnabled before use it here.
EsResource.fillUrlsWithSchema(seeds, httpSslEnabled);
if (properties.containsKey(EsResource.INCLUDE_HIDDEN_INDEX_DEFAULT_VALUE)) {
includeHiddenIndex = EsUtil.getBoolean(properties, EsResource.INCLUDE_HIDDEN_INDEX_DEFAULT_VALUE);
}
tableContext.put("hosts", hosts);
tableContext.put("userName", userName);
tableContext.put("passwd", passwd);
@ -215,6 +222,7 @@ public class EsTable extends Table {
tableContext.put(EsResource.NODES_DISCOVERY, String.valueOf(nodesDiscovery));
tableContext.put(EsResource.HTTP_SSL_ENABLED, String.valueOf(httpSslEnabled));
tableContext.put(EsResource.LIKE_PUSH_DOWN, String.valueOf(likePushDown));
tableContext.put(EsResource.INCLUDE_HIDDEN_INDEX, String.valueOf(includeHiddenIndex));
}
@Override
@ -295,6 +303,8 @@ public class EsTable extends Table {
tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED, EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
likePushDown = Boolean.parseBoolean(
tableContext.getOrDefault(EsResource.LIKE_PUSH_DOWN, EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE));
includeHiddenIndex = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.INCLUDE_HIDDEN_INDEX,
EsResource.INCLUDE_HIDDEN_INDEX_DEFAULT_VALUE));
PartitionType partType = PartitionType.valueOf(Text.readString(in));
if (partType == PartitionType.UNPARTITIONED) {
partitionInfo = SinglePartitionInfo.read(in);

1
fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java vendored
View File

@ -97,6 +97,7 @@ public class EsExternalTable extends ExternalTable {
esTable.setSeeds(esCatalog.getNodes());
esTable.setHosts(String.join(",", esCatalog.getNodes()));
esTable.syncTableMetaData();
esTable.setIncludeHiddenIndex(esCatalog.enableIncludeHiddenIndex());
return esTable;
}
}

7
fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
View File

@ -118,6 +118,11 @@ public class EsExternalCatalog extends ExternalCatalog {
EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE));
}
public boolean enableIncludeHiddenIndex() {
return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.INCLUDE_HIDDEN_INDEX,
EsResource.INCLUDE_HIDDEN_INDEX_DEFAULT_VALUE));
}
@Override
protected void initLocalObjectsImpl() {
esRestClient = new EsRestClient(getNodes(), getUsername(), getPassword(), enableSsl());
@ -136,7 +141,7 @@ public class EsExternalCatalog extends ExternalCatalog {
db.getTables().forEach(table -> names.add(table.getName()));
return names;
} else {
return esRestClient.listTable();
return esRestClient.listTable(enableIncludeHiddenIndex());
}
}

13
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java vendored
View File

@ -144,7 +144,7 @@ public class EsRestClient {
/**
* Get all index.
**/
public List<String> getIndices() {
public List<String> getIndices(boolean includeHiddenIndex) {
String indexes = execute("_cat/indices?h=index&format=json&s=index:asc");
if (indexes == null) {
throw new DorisEsException("get es indexes error");
@ -154,9 +154,14 @@ public class EsRestClient {
jsonNodes.forEach(json -> {
// es 7.17 has .geoip_databases, but _mapping response 400.
String index = json.get("index").asText();
if (!index.startsWith(".")) {
if (includeHiddenIndex) {
ret.add(index);
} else {
if (!index.startsWith(".")) {
ret.add(index);
}
}
});
return ret;
}
@ -186,8 +191,8 @@ public class EsRestClient {
/**
* Returns the merge of index and alias
**/
public List<String> listTable() {
List<String> indices = getIndices().stream().distinct().collect(Collectors.toList());
public List<String> listTable(boolean includeHiddenIndex) {
List<String> indices = getIndices(includeHiddenIndex).stream().distinct().collect(Collectors.toList());
getAliases().entrySet().stream().filter(e -> indices.contains(e.getKey())).flatMap(e -> e.getValue().stream())
.distinct().forEach(indices::add);
return indices;