From e2d09f57afe35521192485f76dd7b934b4ac0081 Mon Sep 17 00:00:00 2001 From: stalary Date: Sat, 2 May 2020 11:31:37 +0800 Subject: [PATCH 01/21] MOD: fix varchar max error --- docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md | 2 +- docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md index 8dfa67cb6e141b..3aa719b7f200ee 100644 --- a/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md +++ b/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md @@ -27,7 +27,7 @@ under the License. # VARCHAR ## Description MARKETING (M) -A variable length string, M represents the length of a variable length string. The range of M is 1-65535. +A variable length string, M represents the length of a variable length string. The range of M is 1-65533. ##keyword VARCHAR diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md index 59416788a3ec14..b8ac7fe597f458 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md @@ -27,7 +27,7 @@ under the License. # VARCHAR ## description VARCHAR(M) - 变长字符串，M代表的是变长字符串的长度。M的范围是1-65535 + 变长字符串，M代表的是变长字符串的长度。M的范围是1-65533 ## keyword From b6ff6b2edfaeea1c0f16b2fd69a27728c97e506d Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 3 May 2020 00:14:12 +0800 Subject: [PATCH 02/21] MOD: Split /_cluster/state to [indexName/_mappings, indexName/_search_shards] --- .../apache/doris/external/EsIndexState.java | 88 +++----- .../apache/doris/external/EsRestClient.java | 76 +++---- .../apache/doris/external/EsShardRouting.java | 4 +- .../apache/doris/external/EsStateStore.java | 201 ++++++----------- .../apache/doris/catalog/CatalogTestUtil.java | 63 ++---- .../org/apache/doris/es/EsStateStoreTest.java | 198 +++------------- fe/src/test/resources/data/es/mappings.json | 25 ++ .../test/resources/data/es/search_shards.json | 213 ++++++++++++++++++ 8 files changed, 422 insertions(+), 446 deletions(-) create mode 100644 fe/src/test/resources/data/es/mappings.json create mode 100644 fe/src/test/resources/data/es/search_shards.json diff --git a/fe/src/main/java/org/apache/doris/external/EsIndexState.java b/fe/src/main/java/org/apache/doris/external/EsIndexState.java index 5dca5db384b12e..12106aec8ef81a 100644 --- a/fe/src/main/java/org/apache/doris/external/EsIndexState.java +++ b/fe/src/main/java/org/apache/doris/external/EsIndexState.java @@ -17,29 +17,21 @@ package org.apache.doris.external; -import org.apache.doris.analysis.PartitionKeyDesc; -import org.apache.doris.analysis.PartitionValue; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; +import java.util.Random; import org.apache.doris.analysis.SingleRangePartitionDesc; -import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.RangePartitionInfo; -import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TNetworkAddress; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; -import java.util.List; -import java.util.Map; -import java.util.Random; - public class EsIndexState { - + private static final Logger LOG = LogManager.getLogger(EsIndexState.class); private final String indexName; @@ -48,7 +40,7 @@ public class EsIndexState { private SingleRangePartitionDesc partitionDesc; private PartitionKey partitionKey; private long partitionId = -1; - + public EsIndexState(String indexName) { this.indexName = indexName; this.shardRoutings = Maps.newHashMap(); @@ -76,66 +68,45 @@ public TNetworkAddress randomAddress(Map nodesInfo) { EsNodeInfo[] nodeInfos = (EsNodeInfo[]) nodesInfo.values().toArray(); return nodeInfos[seed].getPublishAddress(); } - - public static EsIndexState parseIndexStateV55(String indexName, JSONObject indicesRoutingMap, - JSONObject nodesMap, - JSONObject indicesMetaMap, PartitionInfo partitionInfo) throws AnalysisException { + + public static EsIndexState parseIndexState(String indexName, JSONObject nodesMap, + JSONArray shards) { EsIndexState indexState = new EsIndexState(indexName); - JSONObject shardRoutings = indicesRoutingMap.getJSONObject(indexName).getJSONObject("shards"); - for (String shardKey : shardRoutings.keySet()) { + int length = shards.length(); + for (int i = 0; i < length; i++) { List singleShardRouting = Lists.newArrayList(); - JSONArray shardRouting = shardRoutings.getJSONArray(shardKey); - for (int i = 0; i < shardRouting.length(); ++i) { - JSONObject shard = shardRouting.getJSONObject(i); + JSONArray shardsArray = shards.getJSONArray(i); + int arrayLength = shardsArray.length(); + for (int j = 0; j < arrayLength; j++) { + JSONObject shard = shardsArray.getJSONObject(j); String shardState = shard.getString("state"); if ("STARTED".equalsIgnoreCase(shardState)) { try { - singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, - shardKey, shard, nodesMap)); + singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, + String.valueOf(i), shard, nodesMap)); } catch (Exception e) { - LOG.info("errors while parse shard routing from json [{}], ignore this shard", shard.toString(), e); + LOG.info( + "errors while parse shard routing from json [{}], ignore this shard", + shard, e); } - } + } } if (singleShardRouting.isEmpty()) { - LOG.warn("could not find a healthy allocation for [{}][{}]", indexName, shardKey); - } - indexState.addShardRouting(Integer.valueOf(shardKey), singleShardRouting); - } - - // get some meta info from es, could be used to prune index when query - // index.bpack.partition.upperbound: stu_age - if (partitionInfo != null && partitionInfo instanceof RangePartitionInfo) { - JSONObject indexMeta = indicesMetaMap.getJSONObject(indexName); - JSONObject partitionSetting = EsUtil.getJsonObject(indexMeta, "settings.index.bpack.partition", 0); - LOG.debug("index {} range partition setting is {}", indexName, - partitionSetting == null ? "" : partitionSetting.toString()); - if (partitionSetting != null && partitionSetting.has("upperbound")) { - String upperBound = partitionSetting.getString("upperbound"); - List upperValues = Lists.newArrayList(new PartitionValue(upperBound)); - PartitionKeyDesc partitionKeyDesc = new PartitionKeyDesc(upperValues); - // use index name as partition name - SingleRangePartitionDesc desc = new SingleRangePartitionDesc(false, - indexName, partitionKeyDesc, null); - PartitionKey partitionKey = PartitionKey.createPartitionKey( - desc.getPartitionKeyDesc().getUpperValues(), - ((RangePartitionInfo) partitionInfo).getPartitionColumns()); - desc.analyze(((RangePartitionInfo) partitionInfo).getPartitionColumns().size(), null); - indexState.setPartitionDesc(desc); - indexState.setPartitionKey(partitionKey); + LOG.warn("could not find a healthy allocation for [{}][{}]", indexName, i); } + indexState.addShardRouting(i, singleShardRouting); } return indexState; } - + public void addShardRouting(int shardId, List singleShardRouting) { shardRoutings.put(shardId, singleShardRouting); } - + public String getIndexName() { return indexName; } - + public Map> getShardRoutings() { return shardRoutings; } @@ -166,7 +137,8 @@ public void setPartitionId(long partitionId) { @Override public String toString() { - return "EsIndexState [indexName=" + indexName + ", partitionDesc=" + partitionDesc + ", partitionKey=" - + partitionKey + "]"; + return "EsIndexState [indexName=" + indexName + ", partitionDesc=" + partitionDesc + + ", partitionKey=" + + partitionKey + "]"; } } diff --git a/fe/src/main/java/org/apache/doris/external/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/EsRestClient.java index 73759c4a4e1996..3d406f6830e1de 100644 --- a/fe/src/main/java/org/apache/doris/external/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/EsRestClient.java @@ -17,10 +17,16 @@ package org.apache.doris.external; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; import okhttp3.Credentials; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.util.Strings; @@ -29,12 +35,6 @@ import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - public class EsRestClient { private static final Logger LOG = LogManager.getLogger(EsRestClient.class); private ObjectMapper mapper; @@ -49,26 +49,27 @@ public class EsRestClient { .readTimeout(10, TimeUnit.SECONDS) .build(); - private String basicAuth; - - private int nextClient = 0; + private Request.Builder builder; private String[] nodes; private String currentNode; + private int currentNodeIndex = 0; public EsRestClient(String[] nodes, String authUser, String authPassword) { this.nodes = nodes; + builder = new Request.Builder(); if (!Strings.isEmpty(authUser) && !Strings.isEmpty(authPassword)) { - basicAuth = Credentials.basic(authUser, authPassword); + builder.addHeader(HttpHeaders.AUTHORIZATION, Credentials.basic(authUser, authPassword)); } - selectNextNode(); + currentNode = nodes[currentNodeIndex]; } - private boolean selectNextNode() { - if (nextClient >= nodes.length) { - return false; + private void selectNextNode() { + currentNodeIndex++; + // reroute, because the previously failed node may have already been restored + if (currentNodeIndex >= nodes.length) { + currentNodeIndex = 0; } - currentNode = nodes[nextClient++]; - return true; + currentNode = nodes[currentNodeIndex]; } public Map getHttpNodes() throws Exception { @@ -76,21 +77,24 @@ public Map getHttpNodes() throws Exception { if (nodesData == null) { return Collections.emptyMap(); } - Map nodes = new HashMap<>(); + Map nodeMap = new HashMap<>(); for (Map.Entry> entry : nodesData.entrySet()) { EsNodeInfo node = new EsNodeInfo(entry.getKey(), entry.getValue()); if (node.hasHttp()) { - nodes.put(node.getId(), node); + nodeMap.put(node.getId(), node); } } - return nodes; + return nodeMap; } - public String getIndexMetaData(String indexName) { - String path = "_cluster/state?indices=" + indexName - + "&metric=routing_table,nodes,metadata&expand_wildcards=open"; + public String getIndexMapping(String indexName) { + String path = indexName + "/_mapping"; return execute(path); + } + public String getSearchShards(String indexName) { + String path = indexName + "/_search_shards"; + return execute(path); } /** @@ -113,20 +117,14 @@ public EsMajorVersion version() { } /** - * execute request for specific path + * execute request for specific path，it will try again nodes.length times if it fails * * @param path the path must not leading with '/' - * @return + * @return response */ private String execute(String path) { - selectNextNode(); - boolean nextNode; - do { - Request.Builder builder = new Request.Builder(); - if (!Strings.isEmpty(basicAuth)) { - builder.addHeader("Authorization", basicAuth); - } - + int retrySize = nodes.length; + for (int i = 0; i < retrySize; i++) { // maybe should add HTTP schema to the address // actually, at this time we can only process http protocol // NOTE. currentNode may have some spaces. @@ -137,10 +135,9 @@ private String execute(String path) { if (!(currentNode.startsWith("http://") || currentNode.startsWith("https://"))) { currentNode = "http://" + currentNode; } - Request request = builder.get() - .url(currentNode + "/" + path) - .build(); + .url(currentNode + "/" + path) + .build(); LOG.trace("es rest client request URL: {}", currentNode + "/" + path); try { Response response = networkClient.newCall(request).execute(); @@ -150,11 +147,9 @@ private String execute(String path) { } catch (IOException e) { LOG.warn("request node [{}] [{}] failures {}, try next nodes", currentNode, path, e); } - nextNode = selectNextNode(); - if (!nextNode) { - LOG.warn("try all nodes [{}],no other nodes left", nodes); - } - } while (nextNode); + selectNextNode(); + } + LOG.warn("try all nodes [{}],no other nodes left", nodes); return null; } @@ -162,6 +157,7 @@ public T get(String q, String key) { return parseContent(execute(q), key); } + @SuppressWarnings("unchecked") private T parseContent(String response, String key) { Map map = Collections.emptyMap(); try { diff --git a/fe/src/main/java/org/apache/doris/external/EsShardRouting.java b/fe/src/main/java/org/apache/doris/external/EsShardRouting.java index 12afa5f34a39ee..65461d8d8cbe0a 100644 --- a/fe/src/main/java/org/apache/doris/external/EsShardRouting.java +++ b/fe/src/main/java/org/apache/doris/external/EsShardRouting.java @@ -51,10 +51,10 @@ public static EsShardRouting parseShardRoutingV55(String indexName, String shard // In http transport mode, should ignore thrift_port, set address to null TNetworkAddress addr = null; if (!StringUtils.isEmpty(thriftPort)) { - addr = new TNetworkAddress(transportAddr[0], Integer.valueOf(thriftPort)); + addr = new TNetworkAddress(transportAddr[0], Integer.parseInt(thriftPort)); } boolean isPrimary = shardInfo.getBoolean("primary"); - return new EsShardRouting(indexName, Integer.valueOf(shardKey), + return new EsShardRouting(indexName, Integer.parseInt(shardKey), isPrimary, addr, nodeId); } diff --git a/fe/src/main/java/org/apache/doris/external/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/EsStateStore.java index 0ac05f21cabc33..19c20d3c775684 100644 --- a/fe/src/main/java/org/apache/doris/external/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/EsStateStore.java @@ -17,49 +17,36 @@ package org.apache.doris.external; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.EsTable; -import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.SinglePartitionInfo; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.MasterDaemon; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; -import com.google.common.collect.Range; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONArray; import org.json.JSONObject; -import java.io.IOException; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import okhttp3.Authenticator; -import okhttp3.Call; -import okhttp3.Credentials; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.Route; - /** * it is used to call es api to get shard allocation state */ public class EsStateStore extends MasterDaemon { + private static final Logger LOG = LogManager.getLogger(EsStateStore.class); private Map esTables; @@ -81,30 +68,34 @@ public void deRegisterTable(long tableId) { esTables.remove(tableId); LOG.info("deregister table [{}] from sync list", tableId); } - + @Override protected void runAfterCatalogReady() { for (EsTable esTable : esTables.values()) { try { EsRestClient client = new EsRestClient(esTable.getSeeds(), - esTable.getUserName(), esTable.getPasswd()); - // if user not specify the es version, try to get the remote cluster versoin - // in the future, we maybe need this version - String indexMetaData = client.getIndexMetaData(esTable.getIndexName()); - if (indexMetaData == null) { + esTable.getUserName(), esTable.getPasswd()); + + String indexMapping = client.getIndexMapping(esTable.getIndexName()); + if (indexMapping == null) { continue; } - EsTableState esTableState = parseClusterState55(indexMetaData, esTable); + loadEsIndexMapping(indexMapping, esTable); + + String shardLocation = client.getSearchShards(esTable.getIndexName()); + EsTableState esTableState = loadEsSearchShards(shardLocation, esTable); if (esTableState == null) { continue; } + if (EsTable.TRANSPORT_HTTP.equals(esTable.getTransport())) { Map nodesInfo = client.getHttpNodes(); esTableState.addHttpAddress(nodesInfo); } esTable.setEsTableState(esTableState); } catch (Throwable e) { - LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", esTable.getName(), e); + LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", + esTable.getName(), e); } } } @@ -130,74 +121,30 @@ public void loadTableFromCatalog() { } } - private EsTableState loadEsIndexMetadataV55(final EsTable esTable) { - OkHttpClient.Builder clientBuilder = new OkHttpClient.Builder(); - clientBuilder.authenticator(new Authenticator() { - @Override - public Request authenticate(Route route, Response response) throws IOException { - String credential = Credentials.basic(esTable.getUserName(), esTable.getPasswd()); - return response.request().newBuilder().header("Authorization", credential).build(); - } - }); - String[] seeds = esTable.getSeeds(); - for (String seed : seeds) { - String url = seed + "/_cluster/state?indices=" - + esTable.getIndexName() - + "&metric=routing_table,nodes,metadata&expand_wildcards=open"; - String basicAuth = ""; - try { - Request request = new Request.Builder() - .get() - .url(url) - .addHeader("Authorization", basicAuth) - .build(); - Call call = clientBuilder.build().newCall(request); - Response response = call.execute(); - String responseStr = response.body().string(); - if (response.isSuccessful()) { - try { - EsTableState esTableState = parseClusterState55(responseStr, esTable); - if (esTableState != null) { - return esTableState; - } - } catch (Exception e) { - LOG.warn("errors while parse response msg {}", responseStr, e); - } - } else { - LOG.info("errors while call es [{}] to get state info {}", url, responseStr); - } - } catch (Exception e) { - LOG.warn("errors while call es [{}]", url, e); - } - } - return null; - } - - @VisibleForTesting - public EsTableState parseClusterState55(String responseStr, EsTable esTable) - throws DdlException, AnalysisException, ExternalDataSourceException { - JSONObject jsonObject = new JSONObject(responseStr); - String clusterName = jsonObject.getString("cluster_name"); - JSONObject nodesMap = jsonObject.getJSONObject("nodes"); - // we build the doc value context for fields maybe used for scanning - // "properties": { - // "city": { - // "type": "text", // text field does not have docvalue - // "fields": { - // "raw": { - // "type": "keyword" - // } - // } - // } - // } - // then the docvalue context provided the mapping between the select field and real request field : - // {"city": "city.raw"} - JSONObject indicesMetaMap = jsonObject.getJSONObject("metadata").getJSONObject("indices"); - JSONObject indexMetaMap = indicesMetaMap.optJSONObject(esTable.getIndexName()); - if (indexMetaMap != null && (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable())) { - JSONObject mappings = indexMetaMap.optJSONObject("mappings"); + // Configure keyword and doc_values by mapping + public void loadEsIndexMapping(String indexMapping, EsTable esTable) { + JSONObject jsonObject = new JSONObject(indexMapping); + // the indexName use alias takes the first mapping + Iterator keys = jsonObject.keys(); + if ((esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) && keys.hasNext()) { + String docKey = keys.next(); + JSONObject docData = jsonObject.optJSONObject(docKey); + JSONObject mappings = docData.optJSONObject("mappings"); JSONObject rootSchema = mappings.optJSONObject(esTable.getMappingType()); JSONObject schema = rootSchema.optJSONObject("properties"); + // we build the doc value context for fields maybe used for scanning + // "properties": { + // "city": { + // "type": "text", // text field does not have docvalue + // "fields": { + // "raw": { + // "type": "keyword" + // } + // } + // } + // } + // then the docvalue context provided the mapping between the select field and real request field : + // {"city": "city.raw"} List colList = esTable.getFullSchema(); for (Column col : colList) { String colName = col.getName(); @@ -222,13 +169,15 @@ public EsTableState parseClusterState55(String responseStr, EsTable esTable) } } } + if (esTable.isDocValueScanEnable()) { if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { JSONObject fieldsObject = fieldObject.optJSONObject("fields"); if (fieldsObject != null) { for (String key : fieldsObject.keySet()) { JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.optString("type"))) { + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS + .contains(innerTypeObject.optString("type"))) { continue; } if (innerTypeObject.has("doc_values")) { @@ -256,13 +205,19 @@ public EsTableState parseClusterState55(String responseStr, EsTable esTable) } } } + } - JSONObject indicesRoutingMap = jsonObject.getJSONObject("routing_table").getJSONObject("indices"); + public EsTableState loadEsSearchShards(String shardLocation, EsTable esTable) + throws ExternalDataSourceException, DdlException { + JSONObject jsonObject = new JSONObject(shardLocation); + JSONObject indicesRoutingMap = jsonObject.getJSONObject("nodes"); + JSONArray shards = jsonObject.getJSONArray("shards"); EsTableState esTableState = new EsTableState(); - PartitionInfo partitionInfo = null; + RangePartitionInfo partitionInfo = null; if (esTable.getPartitionInfo() != null) { if (esTable.getPartitionInfo() instanceof RangePartitionInfo) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTable.getPartitionInfo(); + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTable + .getPartitionInfo(); partitionInfo = new RangePartitionInfo(rangePartitionInfo.getPartitionColumns()); esTableState.setPartitionInfo(partitionInfo); if (LOG.isDebugEnabled()) { @@ -276,48 +231,40 @@ public EsTableState parseClusterState55(String responseStr, EsTable esTable) idx++; } sb.append(")"); - LOG.debug("begin to parse es table [{}] state from cluster state," - + " with partition info [{}]", esTable.getName(), sb.toString()); + LOG.debug("begin to parse es table [{}] state from search shards," + + " with partition info [{}]", esTable.getName(), sb.toString()); } } else if (esTable.getPartitionInfo() instanceof SinglePartitionInfo) { - LOG.debug("begin to parse es table [{}] state from cluster state, " - + "with no partition info", esTable.getName()); + LOG.debug("begin to parse es table [{}] state from search shards, " + + "with no partition info", esTable.getName()); } else { throw new ExternalDataSourceException("es table only support range partition, " - + "but current partition type is " - + esTable.getPartitionInfo().getType()); + + "but current partition type is " + + esTable.getPartitionInfo().getType()); } } - - for (String indexName : indicesRoutingMap.keySet()) { - EsIndexState indexState = EsIndexState.parseIndexStateV55(indexName, - indicesRoutingMap, nodesMap, - indicesMetaMap, partitionInfo); - esTableState.addIndexState(indexName, indexState); - LOG.debug("add index {} to es table {}", indexState, esTable.getName()); - } - - if (partitionInfo instanceof RangePartitionInfo) { + EsIndexState indexState = EsIndexState.parseIndexState(esTable.getIndexName(), + indicesRoutingMap, shards); + esTableState.addIndexState(esTable.getIndexName(), indexState); + LOG.debug("add index {} to es table {}", indexState, esTable.getName()); + if (partitionInfo != null) { // sort the index state according to partition key and then add to range map - List esIndexStates = esTableState.getPartitionedIndexStates().values() - .stream().collect(Collectors.toList()); - Collections.sort(esIndexStates, new Comparator() { - @Override - public int compare(EsIndexState o1, EsIndexState o2) { - return o1.getPartitionKey().compareTo(o2.getPartitionKey()); - } - }); + List esIndexStates = new ArrayList<>( + esTableState.getPartitionedIndexStates().values()); + esIndexStates.sort(Comparator.comparing(EsIndexState::getPartitionKey)); long partitionId = 0; for (EsIndexState esIndexState : esIndexStates) { - Range range = ((RangePartitionInfo) partitionInfo).handleNewSinglePartitionDesc( - esIndexState.getPartitionDesc(), partitionId, false); + Range range = partitionInfo.handleNewSinglePartitionDesc( + esIndexState.getPartitionDesc(), partitionId, false); esTableState.addPartition(esIndexState.getIndexName(), partitionId); esIndexState.setPartitionId(partitionId); ++partitionId; - LOG.debug("add parition to es table [{}] with range [{}]", esTable.getName(), range); + LOG.debug("add parition to es table [{}] with range [{}]", esTable.getName(), + range); } } return esTableState; } + } diff --git a/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index 29536fae9383f7..e799352027d288 100644 --- a/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -79,10 +79,8 @@ public class CatalogTestUtil { public static String testTxnLable10 = "testTxnLable10"; public static String testTxnLable11 = "testTxnLable11"; public static String testTxnLable12 = "testTxnLable12"; - public static String testPartitionedEsTable1 = "partitionedEsTable1"; - public static long testPartitionedEsTableId1 = 14; - public static String testUnPartitionedEsTable1 = "unpartitionedEsTable1"; - public static long testUnPartitionedEsTableId1 = 15; + public static String testEsTable1 = "partitionedEsTable1"; + public static long testEsTableId1 = 14; public static Catalog createTestCatalog() throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { @@ -229,8 +227,7 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, // add a es table to catalog try { - createPartitionedEsTable(db); - createUnPartitionedEsTable(db); + createEsTable(db); } catch (DdlException e) { // TODO Auto-generated catch block // e.printStackTrace(); @@ -238,21 +235,18 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, return db; } - public static void createPartitionedEsTable(Database db) throws DdlException { + public static void createEsTable(Database db) throws DdlException { // columns - List columns = new ArrayList(); - Column k1 = new Column("k1", PrimitiveType.DATE); - k1.setIsKey(true); - columns.add(k1); - Column k2 = new Column("k2", PrimitiveType.INT); - k2.setIsKey(true); - columns.add(k2); - columns.add(new Column("v", ScalarType.createType(PrimitiveType.DOUBLE), false, AggregateType.SUM, "0", "")); + List columns = new ArrayList<>(); + Column userId = new Column("userId", PrimitiveType.VARCHAR); + columns.add(userId); + columns.add(new Column("time", PrimitiveType.BIGINT)); + columns.add(new Column("type", PrimitiveType.VARCHAR)); // table List partitionColumns = Lists.newArrayList(); List singleRangePartitionDescs = Lists.newArrayList(); - partitionColumns.add(k1); + partitionColumns.add(userId); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", new PartitionKeyDesc(Lists @@ -263,44 +257,15 @@ public static void createPartitionedEsTable(Database db) throws DdlException { Map properties = Maps.newHashMap(); properties.put(EsTable.HOSTS, "xxx"); properties.put(EsTable.INDEX, "indexa"); + properties.put(EsTable.TYPE, "doc"); properties.put(EsTable.PASSWORD, ""); properties.put(EsTable.USER, "root"); - EsTable esTable = new EsTable(testPartitionedEsTableId1, testPartitionedEsTable1, + properties.put(EsTable.DOC_VALUE_SCAN, "true"); + properties.put(EsTable.KEYWORD_SNIFF, "true"); + EsTable esTable = new EsTable(testEsTableId1, testEsTable1, columns, properties, partitionInfo); db.createTable(esTable); } - - public static void createUnPartitionedEsTable(Database db) throws DdlException { - // columns - List columns = new ArrayList(); - Column k1 = new Column("k1", PrimitiveType.DATE); - k1.setIsKey(true); - columns.add(k1); - Column k2 = new Column("k2", PrimitiveType.INT); - k2.setIsKey(true); - columns.add(k2); - columns.add(new Column("v", ScalarType.createType(PrimitiveType.DOUBLE), false, AggregateType.SUM, "0", "")); - - // table - List partitionColumns = Lists.newArrayList(); - List singleRangePartitionDescs = Lists.newArrayList(); - partitionColumns.add(k1); - - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists - .newArrayList(new PartitionValue("100"))), - null)); - - SinglePartitionInfo partitionInfo = new SinglePartitionInfo(); - Map properties = Maps.newHashMap(); - properties.put(EsTable.HOSTS, "xxx"); - properties.put(EsTable.INDEX, "indexa"); - properties.put(EsTable.PASSWORD, ""); - properties.put(EsTable.USER, "root"); - EsTable esTable = new EsTable(testUnPartitionedEsTableId1, testUnPartitionedEsTable1, - columns, properties, partitionInfo); - db.createTable(esTable); - } public static Backend createBackend(long id, String host, int heartPort, int bePort, int httpPort) { Backend backend = new Backend(id, host, heartPort); diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index a7fa438074db78..a6f649b8fbf976 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -18,52 +18,39 @@ package org.apache.doris.es; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import org.apache.doris.analysis.PartitionValue; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.FakeCatalog; import org.apache.doris.catalog.FakeEditLog; -import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.external.EsIndexState; import org.apache.doris.external.EsStateStore; import org.apache.doris.external.EsTableState; +import org.apache.doris.external.ExternalDataSourceException; import org.apache.doris.meta.MetaContext; - -import com.google.common.collect.Lists; -import com.google.common.collect.Range; - import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.InvocationTargetException; -import java.net.URISyntaxException; -import java.util.Map; - public class EsStateStoreTest { private static FakeEditLog fakeEditLog; private static FakeCatalog fakeCatalog; private static Catalog masterCatalog; - private static String clusterStateStr1 = ""; - private static String clusterStateStr2 = ""; - private static String clusterStateStr3 = ""; - private static String clusterStateStr4 = ""; - private static String clusterStateStr5 = ""; + private static String mappingsStr = ""; + private static String searchShardsStr = ""; private EsStateStore esStateStore; @BeforeClass @@ -78,164 +65,35 @@ public static void init() throws IOException, InstantiationException, IllegalAcc metaContext.setThreadLocalInfo(); // masterCatalog.setJournalVersion(FeMetaVersion.VERSION_40); FakeCatalog.setCatalog(masterCatalog); - clusterStateStr1 = loadJsonFromFile("data/es/clusterstate1.json"); - clusterStateStr2 = loadJsonFromFile("data/es/clusterstate2.json"); - clusterStateStr3 = loadJsonFromFile("data/es/clusterstate3.json"); - clusterStateStr4 = loadJsonFromFile("data/es/clusterstate4.json"); - clusterStateStr5 = loadJsonFromFile("data/es/clusterstate5.json"); + mappingsStr = loadJsonFromFile("data/es/mappings.json"); + searchShardsStr = loadJsonFromFile("data/es/search_shards.json"); } @Before public void setUp() { esStateStore = new EsStateStore(); } - - /** - * partitioned es table schema: k1(date), k2(int), v(double) - * @throws AnalysisException - */ + @Test - public void testParsePartitionedClusterState() throws AnalysisException { + public void testLoadEsIndexMapping() { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() - .getDb(CatalogTestUtil.testDb1) - .getTable(CatalogTestUtil.testPartitionedEsTable1); - boolean hasException = false; - EsTableState esTableState = null; - try { - esTableState = esStateStore.parseClusterState55(clusterStateStr1, esTable); - } catch (Exception e) { - e.printStackTrace(); - hasException = true; - } - assertFalse(hasException); - assertNotNull(esTableState); - assertEquals(2, esTableState.getPartitionedIndexStates().size()); - RangePartitionInfo definedPartInfo = (RangePartitionInfo) esTable.getPartitionInfo(); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTableState.getPartitionInfo(); - Map> rangeMap = rangePartitionInfo.getIdToRange(false); - assertEquals(2, rangeMap.size()); - Range part0 = rangeMap.get(new Long(0)); - EsIndexState esIndexState1 = esTableState.getIndexState(0); - assertEquals(5, esIndexState1.getShardRoutings().size()); - assertEquals("index1", esIndexState1.getIndexName()); - PartitionKey lowKey = PartitionKey.createInfinityPartitionKey(definedPartInfo.getPartitionColumns(), false); - PartitionKey upperKey = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("2018-10-01")), - definedPartInfo.getPartitionColumns()); - Range newRange = Range.closedOpen(lowKey, upperKey); - assertEquals(newRange, part0); - Range part1 = rangeMap.get(new Long(1)); - EsIndexState esIndexState2 = esTableState.getIndexState(1); - assertEquals("index2", esIndexState2.getIndexName()); - lowKey = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("2018-10-01")), - definedPartInfo.getPartitionColumns()); - upperKey = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("2018-10-02")), - definedPartInfo.getPartitionColumns()); - newRange = Range.closedOpen(lowKey, upperKey); - assertEquals(newRange, part1); - assertEquals(6, esIndexState2.getShardRoutings().size()); + .getDb(CatalogTestUtil.testDb1) + .getTable(CatalogTestUtil.testEsTableId1); + esStateStore.loadEsIndexMapping(mappingsStr, esTable); + assertEquals("userId.keyword", esTable.fieldsContext().get("userId")); + assertEquals("userId.keyword", esTable.docValueContext().get("userId")); } - - /** - * partitioned es table schema: k1(date), k2(int), v(double) - * scenario desc: - * 2 indices, one with partition desc, the other does not contains partition desc - * @throws AnalysisException - */ + @Test - public void testParsePartitionedClusterStateTwoIndices() throws AnalysisException { + public void testLocalSearchShards() throws ExternalDataSourceException, DdlException { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() - .getDb(CatalogTestUtil.testDb1) - .getTable(CatalogTestUtil.testPartitionedEsTable1); - boolean hasException = false; - EsTableState esTableState = null; - try { - esTableState = esStateStore.parseClusterState55(clusterStateStr3, esTable); - } catch (Exception e) { - e.printStackTrace(); - hasException = true; - } - assertFalse(hasException); + .getDb(CatalogTestUtil.testDb1) + .getTable(CatalogTestUtil.testEsTableId1); + esStateStore.loadEsIndexMapping(mappingsStr, esTable); + EsTableState esTableState = esStateStore.loadEsSearchShards(searchShardsStr, esTable); assertNotNull(esTableState); - - // check - assertEquals(1, esTableState.getPartitionedIndexStates().size()); assertEquals(1, esTableState.getUnPartitionedIndexStates().size()); - - // check partition info - RangePartitionInfo definedPartInfo = (RangePartitionInfo) esTable.getPartitionInfo(); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTableState.getPartitionInfo(); - Map> rangeMap = rangePartitionInfo.getIdToRange(false); - assertEquals(1, rangeMap.size()); - Range part0 = rangeMap.get(new Long(0)); - EsIndexState esIndexState1 = esTableState.getIndexState(0); - assertEquals(5, esIndexState1.getShardRoutings().size()); - assertEquals("index1", esIndexState1.getIndexName()); - PartitionKey lowKey = PartitionKey.createInfinityPartitionKey(definedPartInfo.getPartitionColumns(), false); - PartitionKey upperKey = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("2018-10-01")), - definedPartInfo.getPartitionColumns()); - Range newRange = Range.closedOpen(lowKey, upperKey); - assertEquals(newRange, part0); - - // check index with no partition desc - EsIndexState esIndexState2 = esTableState.getUnPartitionedIndexStates().get("index2"); - assertEquals("index2", esIndexState2.getIndexName()); - assertEquals(6, esIndexState2.getShardRoutings().size()); - } - - /** - * partitioned es table schema: k1(date), k2(int), v(double) - * scenario desc: - * 2 indices, both of them does not contains partition desc and es table does not have partition info - * but cluster state have partition info - * @throws AnalysisException - */ - @Test - public void testParseUnPartitionedClusterStateTwoIndices() throws AnalysisException { - EsTable esTable = (EsTable) Catalog.getCurrentCatalog() - .getDb(CatalogTestUtil.testDb1) - .getTable(CatalogTestUtil.testUnPartitionedEsTableId1); - boolean hasException = false; - EsTableState esTableState = null; - try { - esTableState = esStateStore.parseClusterState55(clusterStateStr4, esTable); - } catch (Exception e) { - e.printStackTrace(); - hasException = true; - } - assertFalse(hasException); - assertNotNull(esTableState); - - // check - assertEquals(0, esTableState.getPartitionedIndexStates().size()); - assertEquals(2, esTableState.getUnPartitionedIndexStates().size()); - - // check index with no partition desc - EsIndexState esIndexState1 = esTableState.getUnPartitionedIndexStates().get("index1"); - assertEquals("index1", esIndexState1.getIndexName()); - EsIndexState esIndexState2 = esTableState.getUnPartitionedIndexStates().get("index2"); - assertEquals("index2", esIndexState2.getIndexName()); - assertEquals(6, esIndexState2.getShardRoutings().size()); - } - - - /** - * partitioned es table schema: k1(date), k2(int), v(double) - * "upperbound": "2018" is not a valid date value, so parsing procedure will fail - */ - @Test - public void testParseInvalidUpperbound() { - EsTable esTable = (EsTable) Catalog.getCurrentCatalog() - .getDb(CatalogTestUtil.testDb1) - .getTable(CatalogTestUtil.testPartitionedEsTable1); - boolean hasException = false; - EsTableState esTableState = null; - try { - esTableState = esStateStore.parseClusterState55(clusterStateStr2, esTable); - } catch (Exception e) { - hasException = true; - } - assertTrue(hasException); - assertTrue(esTableState == null); + assertEquals(5, esTableState.getIndexState("indexa").getShardRoutings().size()); } private static String loadJsonFromFile(String fileName) throws IOException, URISyntaxException { diff --git a/fe/src/test/resources/data/es/mappings.json b/fe/src/test/resources/data/es/mappings.json new file mode 100644 index 00000000000000..1c3f23308ce54f --- /dev/null +++ b/fe/src/test/resources/data/es/mappings.json @@ -0,0 +1,25 @@ +{ + "indexa_2020.05.02": { + "mappings": { + "doc": { + "dynamic": "strict", + "properties": { + "time": { + "type": "long" + }, + "type": { + "type": "keyword" + }, + "userId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/fe/src/test/resources/data/es/search_shards.json b/fe/src/test/resources/data/es/search_shards.json new file mode 100644 index 00000000000000..0a0446cced6771 --- /dev/null +++ b/fe/src/test/resources/data/es/search_shards.json @@ -0,0 +1,213 @@ +{ + "nodes": { + "yoBi15gPSFe7BKDDOhI-0g": { + "name": "test239-9220-9320", + "ephemeral_id": "Th56FehQT36Zda2l0FQewg", + "transport_address": "192.168.0.1:9320", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "PTrurLShRN6POIuNe8WnFg": { + "name": "test057-29210-29310", + "ephemeral_id": "KyIWi165QVa9sqHYvjeJxw", + "transport_address": "192.168.0.1:29310", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "7mjpC74LQkGEgcGks3-dYQ": { + "name": "test239-9230-9330", + "ephemeral_id": "g821RcAARpy3hIOnboqnHA", + "transport_address": "192.168.0.1:9330", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "_DigEpF0SPmV32lPTzZ_QA": { + "name": "test240-9210-9310", + "ephemeral_id": "sgKUhJ59QuyzUvKXxNnzww", + "transport_address": "192.168.0.1:9310", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "zMH8NhsdTy63mnTeNgGILw": { + "name": "test240-9230-9330", + "ephemeral_id": "OEvljKbCSPaQpU3SPavBvQ", + "transport_address": "192.168.0.1:9330", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "5D_18rKKRzOnAlnbZUdSRw": { + "name": "test239-9210-9310", + "ephemeral_id": "ECEwuqcvTY-qAlT5OcWz2Q", + "transport_address": "192.168.0.1:9310", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "8F5Wip4-Qb6og3jhepn5sg": { + "name": "test058-29210-29310", + "ephemeral_id": "QU24dPM_T-S-OPGoDH9VGA", + "transport_address": "192.168.0.1:29310", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + }, + "7k1HfDdMQ1G8Ob4vUDRl1Q": { + "name": "test240-9220-9320", + "ephemeral_id": "IzSWsgStRriKFdYvBz9hNw", + "transport_address": "192.168.0.1:9320", + "attributes": { + "disk": "hdd", + "ml.max_open_jobs": "10", + "ml.enabled": "true" + } + } + }, + "indices": { + "indexa_2020.05.02": { + "aliases": [ + "indexa" + ] + } + }, + "shards": [ + [ + { + "state": "STARTED", + "primary": true, + "node": "7mjpC74LQkGEgcGks3-dYQ", + "relocating_node": null, + "shard": 0, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "Q77NsOEYR0SlzOSOQsyixQ" + } + }, + { + "state": "STARTED", + "primary": false, + "node": "8F5Wip4-Qb6og3jhepn5sg", + "relocating_node": null, + "shard": 0, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "nKsg32sPSyinydeaRyUA8w" + } + } + ], + [ + { + "state": "STARTED", + "primary": false, + "node": "_DigEpF0SPmV32lPTzZ_QA", + "relocating_node": null, + "shard": 1, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "2L5soNwQTDaNz7EdxNV9yQ" + } + }, + { + "state": "STARTED", + "primary": true, + "node": "yoBi15gPSFe7BKDDOhI-0g", + "relocating_node": null, + "shard": 1, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "FJKsGnYyRniK0BexSdJCiQ" + } + } + ], + [ + { + "state": "STARTED", + "primary": false, + "node": "zMH8NhsdTy63mnTeNgGILw", + "relocating_node": null, + "shard": 2, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "qyTTlZYCQ4y0hQPL5uLIaQ" + } + }, + { + "state": "STARTED", + "primary": true, + "node": "5D_18rKKRzOnAlnbZUdSRw", + "relocating_node": null, + "shard": 2, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "SxFS7Vz_S-yxCCgsU4n52g" + } + } + ], + [ + { + "state": "STARTED", + "primary": false, + "node": "8F5Wip4-Qb6og3jhepn5sg", + "relocating_node": null, + "shard": 3, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "TL2QDmCLQCO51MCLAVmBOA" + } + }, + { + "state": "STARTED", + "primary": true, + "node": "PTrurLShRN6POIuNe8WnFg", + "relocating_node": null, + "shard": 3, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "9Mvuikm_RomoWi7020F48A" + } + } + ], + [ + { + "state": "STARTED", + "primary": false, + "node": "7k1HfDdMQ1G8Ob4vUDRl1Q", + "relocating_node": null, + "shard": 4, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "aE4DvgpkQXut2FN-Skv-sw" + } + }, + { + "state": "STARTED", + "primary": true, + "node": "5D_18rKKRzOnAlnbZUdSRw", + "relocating_node": null, + "shard": 4, + "index": "indexa_2020.05.02", + "allocation_id": { + "id": "u8zHP-RwS1SiJD9ZXJL1yw" + } + } + ] + ] +} \ No newline at end of file From d858a7415cb584db5a96bea10f4a1844da891cce Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 3 May 2020 00:21:43 +0800 Subject: [PATCH 03/21] MOD: Mod some details --- .../java/org/apache/doris/external/EsRestClient.java | 12 ++++++------ .../java/org/apache/doris/external/EsStateStore.java | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/EsRestClient.java index 3d406f6830e1de..b7f336bc8b03d3 100644 --- a/fe/src/main/java/org/apache/doris/external/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/EsRestClient.java @@ -56,11 +56,11 @@ public class EsRestClient { public EsRestClient(String[] nodes, String authUser, String authPassword) { this.nodes = nodes; - builder = new Request.Builder(); + this.builder = new Request.Builder(); if (!Strings.isEmpty(authUser) && !Strings.isEmpty(authPassword)) { - builder.addHeader(HttpHeaders.AUTHORIZATION, Credentials.basic(authUser, authPassword)); + this.builder.addHeader(HttpHeaders.AUTHORIZATION, Credentials.basic(authUser, authPassword)); } - currentNode = nodes[currentNodeIndex]; + this.currentNode = nodes[currentNodeIndex]; } private void selectNextNode() { @@ -77,14 +77,14 @@ public Map getHttpNodes() throws Exception { if (nodesData == null) { return Collections.emptyMap(); } - Map nodeMap = new HashMap<>(); + Map nodesMap = new HashMap<>(); for (Map.Entry> entry : nodesData.entrySet()) { EsNodeInfo node = new EsNodeInfo(entry.getKey(), entry.getValue()); if (node.hasHttp()) { - nodeMap.put(node.getId(), node); + nodesMap.put(node.getId(), node); } } - return nodeMap; + return nodesMap; } public String getIndexMapping(String indexName) { diff --git a/fe/src/main/java/org/apache/doris/external/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/EsStateStore.java index 19c20d3c775684..16fc171cd3f721 100644 --- a/fe/src/main/java/org/apache/doris/external/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/EsStateStore.java @@ -210,7 +210,7 @@ public void loadEsIndexMapping(String indexMapping, EsTable esTable) { public EsTableState loadEsSearchShards(String shardLocation, EsTable esTable) throws ExternalDataSourceException, DdlException { JSONObject jsonObject = new JSONObject(shardLocation); - JSONObject indicesRoutingMap = jsonObject.getJSONObject("nodes"); + JSONObject nodesMap = jsonObject.getJSONObject("nodes"); JSONArray shards = jsonObject.getJSONArray("shards"); EsTableState esTableState = new EsTableState(); RangePartitionInfo partitionInfo = null; @@ -245,7 +245,7 @@ public EsTableState loadEsSearchShards(String shardLocation, EsTable esTable) } EsIndexState indexState = EsIndexState.parseIndexState(esTable.getIndexName(), - indicesRoutingMap, shards); + nodesMap, shards); esTableState.addIndexState(esTable.getIndexName(), indexState); LOG.debug("add index {} to es table {}", indexState, esTable.getName()); if (partitionInfo != null) { From a3d29281ef993706ee654aeb7d0853be74f7e499 Mon Sep 17 00:00:00 2001 From: stalary Date: Wed, 6 May 2020 17:02:51 +0800 Subject: [PATCH 04/21] =?UTF-8?q?MOD:=20Cancel=20the=20folding=20line?= =?UTF-8?q?=EF=BC=8CRemove=20the=20changes=20to=20varchar.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md | 2 +- docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md | 2 +- fe/src/main/java/org/apache/doris/external/EsIndexState.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md index 3aa719b7f200ee..8dfa67cb6e141b 100644 --- a/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md +++ b/docs/en/sql-reference/sql-statements/Data Types/VARCHAR.md @@ -27,7 +27,7 @@ under the License. # VARCHAR ## Description MARKETING (M) -A variable length string, M represents the length of a variable length string. The range of M is 1-65533. +A variable length string, M represents the length of a variable length string. The range of M is 1-65535. ##keyword VARCHAR diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md b/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md index b8ac7fe597f458..59416788a3ec14 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Types/VARCHAR.md @@ -27,7 +27,7 @@ under the License. # VARCHAR ## description VARCHAR(M) - 变长字符串，M代表的是变长字符串的长度。M的范围是1-65533 + 变长字符串，M代表的是变长字符串的长度。M的范围是1-65535 ## keyword diff --git a/fe/src/main/java/org/apache/doris/external/EsIndexState.java b/fe/src/main/java/org/apache/doris/external/EsIndexState.java index 12106aec8ef81a..76e55bf61e0d5e 100644 --- a/fe/src/main/java/org/apache/doris/external/EsIndexState.java +++ b/fe/src/main/java/org/apache/doris/external/EsIndexState.java @@ -137,8 +137,7 @@ public void setPartitionId(long partitionId) { @Override public String toString() { - return "EsIndexState [indexName=" + indexName + ", partitionDesc=" + partitionDesc - + ", partitionKey=" + return "EsIndexState [indexName=" + indexName + ", partitionDesc=" + partitionDesc + ", partitionKey=" + partitionKey + "]"; } } From d25b21fbb2287073622f33e5f25c1e4ff407cfc1 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 18 May 2020 22:31:30 +0800 Subject: [PATCH 05/21] MOD: Make some changes based on comments --- .../apache/doris/external/EsIndexState.java | 33 ---- .../apache/doris/external/EsRestClient.java | 68 +++++++- .../apache/doris/external/EsStateStore.java | 163 ++++++++---------- .../org/apache/doris/es/EsStateStoreTest.java | 18 +- 4 files changed, 152 insertions(+), 130 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/EsIndexState.java b/fe/src/main/java/org/apache/doris/external/EsIndexState.java index 76e55bf61e0d5e..9b8b7cfdcd8567 100644 --- a/fe/src/main/java/org/apache/doris/external/EsIndexState.java +++ b/fe/src/main/java/org/apache/doris/external/EsIndexState.java @@ -17,7 +17,6 @@ package org.apache.doris.external; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.List; import java.util.Map; @@ -27,8 +26,6 @@ import org.apache.doris.thrift.TNetworkAddress; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONObject; public class EsIndexState { @@ -69,36 +66,6 @@ public TNetworkAddress randomAddress(Map nodesInfo) { return nodeInfos[seed].getPublishAddress(); } - public static EsIndexState parseIndexState(String indexName, JSONObject nodesMap, - JSONArray shards) { - EsIndexState indexState = new EsIndexState(indexName); - int length = shards.length(); - for (int i = 0; i < length; i++) { - List singleShardRouting = Lists.newArrayList(); - JSONArray shardsArray = shards.getJSONArray(i); - int arrayLength = shardsArray.length(); - for (int j = 0; j < arrayLength; j++) { - JSONObject shard = shardsArray.getJSONObject(j); - String shardState = shard.getString("state"); - if ("STARTED".equalsIgnoreCase(shardState)) { - try { - singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, - String.valueOf(i), shard, nodesMap)); - } catch (Exception e) { - LOG.info( - "errors while parse shard routing from json [{}], ignore this shard", - shard, e); - } - } - } - if (singleShardRouting.isEmpty()) { - LOG.warn("could not find a healthy allocation for [{}][{}]", indexName, i); - } - indexState.addShardRouting(i, singleShardRouting); - } - return indexState; - } - public void addShardRouting(int shardId, List singleShardRouting) { shardRoutings.put(shardId, singleShardRouting); } diff --git a/fe/src/main/java/org/apache/doris/external/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/EsRestClient.java index b7f336bc8b03d3..80686e2ae12b63 100644 --- a/fe/src/main/java/org/apache/doris/external/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/EsRestClient.java @@ -17,9 +17,12 @@ package org.apache.doris.external; +import com.google.common.collect.Lists; import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import okhttp3.Credentials; @@ -34,6 +37,8 @@ import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; +import org.json.JSONArray; +import org.json.JSONObject; public class EsRestClient { private static final Logger LOG = LogManager.getLogger(EsRestClient.class); @@ -87,14 +92,65 @@ public Map getHttpNodes() throws Exception { return nodesMap; } - public String getIndexMapping(String indexName) { + public JSONObject getIndexProperties(String indexName, String mappingType) { String path = indexName + "/_mapping"; - return execute(path); + String indexMapping = execute(path); + if (indexMapping == null) { + return null; + } + return parseProperties(indexMapping, mappingType); + } + + public JSONObject parseProperties(String indexMapping, String mappingType) { + JSONObject jsonObject = new JSONObject(indexMapping); + // the indexName use alias takes the first mapping + Iterator keys = jsonObject.keys(); + String docKey = keys.next(); + JSONObject docData = jsonObject.optJSONObject(docKey); + JSONObject mappings = docData.optJSONObject("mappings"); + JSONObject rootSchema = mappings.optJSONObject(mappingType); + return rootSchema.optJSONObject("properties"); } - public String getSearchShards(String indexName) { + public EsIndexState getIndexState(String indexName) { String path = indexName + "/_search_shards"; - return execute(path); + String shardLocation = execute(path); + if (shardLocation == null) { + return null; + } + return parseIndexState(indexName, shardLocation); + } + + public EsIndexState parseIndexState(String indexName, String shardLocation) { + EsIndexState indexState = new EsIndexState(indexName); + JSONObject jsonObject = new JSONObject(shardLocation); + JSONObject nodesMap = jsonObject.getJSONObject("nodes"); + JSONArray shards = jsonObject.getJSONArray("shards"); + int length = shards.length(); + for (int i = 0; i < length; i++) { + List singleShardRouting = Lists.newArrayList(); + JSONArray shardsArray = shards.getJSONArray(i); + int arrayLength = shardsArray.length(); + for (int j = 0; j < arrayLength; j++) { + JSONObject shard = shardsArray.getJSONObject(j); + String shardState = shard.getString("state"); + if ("STARTED".equalsIgnoreCase(shardState) || "RELOCATING".equalsIgnoreCase(shardState)) { + try { + singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, + String.valueOf(i), shard, nodesMap)); + } catch (Exception e) { + LOG.info( + "errors while parse shard routing from json [{}], ignore this shard", + shard, e); + } + } + } + if (singleShardRouting.isEmpty()) { + LOG.warn("could not find a healthy allocation for [{}][{}]", indexName, i); + } + indexState.addShardRouting(i, singleShardRouting); + } + return indexState; } /** @@ -138,7 +194,9 @@ private String execute(String path) { Request request = builder.get() .url(currentNode + "/" + path) .build(); - LOG.trace("es rest client request URL: {}", currentNode + "/" + path); + if (LOG.isTraceEnabled()) { + LOG.trace("es rest client request URL: {}", currentNode + "/" + path); + } try { Response response = networkClient.newCall(request).execute(); if (response.isSuccessful()) { diff --git a/fe/src/main/java/org/apache/doris/external/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/EsStateStore.java index 16fc171cd3f721..494d81716308c4 100644 --- a/fe/src/main/java/org/apache/doris/external/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/EsStateStore.java @@ -21,7 +21,6 @@ import com.google.common.collect.Range; import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.doris.catalog.Catalog; @@ -38,7 +37,6 @@ import org.apache.doris.common.util.MasterDaemon; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONArray; import org.json.JSONObject; @@ -61,7 +59,7 @@ public void registerTable(EsTable esTable) { return; } esTables.put(esTable.getId(), esTable); - LOG.info("register a new table [{}] to sync list", esTable.toString()); + LOG.info("register a new table [{}] to sync list", esTable); } public void deRegisterTable(long tableId) { @@ -73,17 +71,24 @@ public void deRegisterTable(long tableId) { protected void runAfterCatalogReady() { for (EsTable esTable : esTables.values()) { try { - EsRestClient client = new EsRestClient(esTable.getSeeds(), - esTable.getUserName(), esTable.getPasswd()); + EsRestClient client = new EsRestClient(esTable.getSeeds(), esTable.getUserName(), + esTable.getPasswd()); - String indexMapping = client.getIndexMapping(esTable.getIndexName()); - if (indexMapping == null) { + if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { + JSONObject properties = client + .getIndexProperties(esTable.getIndexName(), esTable.getMappingType()); + if (properties == null) { + continue; + } + setEsTableContext(properties, esTable); + } + + EsIndexState esIndexState = client.getIndexState(esTable.getIndexName()); + if (esIndexState == null) { continue; } - loadEsIndexMapping(indexMapping, esTable); - String shardLocation = client.getSearchShards(esTable.getIndexName()); - EsTableState esTableState = loadEsSearchShards(shardLocation, esTable); + EsTableState esTableState = setTableStatePartitionInfo(esTable, esIndexState); if (esTableState == null) { continue; } @@ -122,96 +127,83 @@ public void loadTableFromCatalog() { } // Configure keyword and doc_values by mapping - public void loadEsIndexMapping(String indexMapping, EsTable esTable) { - JSONObject jsonObject = new JSONObject(indexMapping); - // the indexName use alias takes the first mapping - Iterator keys = jsonObject.keys(); - if ((esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) && keys.hasNext()) { - String docKey = keys.next(); - JSONObject docData = jsonObject.optJSONObject(docKey); - JSONObject mappings = docData.optJSONObject("mappings"); - JSONObject rootSchema = mappings.optJSONObject(esTable.getMappingType()); - JSONObject schema = rootSchema.optJSONObject("properties"); - // we build the doc value context for fields maybe used for scanning - // "properties": { - // "city": { - // "type": "text", // text field does not have docvalue - // "fields": { - // "raw": { - // "type": "keyword" - // } - // } - // } - // } - // then the docvalue context provided the mapping between the select field and real request field : - // {"city": "city.raw"} - List colList = esTable.getFullSchema(); - for (Column col : colList) { - String colName = col.getName(); - if (!schema.has(colName)) { - continue; - } - JSONObject fieldObject = schema.optJSONObject(colName); - String fieldType = fieldObject.optString("type"); - // string-type field used keyword type to generate predicate - if (esTable.isKeywordSniffEnable()) { - // if text field type seen, we should use the `field` keyword type? - if ("text".equals(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); - if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - // just for text type - if ("keyword".equals(innerTypeObject.optString("type"))) { - esTable.addFetchField(colName, colName + "." + key); - } + public void setEsTableContext(JSONObject properties, EsTable esTable) { + // we build the doc value context for fields maybe used for scanning + // "properties": { + // "city": { + // "type": "text", // text field does not have docvalue + // "fields": { + // "raw": { + // "type": "keyword" + // } + // } + // } + // } + // then the docvalue context provided the mapping between the select field and real request field : + // {"city": "city.raw"} + List colList = esTable.getFullSchema(); + for (Column col : colList) { + String colName = col.getName(); + if (!properties.has(colName)) { + continue; + } + JSONObject fieldObject = properties.optJSONObject(colName); + String fieldType = fieldObject.optString("type"); + // string-type field used keyword type to generate predicate + if (esTable.isKeywordSniffEnable()) { + // if text field type seen, we should use the `field` keyword type? + if ("text".equals(fieldType)) { + JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + if (fieldsObject != null) { + for (String key : fieldsObject.keySet()) { + JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + // just for text type + if ("keyword".equals(innerTypeObject.optString("type"))) { + esTable.addFetchField(colName, colName + "." + key); } } } } + } - if (esTable.isDocValueScanEnable()) { - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); - if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS - .contains(innerTypeObject.optString("type"))) { - continue; - } - if (innerTypeObject.has("doc_values")) { - boolean docValue = innerTypeObject.getBoolean("doc_values"); - if (docValue) { - esTable.addDocValueField(colName, colName); - } - } else { - // a : {c : {}} -> a -> a.c - esTable.addDocValueField(colName, colName + "." + key); + if (esTable.isDocValueScanEnable()) { + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { + JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + if (fieldsObject != null) { + for (String key : fieldsObject.keySet()) { + JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS + .contains(innerTypeObject.optString("type"))) { + continue; + } + if (innerTypeObject.has("doc_values")) { + boolean docValue = innerTypeObject.getBoolean("doc_values"); + if (docValue) { + esTable.addDocValueField(colName, colName); } + } else { + // a : {c : {}} -> a -> a.c + esTable.addDocValueField(colName, colName + "." + key); } } - // skip this field - continue; } - // set doc_value = false manually - if (fieldObject.has("doc_values")) { - boolean docValue = fieldObject.optBoolean("doc_values"); - if (!docValue) { - continue; - } + // skip this field + continue; + } + // set doc_value = false manually + if (fieldObject.has("doc_values")) { + boolean docValue = fieldObject.optBoolean("doc_values"); + if (!docValue) { + continue; } - esTable.addDocValueField(colName, colName); } + esTable.addDocValueField(colName, colName); } } } - public EsTableState loadEsSearchShards(String shardLocation, EsTable esTable) + public EsTableState setTableStatePartitionInfo(EsTable esTable, EsIndexState indexState) throws ExternalDataSourceException, DdlException { - JSONObject jsonObject = new JSONObject(shardLocation); - JSONObject nodesMap = jsonObject.getJSONObject("nodes"); - JSONArray shards = jsonObject.getJSONArray("shards"); EsTableState esTableState = new EsTableState(); RangePartitionInfo partitionInfo = null; if (esTable.getPartitionInfo() != null) { @@ -243,9 +235,6 @@ public EsTableState loadEsSearchShards(String shardLocation, EsTable esTable) + esTable.getPartitionInfo().getType()); } } - - EsIndexState indexState = EsIndexState.parseIndexState(esTable.getIndexName(), - nodesMap, shards); esTableState.addIndexState(esTable.getIndexName(), indexState); LOG.debug("add index {} to es table {}", indexState, esTable.getName()); if (partitionInfo != null) { diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index a6f649b8fbf976..040154136d92c5 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -28,6 +28,7 @@ import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; +import java.util.Iterator; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.EsTable; @@ -36,10 +37,13 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.external.EsIndexState; +import org.apache.doris.external.EsRestClient; import org.apache.doris.external.EsStateStore; import org.apache.doris.external.EsTableState; import org.apache.doris.external.ExternalDataSourceException; import org.apache.doris.meta.MetaContext; +import org.json.JSONObject; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -52,6 +56,7 @@ public class EsStateStoreTest { private static String mappingsStr = ""; private static String searchShardsStr = ""; private EsStateStore esStateStore; + private EsRestClient fakeClient; @BeforeClass public static void init() throws IOException, InstantiationException, IllegalAccessException, @@ -72,25 +77,28 @@ public static void init() throws IOException, InstantiationException, IllegalAcc @Before public void setUp() { esStateStore = new EsStateStore(); + fakeClient = new EsRestClient(new String[]{"localhost:9200"}, null, null); } @Test - public void testLoadEsIndexMapping() { + public void testSetEsTableContext() { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); - esStateStore.loadEsIndexMapping(mappingsStr, esTable); + JSONObject properties = fakeClient.parseProperties(mappingsStr, esTable.getMappingType()); + esStateStore.setEsTableContext(properties, esTable); assertEquals("userId.keyword", esTable.fieldsContext().get("userId")); assertEquals("userId.keyword", esTable.docValueContext().get("userId")); } @Test - public void testLocalSearchShards() throws ExternalDataSourceException, DdlException { + public void testSetTableState() throws ExternalDataSourceException, DdlException { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); - esStateStore.loadEsIndexMapping(mappingsStr, esTable); - EsTableState esTableState = esStateStore.loadEsSearchShards(searchShardsStr, esTable); + EsIndexState esIndexState = fakeClient + .parseIndexState(esTable.getIndexName(), searchShardsStr); + EsTableState esTableState = esStateStore.setTableStatePartitionInfo(esTable, esIndexState); assertNotNull(esTableState); assertEquals(1, esTableState.getUnPartitionedIndexStates().size()); assertEquals(5, esTableState.getIndexState("indexa").getShardRoutings().size()); From 407e9131954b456050aad5c2202f2f37a57a3a59 Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 24 May 2020 12:38:20 +0800 Subject: [PATCH 06/21] MOD: make some changes based on the comments --- .../apache/doris/external/EsRestClient.java | 6 +- .../apache/doris/external/EsStateStore.java | 67 +++++------------ .../org/apache/doris/external/EsUtil.java | 72 ++++++++++++++++--- 3 files changed, 83 insertions(+), 62 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/EsRestClient.java index 80686e2ae12b63..249901363233e9 100644 --- a/fe/src/main/java/org/apache/doris/external/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/EsRestClient.java @@ -41,6 +41,7 @@ import org.json.JSONObject; public class EsRestClient { + private static final Logger LOG = LogManager.getLogger(EsRestClient.class); private ObjectMapper mapper; @@ -51,8 +52,8 @@ public class EsRestClient { } private static OkHttpClient networkClient = new OkHttpClient.Builder() - .readTimeout(10, TimeUnit.SECONDS) - .build(); + .readTimeout(10, TimeUnit.SECONDS) + .build(); private Request.Builder builder; private String[] nodes; @@ -154,7 +155,6 @@ public EsIndexState parseIndexState(String indexName, String shardLocation) { } /** - * * Get the Elasticsearch cluster version * * @return diff --git a/fe/src/main/java/org/apache/doris/external/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/EsStateStore.java index 494d81716308c4..976cf3425942fd 100644 --- a/fe/src/main/java/org/apache/doris/external/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/EsStateStore.java @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; @@ -49,9 +50,12 @@ public class EsStateStore extends MasterDaemon { private Map esTables; + private Map esClients; + public EsStateStore() { super("es state store", Config.es_state_sync_interval_second * 1000); esTables = Maps.newConcurrentMap(); + esClients = Maps.newConcurrentMap(); } public void registerTable(EsTable esTable) { @@ -59,11 +63,14 @@ public void registerTable(EsTable esTable) { return; } esTables.put(esTable.getId(), esTable); + esClients.put(esTable.getId(), + new EsRestClient(esTable.getSeeds(), esTable.getUserName(), esTable.getPasswd())); LOG.info("register a new table [{}] to sync list", esTable); } public void deRegisterTable(long tableId) { esTables.remove(tableId); + esClients.remove(tableId); LOG.info("deregister table [{}] from sync list", tableId); } @@ -71,18 +78,16 @@ public void deRegisterTable(long tableId) { protected void runAfterCatalogReady() { for (EsTable esTable : esTables.values()) { try { - EsRestClient client = new EsRestClient(esTable.getSeeds(), esTable.getUserName(), - esTable.getPasswd()); + EsRestClient client = esClients.get(esTable.getId()); if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { - JSONObject properties = client - .getIndexProperties(esTable.getIndexName(), esTable.getMappingType()); + JSONObject properties = client.getIndexProperties(esTable.getIndexName(), esTable.getMappingType()); if (properties == null) { continue; } setEsTableContext(properties, esTable); } - + EsIndexState esIndexState = client.getIndexState(esTable.getIndexName()); if (esIndexState == null) { continue; @@ -148,56 +153,19 @@ public void setEsTableContext(JSONObject properties, EsTable esTable) { continue; } JSONObject fieldObject = properties.optJSONObject(colName); - String fieldType = fieldObject.optString("type"); // string-type field used keyword type to generate predicate if (esTable.isKeywordSniffEnable()) { - // if text field type seen, we should use the `field` keyword type? - if ("text".equals(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); - if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - // just for text type - if ("keyword".equals(innerTypeObject.optString("type"))) { - esTable.addFetchField(colName, colName + "." + key); - } - } - } + String fetchField = EsUtil.getFetchField(fieldObject, colName); + if (StringUtils.isNotEmpty(fetchField)) { + esTable.addFetchField(colName, fetchField); } } if (esTable.isDocValueScanEnable()) { - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); - if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS - .contains(innerTypeObject.optString("type"))) { - continue; - } - if (innerTypeObject.has("doc_values")) { - boolean docValue = innerTypeObject.getBoolean("doc_values"); - if (docValue) { - esTable.addDocValueField(colName, colName); - } - } else { - // a : {c : {}} -> a -> a.c - esTable.addDocValueField(colName, colName + "." + key); - } - } - } - // skip this field - continue; - } - // set doc_value = false manually - if (fieldObject.has("doc_values")) { - boolean docValue = fieldObject.optBoolean("doc_values"); - if (!docValue) { - continue; - } + String docValueField = EsUtil.getDocValueField(fieldObject, colName); + if (StringUtils.isNotEmpty(docValueField)) { + esTable.addDocValueField(colName, docValueField); } - esTable.addDocValueField(colName, colName); } } } @@ -208,8 +176,7 @@ public EsTableState setTableStatePartitionInfo(EsTable esTable, EsIndexState ind RangePartitionInfo partitionInfo = null; if (esTable.getPartitionInfo() != null) { if (esTable.getPartitionInfo() instanceof RangePartitionInfo) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTable - .getPartitionInfo(); + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTable.getPartitionInfo(); partitionInfo = new RangePartitionInfo(rangePartitionInfo.getPartitionColumns()); esTableState.setPartitionInfo(partitionInfo); if (LOG.isDebugEnabled()) { diff --git a/fe/src/main/java/org/apache/doris/external/EsUtil.java b/fe/src/main/java/org/apache/doris/external/EsUtil.java index 4e05d0ebc9f4ee..1b40795a40b0e3 100644 --- a/fe/src/main/java/org/apache/doris/external/EsUtil.java +++ b/fe/src/main/java/org/apache/doris/external/EsUtil.java @@ -17,18 +17,18 @@ package org.apache.doris.external; -import org.json.JSONObject; - import org.apache.doris.analysis.DistributionDesc; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.RangePartitionDesc; +import org.apache.doris.catalog.EsTable; import org.apache.doris.common.AnalysisException; +import org.json.JSONObject; public class EsUtil { - + public static void analyzePartitionAndDistributionDesc(PartitionDesc partitionDesc, - DistributionDesc distributionDesc) - throws AnalysisException { + DistributionDesc distributionDesc) + throws AnalysisException { if (partitionDesc == null && distributionDesc == null) { return; } @@ -48,20 +48,21 @@ public static void analyzePartitionAndDistributionDesc(PartitionDesc partitionDe } private static void analyzePartitionDesc(RangePartitionDesc partDesc) - throws AnalysisException { + throws AnalysisException { if (partDesc.getPartitionColNames() == null || partDesc.getPartitionColNames().isEmpty()) { throw new AnalysisException("No partition columns."); } if (partDesc.getPartitionColNames().size() > 1) { - throw new AnalysisException("Elasticsearch table's parition column could only be a single column"); + throw new AnalysisException( + "Elasticsearch table's parition column could only be a single column"); } } - - + /** * get the json object from specified jsonObject + * * @param jsonObject * @param key * @return @@ -83,4 +84,57 @@ public static JSONObject getJsonObject(JSONObject jsonObject, String key, int fr return null; } } + + public static String getFetchField(JSONObject fieldObject, String colName) { + String fieldType = fieldObject.optString("type"); + // string-type field used keyword type to generate predicate + // if text field type seen, we should use the `field` keyword type? + if ("text".equals(fieldType)) { + JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + if (fieldsObject != null) { + for (String key : fieldsObject.keySet()) { + JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + // just for text type + if ("keyword".equals(innerTypeObject.optString("type"))) { + return colName + "." + key; + } + } + } + } + return null; + } + + public static String getDocValueField(JSONObject fieldObject, String colName) { + String fieldType = fieldObject.optString("type"); + String docValueField = null; + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { + JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + if (fieldsObject != null) { + for (String key : fieldsObject.keySet()) { + JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.optString("type"))) { + continue; + } + if (innerTypeObject.has("doc_values")) { + boolean docValue = innerTypeObject.getBoolean("doc_values"); + if (docValue) { + docValueField = colName; + } + } else { + // a : {c : {}} -> a -> a.c + docValueField = colName + "." + key; + } + } + } + } + // set doc_value = false manually + if (fieldObject.has("doc_values")) { + boolean docValue = fieldObject.optBoolean("doc_values"); + if (!docValue) { + return docValueField; + } + } + docValueField = colName; + return docValueField; + } } From f06f846b98e9cd3c82a11c6827b3680586a8da8d Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 24 May 2020 17:28:25 +0800 Subject: [PATCH 07/21] FIX: fix dovValueField bug --- fe/src/main/java/org/apache/doris/external/EsUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/main/java/org/apache/doris/external/EsUtil.java b/fe/src/main/java/org/apache/doris/external/EsUtil.java index 1b40795a40b0e3..f6340dfa466b6e 100644 --- a/fe/src/main/java/org/apache/doris/external/EsUtil.java +++ b/fe/src/main/java/org/apache/doris/external/EsUtil.java @@ -126,6 +126,7 @@ public static String getDocValueField(JSONObject fieldObject, String colName) { } } } + return docValueField; } // set doc_value = false manually if (fieldObject.has("doc_values")) { From a381b68a0b1c624c56d950f3a3c2aa275a90db47 Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 24 May 2020 17:31:41 +0800 Subject: [PATCH 08/21] MOD: remove useless import --- fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index 040154136d92c5..9e6e4db482b348 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -28,13 +28,11 @@ import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; -import java.util.Iterator; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.FakeCatalog; import org.apache.doris.catalog.FakeEditLog; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.external.EsIndexState; From 8400246919090c5257c2ead4323af9ae28bcc6a3 Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 24 May 2020 17:36:49 +0800 Subject: [PATCH 09/21] MOD --- .../apache/doris/analysis/AggregateTest.java | 202 +++++++++--------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/fe/src/test/java/org/apache/doris/analysis/AggregateTest.java b/fe/src/test/java/org/apache/doris/analysis/AggregateTest.java index 328aa2b9585d95..722a93249d8b8c 100644 --- a/fe/src/test/java/org/apache/doris/analysis/AggregateTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/AggregateTest.java @@ -1,102 +1,102 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.commons.io.filefilter.FalseFileFilter; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FeConstants; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.DorisAssert; -import org.apache.doris.utframe.UtFrameUtils; - -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import javax.validation.constraints.AssertTrue; -import java.util.UUID; - -public class AggregateTest { - - private static String baseDir = "fe"; - private static String runningDir = baseDir + "/mocked/AggregateTest/" - + UUID.randomUUID().toString() + "/"; - private static final String TABLE_NAME = "table1"; - private static final String DB_NAME = "db1"; - private static DorisAssert dorisAssert; - - @BeforeClass - public static void beforeClass() throws Exception{ - FeConstants.runningUnitTest = true; - UtFrameUtils.createMinDorisCluster(runningDir); - dorisAssert = new DorisAssert(); - dorisAssert.withDatabase(DB_NAME).useDatabase(DB_NAME); - String createTableSQL = "create table " + DB_NAME + "." + TABLE_NAME + " (empid int, name varchar, " + - "deptno int, salary int, commission int) " - + "distributed by hash(empid) buckets 3 properties('replication_num' = '1');"; - dorisAssert.withTable(createTableSQL); - } - - /** - * ISSUE-3492 - */ - @Test - public void testCountDisintctAnalysisException() throws Exception { - ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - - // NOT support mix distinct, one DistinctAggregationFunction has one column, the other DistinctAggregationFunction has some columns. - do { - String query = "select count(distinct empid), count(distinct salary), count(distinct empid, salary) from " + DB_NAME + "." + TABLE_NAME; - try { - UtFrameUtils.parseAndAnalyzeStmt(query, ctx); - } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("The query contains multi count distinct or sum distinct, each can't have multi columns.")); - break; - } catch (Exception e) { - Assert.fail("must be AnalysisException."); - } - Assert.fail("must be AnalysisException."); - } while (false); - - // support multi DistinctAggregationFunction, but each DistinctAggregationFunction only has one column - do { - String query = "select count(distinct empid), count(distinct salary) from " + DB_NAME + "." + TABLE_NAME; - try { - UtFrameUtils.parseAndAnalyzeStmt(query, ctx); - } catch (Exception e) { - Assert.fail("should be query, no exception"); - } - } while (false); - - // support 1 DistinctAggregationFunction with some columns - do { - String query = "select count(distinct salary, empid) from " + DB_NAME + "." + TABLE_NAME; - try { - UtFrameUtils.parseAndAnalyzeStmt(query, ctx); - } catch (Exception e) { - Assert.fail("should be query, no exception"); - } - } while (false); - } - - @AfterClass - public static void afterClass() throws Exception { - UtFrameUtils.cleanDorisFeDir(baseDir); - } +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.commons.io.filefilter.FalseFileFilter; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.DorisAssert; +import org.apache.doris.utframe.UtFrameUtils; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.validation.constraints.AssertTrue; +import java.util.UUID; + +public class AggregateTest { + + private static String baseDir = "fe"; + private static String runningDir = baseDir + "/mocked/AggregateTest/" + + UUID.randomUUID().toString() + "/"; + private static final String TABLE_NAME = "table1"; + private static final String DB_NAME = "db1"; + private static DorisAssert dorisAssert; + + @BeforeClass + public static void beforeClass() throws Exception{ + FeConstants.runningUnitTest = true; + UtFrameUtils.createMinDorisCluster(runningDir); + dorisAssert = new DorisAssert(); + dorisAssert.withDatabase(DB_NAME).useDatabase(DB_NAME); + String createTableSQL = "create table " + DB_NAME + "." + TABLE_NAME + " (empid int, name varchar, " + + "deptno int, salary int, commission int) " + + "distributed by hash(empid) buckets 3 properties('replication_num' = '1');"; + dorisAssert.withTable(createTableSQL); + } + + /** + * ISSUE-3492 + */ + @Test + public void testCountDisintctAnalysisException() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + + // NOT support mix distinct, one DistinctAggregationFunction has one column, the other DistinctAggregationFunction has some columns. + do { + String query = "select count(distinct empid), count(distinct salary), count(distinct empid, salary) from " + DB_NAME + "." + TABLE_NAME; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("The query contains multi count distinct or sum distinct, each can't have multi columns.")); + break; + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + Assert.fail("must be AnalysisException."); + } while (false); + + // support multi DistinctAggregationFunction, but each DistinctAggregationFunction only has one column + do { + String query = "select count(distinct empid), count(distinct salary) from " + DB_NAME + "." + TABLE_NAME; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (Exception e) { + Assert.fail("should be query, no exception"); + } + } while (false); + + // support 1 DistinctAggregationFunction with some columns + do { + String query = "select count(distinct salary, empid) from " + DB_NAME + "." + TABLE_NAME; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (Exception e) { + Assert.fail("should be query, no exception"); + } + } while (false); + } + + @AfterClass + public static void afterClass() throws Exception { + UtFrameUtils.cleanDorisFeDir(baseDir); + } } \ No newline at end of file From 451b9a60693e3a2f4c415a4c77a81c12830081c9 Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 26 May 2020 10:47:57 +0800 Subject: [PATCH 10/21] MOD: Supplement Doris on es documentation --- docs/en/extending-doris/doris-on-es.md | 2 +- docs/zh-CN/extending-doris/doris-on-es.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/extending-doris/doris-on-es.md b/docs/en/extending-doris/doris-on-es.md index cf66cabdc77229..f91090b3e99f32 100644 --- a/docs/en/extending-doris/doris-on-es.md +++ b/docs/en/extending-doris/doris-on-es.md @@ -75,7 +75,7 @@ Parameter | description Hosts | ES Cluster Connection Address, which can specify one or more, through which Doris obtains the share distribution information of ES version number and index User | Open the user name of the ES cluster authenticated by basic, you need to ensure that the user has access to: / cluster / state / nodes / HTTP and other path permissions and read permissions for index Password | corresponding user's password information -The index name of the ES corresponding to the table in index | Doris can be alias +The index name of the ES corresponding to the table in index | Doris can be alias | If you use doc_value, it is recommended to use the real index name Type | Specifies the type of index, defaulting to _doc Transport | Internal reservation, default to http diff --git a/docs/zh-CN/extending-doris/doris-on-es.md b/docs/zh-CN/extending-doris/doris-on-es.md index d40fe89be4b14a..994e1087c35a9b 100644 --- a/docs/zh-CN/extending-doris/doris-on-es.md +++ b/docs/zh-CN/extending-doris/doris-on-es.md @@ -75,7 +75,7 @@ PROPERTIES ( hosts | ES集群连接地址，可指定一个或多个，Doris通过这个地址获取到ES版本号、index的shard分布信息 user | 开启basic认证的ES集群的用户名，需要确保该用户有访问: /\_cluster/state/\_nodes/http等路径权限和对index的读权限 password | 对应用户的密码信息 -index | Doris中的表对应的ES的index名字，可以是alias +index | Doris中的表对应的ES的index名字，可以是alias，如果使用doc_value，建议使用真实的index名称 type | 指定index的type，默认是_doc transport | 内部保留，默认为http From 3e768c8b71abfeb6d74f156407d4f12f349d82ed Mon Sep 17 00:00:00 2001 From: stalary Date: Wed, 10 Jun 2020 17:58:08 +0800 Subject: [PATCH 11/21] MOD: refactor code --- .../org/apache/doris/catalog/EsTable.java | 8 ++-- .../apache/doris/external/EsFieldInfo.java | 46 +++++++++++++++++++ .../apache/doris/external/EsRestClient.java | 44 ++++++++++++++---- .../apache/doris/external/EsStateStore.java | 42 ++++++----------- .../org/apache/doris/es/EsStateStoreTest.java | 10 ++-- 5 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 fe/src/main/java/org/apache/doris/external/EsFieldInfo.java diff --git a/fe/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/src/main/java/org/apache/doris/catalog/EsTable.java index 252d962d09bee2..c1c4043bec7b6f 100644 --- a/fe/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/EsTable.java @@ -108,16 +108,16 @@ public EsTable(long id, String name, List schema, validate(properties); } - public void addFetchField(String originName, String replaceName) { - fieldsContext.put(originName, replaceName); + public void addFetchField(Map fetchFieldMap) { + fieldsContext.putAll(fetchFieldMap); } public Map fieldsContext() { return fieldsContext; } - public void addDocValueField(String name, String fieldsName) { - docValueContext.put(name, fieldsName); + public void addDocValueField(Map docValueFieldMap) { + docValueContext.putAll(docValueFieldMap); } public Map docValueContext() { diff --git a/fe/src/main/java/org/apache/doris/external/EsFieldInfo.java b/fe/src/main/java/org/apache/doris/external/EsFieldInfo.java new file mode 100644 index 00000000000000..4fc342419918a6 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/external/EsFieldInfo.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.external; + +import java.util.Map; + +/** + * EsFieldInfo + * + * @author stalary + * @since 2020/06/10 + */ +public class EsFieldInfo { + + private Map fetchFields; + + private Map docValueFields; + + public EsFieldInfo(Map fetchFields, Map docValueFields) { + this.fetchFields = fetchFields; + this.docValueFields = docValueFields; + } + + public Map getFetchFields() { + return fetchFields; + } + + public Map getDocValueFields() { + return docValueFields; + } +} \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/external/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/EsRestClient.java index 249901363233e9..0a55f40e4df053 100644 --- a/fe/src/main/java/org/apache/doris/external/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/EsRestClient.java @@ -17,6 +17,7 @@ package org.apache.doris.external; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import java.io.IOException; import java.util.Collections; @@ -29,6 +30,8 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import org.apache.commons.lang3.StringUtils; +import org.apache.doris.catalog.Column; import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -92,16 +95,39 @@ public Map getHttpNodes() throws Exception { } return nodesMap; } - - public JSONObject getIndexProperties(String indexName, String mappingType) { + + public EsFieldInfo getFieldInfo(String indexName, String mappingType, List colList) { String path = indexName + "/_mapping"; String indexMapping = execute(path); - if (indexMapping == null) { + return indexMapping == null ? null : getFieldInfo(colList, parseProperties(indexMapping, mappingType)); + } + + @VisibleForTesting + public EsFieldInfo getFieldInfo(List colList, JSONObject properties) { + if (properties == null) { return null; } - return parseProperties(indexMapping, mappingType); + Map fetchFieldMap = new HashMap<>(); + Map docValueFieldMap = new HashMap<>(); + for (Column col : colList) { + String colName = col.getName(); + if (!properties.has(colName)) { + continue; + } + JSONObject fieldObject = properties.optJSONObject(colName); + String fetchField = EsUtil.getFetchField(fieldObject, colName); + if (StringUtils.isNotEmpty(fetchField)) { + fetchFieldMap.put(colName, fetchField); + } + String docValueField = EsUtil.getDocValueField(fieldObject, colName); + if (StringUtils.isNotEmpty(docValueField)) { + docValueFieldMap.put(colName, docValueField); + } + } + return new EsFieldInfo(fetchFieldMap, docValueFieldMap); } - + + @VisibleForTesting public JSONObject parseProperties(String indexMapping, String mappingType) { JSONObject jsonObject = new JSONObject(indexMapping); // the indexName use alias takes the first mapping @@ -116,12 +142,10 @@ public JSONObject parseProperties(String indexMapping, String mappingType) { public EsIndexState getIndexState(String indexName) { String path = indexName + "/_search_shards"; String shardLocation = execute(path); - if (shardLocation == null) { - return null; - } - return parseIndexState(indexName, shardLocation); + return shardLocation == null ? null : parseIndexState(indexName, shardLocation); } - + + @VisibleForTesting public EsIndexState parseIndexState(String indexName, String shardLocation) { EsIndexState indexState = new EsIndexState(indexName); JSONObject jsonObject = new JSONObject(shardLocation); diff --git a/fe/src/main/java/org/apache/doris/external/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/EsStateStore.java index 976cf3425942fd..d07a30ca8b780a 100644 --- a/fe/src/main/java/org/apache/doris/external/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/EsStateStore.java @@ -17,13 +17,13 @@ package org.apache.doris.external; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.common.collect.Range; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; @@ -38,7 +38,6 @@ import org.apache.doris.common.util.MasterDaemon; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONObject; /** @@ -81,11 +80,11 @@ protected void runAfterCatalogReady() { EsRestClient client = esClients.get(esTable.getId()); if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { - JSONObject properties = client.getIndexProperties(esTable.getIndexName(), esTable.getMappingType()); - if (properties == null) { + EsFieldInfo fieldInfo = client.getFieldInfo(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); + if (fieldInfo == null) { continue; } - setEsTableContext(properties, esTable); + setEsTableContext(fieldInfo, esTable); } EsIndexState esIndexState = client.getIndexState(esTable.getIndexName()); @@ -93,7 +92,7 @@ protected void runAfterCatalogReady() { continue; } - EsTableState esTableState = setTableStatePartitionInfo(esTable, esIndexState); + EsTableState esTableState = setPartitionInfo(esTable, esIndexState); if (esTableState == null) { continue; } @@ -132,7 +131,7 @@ public void loadTableFromCatalog() { } // Configure keyword and doc_values by mapping - public void setEsTableContext(JSONObject properties, EsTable esTable) { + public void setEsTableContext(EsFieldInfo fieldInfo, EsTable esTable) { // we build the doc value context for fields maybe used for scanning // "properties": { // "city": { @@ -146,31 +145,16 @@ public void setEsTableContext(JSONObject properties, EsTable esTable) { // } // then the docvalue context provided the mapping between the select field and real request field : // {"city": "city.raw"} - List colList = esTable.getFullSchema(); - for (Column col : colList) { - String colName = col.getName(); - if (!properties.has(colName)) { - continue; - } - JSONObject fieldObject = properties.optJSONObject(colName); - // string-type field used keyword type to generate predicate - if (esTable.isKeywordSniffEnable()) { - String fetchField = EsUtil.getFetchField(fieldObject, colName); - if (StringUtils.isNotEmpty(fetchField)) { - esTable.addFetchField(colName, fetchField); - } - } - - if (esTable.isDocValueScanEnable()) { - String docValueField = EsUtil.getDocValueField(fieldObject, colName); - if (StringUtils.isNotEmpty(docValueField)) { - esTable.addDocValueField(colName, docValueField); - } - } + if (esTable.isKeywordSniffEnable() && fieldInfo.getFetchFields() != null) { + esTable.addFetchField(fieldInfo.getFetchFields()); + } + if (esTable.isDocValueScanEnable() && fieldInfo.getDocValueFields() != null) { + esTable.addDocValueField(fieldInfo.getDocValueFields()); } } - public EsTableState setTableStatePartitionInfo(EsTable esTable, EsIndexState indexState) + @VisibleForTesting + public EsTableState setPartitionInfo(EsTable esTable, EsIndexState indexState) throws ExternalDataSourceException, DdlException { EsTableState esTableState = new EsTableState(); RangePartitionInfo partitionInfo = null; diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index 9e6e4db482b348..93da2bdce71ab3 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -28,6 +28,7 @@ import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; +import java.util.Map; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.EsTable; @@ -35,6 +36,7 @@ import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.external.EsFieldInfo; import org.apache.doris.external.EsIndexState; import org.apache.doris.external.EsRestClient; import org.apache.doris.external.EsStateStore; @@ -84,7 +86,8 @@ public void testSetEsTableContext() { .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); JSONObject properties = fakeClient.parseProperties(mappingsStr, esTable.getMappingType()); - esStateStore.setEsTableContext(properties, esTable); + EsFieldInfo fieldInfo = fakeClient.getFieldInfo(esTable.getFullSchema(), properties); + esStateStore.setEsTableContext(fieldInfo, esTable); assertEquals("userId.keyword", esTable.fieldsContext().get("userId")); assertEquals("userId.keyword", esTable.docValueContext().get("userId")); } @@ -94,9 +97,8 @@ public void testSetTableState() throws ExternalDataSourceException, DdlException EsTable esTable = (EsTable) Catalog.getCurrentCatalog() .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); - EsIndexState esIndexState = fakeClient - .parseIndexState(esTable.getIndexName(), searchShardsStr); - EsTableState esTableState = esStateStore.setTableStatePartitionInfo(esTable, esIndexState); + EsIndexState esIndexState = fakeClient.parseIndexState(esTable.getIndexName(), searchShardsStr); + EsTableState esTableState = esStateStore.setPartitionInfo(esTable, esIndexState); assertNotNull(esTableState); assertEquals(1, esTableState.getUnPartitionedIndexStates().size()); assertEquals(5, esTableState.getIndexState("indexa").getShardRoutings().size()); From 73af7d51f1c58354499834ee8fa8f10d5c353d30 Mon Sep 17 00:00:00 2001 From: stalary Date: Wed, 10 Jun 2020 20:05:22 +0800 Subject: [PATCH 12/21] Merge branch 'master' of https://github.com/apache/incubator-doris into stalary_enhancement # Conflicts: # fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java # fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java # fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java --- .../{ => elasticsearch}/EsFieldInfo.java | 2 +- .../external/elasticsearch/EsRestClient.java | 81 +++++++++++-------- .../external/elasticsearch/EsStateStore.java | 63 +++++++-------- .../org/apache/doris/es/EsStateStoreTest.java | 37 +++++---- 4 files changed, 96 insertions(+), 87 deletions(-) rename fe/src/main/java/org/apache/doris/external/{ => elasticsearch}/EsFieldInfo.java (96%) diff --git a/fe/src/main/java/org/apache/doris/external/EsFieldInfo.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java similarity index 96% rename from fe/src/main/java/org/apache/doris/external/EsFieldInfo.java rename to fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java index 4fc342419918a6..75841be5caeca7 100644 --- a/fe/src/main/java/org/apache/doris/external/EsFieldInfo.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external; +package org.apache.doris.external.elasticsearch; import java.util.Map; diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index 0a55f40e4df053..66085341bd332c 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external; +package org.apache.doris.external.elasticsearch; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -44,34 +44,35 @@ import org.json.JSONObject; public class EsRestClient { - + private static final Logger LOG = LogManager.getLogger(EsRestClient.class); private ObjectMapper mapper; - + { mapper = new ObjectMapper(); mapper.configure(DeserializationConfig.Feature.USE_ANNOTATIONS, false); mapper.configure(SerializationConfig.Feature.USE_ANNOTATIONS, false); } - + private static OkHttpClient networkClient = new OkHttpClient.Builder() - .readTimeout(10, TimeUnit.SECONDS) - .build(); - + .readTimeout(10, TimeUnit.SECONDS) + .build(); + private Request.Builder builder; private String[] nodes; private String currentNode; private int currentNodeIndex = 0; - + public EsRestClient(String[] nodes, String authUser, String authPassword) { this.nodes = nodes; this.builder = new Request.Builder(); if (!Strings.isEmpty(authUser) && !Strings.isEmpty(authPassword)) { - this.builder.addHeader(HttpHeaders.AUTHORIZATION, Credentials.basic(authUser, authPassword)); + this.builder.addHeader(HttpHeaders.AUTHORIZATION, + Credentials.basic(authUser, authPassword)); } this.currentNode = nodes[currentNodeIndex]; } - + private void selectNextNode() { currentNodeIndex++; // reroute, because the previously failed node may have already been restored @@ -80,7 +81,7 @@ private void selectNextNode() { } currentNode = nodes[currentNodeIndex]; } - + public Map getHttpNodes() throws Exception { Map> nodesData = get("_nodes/http", "nodes"); if (nodesData == null) { @@ -96,10 +97,13 @@ public Map getHttpNodes() throws Exception { return nodesMap; } - public EsFieldInfo getFieldInfo(String indexName, String mappingType, List colList) { + public EsFieldInfo getFieldInfo(String indexName, String mappingType, List colList) throws Exception { String path = indexName + "/_mapping"; String indexMapping = execute(path); - return indexMapping == null ? null : getFieldInfo(colList, parseProperties(indexMapping, mappingType)); + if (indexMapping == null) { + throw new Exception( "index[" + indexName + "] _mapping not found for the Elasticsearch Cluster"); + } + return getFieldInfo(colList, parseProperties(indexMapping, mappingType)); } @VisibleForTesting @@ -121,7 +125,7 @@ public EsFieldInfo getFieldInfo(List colList, JSONObject properties) { } String docValueField = EsUtil.getDocValueField(fieldObject, colName); if (StringUtils.isNotEmpty(docValueField)) { - docValueFieldMap.put(colName, docValueField); + docValueFieldMap.put(colName, docValueField); } } return new EsFieldInfo(fetchFieldMap, docValueFieldMap); @@ -138,11 +142,14 @@ public JSONObject parseProperties(String indexMapping, String mappingType) { JSONObject rootSchema = mappings.optJSONObject(mappingType); return rootSchema.optJSONObject("properties"); } - - public EsIndexState getIndexState(String indexName) { + + public EsIndexState getIndexState(String indexName) throws Exception { String path = indexName + "/_search_shards"; String shardLocation = execute(path); - return shardLocation == null ? null : parseIndexState(indexName, shardLocation); + if (shardLocation == null) { + throw new Exception( "index[" + indexName + "] _search_shards not found for the Elasticsearch Cluster"); + } + return parseIndexState(indexName, shardLocation); } @VisibleForTesting @@ -159,14 +166,15 @@ public EsIndexState parseIndexState(String indexName, String shardLocation) { for (int j = 0; j < arrayLength; j++) { JSONObject shard = shardsArray.getJSONObject(j); String shardState = shard.getString("state"); - if ("STARTED".equalsIgnoreCase(shardState) || "RELOCATING".equalsIgnoreCase(shardState)) { + if ("STARTED".equalsIgnoreCase(shardState) || + "RELOCATING".equalsIgnoreCase(shardState)) { try { singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, - String.valueOf(i), shard, nodesMap)); + String.valueOf(i), shard, nodesMap)); } catch (Exception e) { LOG.info( - "errors while parse shard routing from json [{}], ignore this shard", - shard, e); + "errors while parse shard routing from json [{}], ignore this shard", + shard, e); } } } @@ -177,15 +185,15 @@ public EsIndexState parseIndexState(String indexName, String shardLocation) { } return indexState; } - + /** * Get the Elasticsearch cluster version * * @return */ - public EsMajorVersion version() { + public EsMajorVersion version () throws Exception { Map versionMap = get("/", "version"); - + EsMajorVersion majorVersion; try { majorVersion = EsMajorVersion.parse(versionMap.get("version")); @@ -195,15 +203,16 @@ public EsMajorVersion version() { } return majorVersion; } - + /** * execute request for specific path，it will try again nodes.length times if it fails * * @param path the path must not leading with '/' * @return response */ - private String execute(String path) { + private String execute (String path) throws Exception { int retrySize = nodes.length; + Exception scratchExceptionForThrow = null; for (int i = 0; i < retrySize; i++) { // maybe should add HTTP schema to the address // actually, at this time we can only process http protocol @@ -216,8 +225,8 @@ private String execute(String path) { currentNode = "http://" + currentNode; } Request request = builder.get() - .url(currentNode + "/" + path) - .build(); + .url(currentNode + "/" + path) + .build(); if (LOG.isTraceEnabled()) { LOG.trace("es rest client request URL: {}", currentNode + "/" + path); } @@ -228,19 +237,23 @@ private String execute(String path) { } } catch (IOException e) { LOG.warn("request node [{}] [{}] failures {}, try next nodes", currentNode, path, e); + scratchExceptionForThrow = e; } selectNextNode(); } LOG.warn("try all nodes [{}],no other nodes left", nodes); + if (scratchExceptionForThrow != null) { + throw scratchExceptionForThrow; + } return null; } - - public T get(String q, String key) { + + public T get(String q, String key) throws Exception { return parseContent(execute(q), key); } - + @SuppressWarnings("unchecked") - private T parseContent(String response, String key) { + private T parseContent(String response, String key) { Map map = Collections.emptyMap(); try { JsonParser jsonParser = mapper.getJsonFactory().createJsonParser(response); @@ -250,5 +263,5 @@ private T parseContent(String response, String key) { } return (T) (key != null ? map.get(key) : map); } - -} + +} \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java index d07a30ca8b780a..a04654fe6d41de 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external; +package org.apache.doris.external.elasticsearch; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; @@ -44,41 +44,41 @@ * it is used to call es api to get shard allocation state */ public class EsStateStore extends MasterDaemon { - + private static final Logger LOG = LogManager.getLogger(EsStateStore.class); - + private Map esTables; - + private Map esClients; - + public EsStateStore() { super("es state store", Config.es_state_sync_interval_second * 1000); esTables = Maps.newConcurrentMap(); esClients = Maps.newConcurrentMap(); } - + public void registerTable(EsTable esTable) { if (Catalog.isCheckpointThread()) { return; } esTables.put(esTable.getId(), esTable); esClients.put(esTable.getId(), - new EsRestClient(esTable.getSeeds(), esTable.getUserName(), esTable.getPasswd())); + new EsRestClient(esTable.getSeeds(), esTable.getUserName(), esTable.getPasswd())); LOG.info("register a new table [{}] to sync list", esTable); } - + public void deRegisterTable(long tableId) { esTables.remove(tableId); esClients.remove(tableId); LOG.info("deregister table [{}] from sync list", tableId); } - + @Override protected void runAfterCatalogReady() { for (EsTable esTable : esTables.values()) { try { EsRestClient client = esClients.get(esTable.getId()); - + if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { EsFieldInfo fieldInfo = client.getFieldInfo(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); if (fieldInfo == null) { @@ -86,33 +86,30 @@ protected void runAfterCatalogReady() { } setEsTableContext(fieldInfo, esTable); } - + EsIndexState esIndexState = client.getIndexState(esTable.getIndexName()); - if (esIndexState == null) { - continue; - } - + EsTableState esTableState = setPartitionInfo(esTable, esIndexState); if (esTableState == null) { continue; } - + if (EsTable.TRANSPORT_HTTP.equals(esTable.getTransport())) { Map nodesInfo = client.getHttpNodes(); esTableState.addHttpAddress(nodesInfo); } esTable.setEsTableState(esTableState); } catch (Throwable e) { - LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", - esTable.getName(), e); + LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", esTable.getName(), e); + esTable.setEsTableState(null); + esTable.setLastMetaDataSyncException(e); } } } - + // should call this method to init the state store after loading image // the rest of tables will be added or removed by replaying edit log // when fe is start to load image, should call this method to init the state store - public void loadTableFromCatalog() { if (Catalog.isCheckpointThread()) { return; @@ -120,7 +117,7 @@ public void loadTableFromCatalog() { List dbIds = Catalog.getCurrentCatalog().getDbIds(); for (Long dbId : dbIds) { Database database = Catalog.getCurrentCatalog().getDb(dbId); - + List tables = database.getTables(); for (Table table : tables) { if (table.getType() == TableType.ELASTICSEARCH) { @@ -129,7 +126,7 @@ public void loadTableFromCatalog() { } } } - + // Configure keyword and doc_values by mapping public void setEsTableContext(EsFieldInfo fieldInfo, EsTable esTable) { // we build the doc value context for fields maybe used for scanning @@ -152,10 +149,10 @@ public void setEsTableContext(EsFieldInfo fieldInfo, EsTable esTable) { esTable.addDocValueField(fieldInfo.getDocValueFields()); } } - + @VisibleForTesting public EsTableState setPartitionInfo(EsTable esTable, EsIndexState indexState) - throws ExternalDataSourceException, DdlException { + throws ExternalDataSourceException, DdlException { EsTableState esTableState = new EsTableState(); RangePartitionInfo partitionInfo = null; if (esTable.getPartitionInfo() != null) { @@ -175,15 +172,15 @@ public EsTableState setPartitionInfo(EsTable esTable, EsIndexState indexState) } sb.append(")"); LOG.debug("begin to parse es table [{}] state from search shards," - + " with partition info [{}]", esTable.getName(), sb.toString()); + + " with partition info [{}]", esTable.getName(), sb.toString()); } } else if (esTable.getPartitionInfo() instanceof SinglePartitionInfo) { LOG.debug("begin to parse es table [{}] state from search shards, " - + "with no partition info", esTable.getName()); + + "with no partition info", esTable.getName()); } else { throw new ExternalDataSourceException("es table only support range partition, " - + "but current partition type is " - + esTable.getPartitionInfo().getType()); + + "but current partition type is " + + esTable.getPartitionInfo().getType()); } } esTableState.addIndexState(esTable.getIndexName(), indexState); @@ -191,20 +188,20 @@ public EsTableState setPartitionInfo(EsTable esTable, EsIndexState indexState) if (partitionInfo != null) { // sort the index state according to partition key and then add to range map List esIndexStates = new ArrayList<>( - esTableState.getPartitionedIndexStates().values()); + esTableState.getPartitionedIndexStates().values()); esIndexStates.sort(Comparator.comparing(EsIndexState::getPartitionKey)); long partitionId = 0; for (EsIndexState esIndexState : esIndexStates) { Range range = partitionInfo.handleNewSinglePartitionDesc( - esIndexState.getPartitionDesc(), partitionId, false); + esIndexState.getPartitionDesc(), partitionId, false); esTableState.addPartition(esIndexState.getIndexName(), partitionId); esIndexState.setPartitionId(partitionId); ++partitionId; LOG.debug("add parition to es table [{}] with range [{}]", esTable.getName(), - range); + range); } } return esTableState; } - -} + +} \ No newline at end of file diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index 93da2bdce71ab3..3db566d43e576d 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -28,7 +28,6 @@ import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; -import java.util.Map; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.EsTable; @@ -36,12 +35,12 @@ import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.external.EsFieldInfo; -import org.apache.doris.external.EsIndexState; -import org.apache.doris.external.EsRestClient; -import org.apache.doris.external.EsStateStore; -import org.apache.doris.external.EsTableState; -import org.apache.doris.external.ExternalDataSourceException; +import org.apache.doris.external.elasticsearch.EsFieldInfo; +import org.apache.doris.external.elasticsearch.EsIndexState; +import org.apache.doris.external.elasticsearch.EsRestClient; +import org.apache.doris.external.elasticsearch.EsStateStore; +import org.apache.doris.external.elasticsearch.EsTableState; +import org.apache.doris.external.elasticsearch.ExternalDataSourceException; import org.apache.doris.meta.MetaContext; import org.json.JSONObject; import org.junit.Before; @@ -49,7 +48,7 @@ import org.junit.Test; public class EsStateStoreTest { - + private static FakeEditLog fakeEditLog; private static FakeCatalog fakeCatalog; private static Catalog masterCatalog; @@ -59,9 +58,9 @@ public class EsStateStoreTest { private EsRestClient fakeClient; @BeforeClass - public static void init() throws IOException, InstantiationException, IllegalAccessException, - IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, - URISyntaxException { + public static void init() throws IOException, InstantiationException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, + URISyntaxException { fakeEditLog = new FakeEditLog(); fakeCatalog = new FakeCatalog(); masterCatalog = CatalogTestUtil.createTestCatalog(); @@ -79,24 +78,24 @@ public void setUp() { esStateStore = new EsStateStore(); fakeClient = new EsRestClient(new String[]{"localhost:9200"}, null, null); } - + @Test public void testSetEsTableContext() { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() - .getDb(CatalogTestUtil.testDb1) - .getTable(CatalogTestUtil.testEsTableId1); + .getDb(CatalogTestUtil.testDb1) + .getTable(CatalogTestUtil.testEsTableId1); JSONObject properties = fakeClient.parseProperties(mappingsStr, esTable.getMappingType()); EsFieldInfo fieldInfo = fakeClient.getFieldInfo(esTable.getFullSchema(), properties); esStateStore.setEsTableContext(fieldInfo, esTable); assertEquals("userId.keyword", esTable.fieldsContext().get("userId")); assertEquals("userId.keyword", esTable.docValueContext().get("userId")); } - + @Test public void testSetTableState() throws ExternalDataSourceException, DdlException { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() - .getDb(CatalogTestUtil.testDb1) - .getTable(CatalogTestUtil.testEsTableId1); + .getDb(CatalogTestUtil.testDb1) + .getTable(CatalogTestUtil.testEsTableId1); EsIndexState esIndexState = fakeClient.parseIndexState(esTable.getIndexName(), searchShardsStr); EsTableState esTableState = esStateStore.setPartitionInfo(esTable, esIndexState); assertNotNull(esTableState); @@ -107,7 +106,7 @@ public void testSetTableState() throws ExternalDataSourceException, DdlException private static String loadJsonFromFile(String fileName) throws IOException, URISyntaxException { File file = new File(EsStateStoreTest.class.getClassLoader().getResource(fileName).toURI()); InputStream is = new FileInputStream(file); - BufferedReader br = new BufferedReader(new InputStreamReader(is)); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); StringBuilder jsonStr = new StringBuilder(); String line = ""; while ((line = br.readLine()) != null) { @@ -117,4 +116,4 @@ private static String loadJsonFromFile(String fileName) throws IOException, URIS is.close(); return jsonStr.toString(); } -} +} \ No newline at end of file From b975c7a8086099b3498b63b43e92e5d7220fbdc0 Mon Sep 17 00:00:00 2001 From: stalary Date: Wed, 10 Jun 2020 22:08:52 +0800 Subject: [PATCH 13/21] MOD: Adjust import order, resolve conflicts --- .../external/elasticsearch/EsFieldInfo.java | 1 + .../external/elasticsearch/EsRestClient.java | 19 +++++++----- .../external/elasticsearch/EsStateStore.java | 6 ++-- .../doris/external/elasticsearch/EsUtil.java | 30 +++++++++---------- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java index 75841be5caeca7..927dd49615b94b 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java @@ -43,4 +43,5 @@ public Map getFetchFields() { public Map getDocValueFields() { return docValueFields; } + } \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index 66085341bd332c..5a4706ec3362db 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -17,8 +17,6 @@ package org.apache.doris.external.elasticsearch; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -26,10 +24,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import okhttp3.Credentials; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; import org.apache.commons.lang3.StringUtils; import org.apache.doris.catalog.Column; import org.apache.http.HttpHeaders; @@ -42,6 +36,12 @@ import org.codehaus.jackson.map.SerializationConfig; import org.json.JSONArray; import org.json.JSONObject; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import okhttp3.Credentials; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; public class EsRestClient { @@ -227,17 +227,22 @@ private String execute (String path) throws Exception { Request request = builder.get() .url(currentNode + "/" + path) .build(); + Response response = null; if (LOG.isTraceEnabled()) { LOG.trace("es rest client request URL: {}", currentNode + "/" + path); } try { - Response response = networkClient.newCall(request).execute(); + response = networkClient.newCall(request).execute(); if (response.isSuccessful()) { return response.body().string(); } } catch (IOException e) { LOG.warn("request node [{}] [{}] failures {}, try next nodes", currentNode, path, e); scratchExceptionForThrow = e; + } finally { + if (response != null) { + response.close(); + } } selectNextNode(); } diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java index a04654fe6d41de..048738cbcd3b1a 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java @@ -17,9 +17,6 @@ package org.apache.doris.external.elasticsearch; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; -import com.google.common.collect.Range; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -38,6 +35,9 @@ import org.apache.doris.common.util.MasterDaemon; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; /** diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index f8d438bc5435b6..16da249f0c16be 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -25,41 +25,40 @@ import org.json.JSONObject; public class EsUtil { - + public static void analyzePartitionAndDistributionDesc(PartitionDesc partitionDesc, - DistributionDesc distributionDesc) - throws AnalysisException { + DistributionDesc distributionDesc) throws AnalysisException { if (partitionDesc == null && distributionDesc == null) { return; } - + if (partitionDesc != null) { if (!(partitionDesc instanceof RangePartitionDesc)) { throw new AnalysisException("Elasticsearch table only permit range partition"); } - + RangePartitionDesc rangePartitionDesc = (RangePartitionDesc) partitionDesc; analyzePartitionDesc(rangePartitionDesc); } - + if (distributionDesc != null) { throw new AnalysisException("could not support distribution clause"); } } - + private static void analyzePartitionDesc(RangePartitionDesc partDesc) - throws AnalysisException { + throws AnalysisException { if (partDesc.getPartitionColNames() == null || partDesc.getPartitionColNames().isEmpty()) { throw new AnalysisException("No partition columns."); } - + if (partDesc.getPartitionColNames().size() > 1) { throw new AnalysisException( - "Elasticsearch table's parition column could only be a single column"); + "Elasticsearch table's parition column could only be a single column"); } } - - + + /** * get the json object from specified jsonObject * @@ -84,7 +83,7 @@ public static JSONObject getJsonObject(JSONObject jsonObject, String key, int fr return null; } } - + public static String getFetchField(JSONObject fieldObject, String colName) { String fieldType = fieldObject.optString("type"); // string-type field used keyword type to generate predicate @@ -103,7 +102,7 @@ public static String getFetchField(JSONObject fieldObject, String colName) { } return null; } - + public static String getDocValueField(JSONObject fieldObject, String colName) { String fieldType = fieldObject.optString("type"); String docValueField = null; @@ -112,7 +111,8 @@ public static String getDocValueField(JSONObject fieldObject, String colName) { if (fieldsObject != null) { for (String key : fieldsObject.keySet()) { JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.optString("type"))) { + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains( + innerTypeObject.optString("type"))) { continue; } if (innerTypeObject.has("doc_values")) { From 3782f4b043182a09870a55ca453ecc80807eb686 Mon Sep 17 00:00:00 2001 From: stalary Date: Thu, 11 Jun 2020 17:39:09 +0800 Subject: [PATCH 14/21] MOD: adjust import order --- .../external/elasticsearch/EsIndexState.java | 8 ++++---- .../external/elasticsearch/EsRestClient.java | 20 +++++++++---------- .../external/elasticsearch/EsStateStore.java | 12 +++++------ .../org/apache/doris/es/EsStateStoreTest.java | 16 +++++++-------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java index af06f4eda3f28d..156611a2da600c 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java @@ -17,15 +17,15 @@ package org.apache.doris.external.elasticsearch; -import com.google.common.collect.Maps; -import java.util.List; -import java.util.Map; -import java.util.Random; import org.apache.doris.analysis.SingleRangePartitionDesc; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.thrift.TNetworkAddress; +import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.List; +import java.util.Map; +import java.util.Random; public class EsIndexState { diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index 5a4706ec3362db..58731b63c9dde0 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -17,15 +17,10 @@ package org.apache.doris.external.elasticsearch; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; import org.apache.doris.catalog.Column; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -36,8 +31,13 @@ import org.codehaus.jackson.map.SerializationConfig; import org.json.JSONArray; import org.json.JSONObject; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import okhttp3.Credentials; import okhttp3.OkHttpClient; import okhttp3.Request; diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java index 048738cbcd3b1a..d53f34d30ead82 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java @@ -17,10 +17,6 @@ package org.apache.doris.external.elasticsearch; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; @@ -33,11 +29,15 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.MasterDaemon; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.common.collect.Range; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; /** diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index 3db566d43e576d..acf71b9faf4a2c 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -20,14 +20,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.InvocationTargetException; -import java.net.URISyntaxException; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.EsTable; @@ -46,6 +38,14 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; public class EsStateStoreTest { From 97b2338fb9f4f5bda2628d194d18dfd5ab6015d4 Mon Sep 17 00:00:00 2001 From: stalary Date: Fri, 12 Jun 2020 22:17:23 +0800 Subject: [PATCH 15/21] MOD: Abstract code --- .../external/elasticsearch/EsFieldInfo.java | 47 ----- .../external/elasticsearch/EsFieldInfos.java | 160 ++++++++++++++++++ .../external/elasticsearch/EsIndexState.java | 41 ++++- .../external/elasticsearch/EsRestClient.java | 111 +----------- .../external/elasticsearch/EsStateStore.java | 19 +-- .../doris/external/elasticsearch/EsUtil.java | 56 ------ .../org/apache/doris/es/EsStateStoreTest.java | 28 ++- .../test/resources/data/es/es7_mappings.json | 23 +++ 8 files changed, 262 insertions(+), 223 deletions(-) delete mode 100644 fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java create mode 100644 fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java create mode 100644 fe/src/test/resources/data/es/es7_mappings.json diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java deleted file mode 100644 index 927dd49615b94b..00000000000000 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfo.java +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.elasticsearch; - -import java.util.Map; - -/** - * EsFieldInfo - * - * @author stalary - * @since 2020/06/10 - */ -public class EsFieldInfo { - - private Map fetchFields; - - private Map docValueFields; - - public EsFieldInfo(Map fetchFields, Map docValueFields) { - this.fetchFields = fetchFields; - this.docValueFields = docValueFields; - } - - public Map getFetchFields() { - return fetchFields; - } - - public Map getDocValueFields() { - return docValueFields; - } - -} \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java new file mode 100644 index 00000000000000..7e1a8713514d31 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.external.elasticsearch; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.EsTable; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class EsFieldInfos { + + private static final Logger LOG = LogManager.getLogger(EsFieldInfos.class); + + private Map fieldsContext; + + private Map docValueContext; + + public EsFieldInfos(Map fieldsContext, Map docValueContext) { + this.fieldsContext = fieldsContext; + this.docValueContext = docValueContext; + } + + public Map getFieldsContext() { + return fieldsContext; + } + + public Map getDocValueContext() { + return docValueContext; + } + + /** + * Parse the required field information from the json + * @param colList table column + * @param indexName indexName(alias or really name) + * @param indexMapping the return value of _mapping + * @param mappingType indexType + * @return fieldsContext and docValueContext + * @throws Exception + */ + public static EsFieldInfos fromMapping(List colList, String indexName, String indexMapping, String mappingType) throws ExternalDataSourceException { + JSONObject jsonObject = new JSONObject(indexMapping); + // the indexName use alias takes the first mapping + Iterator keys = jsonObject.keys(); + String docKey = keys.next(); + JSONObject docData = jsonObject.optJSONObject(docKey); + JSONObject mappings = docData.optJSONObject("mappings"); + JSONObject rootSchema = mappings.optJSONObject(mappingType); + JSONObject properties; + // no type in es7 + if (rootSchema == null) { + properties = mappings.optJSONObject("properties"); + } else { + properties = rootSchema.optJSONObject("properties"); + } + if (properties == null) { + throw new ExternalDataSourceException( "index[" + indexName + "] type[" + mappingType + "] mapping not found for the Elasticsearch Cluster"); + } + return parseProperties(colList, properties); + } + + private static EsFieldInfos parseProperties(List colList, JSONObject properties) { + if (properties == null) { + return null; + } + Map fieldsMap = new HashMap<>(); + Map docValueMap = new HashMap<>(); + for (Column col : colList) { + String colName = col.getName(); + if (!properties.has(colName)) { + continue; + } + JSONObject fieldObject = properties.optJSONObject(colName); + String fetchField = getFieldContext(fieldObject, colName); + if (StringUtils.isNotEmpty(fetchField)) { + fieldsMap.put(colName, fetchField); + } + String docValueField = getDocValueField(fieldObject, colName); + if (StringUtils.isNotEmpty(docValueField)) { + docValueMap.put(colName, docValueField); + } + } + return new EsFieldInfos(fieldsMap, docValueMap); + } + + private static String getFieldContext(JSONObject fieldObject, String colName) { + String fieldType = fieldObject.optString("type"); + // string-type field used keyword type to generate predicate + // if text field type seen, we should use the `field` keyword type? + if ("text".equals(fieldType)) { + JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + if (fieldsObject != null) { + for (String key : fieldsObject.keySet()) { + JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + // just for text type + if ("keyword".equals(innerTypeObject.optString("type"))) { + return colName + "." + key; + } + } + } + } + return null; + } + + private static String getDocValueField(JSONObject fieldObject, String colName) { + String fieldType = fieldObject.optString("type"); + String docValueField = null; + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { + JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + if (fieldsObject != null) { + for (String key : fieldsObject.keySet()) { + JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains( + innerTypeObject.optString("type"))) { + continue; + } + if (innerTypeObject.has("doc_values")) { + boolean docValue = innerTypeObject.getBoolean("doc_values"); + if (docValue) { + docValueField = colName; + } + } else { + // a : {c : {}} -> a -> a.c + docValueField = colName + "." + key; + } + } + } + return docValueField; + } + // set doc_value = false manually + if (fieldObject.has("doc_values")) { + boolean docValue = fieldObject.optBoolean("doc_values"); + if (!docValue) { + return docValueField; + } + } + docValueField = colName; + return docValueField; + } +} diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java index 156611a2da600c..78932252e30fdc 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsIndexState.java @@ -20,9 +20,12 @@ import org.apache.doris.analysis.SingleRangePartitionDesc; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.thrift.TNetworkAddress; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; import java.util.List; import java.util.Map; import java.util.Random; @@ -44,7 +47,43 @@ public EsIndexState(String indexName) { this.partitionDesc = null; this.partitionKey = null; } - + + /** + * Parse shardRoutings from the json + * @param indexName indexName(alias or really name) + * @param shardLocation the return value of _search_shards + * @return shardRoutings is used for searching + */ + public static EsIndexState fromShardLocation(String indexName, String shardLocation) { + EsIndexState indexState = new EsIndexState(indexName); + JSONObject jsonObject = new JSONObject(shardLocation); + JSONObject nodesMap = jsonObject.getJSONObject("nodes"); + JSONArray shards = jsonObject.getJSONArray("shards"); + int length = shards.length(); + for (int i = 0; i < length; i++) { + List singleShardRouting = Lists.newArrayList(); + JSONArray shardsArray = shards.getJSONArray(i); + int arrayLength = shardsArray.length(); + for (int j = 0; j < arrayLength; j++) { + JSONObject shard = shardsArray.getJSONObject(j); + String shardState = shard.getString("state"); + if ("STARTED".equalsIgnoreCase(shardState) || "RELOCATING".equalsIgnoreCase(shardState)) { + try { + singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, String.valueOf(i), shard, nodesMap)); + } catch (Exception e) { + LOG.info( + "errors while parse shard routing from json [{}], ignore this shard", + shard, e); + } + } + } + if (singleShardRouting.isEmpty()) { + LOG.warn("could not find a healthy allocation for [{}][{}]", indexName, i); + } + indexState.addShardRouting(i, singleShardRouting); + } + return indexState; + } public void addHttpAddress(Map nodesInfo) { for (Map.Entry> entry : shardRoutings.entrySet()) { diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index 58731b63c9dde0..42531c90b31c1b 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -18,9 +18,6 @@ package org.apache.doris.external.elasticsearch; import org.apache.doris.catalog.Column; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -29,12 +26,9 @@ import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; -import org.json.JSONArray; -import org.json.JSONObject; import java.io.IOException; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -97,111 +91,23 @@ public Map getHttpNodes() throws Exception { return nodesMap; } - public EsFieldInfo getFieldInfo(String indexName, String mappingType, List colList) throws Exception { + public EsFieldInfos getFieldInfo(String indexName, String mappingType, List colList) throws Exception { String path = indexName + "/_mapping"; String indexMapping = execute(path); if (indexMapping == null) { - throw new Exception( "index[" + indexName + "] _mapping not found for the Elasticsearch Cluster"); + throw new ExternalDataSourceException( "index[" + indexName + "] _mapping not found for the Elasticsearch Cluster"); } - return getFieldInfo(colList, parseProperties(indexMapping, mappingType)); - } - - @VisibleForTesting - public EsFieldInfo getFieldInfo(List colList, JSONObject properties) { - if (properties == null) { - return null; - } - Map fetchFieldMap = new HashMap<>(); - Map docValueFieldMap = new HashMap<>(); - for (Column col : colList) { - String colName = col.getName(); - if (!properties.has(colName)) { - continue; - } - JSONObject fieldObject = properties.optJSONObject(colName); - String fetchField = EsUtil.getFetchField(fieldObject, colName); - if (StringUtils.isNotEmpty(fetchField)) { - fetchFieldMap.put(colName, fetchField); - } - String docValueField = EsUtil.getDocValueField(fieldObject, colName); - if (StringUtils.isNotEmpty(docValueField)) { - docValueFieldMap.put(colName, docValueField); - } - } - return new EsFieldInfo(fetchFieldMap, docValueFieldMap); - } - - @VisibleForTesting - public JSONObject parseProperties(String indexMapping, String mappingType) { - JSONObject jsonObject = new JSONObject(indexMapping); - // the indexName use alias takes the first mapping - Iterator keys = jsonObject.keys(); - String docKey = keys.next(); - JSONObject docData = jsonObject.optJSONObject(docKey); - JSONObject mappings = docData.optJSONObject("mappings"); - JSONObject rootSchema = mappings.optJSONObject(mappingType); - return rootSchema.optJSONObject("properties"); + return EsFieldInfos.fromMapping(colList, indexName, indexMapping, mappingType); } + public EsIndexState getIndexState(String indexName) throws Exception { String path = indexName + "/_search_shards"; String shardLocation = execute(path); if (shardLocation == null) { - throw new Exception( "index[" + indexName + "] _search_shards not found for the Elasticsearch Cluster"); - } - return parseIndexState(indexName, shardLocation); - } - - @VisibleForTesting - public EsIndexState parseIndexState(String indexName, String shardLocation) { - EsIndexState indexState = new EsIndexState(indexName); - JSONObject jsonObject = new JSONObject(shardLocation); - JSONObject nodesMap = jsonObject.getJSONObject("nodes"); - JSONArray shards = jsonObject.getJSONArray("shards"); - int length = shards.length(); - for (int i = 0; i < length; i++) { - List singleShardRouting = Lists.newArrayList(); - JSONArray shardsArray = shards.getJSONArray(i); - int arrayLength = shardsArray.length(); - for (int j = 0; j < arrayLength; j++) { - JSONObject shard = shardsArray.getJSONObject(j); - String shardState = shard.getString("state"); - if ("STARTED".equalsIgnoreCase(shardState) || - "RELOCATING".equalsIgnoreCase(shardState)) { - try { - singleShardRouting.add(EsShardRouting.parseShardRoutingV55(shardState, - String.valueOf(i), shard, nodesMap)); - } catch (Exception e) { - LOG.info( - "errors while parse shard routing from json [{}], ignore this shard", - shard, e); - } - } - } - if (singleShardRouting.isEmpty()) { - LOG.warn("could not find a healthy allocation for [{}][{}]", indexName, i); - } - indexState.addShardRouting(i, singleShardRouting); - } - return indexState; - } - - /** - * Get the Elasticsearch cluster version - * - * @return - */ - public EsMajorVersion version () throws Exception { - Map versionMap = get("/", "version"); - - EsMajorVersion majorVersion; - try { - majorVersion = EsMajorVersion.parse(versionMap.get("version")); - } catch (Exception e) { - LOG.warn("detect es version failure on node [{}]", currentNode); - return EsMajorVersion.V_5_X; + throw new ExternalDataSourceException( "index[" + indexName + "] _search_shards not found for the Elasticsearch Cluster"); } - return majorVersion; + return EsIndexState.fromShardLocation(indexName, shardLocation); } /** @@ -210,7 +116,7 @@ public EsMajorVersion version () throws Exception { * @param path the path must not leading with '/' * @return response */ - private String execute (String path) throws Exception { + private String execute(String path) throws Exception { int retrySize = nodes.length; Exception scratchExceptionForThrow = null; for (int i = 0; i < retrySize; i++) { @@ -268,5 +174,4 @@ private T parseContent(String response, String key) { } return (T) (key != null ? map.get(key) : map); } - -} \ No newline at end of file +} diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java index d53f34d30ead82..05c224a9a3c1ae 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java @@ -80,11 +80,11 @@ protected void runAfterCatalogReady() { EsRestClient client = esClients.get(esTable.getId()); if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { - EsFieldInfo fieldInfo = client.getFieldInfo(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); - if (fieldInfo == null) { + EsFieldInfos fieldInfos = client.getFieldInfo(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); + if (fieldInfos == null) { continue; } - setEsTableContext(fieldInfo, esTable); + setEsTableContext(fieldInfos, esTable); } EsIndexState esIndexState = client.getIndexState(esTable.getIndexName()); @@ -128,7 +128,7 @@ public void loadTableFromCatalog() { } // Configure keyword and doc_values by mapping - public void setEsTableContext(EsFieldInfo fieldInfo, EsTable esTable) { + public void setEsTableContext(EsFieldInfos fieldInfos, EsTable esTable) { // we build the doc value context for fields maybe used for scanning // "properties": { // "city": { @@ -142,11 +142,11 @@ public void setEsTableContext(EsFieldInfo fieldInfo, EsTable esTable) { // } // then the docvalue context provided the mapping between the select field and real request field : // {"city": "city.raw"} - if (esTable.isKeywordSniffEnable() && fieldInfo.getFetchFields() != null) { - esTable.addFetchField(fieldInfo.getFetchFields()); + if (esTable.isKeywordSniffEnable() && fieldInfos.getFieldsContext() != null) { + esTable.addFetchField(fieldInfos.getFieldsContext()); } - if (esTable.isDocValueScanEnable() && fieldInfo.getDocValueFields() != null) { - esTable.addDocValueField(fieldInfo.getDocValueFields()); + if (esTable.isDocValueScanEnable() && fieldInfos.getDocValueContext() != null) { + esTable.addDocValueField(fieldInfos.getDocValueContext()); } } @@ -203,5 +203,4 @@ public EsTableState setPartitionInfo(EsTable esTable, EsIndexState indexState) } return esTableState; } - -} \ No newline at end of file +} diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index 16da249f0c16be..085d9a1585cd60 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.DistributionDesc; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.RangePartitionDesc; -import org.apache.doris.catalog.EsTable; import org.apache.doris.common.AnalysisException; import org.json.JSONObject; @@ -83,59 +82,4 @@ public static JSONObject getJsonObject(JSONObject jsonObject, String key, int fr return null; } } - - public static String getFetchField(JSONObject fieldObject, String colName) { - String fieldType = fieldObject.optString("type"); - // string-type field used keyword type to generate predicate - // if text field type seen, we should use the `field` keyword type? - if ("text".equals(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); - if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - // just for text type - if ("keyword".equals(innerTypeObject.optString("type"))) { - return colName + "." + key; - } - } - } - } - return null; - } - - public static String getDocValueField(JSONObject fieldObject, String colName) { - String fieldType = fieldObject.optString("type"); - String docValueField = null; - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); - if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains( - innerTypeObject.optString("type"))) { - continue; - } - if (innerTypeObject.has("doc_values")) { - boolean docValue = innerTypeObject.getBoolean("doc_values"); - if (docValue) { - docValueField = colName; - } - } else { - // a : {c : {}} -> a -> a.c - docValueField = colName + "." + key; - } - } - } - return docValueField; - } - // set doc_value = false manually - if (fieldObject.has("doc_values")) { - boolean docValue = fieldObject.optBoolean("doc_values"); - if (!docValue) { - return docValueField; - } - } - docValueField = colName; - return docValueField; - } } diff --git a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java index acf71b9faf4a2c..3ef231fe4a7adc 100644 --- a/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/es/EsStateStoreTest.java @@ -27,7 +27,7 @@ import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.external.elasticsearch.EsFieldInfo; +import org.apache.doris.external.elasticsearch.EsFieldInfos; import org.apache.doris.external.elasticsearch.EsIndexState; import org.apache.doris.external.elasticsearch.EsRestClient; import org.apache.doris.external.elasticsearch.EsStateStore; @@ -53,6 +53,7 @@ public class EsStateStoreTest { private static FakeCatalog fakeCatalog; private static Catalog masterCatalog; private static String mappingsStr = ""; + private static String es7MappingsStr = ""; private static String searchShardsStr = ""; private EsStateStore esStateStore; private EsRestClient fakeClient; @@ -70,6 +71,7 @@ public static void init() throws IOException, InstantiationException, IllegalAcc // masterCatalog.setJournalVersion(FeMetaVersion.VERSION_40); FakeCatalog.setCatalog(masterCatalog); mappingsStr = loadJsonFromFile("data/es/mappings.json"); + es7MappingsStr = loadJsonFromFile("data/es/es7_mappings.json"); searchShardsStr = loadJsonFromFile("data/es/search_shards.json"); } @@ -80,15 +82,29 @@ public void setUp() { } @Test - public void testSetEsTableContext() { + public void testSetEsTableContext() throws Exception { EsTable esTable = (EsTable) Catalog.getCurrentCatalog() .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); - JSONObject properties = fakeClient.parseProperties(mappingsStr, esTable.getMappingType()); - EsFieldInfo fieldInfo = fakeClient.getFieldInfo(esTable.getFullSchema(), properties); - esStateStore.setEsTableContext(fieldInfo, esTable); + // es5 + EsFieldInfos fieldInfos = EsFieldInfos.fromMapping(esTable.getFullSchema(), esTable.getIndexName(), mappingsStr, esTable.getMappingType()); + esStateStore.setEsTableContext(fieldInfos, esTable); assertEquals("userId.keyword", esTable.fieldsContext().get("userId")); assertEquals("userId.keyword", esTable.docValueContext().get("userId")); + // es7 + EsFieldInfos fieldInfos7 = EsFieldInfos.fromMapping(esTable.getFullSchema(), esTable.getIndexName(), es7MappingsStr, ""); + assertEquals("userId.keyword", fieldInfos7.getFieldsContext().get("userId")); + assertEquals("userId.keyword", fieldInfos7.getDocValueContext().get("userId")); + + } + + @Test(expected = ExternalDataSourceException.class) + public void testSetErrorType() throws Exception { + EsTable esTable = (EsTable) Catalog.getCurrentCatalog() + .getDb(CatalogTestUtil.testDb1) + .getTable(CatalogTestUtil.testEsTableId1); + // error type + EsFieldInfos.fromMapping(esTable.getFullSchema(), esTable.getIndexName(), mappingsStr, "errorType"); } @Test @@ -96,7 +112,7 @@ public void testSetTableState() throws ExternalDataSourceException, DdlException EsTable esTable = (EsTable) Catalog.getCurrentCatalog() .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); - EsIndexState esIndexState = fakeClient.parseIndexState(esTable.getIndexName(), searchShardsStr); + EsIndexState esIndexState = EsIndexState.fromShardLocation(esTable.getIndexName(), searchShardsStr); EsTableState esTableState = esStateStore.setPartitionInfo(esTable, esIndexState); assertNotNull(esTableState); assertEquals(1, esTableState.getUnPartitionedIndexStates().size()); diff --git a/fe/src/test/resources/data/es/es7_mappings.json b/fe/src/test/resources/data/es/es7_mappings.json new file mode 100644 index 00000000000000..cf44130f68b5e9 --- /dev/null +++ b/fe/src/test/resources/data/es/es7_mappings.json @@ -0,0 +1,23 @@ +{ + "indexa_2020.05.02": { + "mappings": { + "dynamic": "strict", + "properties": { + "time": { + "type": "long" + }, + "type": { + "type": "keyword" + }, + "userId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + } + } +} \ No newline at end of file From 95b51173ac8709c1c28704f73a64d597dd2693aa Mon Sep 17 00:00:00 2001 From: stalary Date: Sat, 20 Jun 2020 10:43:21 +0800 Subject: [PATCH 16/21] MOD: Refactor some places --- docs/en/extending-doris/doris-on-es.md | 2 +- docs/zh-CN/extending-doris/doris-on-es.md | 2 +- .../org/apache/doris/catalog/Catalog.java | 16 +- .../org/apache/doris/catalog/Database.java | 2 +- .../org/apache/doris/catalog/EsTable.java | 28 +-- .../common/proc/EsPartitionsProcDir.java | 6 +- .../doris/common/proc/EsShardProcDir.java | 2 +- .../external/elasticsearch/EsFieldInfos.java | 58 +++-- .../external/elasticsearch/EsRepository.java | 118 ++++++++++ .../external/elasticsearch/EsRestClient.java | 2 +- .../external/elasticsearch/EsStateStore.java | 206 ------------------ .../elasticsearch/EsTablePartitions.java | 172 +++++++++++++++ .../external/elasticsearch/EsTableState.java | 105 --------- .../org/apache/doris/planner/EsScanNode.java | 14 +- ...teStoreTest.java => EsRepositoryTest.java} | 18 +- 15 files changed, 381 insertions(+), 370 deletions(-) create mode 100644 fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java delete mode 100644 fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java create mode 100644 fe/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java delete mode 100644 fe/src/main/java/org/apache/doris/external/elasticsearch/EsTableState.java rename fe/src/test/java/org/apache/doris/external/elasticsearch/{EsStateStoreTest.java => EsRepositoryTest.java} (90%) diff --git a/docs/en/extending-doris/doris-on-es.md b/docs/en/extending-doris/doris-on-es.md index b34b678a0253dc..d79505ffe63744 100644 --- a/docs/en/extending-doris/doris-on-es.md +++ b/docs/en/extending-doris/doris-on-es.md @@ -130,7 +130,7 @@ The following parameters are accepted by ES table: Parameter | Description ---|--- **hosts** | ES Cluster Connection Address, maybe one or more node, load-balance is also accepted -**index** | the related ES index name, can be alias, If you use doc_value, it is recommended to use the real index name +**index** | the related ES index name, alias is supported, and if you use doc_value, you need to use the real name **type** | the type for this index, If not specified, `_doc` will be used **user** | username for ES **password** | password for the user diff --git a/docs/zh-CN/extending-doris/doris-on-es.md b/docs/zh-CN/extending-doris/doris-on-es.md index 643aa7fcacfafc..36319937f6e046 100644 --- a/docs/zh-CN/extending-doris/doris-on-es.md +++ b/docs/zh-CN/extending-doris/doris-on-es.md @@ -128,7 +128,7 @@ PROPERTIES ( 参数 | 说明 ---|--- **hosts** | ES集群地址，可以是一个或多个，也可以是ES前端的负载均衡地址 -**index** | 对应的ES的index名字，可以是alias，如果使用doc_value，建议使用真实的index名称 +**index** | 对应的ES的index名字，支持alias，如果使用doc_value，需要使用真实的名称 **type** | index的type，不指定的情况会使用_doc **user** | ES集群用户名 **password** | 对应用户的密码信息 diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index da9906c3afc1aa..b659374f3c646d 100755 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -127,7 +127,7 @@ import org.apache.doris.deploy.impl.AmbariDeployManager; import org.apache.doris.deploy.impl.K8sDeployManager; import org.apache.doris.deploy.impl.LocalFileDeployManager; -import org.apache.doris.external.elasticsearch.EsStateStore; +import org.apache.doris.external.elasticsearch.EsRepository; import org.apache.doris.ha.BDBHA; import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.ha.HAProtocol; @@ -299,7 +299,7 @@ public class Catalog { private Daemon replayer; private Daemon timePrinter; private Daemon listener; - private EsStateStore esStateStore; // it is a daemon, so add it here + private EsRepository esRepository; // it is a daemon, so add it here private boolean isFirstTimeStartUp = false; private boolean isElectable; @@ -506,7 +506,7 @@ private Catalog() { this.auth = new PaloAuth(); this.domainResolver = new DomainResolver(auth); - this.esStateStore = new EsStateStore(); + this.esRepository = new EsRepository(); this.metaContext = new MetaContext(); this.metaContext.setThreadLocalInfo(); @@ -1261,7 +1261,7 @@ private void startNonMasterDaemonThreads() { // load and export job label cleaner thread labelCleaner.start(); // ES state store - esStateStore.start(); + esRepository.start(); // domain resolver domainResolver.start(); } @@ -1423,7 +1423,7 @@ public void loadImage(String imageDir) throws IOException, DdlException { // ATTN: this should be done after load Db, and before loadAlterJob recreateTabletInvertIndex(); // rebuild es state state - esStateStore.loadTableFromCatalog(); + esRepository.loadTableFromCatalog(); checksum = loadLoadJob(dis, checksum); checksum = loadAlterJob(dis, checksum); @@ -4347,7 +4347,7 @@ public boolean unprotectDropTable(Database db, long tableId) { } if (table.getType() == TableType.ELASTICSEARCH) { - esStateStore.deRegisterTable(tableId); + esRepository.deRegisterTable(tableId); } else if (table.getType() == TableType.OLAP) { // drop all temp partitions of this table, so that there is no temp partitions in recycle bin, // which make things easier. @@ -4799,8 +4799,8 @@ public String getMasterIp() { return this.masterIp; } - public EsStateStore getEsStateStore() { - return this.esStateStore; + public EsRepository getEsRepository() { + return this.esRepository; } public void setMaster(MasterInfo info) { diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java b/fe/src/main/java/org/apache/doris/catalog/Database.java index 5333d3177ae918..b287ada07e8959 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/src/main/java/org/apache/doris/catalog/Database.java @@ -304,7 +304,7 @@ public boolean createTableWithLock(Table table, boolean isReplay, boolean setIfN Catalog.getCurrentCatalog().getEditLog().logCreateTable(info); } if (table.getType() == TableType.ELASTICSEARCH) { - Catalog.getCurrentCatalog().getEsStateStore().registerTable((EsTable)table); + Catalog.getCurrentCatalog().getEsRepository().registerTable((EsTable)table); } } return result; diff --git a/fe/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/src/main/java/org/apache/doris/catalog/EsTable.java index 4d62f2922d4b07..7c25ebf9b919ed 100644 --- a/fe/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/EsTable.java @@ -20,8 +20,9 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; +import org.apache.doris.external.elasticsearch.EsFieldInfos; import org.apache.doris.external.elasticsearch.EsMajorVersion; -import org.apache.doris.external.elasticsearch.EsTableState; +import org.apache.doris.external.elasticsearch.EsTablePartitions; import org.apache.doris.thrift.TEsTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -71,7 +72,7 @@ public class EsTable extends Table { // only save the partition definition, save the partition key, // partition list is got from es cluster dynamically and is saved in esTableState private PartitionInfo partitionInfo; - private EsTableState esTableState; + private EsTablePartitions esTablePartitions; private boolean enableDocValueScan = false; private boolean enableKeywordSniff = true; @@ -98,7 +99,7 @@ public class EsTable extends Table { // use select city from table, if enable the docvalue, we will fetch the `city` field value from `city.raw` private Map docValueContext = new HashMap<>(); - private Map fieldsContext = new HashMap<>(); + private Map fieldsContext= new HashMap<>(); // record the latest and recently exception when sync ES table metadata (mapping, shard location) private Throwable lastMetaDataSyncException = null; @@ -114,18 +115,19 @@ public EsTable(long id, String name, List schema, validate(properties); } - public void addFetchField(Map fetchFieldMap) { - fieldsContext.putAll(fetchFieldMap); + public void addFieldInfos(EsFieldInfos esFieldInfos) { + if (enableKeywordSniff && esFieldInfos.getFieldsContext() != null) { + fieldsContext = esFieldInfos.getFieldsContext(); + } + if (enableDocValueScan && esFieldInfos.getDocValueContext() != null) { + docValueContext = esFieldInfos.getDocValueContext(); + } } public Map fieldsContext() { return fieldsContext; } - public void addDocValueField(Map docValueFieldMap) { - docValueContext.putAll(docValueFieldMap); - } - public Map docValueContext() { return docValueContext; } @@ -386,12 +388,12 @@ public PartitionInfo getPartitionInfo() { return partitionInfo; } - public EsTableState getEsTableState() { - return esTableState; + public EsTablePartitions getEsTablePartitions() { + return esTablePartitions; } - public void setEsTableState(EsTableState esTableState) { - this.esTableState = esTableState; + public void setEsTablePartitions(EsTablePartitions esTablePartitions) { + this.esTablePartitions = esTablePartitions; } public Throwable getLastMetaDataSyncException() { diff --git a/fe/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java index ec70fa32f34c9a..5b470e991ec3ff 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java @@ -66,11 +66,11 @@ public ProcResult fetchResult() throws AnalysisException { try { RangePartitionInfo rangePartitionInfo = null; if (esTable.getPartitionInfo().getType() == PartitionType.RANGE) { - rangePartitionInfo = (RangePartitionInfo) esTable.getEsTableState().getPartitionInfo(); + rangePartitionInfo = (RangePartitionInfo) esTable.getEsTablePartitions().getPartitionInfo(); } Joiner joiner = Joiner.on(", "); - Map unPartitionedIndices = esTable.getEsTableState().getUnPartitionedIndexStates(); - Map partitionedIndices = esTable.getEsTableState().getPartitionedIndexStates(); + Map unPartitionedIndices = esTable.getEsTablePartitions().getUnPartitionedIndexStates(); + Map partitionedIndices = esTable.getEsTablePartitions().getPartitionedIndexStates(); for (EsShardPartitions esShardPartitions : unPartitionedIndices.values()) { List partitionInfo = new ArrayList(); partitionInfo.add(esShardPartitions.getIndexName()); diff --git a/fe/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java index 89f8c2d8ade9d5..45e34ffaea7bd0 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java @@ -55,7 +55,7 @@ public ProcResult fetchResult() { db.readLock(); try { // get infos - EsShardPartitions esShardPartitions = esTable.getEsTableState().getIndexState(indexName); + EsShardPartitions esShardPartitions = esTable.getEsTablePartitions().getEsShardPartitions(indexName); for (int shardId : esShardPartitions.getShardRoutings().keySet()) { List shardRoutings = esShardPartitions.getShardRoutings().get(shardId); if (shardRoutings != null && shardRoutings.size() > 0) { diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java index 7e1a8713514d31..59d251118d471f 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsFieldInfos.java @@ -28,12 +28,18 @@ import java.util.List; import java.util.Map; +/** + * It is used to hold the field information obtained from es, currently including the fields and docValue, + * it will eventually be added to the EsTable + **/ public class EsFieldInfos { private static final Logger LOG = LogManager.getLogger(EsFieldInfos.class); - + + // userId => userId.keyword private Map fieldsContext; - + + // city => city.raw private Map docValueContext; public EsFieldInfos(Map fieldsContext, Map docValueContext) { @@ -54,18 +60,41 @@ public Map getDocValueContext() { * @param colList table column * @param indexName indexName(alias or really name) * @param indexMapping the return value of _mapping - * @param mappingType indexType + * @param docType The docType used by the index * @return fieldsContext and docValueContext * @throws Exception */ - public static EsFieldInfos fromMapping(List colList, String indexName, String indexMapping, String mappingType) throws ExternalDataSourceException { + public static EsFieldInfos fromMapping(List colList, String indexName, String indexMapping, String docType) throws ExternalDataSourceException { JSONObject jsonObject = new JSONObject(indexMapping); // the indexName use alias takes the first mapping Iterator keys = jsonObject.keys(); String docKey = keys.next(); JSONObject docData = jsonObject.optJSONObject(docKey); + //{ + // "mappings": { + // "doc": { + // "dynamic": "strict", + // "properties": { + // "time": { + // "type": "long" + // }, + // "type": { + // "type": "keyword" + // }, + // "userId": { + // "type": "text", + // "fields": { + // "keyword": { + // "type": "keyword" + // } + // } + // } + // } + // } + // } + //} JSONObject mappings = docData.optJSONObject("mappings"); - JSONObject rootSchema = mappings.optJSONObject(mappingType); + JSONObject rootSchema = mappings.optJSONObject(docType); JSONObject properties; // no type in es7 if (rootSchema == null) { @@ -74,11 +103,12 @@ public static EsFieldInfos fromMapping(List colList, String indexName, S properties = rootSchema.optJSONObject("properties"); } if (properties == null) { - throw new ExternalDataSourceException( "index[" + indexName + "] type[" + mappingType + "] mapping not found for the Elasticsearch Cluster"); + throw new ExternalDataSourceException( "index[" + indexName + "] type[" + docType + "] mapping not found for the Elasticsearch Cluster"); } return parseProperties(colList, properties); } - + + // get fields information in properties private static EsFieldInfos parseProperties(List colList, JSONObject properties) { if (properties == null) { return null; @@ -91,9 +121,9 @@ private static EsFieldInfos parseProperties(List colList, JSONObject pro continue; } JSONObject fieldObject = properties.optJSONObject(colName); - String fetchField = getFieldContext(fieldObject, colName); - if (StringUtils.isNotEmpty(fetchField)) { - fieldsMap.put(colName, fetchField); + String keywordField = getKeywordField(fieldObject, colName); + if (StringUtils.isNotEmpty(keywordField)) { + fieldsMap.put(colName, keywordField); } String docValueField = getDocValueField(fieldObject, colName); if (StringUtils.isNotEmpty(docValueField)) { @@ -102,8 +132,9 @@ private static EsFieldInfos parseProperties(List colList, JSONObject pro } return new EsFieldInfos(fieldsMap, docValueMap); } - - private static String getFieldContext(JSONObject fieldObject, String colName) { + + // get a field of keyword type in the fields + private static String getKeywordField(JSONObject fieldObject, String colName) { String fieldType = fieldObject.optString("type"); // string-type field used keyword type to generate predicate // if text field type seen, we should use the `field` keyword type? @@ -130,8 +161,7 @@ private static String getDocValueField(JSONObject fieldObject, String colName) { if (fieldsObject != null) { for (String key : fieldsObject.keySet()) { JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains( - innerTypeObject.optString("type"))) { + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.optString("type"))) { continue; } if (innerTypeObject.has("doc_values")) { diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java new file mode 100644 index 00000000000000..90360c368cf3b5 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.external.elasticsearch; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.EsTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Table.TableType; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.MasterDaemon; +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; + + +/** + * It is used to call es api to get shard allocation state + */ +public class EsRepository extends MasterDaemon { + + private static final Logger LOG = LogManager.getLogger(EsRepository.class); + + private Map esTables; + + private Map esClients; + + public EsRepository() { + super("es state store", Config.es_state_sync_interval_second * 1000); + esTables = Maps.newConcurrentMap(); + esClients = Maps.newConcurrentMap(); + } + + public void registerTable(EsTable esTable) { + if (Catalog.isCheckpointThread()) { + return; + } + esTables.put(esTable.getId(), esTable); + esClients.put(esTable.getId(), + new EsRestClient(esTable.getSeeds(), esTable.getUserName(), esTable.getPasswd())); + LOG.info("register a new table [{}] to sync list", esTable); + } + + public void deRegisterTable(long tableId) { + esTables.remove(tableId); + esClients.remove(tableId); + LOG.info("deregister table [{}] from sync list", tableId); + } + + @Override + protected void runAfterCatalogReady() { + for (EsTable esTable : esTables.values()) { + try { + EsRestClient client = esClients.get(esTable.getId()); + + if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { + EsFieldInfos fieldInfos = client.getFieldInfos(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); + if (fieldInfos == null) { + continue; + } + esTable.addFieldInfos(fieldInfos); + } + + EsShardPartitions esShardPartitions = client.getShardPartitions(esTable.getIndexName()); + + EsTablePartitions esTablePartitions = EsTablePartitions.fromShardPartitions(esTable, esShardPartitions); + + if (EsTable.TRANSPORT_HTTP.equals(esTable.getTransport())) { + Map nodesInfo = client.getHttpNodes(); + esTablePartitions.addHttpAddress(nodesInfo); + } + esTable.setEsTablePartitions(esTablePartitions); + } catch (Throwable e) { + LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", esTable.getName(), e); + esTable.setEsTablePartitions(null); + esTable.setLastMetaDataSyncException(e); + } + } + } + + // should call this method to init the state store after loading image + // the rest of tables will be added or removed by replaying edit log + // when fe is start to load image, should call this method to init the state store + public void loadTableFromCatalog() { + if (Catalog.isCheckpointThread()) { + return; + } + List dbIds = Catalog.getCurrentCatalog().getDbIds(); + for (Long dbId : dbIds) { + Database database = Catalog.getCurrentCatalog().getDb(dbId); + + List
tables = database.getTables(); + for (Table table : tables) { + if (table.getType() == TableType.ELASTICSEARCH) { + esTables.put(table.getId(), (EsTable) table); + } + } + } + } +} diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index b72c133b3a9234..ce354cc04efeb4 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -91,7 +91,7 @@ public Map getHttpNodes() throws Exception { return nodesMap; } - public EsFieldInfos getFieldInfo(String indexName, String mappingType, List colList) throws Exception { + public EsFieldInfos getFieldInfos(String indexName, String mappingType, List colList) throws Exception { String path = indexName + "/_mapping"; String indexMapping = execute(path); if (indexMapping == null) { diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java deleted file mode 100644 index d371b6e8f01817..00000000000000 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsStateStore.java +++ /dev/null @@ -1,206 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.elasticsearch; - -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.EsTable; -import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.RangePartitionInfo; -import org.apache.doris.catalog.SinglePartitionInfo; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.MasterDaemon; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; -import com.google.common.collect.Range; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - - -/** - * it is used to call es api to get shard allocation state - */ -public class EsStateStore extends MasterDaemon { - - private static final Logger LOG = LogManager.getLogger(EsStateStore.class); - - private Map esTables; - - private Map esClients; - - public EsStateStore() { - super("es state store", Config.es_state_sync_interval_second * 1000); - esTables = Maps.newConcurrentMap(); - esClients = Maps.newConcurrentMap(); - } - - public void registerTable(EsTable esTable) { - if (Catalog.isCheckpointThread()) { - return; - } - esTables.put(esTable.getId(), esTable); - esClients.put(esTable.getId(), - new EsRestClient(esTable.getSeeds(), esTable.getUserName(), esTable.getPasswd())); - LOG.info("register a new table [{}] to sync list", esTable); - } - - public void deRegisterTable(long tableId) { - esTables.remove(tableId); - esClients.remove(tableId); - LOG.info("deregister table [{}] from sync list", tableId); - } - - @Override - protected void runAfterCatalogReady() { - for (EsTable esTable : esTables.values()) { - try { - EsRestClient client = esClients.get(esTable.getId()); - - if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { - EsFieldInfos fieldInfos = client.getFieldInfo(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); - if (fieldInfos == null) { - continue; - } - setEsTableContext(fieldInfos, esTable); - } - - EsShardPartitions esShardPartitions = client.getShardPartitions(esTable.getIndexName()); - - EsTableState esTableState = setPartitionInfo(esTable, esShardPartitions); - if (esTableState == null) { - continue; - } - - if (EsTable.TRANSPORT_HTTP.equals(esTable.getTransport())) { - Map nodesInfo = client.getHttpNodes(); - esTableState.addHttpAddress(nodesInfo); - } - esTable.setEsTableState(esTableState); - } catch (Throwable e) { - LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", esTable.getName(), e); - esTable.setEsTableState(null); - esTable.setLastMetaDataSyncException(e); - } - } - } - - // should call this method to init the state store after loading image - // the rest of tables will be added or removed by replaying edit log - // when fe is start to load image, should call this method to init the state store - public void loadTableFromCatalog() { - if (Catalog.isCheckpointThread()) { - return; - } - List dbIds = Catalog.getCurrentCatalog().getDbIds(); - for (Long dbId : dbIds) { - Database database = Catalog.getCurrentCatalog().getDb(dbId); - - List
tables = database.getTables(); - for (Table table : tables) { - if (table.getType() == TableType.ELASTICSEARCH) { - esTables.put(table.getId(), (EsTable) table); - } - } - } - } - - // Configure keyword and doc_values by mapping - public void setEsTableContext(EsFieldInfos fieldInfos, EsTable esTable) { - // we build the doc value context for fields maybe used for scanning - // "properties": { - // "city": { - // "type": "text", // text field does not have docvalue - // "fields": { - // "raw": { - // "type": "keyword" - // } - // } - // } - // } - // then the docvalue context provided the mapping between the select field and real request field : - // {"city": "city.raw"} - if (esTable.isKeywordSniffEnable() && fieldInfos.getFieldsContext() != null) { - esTable.addFetchField(fieldInfos.getFieldsContext()); - } - if (esTable.isDocValueScanEnable() && fieldInfos.getDocValueContext() != null) { - esTable.addDocValueField(fieldInfos.getDocValueContext()); - } - } - - @VisibleForTesting - public EsTableState setPartitionInfo(EsTable esTable, EsShardPartitions indexState) - throws ExternalDataSourceException, DdlException { - EsTableState esTableState = new EsTableState(); - RangePartitionInfo partitionInfo = null; - if (esTable.getPartitionInfo() != null) { - if (esTable.getPartitionInfo() instanceof RangePartitionInfo) { - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTable.getPartitionInfo(); - partitionInfo = new RangePartitionInfo(rangePartitionInfo.getPartitionColumns()); - esTableState.setPartitionInfo(partitionInfo); - if (LOG.isDebugEnabled()) { - StringBuilder sb = new StringBuilder(); - int idx = 0; - for (Column column : rangePartitionInfo.getPartitionColumns()) { - if (idx != 0) { - sb.append(", "); - } - sb.append("`").append(column.getName()).append("`"); - idx++; - } - sb.append(")"); - LOG.debug("begin to parse es table [{}] state from search shards," - + " with partition info [{}]", esTable.getName(), sb.toString()); - } - } else if (esTable.getPartitionInfo() instanceof SinglePartitionInfo) { - LOG.debug("begin to parse es table [{}] state from search shards, " - + "with no partition info", esTable.getName()); - } else { - throw new ExternalDataSourceException("es table only support range partition, " - + "but current partition type is " - + esTable.getPartitionInfo().getType()); - } - } - esTableState.addIndexState(esTable.getIndexName(), indexState); - LOG.debug("add index {} to es table {}", indexState, esTable.getName()); - if (partitionInfo != null) { - // sort the index state according to partition key and then add to range map - List esShardPartitionsList = new ArrayList<>( - esTableState.getPartitionedIndexStates().values()); - esShardPartitionsList.sort(Comparator.comparing(EsShardPartitions::getPartitionKey)); - long partitionId = 0; - for (EsShardPartitions esShardPartitions : esShardPartitionsList) { - Range range = partitionInfo.handleNewSinglePartitionDesc( - esShardPartitions.getPartitionDesc(), partitionId, false); - esTableState.addPartition(esShardPartitions.getIndexName(), partitionId); - esShardPartitions.setPartitionId(partitionId); - ++partitionId; - LOG.debug("add parition to es table [{}] with range [{}]", esTable.getName(), - range); - } - } - return esTableState; - } -} diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java new file mode 100644 index 00000000000000..52630868edaa7e --- /dev/null +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.external.elasticsearch; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.EsTable; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.SinglePartitionInfo; +import org.apache.doris.common.DdlException; +import org.apache.doris.thrift.TNetworkAddress; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * save the dynamic info parsed from es cluster state such as shard routing, partition info + */ +public class EsTablePartitions { + + private static final Logger LOG = LogManager.getLogger(EsTablePartitions.class); + + private PartitionInfo partitionInfo; + private Map partitionIdToIndices; + private Map partitionedIndexStates; + private Map unPartitionedIndexStates; + + public EsTablePartitions() { + partitionInfo = null; + partitionIdToIndices = Maps.newHashMap(); + partitionedIndexStates = Maps.newHashMap(); + unPartitionedIndexStates = Maps.newHashMap(); + } + + public static EsTablePartitions fromShardPartitions(EsTable esTable, EsShardPartitions shardPartitions) + throws ExternalDataSourceException, DdlException { + EsTablePartitions esTablePartitions = new EsTablePartitions(); + RangePartitionInfo partitionInfo = null; + if (esTable.getPartitionInfo() != null) { + if (esTable.getPartitionInfo() instanceof RangePartitionInfo) { + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) esTable.getPartitionInfo(); + partitionInfo = new RangePartitionInfo(rangePartitionInfo.getPartitionColumns()); + esTablePartitions.setPartitionInfo(partitionInfo); + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder(); + int idx = 0; + for (Column column : rangePartitionInfo.getPartitionColumns()) { + if (idx != 0) { + sb.append(", "); + } + sb.append("`").append(column.getName()).append("`"); + idx++; + } + sb.append(")"); + LOG.debug("begin to parse es table [{}] state from search shards," + + " with partition info [{}]", esTable.getName(), sb.toString()); + } + } else if (esTable.getPartitionInfo() instanceof SinglePartitionInfo) { + LOG.debug("begin to parse es table [{}] state from search shards, " + + "with no partition info", esTable.getName()); + } else { + throw new ExternalDataSourceException("es table only support range partition, " + + "but current partition type is " + + esTable.getPartitionInfo().getType()); + } + } + esTablePartitions.addIndexState(esTable.getIndexName(), shardPartitions); + LOG.debug("add index {} to es table {}", shardPartitions, esTable.getName()); + if (partitionInfo != null) { + // sort the index state according to partition key and then add to range map + List esShardPartitionsList = new ArrayList<>( + esTablePartitions.getPartitionedIndexStates().values()); + esShardPartitionsList.sort(Comparator.comparing(EsShardPartitions::getPartitionKey)); + long partitionId = 0; + for (EsShardPartitions esShardPartitions : esShardPartitionsList) { + Range range = partitionInfo.handleNewSinglePartitionDesc( + esShardPartitions.getPartitionDesc(), partitionId, false); + esTablePartitions.addPartition(esShardPartitions.getIndexName(), partitionId); + esShardPartitions.setPartitionId(partitionId); + ++partitionId; + LOG.debug("add parition to es table [{}] with range [{}]", esTable.getName(), + range); + } + } + return esTablePartitions; + } + + public void addHttpAddress(Map nodesInfo) { + for (EsShardPartitions indexState : partitionedIndexStates.values()) { + indexState.addHttpAddress(nodesInfo); + } + for (EsShardPartitions indexState : unPartitionedIndexStates.values()) { + indexState.addHttpAddress(nodesInfo); + } + + } + + public TNetworkAddress randomAddress(Map nodesInfo) { + int seed = new Random().nextInt() % nodesInfo.size(); + EsNodeInfo[] nodeInfos = (EsNodeInfo[]) nodesInfo.values().toArray(); + return nodeInfos[seed].getPublishAddress(); + } + + public PartitionInfo getPartitionInfo() { + return partitionInfo; + } + + public void setPartitionInfo(PartitionInfo partitionInfo) { + this.partitionInfo = partitionInfo; + } + + public Map getPartitionIdToIndices() { + return partitionIdToIndices; + } + + public void addPartition(String indexName, long partitionId) { + partitionIdToIndices.put(partitionId, indexName); + } + + public void addIndexState(String indexName, EsShardPartitions indexState) { + if (indexState.getPartitionDesc() != null) { + partitionedIndexStates.put(indexName, indexState); + } else { + unPartitionedIndexStates.put(indexName, indexState); + } + } + + public Map getPartitionedIndexStates() { + return partitionedIndexStates; + } + + public Map getUnPartitionedIndexStates() { + return unPartitionedIndexStates; + } + + public EsShardPartitions getEsShardPartitions(long partitionId) { + if (partitionIdToIndices.containsKey(partitionId)) { + return partitionedIndexStates.get(partitionIdToIndices.get(partitionId)); + } + return null; + } + + public EsShardPartitions getEsShardPartitions(String indexName) { + if (partitionedIndexStates.containsKey(indexName)) { + return partitionedIndexStates.get(indexName); + } + return unPartitionedIndexStates.get(indexName); + } +} diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsTableState.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsTableState.java deleted file mode 100644 index 42c3b4f6a22472..00000000000000 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsTableState.java +++ /dev/null @@ -1,105 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.elasticsearch; - -import java.util.Map; -import java.util.Random; - -import org.apache.doris.catalog.PartitionInfo; -import com.google.common.collect.Maps; -import org.apache.doris.thrift.TNetworkAddress; - -/** - * save the dynamic info parsed from es cluster state such as shard routing, partition info - */ -public class EsTableState { - - private PartitionInfo partitionInfo; - private Map partitionIdToIndices; - private Map partitionedIndexStates; - private Map unPartitionedIndexStates; - - public EsTableState() { - partitionInfo = null; - partitionIdToIndices = Maps.newHashMap(); - partitionedIndexStates = Maps.newHashMap(); - unPartitionedIndexStates = Maps.newHashMap(); - } - - public void addHttpAddress(Map nodesInfo) { - for (EsShardPartitions indexState : partitionedIndexStates.values()) { - indexState.addHttpAddress(nodesInfo); - } - for (EsShardPartitions indexState : unPartitionedIndexStates.values()) { - indexState.addHttpAddress(nodesInfo); - } - - } - - public TNetworkAddress randomAddress(Map nodesInfo) { - int seed = new Random().nextInt() % nodesInfo.size(); - EsNodeInfo[] nodeInfos = (EsNodeInfo[]) nodesInfo.values().toArray(); - return nodeInfos[seed].getPublishAddress(); - } - - public PartitionInfo getPartitionInfo() { - return partitionInfo; - } - - public void setPartitionInfo(PartitionInfo partitionInfo) { - this.partitionInfo = partitionInfo; - } - - public Map getPartitionIdToIndices() { - return partitionIdToIndices; - } - - public void addPartition(String indexName, long partitionId) { - partitionIdToIndices.put(partitionId, indexName); - } - - public void addIndexState(String indexName, EsShardPartitions indexState) { - if (indexState.getPartitionDesc() != null) { - partitionedIndexStates.put(indexName, indexState); - } else { - unPartitionedIndexStates.put(indexName, indexState); - } - } - - public Map getPartitionedIndexStates() { - return partitionedIndexStates; - } - - public Map getUnPartitionedIndexStates() { - return unPartitionedIndexStates; - } - - public EsShardPartitions getIndexState(long partitionId) { - if (partitionIdToIndices.containsKey(partitionId)) { - return partitionedIndexStates.get(partitionIdToIndices.get(partitionId)); - } - return null; - } - - public EsShardPartitions getIndexState(String indexName) { - if (partitionedIndexStates.containsKey(indexName)) { - return partitionedIndexStates.get(indexName); - } - return unPartitionedIndexStates.get(indexName); - } -} diff --git a/fe/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/src/main/java/org/apache/doris/planner/EsScanNode.java index 9bbaeb5eada5cd..aa04ec66164b9c 100644 --- a/fe/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -28,7 +28,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.external.elasticsearch.EsShardPartitions; import org.apache.doris.external.elasticsearch.EsShardRouting; -import org.apache.doris.external.elasticsearch.EsTableState; +import org.apache.doris.external.elasticsearch.EsTablePartitions; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TEsScanNode; import org.apache.doris.thrift.TEsScanRange; @@ -65,7 +65,7 @@ public class EsScanNode extends ScanNode { private final Random random = new Random(System.currentTimeMillis()); private Multimap backendMap; private List backendList; - private EsTableState esTableState; + private EsTablePartitions esTablePartitions; private List shardScanRanges = Lists.newArrayList(); private EsTable table; @@ -74,7 +74,7 @@ public class EsScanNode extends ScanNode { public EsScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) { super(id, desc, planNodeName); table = (EsTable) (desc.getTable()); - esTableState = table.getEsTableState(); + esTablePartitions = table.getEsTablePartitions(); } @Override @@ -147,23 +147,23 @@ private void assignBackends() throws UserException { // only do partition(es index level) prune private List getShardLocations() throws UserException { // has to get partition info from es state not from table because the partition info is generated from es cluster state dynamically - if (esTableState == null) { + if (esTablePartitions == null) { if (table.getLastMetaDataSyncException() != null) { throw new UserException("fetch es table [" + table.getName() + "] metadata failure: " + table.getLastMetaDataSyncException().getLocalizedMessage()); } throw new UserException("EsTable metadata has not been synced, Try it later"); } - Collection partitionIds = partitionPrune(esTableState.getPartitionInfo()); + Collection partitionIds = partitionPrune(esTablePartitions.getPartitionInfo()); List selectedIndex = Lists.newArrayList(); ArrayList unPartitionedIndices = Lists.newArrayList(); ArrayList partitionedIndices = Lists.newArrayList(); - for (EsShardPartitions esShardPartitions : esTableState.getUnPartitionedIndexStates().values()) { + for (EsShardPartitions esShardPartitions : esTablePartitions.getUnPartitionedIndexStates().values()) { selectedIndex.add(esShardPartitions); unPartitionedIndices.add(esShardPartitions.getIndexName()); } if (partitionIds != null) { for (Long partitionId : partitionIds) { - EsShardPartitions indexState = esTableState.getIndexState(partitionId); + EsShardPartitions indexState = esTablePartitions.getEsShardPartitions(partitionId); selectedIndex.add(indexState); partitionedIndices.add(indexState.getIndexName()); } diff --git a/fe/src/test/java/org/apache/doris/external/elasticsearch/EsStateStoreTest.java b/fe/src/test/java/org/apache/doris/external/elasticsearch/EsRepositoryTest.java similarity index 90% rename from fe/src/test/java/org/apache/doris/external/elasticsearch/EsStateStoreTest.java rename to fe/src/test/java/org/apache/doris/external/elasticsearch/EsRepositoryTest.java index a1f6abe6ab10ba..a7ce9610de2e72 100644 --- a/fe/src/test/java/org/apache/doris/external/elasticsearch/EsStateStoreTest.java +++ b/fe/src/test/java/org/apache/doris/external/elasticsearch/EsRepositoryTest.java @@ -40,7 +40,7 @@ import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; -public class EsStateStoreTest { +public class EsRepositoryTest { private static FakeEditLog fakeEditLog; private static FakeCatalog fakeCatalog; @@ -48,7 +48,7 @@ public class EsStateStoreTest { private static String mappingsStr = ""; private static String es7MappingsStr = ""; private static String searchShardsStr = ""; - private EsStateStore esStateStore; + private EsRepository esRepository; private EsRestClient fakeClient; @BeforeClass @@ -70,7 +70,7 @@ public static void init() throws IOException, InstantiationException, IllegalAcc @Before public void setUp() { - esStateStore = new EsStateStore(); + esRepository = new EsRepository(); fakeClient = new EsRestClient(new String[]{"localhost:9200"}, null, null); } @@ -81,7 +81,7 @@ public void testSetEsTableContext() throws Exception { .getTable(CatalogTestUtil.testEsTableId1); // es5 EsFieldInfos fieldInfos = EsFieldInfos.fromMapping(esTable.getFullSchema(), esTable.getIndexName(), mappingsStr, esTable.getMappingType()); - esStateStore.setEsTableContext(fieldInfos, esTable); + esTable.addFieldInfos(fieldInfos); assertEquals("userId.keyword", esTable.fieldsContext().get("userId")); assertEquals("userId.keyword", esTable.docValueContext().get("userId")); // es7 @@ -106,14 +106,14 @@ public void testSetTableState() throws ExternalDataSourceException, DdlException .getDb(CatalogTestUtil.testDb1) .getTable(CatalogTestUtil.testEsTableId1); EsShardPartitions esShardPartitions = EsShardPartitions.findShardPartitions(esTable.getIndexName(), searchShardsStr); - EsTableState esTableState = esStateStore.setPartitionInfo(esTable, esShardPartitions); - assertNotNull(esTableState); - assertEquals(1, esTableState.getUnPartitionedIndexStates().size()); - assertEquals(5, esTableState.getIndexState("indexa").getShardRoutings().size()); + EsTablePartitions esTablePartitions = EsTablePartitions.fromShardPartitions(esTable, esShardPartitions); + assertNotNull(esTablePartitions); + assertEquals(1, esTablePartitions.getUnPartitionedIndexStates().size()); + assertEquals(5, esTablePartitions.getEsShardPartitions("indexa").getShardRoutings().size()); } private static String loadJsonFromFile(String fileName) throws IOException, URISyntaxException { - File file = new File(EsStateStoreTest.class.getClassLoader().getResource(fileName).toURI()); + File file = new File(EsRepositoryTest.class.getClassLoader().getResource(fileName).toURI()); InputStream is = new FileInputStream(file); BufferedReader br = new BufferedReader(new InputStreamReader(is)); StringBuilder jsonStr = new StringBuilder(); From 5e42b5b4a51fc25f522efbc573cb06f39b32824b Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 22 Jun 2020 10:39:42 +0800 Subject: [PATCH 17/21] MOD: EsRepository logic moves to EsTable --- .../org/apache/doris/catalog/EsTable.java | 26 +++++++++++++ .../external/elasticsearch/EsRepository.java | 38 ++++++------------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/src/main/java/org/apache/doris/catalog/EsTable.java index 7c25ebf9b919ed..d6b80e49c07910 100644 --- a/fe/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/EsTable.java @@ -22,6 +22,8 @@ import org.apache.doris.common.io.Text; import org.apache.doris.external.elasticsearch.EsFieldInfos; import org.apache.doris.external.elasticsearch.EsMajorVersion; +import org.apache.doris.external.elasticsearch.EsNodeInfo; +import org.apache.doris.external.elasticsearch.EsShardPartitions; import org.apache.doris.external.elasticsearch.EsTablePartitions; import org.apache.doris.thrift.TEsTable; import org.apache.doris.thrift.TTableDescriptor; @@ -403,4 +405,28 @@ public Throwable getLastMetaDataSyncException() { public void setLastMetaDataSyncException(Throwable lastMetaDataSyncException) { this.lastMetaDataSyncException = lastMetaDataSyncException; } + + /** + * set es meta from remote + * @param fieldInfos contains docValue and fields + * @param esShardPartitions shard partitions for search + * @param nodesInfo http-nodes info + */ + public void setRemoteMeta(EsFieldInfos fieldInfos, EsShardPartitions esShardPartitions, Map nodesInfo) { + try { + if (this.enableKeywordSniff || this.enableDocValueScan) { + addFieldInfos(fieldInfos); + } + + this.esTablePartitions = EsTablePartitions.fromShardPartitions(this, esShardPartitions); + + if (EsTable.TRANSPORT_HTTP.equals(getTransport())) { + this.esTablePartitions.addHttpAddress(nodesInfo); + } + } catch (Throwable e) { + LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", this.name, e); + this.esTablePartitions = null; + this.lastMetaDataSyncException = e; + } + } } diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java index 90360c368cf3b5..be11487fb9846f 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java @@ -36,19 +36,19 @@ * It is used to call es api to get shard allocation state */ public class EsRepository extends MasterDaemon { - + private static final Logger LOG = LogManager.getLogger(EsRepository.class); - + private Map esTables; - + private Map esClients; - + public EsRepository() { super("es state store", Config.es_state_sync_interval_second * 1000); esTables = Maps.newConcurrentMap(); esClients = Maps.newConcurrentMap(); } - + public void registerTable(EsTable esTable) { if (Catalog.isCheckpointThread()) { return; @@ -58,36 +58,22 @@ public void registerTable(EsTable esTable) { new EsRestClient(esTable.getSeeds(), esTable.getUserName(), esTable.getPasswd())); LOG.info("register a new table [{}] to sync list", esTable); } - + public void deRegisterTable(long tableId) { esTables.remove(tableId); esClients.remove(tableId); LOG.info("deregister table [{}] from sync list", tableId); } - + @Override protected void runAfterCatalogReady() { for (EsTable esTable : esTables.values()) { try { EsRestClient client = esClients.get(esTable.getId()); - - if (esTable.isKeywordSniffEnable() || esTable.isDocValueScanEnable()) { - EsFieldInfos fieldInfos = client.getFieldInfos(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); - if (fieldInfos == null) { - continue; - } - esTable.addFieldInfos(fieldInfos); - } - + EsFieldInfos fieldInfos = client.getFieldInfos(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); EsShardPartitions esShardPartitions = client.getShardPartitions(esTable.getIndexName()); - - EsTablePartitions esTablePartitions = EsTablePartitions.fromShardPartitions(esTable, esShardPartitions); - - if (EsTable.TRANSPORT_HTTP.equals(esTable.getTransport())) { - Map nodesInfo = client.getHttpNodes(); - esTablePartitions.addHttpAddress(nodesInfo); - } - esTable.setEsTablePartitions(esTablePartitions); + Map nodesInfo = client.getHttpNodes(); + esTable.setRemoteMeta(fieldInfos, esShardPartitions, nodesInfo); } catch (Throwable e) { LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", esTable.getName(), e); esTable.setEsTablePartitions(null); @@ -95,7 +81,7 @@ protected void runAfterCatalogReady() { } } } - + // should call this method to init the state store after loading image // the rest of tables will be added or removed by replaying edit log // when fe is start to load image, should call this method to init the state store @@ -106,7 +92,7 @@ public void loadTableFromCatalog() { List dbIds = Catalog.getCurrentCatalog().getDbIds(); for (Long dbId : dbIds) { Database database = Catalog.getCurrentCatalog().getDb(dbId); - + List
tables = database.getTables(); for (Table table : tables) { if (table.getType() == TableType.ELASTICSEARCH) { From a7ba9d1eb8838a7201a359d400b455ef7dacaa64 Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 23 Jun 2020 14:00:27 +0800 Subject: [PATCH 18/21] MOD: Abstract code, change the function name --- .../main/java/org/apache/doris/catalog/EsTable.java | 12 +++++++----- .../doris/external/elasticsearch/EsRepository.java | 7 ++----- .../doris/external/elasticsearch/EsRestClient.java | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/src/main/java/org/apache/doris/catalog/EsTable.java index d6b80e49c07910..8c34445e352058 100644 --- a/fe/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/EsTable.java @@ -23,6 +23,7 @@ import org.apache.doris.external.elasticsearch.EsFieldInfos; import org.apache.doris.external.elasticsearch.EsMajorVersion; import org.apache.doris.external.elasticsearch.EsNodeInfo; +import org.apache.doris.external.elasticsearch.EsRestClient; import org.apache.doris.external.elasticsearch.EsShardPartitions; import org.apache.doris.external.elasticsearch.EsTablePartitions; import org.apache.doris.thrift.TEsTable; @@ -407,13 +408,14 @@ public void setLastMetaDataSyncException(Throwable lastMetaDataSyncException) { } /** - * set es meta from remote - * @param fieldInfos contains docValue and fields - * @param esShardPartitions shard partitions for search - * @param nodesInfo http-nodes info + * sync es index meta from remote + * @param client esRestClient */ - public void setRemoteMeta(EsFieldInfos fieldInfos, EsShardPartitions esShardPartitions, Map nodesInfo) { + public void syncESIndexMeta(EsRestClient client) { try { + EsFieldInfos fieldInfos = client.getFieldInfos(this.indexName, this.mappingType, this.fullSchema); + EsShardPartitions esShardPartitions = client.getShardPartitions(this.indexName); + Map nodesInfo = client.getHttpNodes(); if (this.enableKeywordSniff || this.enableDocValueScan) { addFieldInfos(fieldInfos); } diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java index be11487fb9846f..fe574ce5b73554 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java @@ -44,7 +44,7 @@ public class EsRepository extends MasterDaemon { private Map esClients; public EsRepository() { - super("es state store", Config.es_state_sync_interval_second * 1000); + super("es repository", Config.es_state_sync_interval_second * 1000); esTables = Maps.newConcurrentMap(); esClients = Maps.newConcurrentMap(); } @@ -70,10 +70,7 @@ protected void runAfterCatalogReady() { for (EsTable esTable : esTables.values()) { try { EsRestClient client = esClients.get(esTable.getId()); - EsFieldInfos fieldInfos = client.getFieldInfos(esTable.getIndexName(), esTable.getMappingType(), esTable.getFullSchema()); - EsShardPartitions esShardPartitions = client.getShardPartitions(esTable.getIndexName()); - Map nodesInfo = client.getHttpNodes(); - esTable.setRemoteMeta(fieldInfos, esShardPartitions, nodesInfo); + esTable.syncESIndexMeta(client); } catch (Throwable e) { LOG.warn("Exception happens when fetch index [{}] meta data from remote es cluster", esTable.getName(), e); esTable.setEsTablePartitions(null); diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index ce354cc04efeb4..97585bb604bfdd 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -91,13 +91,13 @@ public Map getHttpNodes() throws Exception { return nodesMap; } - public EsFieldInfos getFieldInfos(String indexName, String mappingType, List colList) throws Exception { + public EsFieldInfos getFieldInfos(String indexName, String docType, List colList) throws Exception { String path = indexName + "/_mapping"; String indexMapping = execute(path); if (indexMapping == null) { throw new ExternalDataSourceException( "index[" + indexName + "] _mapping not found for the Elasticsearch Cluster"); } - return EsFieldInfos.fromMapping(colList, indexName, indexMapping, mappingType); + return EsFieldInfos.fromMapping(colList, indexName, indexMapping, docType); } From 303d0a2f9627c0e6b9efe1c3000fd4571a07cab8 Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 23 Jun 2020 14:19:12 +0800 Subject: [PATCH 19/21] MOD: format --- .../org/apache/doris/external/elasticsearch/EsRestClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index 97585bb604bfdd..5f7f75cc3d1d89 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -159,12 +159,12 @@ private String execute(String path) throws Exception { return null; } - public T get(String q, String key) throws Exception { + public T get(String q, String key) throws Exception { return parseContent(execute(q), key); } @SuppressWarnings("unchecked") - private T parseContent(String response, String key) { + private T parseContent(String response, String key) { Map map = Collections.emptyMap(); try { JsonParser jsonParser = mapper.getJsonFactory().createJsonParser(response); From 15ba2d390b1d8f7de5f9978a7cc63eeb0f571f2d Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 23 Jun 2020 19:53:43 +0800 Subject: [PATCH 20/21] MOD: mod error info --- .../org/apache/doris/external/elasticsearch/EsRestClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java index 5f7f75cc3d1d89..adfc294faa34b9 100644 --- a/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java @@ -95,7 +95,7 @@ public EsFieldInfos getFieldInfos(String indexName, String docType, List String path = indexName + "/_mapping"; String indexMapping = execute(path); if (indexMapping == null) { - throw new ExternalDataSourceException( "index[" + indexName + "] _mapping not found for the Elasticsearch Cluster"); + throw new ExternalDataSourceException( "index[" + indexName + "] not found for the Elasticsearch Cluster"); } return EsFieldInfos.fromMapping(colList, indexName, indexMapping, docType); } @@ -105,7 +105,7 @@ public EsShardPartitions getShardPartitions(String indexName) throws Exception { String path = indexName + "/_search_shards"; String searchShards = execute(path); if (searchShards == null) { - throw new ExternalDataSourceException( "index[" + indexName + "] _search_shards not found for the Elasticsearch Cluster"); + throw new ExternalDataSourceException( "index[" + indexName + "] search_shards not found for the Elasticsearch Cluster"); } return EsShardPartitions.findShardPartitions(indexName, searchShards); } From 5fbae715aa676d0d39f5906e840ca593e3ba1da2 Mon Sep 17 00:00:00 2001 From: stalary Date: Wed, 24 Jun 2020 16:57:43 +0800 Subject: [PATCH 21/21] MOD: format code --- fe/src/main/java/org/apache/doris/catalog/EsTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/src/main/java/org/apache/doris/catalog/EsTable.java index 8c34445e352058..2a26d51b71a2fc 100644 --- a/fe/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/EsTable.java @@ -102,7 +102,7 @@ public class EsTable extends Table { // use select city from table, if enable the docvalue, we will fetch the `city` field value from `city.raw` private Map docValueContext = new HashMap<>(); - private Map fieldsContext= new HashMap<>(); + private Map fieldsContext = new HashMap<>(); // record the latest and recently exception when sync ES table metadata (mapping, shard location) private Throwable lastMetaDataSyncException = null;