From 6efb00049289c7ef8bb325b46f1144683af7f2b0 Mon Sep 17 00:00:00 2001 From: Salieri1969 Date: Wed, 19 Dec 2018 15:41:20 +0800 Subject: [PATCH 1/7] add es scan node --- be/src/exec/es_scan_node.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index 39492fd2e77eb7..d83db6a4d3c6fc 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -520,7 +520,6 @@ bool EsScanNode::get_disjuncts(ExprContext* context, Expr* conjunct, } } -bool EsScanNode::is_match_func(Expr* conjunct) { if (TExprNodeType::FUNCTION_CALL == conjunct->node_type() && conjunct->fn().name.function_name == "esquery") { return true; From 316c8240d6e78510389a7d5048ea86ea5bc9201d Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 14 Jan 2019 16:38:18 +0800 Subject: [PATCH 2/7] Add esquery function support --- be/src/exec/es_scan_node.cpp | 1 + be/src/exec/es_scan_node.h | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index d83db6a4d3c6fc..7bb261433e40f6 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -350,6 +350,7 @@ Status EsScanNode::open_es(TNetworkAddress& address, TExtOpenResult& result, TEx } try { + VLOG(1) << "es open param=" << apache::thrift::ThriftDebugString(params); client->open(result, params); } catch (apache::thrift::transport::TTransportException& e) { LOG(WARNING) << "es open retrying, because: " << e.what(); diff --git a/be/src/exec/es_scan_node.h b/be/src/exec/es_scan_node.h index 18640a006d40b8..2b68b819d6ccd2 100644 --- a/be/src/exec/es_scan_node.h +++ b/be/src/exec/es_scan_node.h @@ -68,6 +68,12 @@ class EsScanNode : public ScanNode { // 1. check if left conjuncts contain "match" function, since match function could only be executed on es bool check_left_conjuncts(Expr* conjunct); + bool is_match_func(Expr* conjunct); + + // check if open result meets condition + // 1. check if left conjuncts contain "match" function, since match function could only be executed on es + bool check_left_conjuncts(Expr* conjunct); + private: TupleId _tuple_id; std::map _properties; From bee4d54b586257b3338bd7749347a0ab6eb78c40 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Wed, 16 Jan 2019 17:36:52 +0800 Subject: [PATCH 3/7] Add support for in predicate --- be/src/exec/es_scan_node.cpp | 1 - be/src/exec/es_scan_node.h | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index 7bb261433e40f6..d83db6a4d3c6fc 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -350,7 +350,6 @@ Status EsScanNode::open_es(TNetworkAddress& address, TExtOpenResult& result, TEx } try { - VLOG(1) << "es open param=" << apache::thrift::ThriftDebugString(params); client->open(result, params); } catch (apache::thrift::transport::TTransportException& e) { LOG(WARNING) << "es open retrying, because: " << e.what(); diff --git a/be/src/exec/es_scan_node.h b/be/src/exec/es_scan_node.h index 2b68b819d6ccd2..001c577ecb0879 100644 --- a/be/src/exec/es_scan_node.h +++ b/be/src/exec/es_scan_node.h @@ -70,6 +70,8 @@ class EsScanNode : public ScanNode { bool is_match_func(Expr* conjunct); + SlotDescriptor* get_slot_desc(SlotRef* slotRef); + // check if open result meets condition // 1. check if left conjuncts contain "match" function, since match function could only be executed on es bool check_left_conjuncts(Expr* conjunct); From c91abd9150ee8817cad333d39a993defff5078dc Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 18 Jan 2019 11:57:18 +0800 Subject: [PATCH 4/7] EsScanNode should return instance num --- be/src/exec/es_scan_node.cpp | 1 - .../java/org/apache/doris/planner/EsScanNode.java | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index d83db6a4d3c6fc..71ee5624dc6785 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -217,7 +217,6 @@ Status EsScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) // get batch TExtGetNextResult result; RETURN_IF_ERROR(get_next_from_es(result)); - VLOG(1) << "es get next success: result=" << apache::thrift::ThriftDebugString(result); _offsets[_scan_range_idx] += result.rows.num_rows; // convert diff --git a/fe/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/src/main/java/org/apache/doris/planner/EsScanNode.java index cc5079eeef0301..6eb6eae2ceae52 100644 --- a/fe/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -100,15 +100,15 @@ public void init(Analyzer analyzer) throws UserException { assignBackends(); } + + @Override + public int getNumInstances() { + return shardScanRanges.size(); + } @Override public List getScanRangeLocations(long maxScanRangeLength) { - try { - return getShardLocations(); - } catch (UserException e) { - LOG.error("errors while get es shard locations", e); - } - return null; + return shardScanRanges; } @Override From fc5747b4d24cfe9ce6315823a334a07fa504576b Mon Sep 17 00:00:00 2001 From: Salieri1969 Date: Mon, 28 Jan 2019 13:10:27 +0800 Subject: [PATCH 5/7] Add largeint support --- be/src/exec/es_scan_node.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index 71ee5624dc6785..1c25bdbfe5e702 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -746,6 +746,12 @@ Status EsScanNode::materialize_row(MemPool* tuple_pool, Tuple* tuple, } *reinterpret_cast(slot) = col.long_vals[val_idx]; break; + case TYPE_LARGEINT: + if (val_idx >= col.long_vals.size()) { + return Status(strings::Substitute(ERROR_INVALID_COL_DATA, "LARGEINT")); + } + *reinterpret_cast(slot) = col.long_vals[val_idx]; + break; case TYPE_DOUBLE: if (val_idx >= col.double_vals.size()) { return Status(strings::Substitute(ERROR_INVALID_COL_DATA, "DOUBLE")); From e6fd83145e2753b9dda5ebcee349d910bf05bf4a Mon Sep 17 00:00:00 2001 From: Salieri1969 Date: Tue, 29 Jan 2019 17:21:27 +0800 Subject: [PATCH 6/7] Fix date display --- be/src/exec/es_scan_node.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index 1c25bdbfe5e702..ac49e03c6973c9 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -771,10 +771,16 @@ Status EsScanNode::materialize_row(MemPool* tuple_pool, Tuple* tuple, *reinterpret_cast(slot) = col.bool_vals[val_idx]; break; case TYPE_DATE: + if (val_idx >= col.long_vals.size() || + !reinterpret_cast(slot)->from_unixtime(col.long_vals[val_idx])) { + return Status(strings::Substitute(ERROR_INVALID_COL_DATA, "TYPE_DATE")); + } + reinterpret_cast(slot)->cast_to_date(); + break; case TYPE_DATETIME: { if (val_idx >= col.long_vals.size() || !reinterpret_cast(slot)->from_unixtime(col.long_vals[val_idx])) { - return Status(strings::Substitute(ERROR_INVALID_COL_DATA, "TYPE_DATE|TYPE_DATETIME")); + return Status(strings::Substitute(ERROR_INVALID_COL_DATA, "TYPE_DATETIME")); } break; } From a98dd83ddb948e53b0fb63f20e93cedaacd15097 Mon Sep 17 00:00:00 2001 From: Salieri1969 Date: Mon, 25 Feb 2019 16:13:55 +0800 Subject: [PATCH 7/7] Fix merge conflicts --- be/src/exec/es_scan_node.cpp | 1 + be/src/exec/es_scan_node.h | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/be/src/exec/es_scan_node.cpp b/be/src/exec/es_scan_node.cpp index ac49e03c6973c9..b4ae0a8e5a499c 100644 --- a/be/src/exec/es_scan_node.cpp +++ b/be/src/exec/es_scan_node.cpp @@ -519,6 +519,7 @@ bool EsScanNode::get_disjuncts(ExprContext* context, Expr* conjunct, } } +bool EsScanNode::is_match_func(Expr* conjunct) { if (TExprNodeType::FUNCTION_CALL == conjunct->node_type() && conjunct->fn().name.function_name == "esquery") { return true; diff --git a/be/src/exec/es_scan_node.h b/be/src/exec/es_scan_node.h index 001c577ecb0879..64c3abc0edb57c 100644 --- a/be/src/exec/es_scan_node.h +++ b/be/src/exec/es_scan_node.h @@ -60,13 +60,6 @@ class EsScanNode : public ScanNode { bool get_disjuncts(ExprContext* context, Expr* conjunct, vector& disjuncts); bool to_ext_literal(ExprContext* context, Expr* expr, TExtLiteral* literal); - bool is_match_func(Expr* conjunct); - - SlotDescriptor* get_slot_desc(SlotRef* slotRef); - - // check if open result meets condition - // 1. check if left conjuncts contain "match" function, since match function could only be executed on es - bool check_left_conjuncts(Expr* conjunct); bool is_match_func(Expr* conjunct);