[fix](ES Catalog)Do not extract doc_values of field with ignore_above setting (#40314)

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json

@@ -0,0 +1,32 @@
+{
+  "test1": "string_ignore_above_10",
+  "test2": "text_ignore_above_10",
+  "test3": 5.0,
+  "test4": "2022-08-08",
+  "test5": 3333.22,
+  "test6": "2022-08-08T12:10:10.151",
+  "c_bool": [true, false, true, true],
+  "c_byte": [1, -2, -3, 4],
+  "c_short": [128, 129, -129, -130],
+  "c_integer": [32768, 32769, -32769, -32770],
+  "c_long": [-1, 0, 1, 2],
+  "c_unsigned_long": [0, 1, 2, 3],
+  "c_float": [1.0, 1.1, 1.2, 1.3],
+  "c_half_float": [1, 2, 3, 4],
+  "c_double": [1, 2, 3, 4],
+  "c_scaled_float": [1, 2, 3, 4],
+  "c_date": ["2020-01-01", "2020-01-02"],
+  "c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
+  "c_keyword": ["a", "b", "c"],
+  "c_text": ["d", "e", "f"],
+  "c_ip": ["192.168.0.1", "127.0.0.1"],
+  "c_person": [
+    {"name": "Andy", "age": 18},
+    {"name": "Tim", "age": 28}
+  ],
+  "message": "I'm not null or empty",
+  "c_user": [
+    {"first": "John", "last":  "Smith"},
+    {"first": "Alice", "last":  "White"}
+  ]
+}

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json

@@ -1,4 +1,35 @@
 {
-  "field1": "value1",
-  "field2": "value2"
+  "test1": "string_ignore_above_10",
+  "test2": "text_ignore_above_10",
+  "test3": 6.0,
+  "test4": "2022-08-08",
+  "test5": "2022-08-11 12:10:10",
+  "test6": 1660191010000,
+  "test7": "2022-08-11 12:10:10",
+  "test8": "2022-08-11T12:10:10+09:00",
+  "test9": "4444.22",
+  "test10": "2022-08-08T12:10:10.151",
+  "c_bool": [true, false, true, true],
+  "c_byte": [1, -2, -3, 4],
+  "c_short": [128, 129, -129, -130],
+  "c_integer": [32768, 32769, -32769, -32770],
+  "c_long": [-1, 0, 1, 2],
+  "c_unsigned_long": [0, 1, 2, 3],
+  "c_float": [1.0, 1.1, 1.2, 1.3],
+  "c_half_float": [1, 2, 3, 4],
+  "c_double": [1, 2, 3, 4],
+  "c_scaled_float": [1, 2, 3, 4],
+  "c_date": ["2020-01-01", "2020-01-02"],
+  "c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"],
+  "c_keyword": ["a", "b", "c"],
+  "c_text": ["d", "e", "f"],
+  "c_ip": ["192.168.0.1", "127.0.0.1"],
+  "c_person": [
+    {"name": "Andy", "age": 18},
+    {"name": "Tim", "age": 28}
+  ],
+  "c_user": [
+    {"first": "John", "last":  "Smith"},
+    {"first": "Alice", "last":  "White"}
+  ]
 }

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data6.json

@@ -0,0 +1,4 @@
+{
+  "field1": "value1",
+  "field2": "value2"
+}

docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh

@@ -53,6 +53,7 @@ curl "http://${ES_5_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -
 curl "http://${ES_5_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
 # only difference between es5 and es6
 curl "http://${ES_5_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es5.json'
+curl "http://${ES_5_HOST}:9200/test1/doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4_es6.json'
 # put data for test2_20220808
 curl "http://${ES_5_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json'
 curl "http://${ES_5_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
@@ -88,6 +89,7 @@ curl "http://${ES_6_HOST}:9200/test2_20220809" -H "Content-Type:application/json
 curl "http://${ES_6_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json'
 curl "http://${ES_6_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
 curl "http://${ES_6_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es6.json'
+curl "http://${ES_6_HOST}:9200/test1/doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4_es6.json'
 # put data for test2_20220808
 curl "http://${ES_6_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json'
 curl "http://${ES_6_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json'
@@ -125,6 +127,7 @@ curl "http://${ES_7_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json"
 curl "http://${ES_7_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_7_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
 curl "http://${ES_7_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
+curl "http://${ES_7_HOST}:9200/test1/_doc/5" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
 # put data for test2_20220808
 curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
@@ -136,7 +139,7 @@ curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/2" -H "Content-Type:applicati
 curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
 curl "http://${ES_7_HOST}:9200/test2_20220809/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 # put data for test3_20231005
-curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
+curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data6.json'
 
 # put _meta for array
 curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
@@ -170,6 +173,7 @@ curl "http://${ES_8_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json"
 curl "http://${ES_8_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_8_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
 curl "http://${ES_8_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
+curl "http://${ES_8_HOST}:9200/test1/_doc/5" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
 # put data for test2_20220808
 curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
@@ -181,7 +185,7 @@ curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/2" -H "Content-Type:applicati
 curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
 curl "http://${ES_8_HOST}:9200/test2_20220809/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 # put data for test3_20231005
-curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
+curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data6.json'
 
 # put _meta for array
 curl "http://${ES_8_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"

docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json

@@ -7,14 +7,15 @@
     "doc": {
       "properties": {
         "test1": {
-          "type": "keyword"
+          "type": "keyword",
+          "ignore_above": 10
         },
         "test2": {
           "type": "text",
           "fields": {
             "keyword": {
               "type": "keyword",
-              "ignore_above": 256
+              "ignore_above": 10
             }
           }
         },

docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json

@@ -6,14 +6,15 @@
   "mappings": {
     "properties": {
       "test1": {
-        "type": "keyword"
+        "type": "keyword",
+        "ignore_above": 10
       },
       "test2": {
         "type": "text",
         "fields": {
           "keyword": {
             "type": "keyword",
-            "ignore_above": 256
+            "ignore_above": 10
           }
         }
       },

fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java

@@ -38,7 +38,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -51,7 +51,10 @@
 @Getter
 @Setter
 public class EsTable extends Table {
-    public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new HashSet<>(Collections.singletonList("text"));
+    // reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/doc-values.html
+    // https://www.elastic.co/guide/en/elasticsearch/reference/current/text.html
+    public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS =
+            new HashSet<>(Arrays.asList("text", "annotated_text", "match_only_text"));
 
     private static final Logger LOG = LogManager.getLogger(EsTable.class);
     // Solr doc_values vs stored_fields performance-smackdown indicate:

fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java

@@ -130,6 +130,16 @@ private static void resolveDocValuesFields(SearchContext searchContext, ObjectNo
                         if (docValue) {
                             docValueField = colName;
                         }
+                    } else if (innerTypeObject.has("ignore_above")) {
+                        // reference:
+                        // https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html#keyword-params
+                        // > ignore_above
+                        // > Do not index any string longer than this value. Defaults to 2147483647 so that all values
+                        // > would be accepted. Please however note that default dynamic mapping rules create a sub
+                        // > keyword field that overrides this default by setting ignore_above: 256.
+                        // this field has `ignore_above` param
+                        // Strings longer than the ignore_above setting will not be indexed or stored
+                        // so we cannot rely on its doc_values
                     } else {
                         // a : {c : {}} -> a -> a.c
                         docValueField = colName + "." + fieldName;
@@ -146,6 +156,17 @@ private static void resolveDocValuesFields(SearchContext searchContext, ObjectNo
             } else if (fieldType == null || "nested".equals(fieldType)) {
                 // The object field has no type, and nested not support doc value.
                 return;
+            } else if (fieldObject.has("ignore_above")) {
+                // reference:
+                // https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html#keyword-params
+                // > ignore_above
+                // > Do not index any string longer than this value. Defaults to 2147483647 so that all values
+                // > would be accepted. Please however note that default dynamic mapping rules create a sub
+                // > keyword field that overrides this default by setting ignore_above: 256.
+                // this field has `ignore_above` param
+                // Strings longer than the ignore_above setting will not be indexed or stored
+                // so we cannot rely on its doc_values
+                return;
             }
             docValueField = colName;
         }