[enhancement](nereids)support null partition for list partition

be/src/exec/tablet_info.cpp

@@ -521,6 +521,11 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr,
         column->insert_data(reinterpret_cast<const char*>(&t_expr.bool_literal.value), 0);
         break;
     }
+    case TExprNodeType::NULL_LITERAL: {
+        // insert a null literal
+        column->insert_data(nullptr, 0);
+        break;
+    }
     default: {
         return Status::InternalError("unsupported partition column node type, type={}",
                                      t_expr.node_type);

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4

@@ -561,7 +561,7 @@ constantSeq
     ;
 
 partitionValueDef
-    : INTEGER_VALUE | STRING_LITERAL | MAXVALUE
+    : INTEGER_VALUE | STRING_LITERAL | MAXVALUE | NULL
     ;
 
 rollupDefs

fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java

@@ -208,9 +208,6 @@ public void analyze(List<ColumnDef> columnDefs, Map<String, String> otherPropert
                         throw new AnalysisException(
                                 "The partition column must be NOT NULL with allow_partition_column_nullable OFF");
                     }
-                    if (this instanceof ListPartitionDesc && columnDef.isAllowNull()) {
-                        throw new AnalysisException("The list partition column must be NOT NULL");
-                    }
                     if (this instanceof RangePartitionDesc && partitionExprs != null) {
                         if (partitionExprs.get(0) instanceof FunctionCallExpr) {
                             if (!columnDef.getType().isDateType()) {

fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java

@@ -27,6 +27,7 @@ public class PartitionValue {
 
     private String value;
     private boolean isHiveDefaultPartition;
+    private boolean isNullPartition;
 
     private PartitionValue() {
 
@@ -45,6 +46,12 @@ public PartitionValue(String value, boolean isHiveDefaultPartition) {
         this.isHiveDefaultPartition = isHiveDefaultPartition;
     }
 
+    public PartitionValue(String value, boolean isNullPartition, boolean isHiveDefaultPartition) {
+        this.value = value;
+        this.isNullPartition = isNullPartition;
+        this.isHiveDefaultPartition = isHiveDefaultPartition;
+    }
+
     public LiteralExpr getValue(Type type) throws AnalysisException {
         if (isHiveDefaultPartition) {
             return new StringLiteral(value);
@@ -81,12 +88,16 @@ public boolean equals(Object o) {
             return false;
         }
         PartitionValue that = (PartitionValue) o;
-        return isHiveDefaultPartition == that.isHiveDefaultPartition
+        return isHiveDefaultPartition == that.isHiveDefaultPartition && isNullPartition == that.isNullPartition
                 && Objects.equal(value, that.value);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hashCode(value, isHiveDefaultPartition);
+        return Objects.hashCode(value, isHiveDefaultPartition, isNullPartition);
+    }
+
+    public boolean isNullPartition() {
+        return isNullPartition;
     }
 }

fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java

@@ -23,6 +23,7 @@
 import org.apache.doris.analysis.LargeIntLiteral;
 import org.apache.doris.analysis.LiteralExpr;
 import org.apache.doris.analysis.MaxLiteral;
+import org.apache.doris.analysis.NullLiteral;
 import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.analysis.StringLiteral;
 import org.apache.doris.common.AnalysisException;
@@ -137,7 +138,12 @@ public static PartitionKey createListPartitionKeyWithTypes(List<PartitionValue>
 
         PartitionKey partitionKey = new PartitionKey();
         for (int i = 0; i < values.size(); i++) {
-            partitionKey.keys.add(values.get(i).getValue(types.get(i)));
+            if (values.get(i).isNullPartition()) {
+                partitionKey.keys.add(NullLiteral.create(types.get(i)));
+            } else {
+                partitionKey.keys.add(values.get(i).getValue(types.get(i)));
+            }
+
             if (isHive) {
                 partitionKey.originHiveKeys.add(values.get(i).getStringValue());
             }
@@ -267,7 +273,7 @@ public String toSql() {
         int i = 0;
         for (LiteralExpr expr : keys) {
             Object value = null;
-            if (expr == MaxLiteral.MAX_VALUE) {
+            if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
                 value = expr.toSql();
                 sb.append(value);
                 continue;
@@ -312,7 +318,7 @@ public static String toString(List<LiteralExpr> keys) {
         int i = 0;
         for (LiteralExpr expr : keys) {
             Object value = null;
-            if (expr == MaxLiteral.MAX_VALUE) {
+            if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
                 value = expr.toSql();
             } else {
                 value = expr.getRealValue();
@@ -341,9 +347,17 @@ public void write(DataOutput out) throws IOException {
         out.writeInt(count);
         for (int i = 0; i < count; i++) {
             PrimitiveType type = types.get(i);
-            Text.writeString(out, type.toString());
+            if (keys.get(i).isNullLiteral()) {
+                Text.writeString(out, Type.NULL.toString());
+            } else {
+                Text.writeString(out, type.toString());
+            }
+
             if (keys.get(i) == MaxLiteral.MAX_VALUE) {
                 out.writeBoolean(true);
+            } else if (keys.get(i).isNullLiteral()) {
+                out.writeBoolean(false);
+                Text.writeString(out, type.toString());
             } else {
                 out.writeBoolean(false);
                 keys.get(i).write(out);
@@ -355,10 +369,16 @@ public void readFields(DataInput in) throws IOException {
         int count = in.readInt();
         for (int i = 0; i < count; i++) {
             PrimitiveType type = PrimitiveType.valueOf(Text.readString(in));
-            types.add(type);
-
-            LiteralExpr literal = null;
             boolean isMax = in.readBoolean();
+            if (type == PrimitiveType.NULL_TYPE) {
+                String realType = StringLiteral.read(in).getStringValue();
+                type = PrimitiveType.valueOf(realType);
+                types.add(type);
+                keys.add(NullLiteral.create(Type.fromPrimitiveType(type)));
+                continue;
+            }
+            LiteralExpr literal = null;
+            types.add(type);
             if (isMax) {
                 literal = MaxLiteral.MAX_VALUE;
             } else {
@@ -480,47 +500,54 @@ public JsonElement serialize(PartitionKey partitionKey, java.lang.reflect.Type r
             for (int i = 0; i < count; i++) {
                 JsonArray typeAndKey = new JsonArray();
                 PrimitiveType type = types.get(i);
-                typeAndKey.add(new JsonPrimitive(type.toString()));
-
-                if (keys.get(i) == MaxLiteral.MAX_VALUE) {
-                    typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
+                if (keys.get(i).isNullLiteral()) {
+                    // save NULL_TYPE as type and real type as key
+                    typeAndKey.add(new JsonPrimitive(PrimitiveType.NULL_TYPE.toString()));
+                    typeAndKey.add(new JsonPrimitive(type.toString()));
                 } else {
-                    switch (type) {
-                        case TINYINT:
-                        case SMALLINT:
-                        case INT:
-                        case BIGINT: {
-                            IntLiteral key = (IntLiteral) keys.get(i);
-                            typeAndKey.add(new JsonPrimitive(key.getLongValue()));
-                        }
-                            break;
-                        case LARGEINT: {
-                            LargeIntLiteral key = (LargeIntLiteral) keys.get(i);
-                            typeAndKey.add(new JsonPrimitive(key.getRealValue().toString()));
-                        }
-                            break;
-                        case DATE:
-                        case DATETIME:
-                        case DATEV2:
-                        case DATETIMEV2: {
-                            DateLiteral key = (DateLiteral) keys.get(i);
-                            typeAndKey.add(new JsonPrimitive(key.convertToString(type)));
-                        }
-                            break;
-                        case CHAR:
-                        case VARCHAR:
-                        case STRING: {
-                            StringLiteral key = (StringLiteral) keys.get(i);
-                            typeAndKey.add(new JsonPrimitive(key.getValue()));
-                        }
-                            break;
-                        case BOOLEAN: {
-                            BoolLiteral key = (BoolLiteral) keys.get(i);
-                            typeAndKey.add(new JsonPrimitive(key.getValue()));
+                    typeAndKey.add(new JsonPrimitive(type.toString()));
+
+                    if (keys.get(i) == MaxLiteral.MAX_VALUE) {
+                        typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
+                    } else {
+                        switch (type) {
+                            case TINYINT:
+                            case SMALLINT:
+                            case INT:
+                            case BIGINT: {
+                                IntLiteral key = (IntLiteral) keys.get(i);
+                                typeAndKey.add(new JsonPrimitive(key.getLongValue()));
+                            }
+                                break;
+                            case LARGEINT: {
+                                LargeIntLiteral key = (LargeIntLiteral) keys.get(i);
+                                typeAndKey.add(new JsonPrimitive(key.getRealValue().toString()));
+                            }
+                                break;
+                            case DATE:
+                            case DATETIME:
+                            case DATEV2:
+                            case DATETIMEV2: {
+                                DateLiteral key = (DateLiteral) keys.get(i);
+                                typeAndKey.add(new JsonPrimitive(key.convertToString(type)));
+                            }
+                                break;
+                            case CHAR:
+                            case VARCHAR:
+                            case STRING: {
+                                StringLiteral key = (StringLiteral) keys.get(i);
+                                typeAndKey.add(new JsonPrimitive(key.getValue()));
+                            }
+                                break;
+                            case BOOLEAN: {
+                                BoolLiteral key = (BoolLiteral) keys.get(i);
+                                typeAndKey.add(new JsonPrimitive(key.getValue()));
+                            }
+                                break;
+                            default:
+                                throw new JsonParseException(
+                                        "type[" + type.name() + "] not supported: ");
                         }
-                            break;
-                        default:
-                            throw new JsonParseException("type[" + type.name() + "] not supported: ");
                     }
                 }
 

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java

@@ -2673,6 +2673,8 @@ public Expression visitPartitionValueDef(PartitionValueDefContext ctx) {
             return Literal.of(toStringValue(ctx.STRING_LITERAL().getText()));
         } else if (ctx.MAXVALUE() != null) {
             return MaxValue.INSTANCE;
+        } else if (ctx.NULL() != null) {
+            return Literal.of(null);
         }
         throw new AnalysisException("Unsupported partition value: " + ctx.getText());
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java

@@ -291,6 +291,8 @@ public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) {
         DataType dataType = DataType.fromCatalogType(type);
         if (literalExpr instanceof org.apache.doris.analysis.MaxLiteral) {
             return new MaxLiteral(dataType);
+        } else if (literalExpr instanceof org.apache.doris.analysis.NullLiteral) {
+            return new NullLiteral(dataType);
         }
         String stringValue = literalExpr.getStringValue();
         if (dataType.isBooleanType()) {

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java

@@ -666,9 +666,6 @@ private void validatePartitionColumn(ColumnDefinition column, ConnectContext ctx
             throw new AnalysisException(
                     "The partition column must be NOT NULL with allow_partition_column_nullable OFF");
         }
-        if (partitionType.equalsIgnoreCase(PartitionType.LIST.name()) && column.isNullable()) {
-            throw new AnalysisException("The list partition column must be NOT NULL");
-        }
     }
 
     // if auto bucket auto bucket enable, rewrite distribution bucket num &&

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java

@@ -161,7 +161,7 @@ public void setPartitionTypes(List<DataType> partitionTypes) {
      */
     protected PartitionValue toLegacyPartitionValueStmt(Expression e) {
         if (e.isLiteral()) {
-            return new PartitionValue(((Literal) e).getStringValue());
+            return new PartitionValue(((Literal) e).getStringValue(), e.isNullLiteral(), false);
         } else if (e instanceof MaxValue) {
             return PartitionValue.MAX_VALUE;
         }

fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java

@@ -20,6 +20,8 @@
 import org.apache.doris.alter.SchemaChangeHandler;
 import org.apache.doris.analysis.Analyzer;
 import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.NullLiteral;
 import org.apache.doris.analysis.SlotDescriptor;
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.catalog.Column;
@@ -475,7 +477,12 @@ public static void setPartitionKeys(TOlapTablePartition tPartition, PartitionIte
             for (PartitionKey partitionKey : partitionKeys) {
                 List<TExprNode> tExprNodes = new ArrayList<>();
                 for (int i = 0; i < partColNum; i++) {
-                    tExprNodes.add(partitionKey.getKeys().get(i).treeToThrift().getNodes().get(0));
+                    LiteralExpr literalExpr = partitionKey.getKeys().get(i);
+                    if (literalExpr.isNullLiteral()) {
+                        tExprNodes.add(NullLiteral.create(literalExpr.getType()).treeToThrift().getNodes().get(0));
+                    } else {
+                        tExprNodes.add(literalExpr.treeToThrift().getNodes().get(0));
+                    }
                 }
                 tPartition.addToInKeys(tExprNodes);
                 tPartition.setIsDefaultPartition(partitionItem.isDefaultPartition());

regression-test/data/nereids_syntax_p0/list_partition_with_null.out

@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+\N
+\N
+
+-- !select2 --
+\N
+\N
+
+-- !select3 --
+SHANGHAI
+
+-- !select4 --
+1
+

regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("list_partition_with_null") {
+    sql "SET enable_fallback_to_original_planner=false"
+    sql "SET enable_nereids_planner=true"
+
+    sql """DROP TABLE IF EXISTS table_null_list_parition;"""
+
+    sql """
+        CREATE TABLE table_null_list_parition
+        (
+            `USER_ID` LARGEINT  COMMENT "用户ID",
+            `CITY` VARCHAR(20)  COMMENT "用户所在城市"
+        )
+        ENGINE=OLAP
+        DUPLICATE KEY(`USER_ID`, `CITY`)
+        PARTITION BY LIST(`USER_ID`, `CITY`)
+        (
+            PARTITION `P1_CITY` VALUES IN (("1", null), ("1", "SHANGHAI")),
+            PARTITION `P2_CITY` VALUES IN (("2", "BEIJING"), (null, "SHANGHAI")),
+            PARTITION `P3_CITY` VALUES IN ((null,null), ("3", "SHANGHAI"))
+        )
+        DISTRIBUTED BY HASH(`USER_ID`) BUCKETS 16
+        PROPERTIES
+        (
+            "replication_num" = "1"
+        );
+    """
+    sql """insert into table_null_list_parition values(1,null),(null,null),(null,'SHANGHAI');"""
+    explain {
+            sql("select * from table_null_list_parition where city is null;")
+            verbose true
+            contains("partitions=2/3 (P1_CITY,P3_CITY)")
+    }
+
+    explain {
+            sql("select * from table_null_list_parition where user_id is null;")
+            verbose true
+            contains("partitions=2/3 (P2_CITY,P3_CITY)")
+    }
+    explain {
+            sql("select * from table_null_list_parition where city is not null;")
+            verbose true
+            contains("partitions=3/3 (P1_CITY,P2_CITY,P3_CITY)")
+    }
+
+    explain {
+            sql("select * from table_null_list_parition where user_id is not null;")
+            verbose true
+            contains("partitions=3/3 (P1_CITY,P2_CITY,P3_CITY)")
+    }
+    qt_select1 """select city from table_null_list_parition where city is null order by city;"""
+    qt_select2 """select user_id from table_null_list_parition where user_id is null;"""
+    qt_select3 """select city from table_null_list_parition where city is not null;"""
+    qt_select4 """select user_id from table_null_list_parition where user_id is not null;"""
+}