diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index 6cc463bf658ef0..0e792563acf14a 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -104,7 +104,9 @@ class PredicateFunctionCreator { M(TYPE_DECIMAL32) \ M(TYPE_DECIMAL64) \ M(TYPE_DECIMAL128I) \ - M(TYPE_DECIMAL256) + M(TYPE_DECIMAL256) \ + M(TYPE_IPV4) \ + M(TYPE_IPV6) template typename Traits::BasePtr create_predicate_function(PrimitiveType type) { diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index d9a0b7b43ce432..67100e680dca2c 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -213,6 +213,10 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column, return valid_datetime(value_str, column.frac()); case FieldType::OLAP_FIELD_TYPE_BOOL: return valid_bool(value_str); + case FieldType::OLAP_FIELD_TYPE_IPV4: + return valid_ipv4(value_str); + case FieldType::OLAP_FIELD_TYPE_IPV6: + return valid_ipv6(value_str); default: LOG(WARNING) << "unknown field type. [type=" << int(field_type) << "]"; } diff --git a/be/src/olap/key_coder.cpp b/be/src/olap/key_coder.cpp index 168117117d9116..7f7fa9ea4a9d74 100644 --- a/be/src/olap/key_coder.cpp +++ b/be/src/olap/key_coder.cpp @@ -81,6 +81,8 @@ class KeyCoderResolver { add_mapping(); add_mapping(); add_mapping(); + add_mapping(); + add_mapping(); } template diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 3811aab378f6d8..07a6d2964814de 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -145,6 +145,8 @@ enum class FieldType { OLAP_FIELD_TYPE_VARIANT = 35, OLAP_FIELD_TYPE_AGG_STATE = 36, OLAP_FIELD_TYPE_DECIMAL256 = 37, + OLAP_FIELD_TYPE_IPV4 = 38, + OLAP_FIELD_TYPE_IPV6 = 39, }; // Define all aggregation methods supported by Field diff --git a/be/src/olap/predicate_creator.h b/be/src/olap/predicate_creator.h index dd9fded40eef2a..742336ec777480 100644 --- a/be/src/olap/predicate_creator.h +++ b/be/src/olap/predicate_creator.h @@ -237,6 +237,24 @@ std::unique_ptr> get_creator(const FieldType& ty return value; }); } + case FieldType::OLAP_FIELD_TYPE_IPV4: { + return std::make_unique>( + [](const std::string& condition) { + vectorized::IPv4 value; + bool res = IPv4Value::from_string(value, condition); + DCHECK(res); + return value; + }); + } + case FieldType::OLAP_FIELD_TYPE_IPV6: { + return std::make_unique>( + [](const std::string& condition) { + vectorized::IPv6 value; + bool res = IPv6Value::from_string(value, condition); + DCHECK(res); + return value; + }); + } default: return nullptr; } diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp index 462b5bdf51c2a4..db41b15e3ded0f 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.cpp +++ b/be/src/olap/rowset/segment_v2/encoding_info.cpp @@ -325,6 +325,12 @@ EncodingInfoResolver::EncodingInfoResolver() { _add_map(); _add_map(); + _add_map(); + + _add_map(); + _add_map(); + _add_map(); + _add_map(); _add_map(); diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index 40f755654ef699..0140ba9cb57497 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -196,6 +196,8 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, M(TYPE_DATETIME) \ M(TYPE_DATEV2) \ M(TYPE_DATETIMEV2) \ + M(TYPE_IPV4) \ + M(TYPE_IPV6) \ M(TYPE_VARCHAR) \ M(TYPE_STRING) \ M(TYPE_DECIMAL32) \ diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp index e55b1dcf2aa1c1..8f08516384bc98 100644 --- a/be/src/olap/schema.cpp +++ b/be/src/olap/schema.cpp @@ -202,6 +202,12 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(const Field& fi case FieldType::OLAP_FIELD_TYPE_DECIMAL256: ptr = doris::vectorized::PredicateColumnType::create(); break; + case FieldType::OLAP_FIELD_TYPE_IPV4: + ptr = doris::vectorized::PredicateColumnType::create(); + break; + case FieldType::OLAP_FIELD_TYPE_IPV6: + ptr = doris::vectorized::PredicateColumnType::create(); + break; default: LOG(FATAL) << "Unexpected type when choosing predicate column, type=" << int(field.type()); } diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 6f3e5260f72c75..7b7f5461b80ae3 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -70,6 +70,10 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT; } else if (0 == upper_type_str.compare("UNSIGNED_BIGINT")) { type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT; + } else if (0 == upper_type_str.compare("IPV4")) { + type = FieldType::OLAP_FIELD_TYPE_IPV4; + } else if (0 == upper_type_str.compare("IPV6")) { + type = FieldType::OLAP_FIELD_TYPE_IPV6; } else if (0 == upper_type_str.compare("FLOAT")) { type = FieldType::OLAP_FIELD_TYPE_FLOAT; } else if (0 == upper_type_str.compare("DISCRETE_DOUBLE")) { @@ -192,6 +196,12 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) { case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT: return "UNSIGNED_BIGINT"; + case FieldType::OLAP_FIELD_TYPE_IPV4: + return "IPV4"; + + case FieldType::OLAP_FIELD_TYPE_IPV6: + return "IPV6"; + case FieldType::OLAP_FIELD_TYPE_FLOAT: return "FLOAT"; @@ -316,6 +326,10 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3 return 8; case TPrimitiveType::LARGEINT: return 16; + case TPrimitiveType::IPV4: + return 4; + case TPrimitiveType::IPV6: + return 16; case TPrimitiveType::DATE: return 3; case TPrimitiveType::DATEV2: diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp index b095d830e7079f..5ed7984bcc6b7e 100644 --- a/be/src/olap/types.cpp +++ b/be/src/olap/types.cpp @@ -99,6 +99,8 @@ const TypeInfo* get_scalar_type_info(FieldType field_type) { get_scalar_type_info(), get_scalar_type_info(), get_scalar_type_info(), + get_scalar_type_info(), + get_scalar_type_info(), nullptr}; return field_type_array[int(field_type)]; } diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 542860362b071d..40c68a3e369e3c 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -83,7 +83,7 @@ class TypeInfo { virtual void direct_copy(void* dest, const void* src) const = 0; - // Use only in zone map to cut data. + // Use only in zone map to cut data.StringParser::string_to_unsigned_int virtual void direct_copy_may_cut(void* dest, const void* src) const = 0; virtual Status from_string(void* buf, const std::string& scan_key, const int precision = 0, @@ -722,6 +722,16 @@ struct CppTypeTraits { using UnsignedCppType = uint64_t; }; template <> +struct CppTypeTraits { + using CppType = uint32_t; + using UnsignedCppType = uint32_t; +}; +template <> +struct CppTypeTraits { + using CppType = uint128_t; + using UnsignedCppType = uint128_t; +}; +template <> struct CppTypeTraits { using CppType = Slice; }; @@ -778,6 +788,8 @@ struct BaseFieldtypeTraits : public CppTypeTraits { static inline CppType get_cpp_type_value(const void* address) { if constexpr (field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT) { return get_int128_from_unalign(address); + } else if constexpr (field_type == FieldType::OLAP_FIELD_TYPE_IPV6) { + return get_uint128_from_unalign(address); } return *reinterpret_cast(address); } @@ -964,6 +976,101 @@ struct FieldTypeTraits } }; +template <> +struct FieldTypeTraits + : public BaseFieldtypeTraits { + static Status from_string(void* buf, const std::string& scan_key, const int precision, + const int scale) { + StringParser::ParseResult result = StringParser::PARSE_SUCCESS; + uint32_t value = StringParser::string_to_unsigned_int(scan_key.c_str(), + scan_key.size(), &result); + + if (result == StringParser::PARSE_FAILURE) { + return Status::Error( + "FieldTypeTraits::from_string meet PARSE_FAILURE"); + } + *reinterpret_cast(buf) = value; + return Status::OK(); + } + + static std::string to_string(const void* src) { + uint32_t value = *reinterpret_cast(src); + std::stringstream ss; + ss << ((value >> 24) & 0xFF) << '.' << ((value >> 16) & 0xFF) << '.' + << ((value >> 8) & 0xFF) << '.' << (value & 0xFF); + return ss.str(); + } +}; + +template <> +struct FieldTypeTraits + : public BaseFieldtypeTraits { + static Status from_string(void* buf, const std::string& scan_key, const int precision, + const int scale) { + std::istringstream iss(scan_key); + std::string token; + uint128_t result = 0; + int count = 0; + + while (std::getline(iss, token, ':')) { + if (token.empty()) { + count += 8 - count; + break; + } + + if (count > 8) { + return Status::Error( + "FieldTypeTraits::from_string meet PARSE_FAILURE"); + } + + uint16_t value = 0; + std::istringstream ss(token); + if (!(ss >> std::hex >> value)) { + return Status::Error( + "FieldTypeTraits::from_string meet PARSE_FAILURE"); + } + + result = (result << 16) | value; + count++; + } + + if (count < 8) { + return Status::Error( + "FieldTypeTraits::from_string meet PARSE_FAILURE"); + } + + *reinterpret_cast(buf) = result; + return Status::OK(); + } + + static std::string to_string(const void* src) { + std::stringstream result; + uint128_t ipv6 = *reinterpret_cast(src); + + for (int i = 0; i < 8; i++) { + uint16_t part = static_cast((ipv6 >> (112 - i * 16)) & 0xFFFF); + result << std::to_string(part); + if (i != 7) { + result << ":"; + } + } + + return result.str(); + } + + static void set_to_max(void* buf) { + *reinterpret_cast(buf) = + static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll; + } + + static void set_to_min(void* buf) { + *reinterpret_cast(buf) = + -(static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll); + } +}; + template <> struct FieldTypeTraits : public NumericFieldtypeTraits { diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp index f0c72d335b3813..9da96f29628749 100644 --- a/be/src/olap/utils.cpp +++ b/be/src/olap/utils.cpp @@ -49,6 +49,8 @@ #include "io/fs/local_file_system.h" #include "olap/olap_common.h" #include "util/string_parser.hpp" +#include "vec/runtime/ipv4_value.h" +#include "vec/runtime/ipv6_value.h" namespace doris { using namespace ErrorCode; @@ -648,6 +650,24 @@ bool valid_bool(const std::string& value_str) { return result == StringParser::PARSE_SUCCESS; } +bool valid_ipv4(const std::string& value_str) { + if (value_str.size() == 0) { + return false; + } + + vectorized::IPv4 value = 0; + return IPv4Value::from_string(value, value_str); +} + +bool valid_ipv6(const std::string& value_str) { + if (value_str.size() == 0) { + return false; + } + + vectorized::IPv6 value; + return IPv6Value::from_string(value, value_str); +} + void write_log_info(char* buf, size_t buf_len, const char* fmt, ...) { va_list args; va_start(args, fmt); diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index f8e2e1fbe9936e..0e887e96394a12 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -232,6 +232,10 @@ bool valid_datetime(const std::string& value_str, const uint32_t scale); bool valid_bool(const std::string& value_str); +bool valid_ipv4(const std::string& value_str); + +bool valid_ipv6(const std::string& value_str); + constexpr bool is_string_type(const FieldType& field_type) { return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR || field_type == FieldType::OLAP_FIELD_TYPE_CHAR || diff --git a/be/src/runtime/define_primitive_type.h b/be/src/runtime/define_primitive_type.h index 44a0f2c38ed13e..6e7d4cf69d6f95 100644 --- a/be/src/runtime/define_primitive_type.h +++ b/be/src/runtime/define_primitive_type.h @@ -64,6 +64,8 @@ enum PrimitiveType : PrimitiveNative { TYPE_LAMBDA_FUNCTION, /* 33 */ TYPE_AGG_STATE, /* 34 */ TYPE_DECIMAL256, /* 35 */ + TYPE_IPV4, /* 36 */ + TYPE_IPV6 /* 37 */ }; constexpr PrimitiveNative BEGIN_OF_PRIMITIVE_TYPE = INVALID_TYPE; diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp index 91f2849901e736..5a95ef2993e78a 100644 --- a/be/src/runtime/primitive_type.cpp +++ b/be/src/runtime/primitive_type.cpp @@ -113,6 +113,12 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::BINARY: return TYPE_BINARY; + case TPrimitiveType::IPV4: + return TYPE_IPV4; + + case TPrimitiveType::IPV6: + return TYPE_IPV6; + case TPrimitiveType::DECIMALV2: return TYPE_DECIMALV2; @@ -229,6 +235,12 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_BINARY: return TPrimitiveType::BINARY; + case TYPE_IPV4: + return TPrimitiveType::IPV4; + + case TYPE_IPV6: + return TPrimitiveType::IPV6; + case TYPE_DECIMALV2: return TPrimitiveType::DECIMALV2; @@ -336,6 +348,12 @@ std::string type_to_string(PrimitiveType t) { case TYPE_BINARY: return "BINARY"; + case TYPE_IPV4: + return "IPV4"; + + case TYPE_IPV6: + return "IPV6"; + case TYPE_DECIMALV2: return "DECIMALV2"; @@ -448,6 +466,12 @@ std::string type_to_odbc_string(PrimitiveType t) { case TYPE_BINARY: return "binary"; + case TYPE_IPV4: + return "ipv4"; + + case TYPE_IPV6: + return "ipv6"; + case TYPE_DECIMALV2: return "decimalv2"; diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 5eff1541d6914e..06e6842363b889 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -73,6 +73,8 @@ constexpr bool is_enumeration_type(PrimitiveType type) { case TYPE_LARGEINT: case TYPE_DATE: case TYPE_DATEV2: + case TYPE_IPV4: + case TYPE_IPV6: return true; case INVALID_TYPE: @@ -216,6 +218,16 @@ struct PrimitiveTypeTraits { using ColumnType = vectorized::ColumnInt128; }; template <> +struct PrimitiveTypeTraits { + using CppType = vectorized::IPv4; + using ColumnType = vectorized::ColumnIPv4; +}; +template <> +struct PrimitiveTypeTraits { + using CppType = vectorized::IPv6; + using ColumnType = vectorized::ColumnIPv6; +}; +template <> struct PrimitiveTypeTraits { using CppType = StringRef; using ColumnType = vectorized::ColumnString; @@ -225,19 +237,16 @@ struct PrimitiveTypeTraits { using CppType = StringRef; using ColumnType = vectorized::ColumnString; }; - template <> struct PrimitiveTypeTraits { using CppType = StringRef; using ColumnType = vectorized::ColumnString; }; - template <> struct PrimitiveTypeTraits { using CppType = StringRef; using ColumnType = vectorized::ColumnString; }; - template <> struct PrimitiveTypeTraits { using CppType = JsonBinaryValue; diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index 978ca0035400d0..ad5e287803fd25 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -34,6 +34,8 @@ #include "runtime/decimalv2_value.h" #include "runtime/large_int_value.h" #include "util/mysql_global.h" +#include "vec/runtime/ipv4_value.h" +#include "vec/runtime/ipv6_value.h" #include "vec/runtime/vdatetime_value.h" // IWYU pragma: keep namespace doris { @@ -524,6 +526,18 @@ int MysqlRowBuffer::push_decimal(const DecimalV2Value& data, i return 0; } +template +int MysqlRowBuffer::push_ipv4(const IPv4Value& ipv4_val) { + auto ipv4_str = ipv4_val.to_string(); + return push_string(ipv4_str.c_str(), ipv4_str.length()); +} + +template +int MysqlRowBuffer::push_ipv6(const IPv6Value& ipv6_val) { + auto ipv6_str = ipv6_val.to_string(); + return push_string(ipv6_str.c_str(), ipv6_str.length()); +} + template int MysqlRowBuffer::push_string(const char* str, int64_t length) { if (is_binary_format && !_dynamic_mode) { diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h index eec21feeadb8a2..394d4d0bdd1d5e 100644 --- a/be/src/util/mysql_row_buffer.h +++ b/be/src/util/mysql_row_buffer.h @@ -49,6 +49,8 @@ namespace doris { */ using int128_t = __int128; class DecimalV2Value; +class IPv4Value; +class IPv6Value; template class MysqlRowBuffer { @@ -76,6 +78,8 @@ class MysqlRowBuffer { template int push_datetime(const DateType& data); int push_decimal(const DecimalV2Value& data, int round_scale); + int push_ipv4(const IPv4Value& ipv4_val); + int push_ipv6(const IPv6Value& ipv6_val); int push_string(const char* str, int64_t length); int push_null(); diff --git a/be/src/util/types.h b/be/src/util/types.h index bd6a9812ab3a3e..b04c0644f6fa7f 100644 --- a/be/src/util/types.h +++ b/be/src/util/types.h @@ -43,4 +43,10 @@ inline int128_t get_int128_from_unalign(const void* address) { memcpy(&value, address, sizeof(int128_t)); return value; } + +inline uint128_t get_uint128_from_unalign(const void* address) { + uint128_t value = 0; + memcpy(&value, address, sizeof(uint128_t)); + return value; +} } // namespace doris diff --git a/be/src/vec/columns/columns_number.h b/be/src/vec/columns/columns_number.h index a9d3c9ac16c785..78311fe218ab9c 100644 --- a/be/src/vec/columns/columns_number.h +++ b/be/src/vec/columns/columns_number.h @@ -55,6 +55,9 @@ using ColumnDecimal128 = ColumnDecimal; using ColumnDecimal128I = ColumnDecimal; using ColumnDecimal256 = ColumnDecimal; +using ColumnIPv4 = ColumnVector; +using ColumnIPv6 = ColumnVector; + template struct IsFixLenColumnType { static constexpr bool value = false; diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h index c6a085acd6a486..3aec3b1540f229 100644 --- a/be/src/vec/columns/predicate_column.h +++ b/be/src/vec/columns/predicate_column.h @@ -469,6 +469,14 @@ class PredicateColumnType final : public COWHelper) { insert_byte_to_res_column(sel, sel_size, col_ptr); + } else if constexpr (std::is_same_v) { + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); + } else if constexpr (std::is_same_v) { + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); } else { return Status::NotSupported("not supported output type in predicate_column, type={}", type_to_string(Type)); diff --git a/be/src/vec/common/format_ip.h b/be/src/vec/common/format_ip.h index 76091e39fbf085..ce61362d286a4d 100644 --- a/be/src/vec/common/format_ip.h +++ b/be/src/vec/common/format_ip.h @@ -28,7 +28,8 @@ #include constexpr size_t IPV4_BINARY_LENGTH = 4; -constexpr size_t IPV4_MAX_TEXT_LENGTH = 15; /// Does not count tail zero byte. +constexpr size_t IPV4_MAX_TEXT_LENGTH = 15; /// Does not count tail zero byte. +constexpr size_t IPV6_MAX_TEXT_LENGTH = 39; constexpr size_t IPV4_MIN_NUM_VALUE = 0; //num value of '0.0.0.0' constexpr size_t IPV4_MAX_NUM_VALUE = 4294967295; //num value of '255.255.255.255' diff --git a/be/src/vec/common/uint128.h b/be/src/vec/common/uint128.h index 523e74ab7404f1..540323c264d1df 100644 --- a/be/src/vec/common/uint128.h +++ b/be/src/vec/common/uint128.h @@ -103,6 +103,12 @@ struct UInt128 { high = 0; return *this; } + + operator uint128_t() const { + uint128_t value = static_cast(high) << 64; + value |= low; + return value; + } }; template <> diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h index ecc595f88ad359..30ad31f3560286 100644 --- a/be/src/vec/core/call_on_type_index.h +++ b/be/src/vec/core/call_on_type_index.h @@ -173,6 +173,8 @@ class DataTypeDate; class DataTypeDateV2; class DataTypeDateTimeV2; class DataTypeDateTime; +class DataTypeIPv4; +class DataTypeIPv6; class DataTypeString; template class DataTypeEnum; @@ -232,6 +234,11 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) { case TypeIndex::DateTime: return f(TypePair()); + case TypeIndex::IPv4: + return f(TypePair()); + case TypeIndex::IPv6: + return f(TypePair()); + case TypeIndex::String: return f(TypePair()); diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h index f24a85b1f35825..861126a030ed42 100644 --- a/be/src/vec/core/types.h +++ b/be/src/vec/core/types.h @@ -99,6 +99,8 @@ enum class TypeIndex { Time = 43, AggState = 44, Decimal256 = 45, + IPv4 = 46, + IPv6 = 47, Int256 }; @@ -277,6 +279,10 @@ struct TypeId { using Strings = std::vector; using Int128 = __int128; + +using IPv4 = UInt32; +using IPv6 = Int128; + template <> inline constexpr bool IsNumber = true; template <> @@ -1030,6 +1036,10 @@ inline const char* getTypeName(TypeIndex idx) { return "DateTimeV2"; case TypeIndex::TimeV2: return "TimeV2"; + case TypeIndex::IPv4: + return "IPv4"; + case TypeIndex::IPv6: + return "IPv6"; case TypeIndex::String: return TypeName::get(); case TypeIndex::FixedString: diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp index 48d37b38c397a2..0db189262b58e5 100644 --- a/be/src/vec/data_types/data_type.cpp +++ b/be/src/vec/data_types/data_type.cpp @@ -127,6 +127,10 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::INT64; case TypeIndex::Int128: return PGenericType::INT128; + case TypeIndex::IPv4: + return PGenericType::IPV4; + case TypeIndex::IPv6: + return PGenericType::IPV6; case TypeIndex::Float32: return PGenericType::FLOAT; case TypeIndex::Float64: diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index b5700cb7f0bc1b..8b86fa14034c4f 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -53,6 +53,8 @@ #include "vec/data_types/data_type_decimal.h" #include "vec/data_types/data_type_fixed_length_object.h" #include "vec/data_types/data_type_hll.h" +#include "vec/data_types/data_type_ipv4.h" +#include "vec/data_types/data_type_ipv6.h" #include "vec/data_types/data_type_jsonb.h" #include "vec/data_types/data_type_map.h" #include "vec/data_types/data_type_nullable.h" @@ -134,6 +136,12 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo case TYPE_LARGEINT: nested = std::make_shared(); break; + case TYPE_IPV4: + nested = std::make_shared(); + break; + case TYPE_IPV6: + nested = std::make_shared(); + break; case TYPE_DATE: nested = std::make_shared(); break; @@ -266,6 +274,12 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeIndex& type_index, bool case TypeIndex::Int128: nested = std::make_shared(); break; + case TypeIndex::IPv4: + nested = std::make_shared(); + break; + case TypeIndex::IPv6: + nested = std::make_shared(); + break; case TypeIndex::Float32: nested = std::make_shared(); break; @@ -359,6 +373,12 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, case FieldType::OLAP_FIELD_TYPE_LARGEINT: result = std::make_shared(); break; + case FieldType::OLAP_FIELD_TYPE_IPV4: + result = std::make_shared(); + break; + case FieldType::OLAP_FIELD_TYPE_IPV6: + result = std::make_shared(); + break; case FieldType::OLAP_FIELD_TYPE_DATE: result = std::make_shared(); break; @@ -449,6 +469,12 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { case PGenericType::DOUBLE: nested = std::make_shared(); break; + case PGenericType::IPV4: + nested = std::make_shared(); + break; + case PGenericType::IPV6: + nested = std::make_shared(); + break; case PGenericType::STRING: nested = std::make_shared(); break; diff --git a/be/src/vec/data_types/data_type_ipv4.cpp b/be/src/vec/data_types/data_type_ipv4.cpp new file mode 100644 index 00000000000000..90a88aa6fc7ee9 --- /dev/null +++ b/be/src/vec/data_types/data_type_ipv4.cpp @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/data_types/data_type_ipv4.h" + +#include "util/binary_cast.hpp" +#include "util/string_parser.hpp" +#include "vec/columns/column.h" +#include "vec/columns/column_const.h" +#include "vec/columns/column_vector.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_buffer.hpp" +#include "vec/data_types/data_type.h" +#include "vec/io/io_helper.h" +#include "vec/io/reader_buffer.h" + +namespace doris::vectorized { +bool DataTypeIPv4::equals(const IDataType& rhs) const { + return typeid(rhs) == typeid(*this); +} + +std::string DataTypeIPv4::to_string(const IColumn& column, size_t row_num) const { + auto result = check_column_const_set_readability(column, row_num); + ColumnPtr ptr = result.first; + row_num = result.second; + IPv4 value = assert_cast(*ptr).get_element(row_num); + return convert_ipv4_to_string(value); +} + +void DataTypeIPv4::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const { + std::string value = to_string(column, row_num); + ostr.write(value.data(), value.size()); +} + +Status DataTypeIPv4::from_string(ReadBuffer& rb, IColumn* column) const { + auto* column_data = static_cast(column); + StringParser::ParseResult result; + IPv4 val = StringParser::string_to_unsigned_int(rb.position(), rb.count(), &result); + column_data->insert_value(val); + return Status::OK(); +} + +std::string DataTypeIPv4::convert_ipv4_to_string(IPv4 ipv4) { + std::stringstream ss; + ss << ((ipv4 >> 24) & 0xFF) << '.' << ((ipv4 >> 16) & 0xFF) << '.' << ((ipv4 >> 8) & 0xFF) + << '.' << (ipv4 & 0xFF); + return ss.str(); +} + +bool DataTypeIPv4::convert_string_to_ipv4(IPv4& x, std::string ipv4) { + const static int IPV4_PARTS_NUM = 4; + IPv4 parts[IPV4_PARTS_NUM]; + int part_index = 0; + std::stringstream ss(ipv4); + std::string part; + StringParser::ParseResult result; + + while (std::getline(ss, part, '.')) { + IPv4 val = StringParser::string_to_unsigned_int(part.data(), part.size(), &result); + if (UNLIKELY(result != StringParser::PARSE_SUCCESS) || val > 255) { + return false; + } + parts[part_index++] = val; + } + + if (part_index != 4) { + return false; + } + + x = (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]; + return true; +} + +MutableColumnPtr DataTypeIPv4::create_column() const { + return ColumnIPv4::create(); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_ipv4.h b/be/src/vec/data_types/data_type_ipv4.h new file mode 100644 index 00000000000000..066345826e0dd7 --- /dev/null +++ b/be/src/vec/data_types/data_type_ipv4.h @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include + +#include "common/status.h" +#include "runtime/define_primitive_type.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_number_base.h" +#include "vec/data_types/serde/data_type_ipv4_serde.h" + +namespace doris { +namespace vectorized { +class BufferWritable; +class ReadBuffer; +class IColumn; +} // namespace vectorized +} // namespace doris + +namespace doris::vectorized { + +class DataTypeIPv4 final : public DataTypeNumberBase { +public: + TypeIndex get_type_id() const override { return TypeIndex::IPv4; } + TPrimitiveType::type get_type_as_tprimitive_type() const override { + return TPrimitiveType::IPV4; + } + const char* get_family_name() const override { return "IPv4"; } + std::string do_get_name() const override { return "IPv4"; } + + bool can_be_inside_nullable() const override { return true; } + + bool equals(const IDataType& rhs) const override; + std::string to_string(const IColumn& column, size_t row_num) const override; + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; + Status from_string(ReadBuffer& rb, IColumn* column) const override; + + static std::string convert_ipv4_to_string(IPv4 ipv4); + static bool convert_string_to_ipv4(IPv4& x, std::string ipv4); + + Field get_field(const TExprNode& node) const override { return (IPv4)node.ipv4_literal.value; } + + MutableColumnPtr create_column() const override; + + DataTypeSerDeSPtr get_serde() const override { return std::make_shared(); } +}; + +template +constexpr bool IsIPv4Type = false; +template <> +inline constexpr bool IsIPv4Type = true; + +} // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_ipv6.cpp b/be/src/vec/data_types/data_type_ipv6.cpp new file mode 100755 index 00000000000000..d54a0f48464ba6 --- /dev/null +++ b/be/src/vec/data_types/data_type_ipv6.cpp @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/data_types/data_type_ipv6.h" + +#include "util/binary_cast.hpp" +#include "vec/columns/column.h" +#include "vec/columns/column_const.h" +#include "vec/columns/column_vector.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_buffer.hpp" +#include "vec/data_types/data_type.h" +#include "vec/io/io_helper.h" +#include "vec/io/reader_buffer.h" +#include "vec/runtime/ipv6_value.h" + +namespace doris::vectorized { +bool DataTypeIPv6::equals(const IDataType& rhs) const { + return typeid(rhs) == typeid(*this); +} + +std::string DataTypeIPv6::to_string(const IColumn& column, size_t row_num) const { + auto result = check_column_const_set_readability(column, row_num); + ColumnPtr ptr = result.first; + row_num = result.second; + IPv6 value = assert_cast(*ptr).get_element(row_num); + return convert_ipv6_to_string(value); +} + +void DataTypeIPv6::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const { + std::string value = to_string(column, row_num); + ostr.write(value.data(), value.size()); +} + +Status DataTypeIPv6::from_string(ReadBuffer& rb, IColumn* column) const { + auto* column_data = static_cast(column); + IPv6 value; + if (!convert_string_to_ipv6(value, rb.to_string())) { + throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, + "Invalid value: {} for type IPv6", rb.to_string()); + } + column_data->insert_value(value); + return Status::OK(); +} + +std::string DataTypeIPv6::convert_ipv6_to_string(IPv6 ipv6) { + return IPv6Value::to_string(ipv6); +} + +bool DataTypeIPv6::convert_string_to_ipv6(IPv6& x, std::string ipv6) { + return IPv6Value::from_string(x, ipv6); +} + +MutableColumnPtr DataTypeIPv6::create_column() const { + return ColumnIPv6::create(); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_ipv6.h b/be/src/vec/data_types/data_type_ipv6.h new file mode 100755 index 00000000000000..992627cff32cf3 --- /dev/null +++ b/be/src/vec/data_types/data_type_ipv6.h @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include + +#include "common/status.h" +#include "runtime/define_primitive_type.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_ipv4.h" +#include "vec/data_types/data_type_number_base.h" +#include "vec/data_types/serde/data_type_ipv6_serde.h" + +namespace doris { +namespace vectorized { +class BufferWritable; +class ReadBuffer; +class IColumn; +} // namespace vectorized +} // namespace doris + +namespace doris::vectorized { + +class DataTypeIPv6 final : public DataTypeNumberBase { +public: + TypeIndex get_type_id() const override { return TypeIndex::IPv6; } + TPrimitiveType::type get_type_as_tprimitive_type() const override { + return TPrimitiveType::IPV6; + } + const char* get_family_name() const override { return "IPv6"; } + std::string do_get_name() const override { return "IPv6"; } + + bool can_be_inside_nullable() const override { return true; } + + bool equals(const IDataType& rhs) const override; + std::string to_string(const IColumn& column, size_t row_num) const override; + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; + Status from_string(ReadBuffer& rb, IColumn* column) const override; + + static std::string convert_ipv6_to_string(IPv6 ipv6); + static bool convert_string_to_ipv6(IPv6& x, std::string ipv6); + + Field get_field(const TExprNode& node) const override { + IPv6 value; + if (!convert_string_to_ipv6(value, node.ipv6_literal.value)) { + throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, + "Invalid value: {} for type IPv6", node.ipv6_literal.value); + } + return value; + } + + MutableColumnPtr create_column() const override; + + DataTypeSerDeSPtr get_serde() const override { return std::make_shared(); } +}; + +template +constexpr bool IsIPv6Type = false; +template <> +inline constexpr bool IsIPv6Type = true; + +template +constexpr bool IsIPType = IsIPv4Type || IsIPv6Type; + +} // namespace doris::vectorized diff --git a/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp new file mode 100644 index 00000000000000..ea9087bae80fce --- /dev/null +++ b/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "data_type_ipv4_serde.h" + +#include + +#include "vec/columns/column_const.h" + +namespace doris { +namespace vectorized { + +template +Status DataTypeIPv4SerDe::_write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& result, + int row_idx, bool col_const) const { + auto& data = assert_cast&>(column).get_data(); + auto col_index = index_check_const(row_idx, col_const); + IPv4Value ipv4_val(data[col_index]); + if (UNLIKELY(0 != result.push_ipv4(ipv4_val))) { + return Status::InternalError("pack mysql buffer failed."); + } + return Status::OK(); +} + +Status DataTypeIPv4SerDe::write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& row_buffer, int row_idx, + bool col_const) const { + return _write_column_to_mysql(column, row_buffer, row_idx, col_const); +} + +Status DataTypeIPv4SerDe::write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& row_buffer, int row_idx, + bool col_const) const { + return _write_column_to_mysql(column, row_buffer, row_idx, col_const); +} + +} // namespace vectorized +} // namespace doris \ No newline at end of file diff --git a/be/src/vec/data_types/serde/data_type_ipv4_serde.h b/be/src/vec/data_types/serde/data_type_ipv4_serde.h new file mode 100644 index 00000000000000..4b4eedc4da13a2 --- /dev/null +++ b/be/src/vec/data_types/serde/data_type_ipv4_serde.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include + +#include "common/status.h" +#include "data_type_number_serde.h" +#include "olap/olap_common.h" +#include "util/jsonb_document.h" +#include "util/jsonb_writer.h" +#include "vec/columns/column.h" +#include "vec/columns/column_vector.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/runtime/ipv4_value.h" + +namespace doris { +namespace vectorized { + +class DataTypeIPv4SerDe : public DataTypeNumberSerDe { + Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, + int row_idx, bool col_const) const override; + Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, + int row_idx, bool col_const) const override; + +private: + template + Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer& result, + int row_idx, bool col_const) const; +}; +} // namespace vectorized +} // namespace doris \ No newline at end of file diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp new file mode 100644 index 00000000000000..6e05252b9861c0 --- /dev/null +++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "data_type_ipv6_serde.h" + +#include + +#include "vec/columns/column_const.h" + +namespace doris { +namespace vectorized { + +template +Status DataTypeIPv6SerDe::_write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& result, + int row_idx, bool col_const) const { + auto& data = assert_cast&>(column).get_data(); + auto col_index = index_check_const(row_idx, col_const); + IPv6Value ipv6_val(data[col_index]); + if (UNLIKELY(0 != result.push_ipv6(ipv6_val))) { + return Status::InternalError("pack mysql buffer failed."); + } + return Status::OK(); +} + +Status DataTypeIPv6SerDe::write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& row_buffer, int row_idx, + bool col_const) const { + return _write_column_to_mysql(column, row_buffer, row_idx, col_const); +} + +Status DataTypeIPv6SerDe::write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& row_buffer, int row_idx, + bool col_const) const { + return _write_column_to_mysql(column, row_buffer, row_idx, col_const); +} + +} // namespace vectorized +} // namespace doris \ No newline at end of file diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.h b/be/src/vec/data_types/serde/data_type_ipv6_serde.h new file mode 100644 index 00000000000000..4f324acc153f50 --- /dev/null +++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include + +#include "common/status.h" +#include "data_type_number_serde.h" +#include "olap/olap_common.h" +#include "util/jsonb_document.h" +#include "util/jsonb_writer.h" +#include "vec/columns/column.h" +#include "vec/columns/column_vector.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/runtime/ipv6_value.h" + +namespace doris { +class JsonbOutStream; + +namespace vectorized { +class Arena; + +class DataTypeIPv6SerDe : public DataTypeNumberSerDe { + Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, + int row_idx, bool col_const) const override; + Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, + int row_idx, bool col_const) const override; + +private: + template + Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer& result, + int row_idx, bool col_const) const; +}; +} // namespace vectorized +} // namespace doris \ No newline at end of file diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index 109ef5e77d324b..f8297a58651a39 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -225,6 +225,8 @@ Status VExpr::create_expr(const TExprNode& expr_node, VExprSPtr& expr) { case TExprNodeType::BOOL_LITERAL: case TExprNodeType::INT_LITERAL: case TExprNodeType::LARGE_INT_LITERAL: + case TExprNodeType::IPV4_LITERAL: + case TExprNodeType::IPV6_LITERAL: case TExprNodeType::FLOAT_LITERAL: case TExprNodeType::DECIMAL_LITERAL: case TExprNodeType::DATE_LITERAL: diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index 5f481aeef399a7..db4ea568ffb736 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -699,6 +699,10 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum M(DateV2, ColumnUInt32) \ M(DateTimeV2, ColumnUInt64) +#define IP_TYPE_TO_COLUMN_TYPE(M) \ + M(IPv4, ColumnIPv4) \ + M(IPv6, ColumnIPv6) + #define COMPLEX_TYPE_TO_COLUMN_TYPE(M) \ M(Array, ColumnArray) \ M(Map, ColumnMap) \ @@ -710,7 +714,8 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum NUMERIC_TYPE_TO_COLUMN_TYPE(M) \ DECIMAL_TYPE_TO_COLUMN_TYPE(M) \ STRING_TYPE_TO_COLUMN_TYPE(M) \ - TIME_TYPE_TO_COLUMN_TYPE(M) + TIME_TYPE_TO_COLUMN_TYPE(M) \ + IP_TYPE_TO_COLUMN_TYPE(M) #define TYPE_TO_COLUMN_TYPE(M) \ TYPE_TO_BASIC_COLUMN_TYPE(M) \ diff --git a/be/src/vec/functions/function_case.h b/be/src/vec/functions/function_case.h index 86a3322d201c0a..1fe306334506db 100644 --- a/be/src/vec/functions/function_case.h +++ b/be/src/vec/functions/function_case.h @@ -258,7 +258,9 @@ class FunctionCase : public IFunction { std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v || + std::is_same_v) { // result_column and all then_column is not nullable. // can't simd when type is string. update_result_normal(result_column_ptr, then_idx, diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 55fc8d178b0f88..8872f373b440ea 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -74,6 +74,8 @@ #include "vec/data_types/data_type_date_time.h" #include "vec/data_types/data_type_decimal.h" #include "vec/data_types/data_type_hll.h" +#include "vec/data_types/data_type_ipv4.h" +#include "vec/data_types/data_type_ipv6.h" #include "vec/data_types/data_type_jsonb.h" #include "vec/data_types/data_type_map.h" #include "vec/data_types/data_type_nullable.h" @@ -853,6 +855,12 @@ struct NameToFloat32 { struct NameToFloat64 { static constexpr auto name = "toFloat64"; }; +struct NameToIPv4 { + static constexpr auto name = "toIPv4"; +}; +struct NameToIPv6 { + static constexpr auto name = "toIPv6"; +}; struct NameToDate { static constexpr auto name = "toDate"; }; @@ -881,6 +889,14 @@ bool try_parse_impl(typename DataType::FieldType& x, ReadBuffer& rb, return try_read_datetime_v2_text(x, rb, local_time_zone, scale); } + if constexpr (IsIPv4Type) { + return try_read_ipv4_text(x, rb); + } + + if constexpr (IsIPv6Type) { + return try_read_ipv6_text(x, rb); + } + if constexpr (std::is_same_v && std::is_same_v) { // cast from string to time(float64) @@ -1211,6 +1227,8 @@ using FunctionToDecimal128I = FunctionConvert, NameToDecimal128I, UnknownMonotonicity>; using FunctionToDecimal256 = FunctionConvert, NameToDecimal256, UnknownMonotonicity>; +using FunctionToIPv4 = FunctionConvert; +using FunctionToIPv6 = FunctionConvert; using FunctionToDate = FunctionConvert; using FunctionToDateTime = FunctionConvert; using FunctionToDateV2 = FunctionConvert; @@ -1285,6 +1303,14 @@ struct FunctionTo> { using Type = FunctionToDecimal256; }; template <> +struct FunctionTo { + using Type = FunctionToIPv4; +}; +template <> +struct FunctionTo { + using Type = FunctionToIPv6; +}; +template <> struct FunctionTo { using Type = FunctionToDate; }; @@ -1445,6 +1471,12 @@ struct ConvertImpl, Name> template struct ConvertImpl, Name> : ConvertThroughParsing, Name> {}; +template +struct ConvertImpl + : ConvertThroughParsing {}; +template +struct ConvertImpl + : ConvertThroughParsing {}; template class FunctionConvertFromString : public IFunction { @@ -2099,7 +2131,10 @@ class FunctionCast final : public IFunctionBase { std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { ret = create_wrapper(from_type, check_and_get_data_type(to_type.get()), requested_result_is_nullable); return true; diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h index 3b58f21c406d9f..598bf01bfdd63a 100644 --- a/be/src/vec/functions/functions_comparison.h +++ b/be/src/vec/functions/functions_comparison.h @@ -634,4 +634,4 @@ class FunctionComparison : public IFunction { } }; -} // namespace doris::vectorized +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h index f1e07ff1ccc71d..c9e52fe87a1312 100644 --- a/be/src/vec/io/io_helper.h +++ b/be/src/vec/io/io_helper.h @@ -32,6 +32,8 @@ #include "vec/core/types.h" #include "vec/io/reader_buffer.h" #include "vec/io/var_int.h" +#include "vec/runtime/ipv4_value.h" +#include "vec/runtime/ipv6_value.h" #include "vec/runtime/vdatetime_value.h" namespace doris::vectorized { @@ -292,6 +294,22 @@ bool read_date_text_impl(T& x, ReadBuffer& buf, const cctz::time_zone& local_tim return ans; } +template +bool read_ipv4_text_impl(T& x, ReadBuffer& buf) { + static_assert(std::is_same_v); + bool res = IPv4Value::from_string(x, buf.to_string()); + buf.position() = buf.end(); + return res; +} + +template +bool read_ipv6_text_impl(T& x, ReadBuffer& buf) { + static_assert(std::is_same_v); + bool res = IPv6Value::from_string(x, buf.to_string()); + buf.position() = buf.end(); + return res; +} + template bool read_datetime_text_impl(T& x, ReadBuffer& buf) { static_assert(std::is_same_v); @@ -444,6 +462,16 @@ StringParser::ParseResult try_read_decimal_text(T& x, ReadBuffer& in, UInt32 pre return read_decimal_text_impl(x, in, precision, scale); } +template +bool try_read_ipv4_text(T& x, ReadBuffer& in) { + return read_ipv4_text_impl(x, in); +} + +template +bool try_read_ipv6_text(T& x, ReadBuffer& in) { + return read_ipv6_text_impl(x, in); +} + template bool try_read_datetime_text(T& x, ReadBuffer& in, const cctz::time_zone& local_time_zone) { return read_datetime_text_impl(x, in, local_time_zone); diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index e15c3926a94881..ef903f70a3d254 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -114,11 +114,9 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co } case FieldType::OLAP_FIELD_TYPE_DATEV2: { return std::make_unique(); - break; } case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { return std::make_unique(); - break; } case FieldType::OLAP_FIELD_TYPE_DECIMAL: { return std::make_unique(); @@ -156,6 +154,12 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co case FieldType::OLAP_FIELD_TYPE_LARGEINT: { return std::make_unique>(); } + case FieldType::OLAP_FIELD_TYPE_IPV4: { + return std::make_unique>(); + } + case FieldType::OLAP_FIELD_TYPE_IPV6: { + return std::make_unique>(); + } case FieldType::OLAP_FIELD_TYPE_FLOAT: { return std::make_unique>(); } diff --git a/be/src/vec/runtime/ipv4_value.h b/be/src/vec/runtime/ipv4_value.h new file mode 100644 index 00000000000000..fb42ed66f9ca5c --- /dev/null +++ b/be/src/vec/runtime/ipv4_value.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "util/string_parser.hpp" + +namespace doris { + +class IPv4Value { +public: + IPv4Value() = default; + + explicit IPv4Value(vectorized::IPv4 ipv4) { _value = ipv4; } + + explicit IPv4Value(std::string ipv4) {} + + [[nodiscard]] const vectorized::IPv4& value() const { return _value; } + + vectorized::IPv4& value() { return _value; } + + void set_value(vectorized::IPv4 ipv4) { _value = ipv4; } + + bool from_string(std::string ipv4) { return from_string(_value, ipv4); } + + [[nodiscard]] std::string to_string() const { return to_string(_value); } + + static bool from_string(vectorized::IPv4& value, std::string ipv4) { + remove_ipv4_space(ipv4); + + // shortest ipv4 string is `0.0.0.0` whose length is 7 + if (ipv4.size() < 7 || !is_valid_string(ipv4)) { + return false; + } + + vectorized::IPv4 octets[4] = {0}; + std::istringstream iss(ipv4); + std::string octet; + uint8_t octet_index = 0; + + while (getline(iss, octet, '.')) { + if (octet_index >= 4) { + return false; + } + + StringParser::ParseResult result; + vectorized::IPv4 val = StringParser::string_to_unsigned_int( + octet.c_str(), octet.length(), &result); + if (result != StringParser::PARSE_SUCCESS || val > 255) { + return false; + } + + octets[octet_index++] = val; + } + + if (octet_index != 4) { + return false; + } + + value = (octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]; + return true; + } + + static std::string to_string(vectorized::IPv4 value) { + std::stringstream ss; + ss << ((value >> 24) & 0xFF) << '.' << ((value >> 16) & 0xFF) << '.' + << ((value >> 8) & 0xFF) << '.' << (value & 0xFF); + return ss.str(); + } + + static void remove_ipv4_space(std::string& ipv4) { + if (ipv4.empty()) { + return; + } + + std::string special_chars = "\r\n\t "; + + size_t pos = ipv4.find_first_not_of(special_chars); + if (pos != std::string::npos) { + ipv4.erase(0, pos); + } + + pos = ipv4.find_last_not_of(special_chars); + if (pos != std::string::npos) { + ipv4.erase(pos + 1); + } + } + + static bool is_valid_string(std::string ipv4) { + static std::regex IPV4_STD_REGEX( + "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-" + "9]?)$"); + if (ipv4.size() > 15 || !std::regex_match(ipv4, IPV4_STD_REGEX)) { + return false; + } + return true; + } + +private: + vectorized::IPv4 _value; +}; + +} // namespace doris diff --git a/be/src/vec/runtime/ipv6_value.h b/be/src/vec/runtime/ipv6_value.h new file mode 100644 index 00000000000000..8839ab4e2b06d2 --- /dev/null +++ b/be/src/vec/runtime/ipv6_value.h @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_number_base.h" + +namespace doris { + +class IPv6Value { +public: + IPv6Value() { _value = 0; } + + explicit IPv6Value(vectorized::IPv6 ipv6) { _value = ipv6; } + + [[nodiscard]] const vectorized::IPv6& value() const { return _value; } + + vectorized::IPv6& value() { return _value; } + + void set_value(vectorized::IPv6 ipv6) { _value = ipv6; } + + bool from_string(std::string ipv6) { return from_string(_value, ipv6); } + + bool from_binary_string(std::string ipv6_binary) { + return from_binary_string(_value, ipv6_binary); + } + + static bool from_string(vectorized::IPv6& x, std::string ipv6) { + remove_ipv6_space(ipv6); + + if (ipv6.empty() || !is_valid_string(ipv6)) { + return false; + } + + std::transform(ipv6.begin(), ipv6.end(), ipv6.begin(), + [](unsigned char ch) { return std::tolower(ch); }); + std::istringstream iss(ipv6); + std::string field; + uint16_t fields[8] = {0}; + uint8_t zero_index = 0; + uint8_t num_field = 0; + uint8_t right_field_num = 0; + + while (num_field < 8) { + if (!getline(iss, field, ':')) { + break; + } + + if (field.empty()) { + zero_index = num_field; + fields[num_field++] = 0; + } else { + try { + if (field.size() > 4 || field > "ffff") { + return false; + } + + fields[num_field++] = std::stoi(field, nullptr, 16); + } catch (const std::exception& /*e*/) { + return false; + } + } + } + + if (zero_index != 0) { + right_field_num = num_field - zero_index - 1; + + for (uint8_t i = 7; i > 7 - right_field_num; --i) { + fields[i] = fields[zero_index + right_field_num + i - 7]; + fields[zero_index + right_field_num + i - 7] = 0; + } + } + + uint64_t high = (static_cast(fields[0]) << 48) | + (static_cast(fields[1]) << 32) | + (static_cast(fields[2]) << 16) | static_cast(fields[3]); + uint64_t low = (static_cast(fields[4]) << 48) | + (static_cast(fields[5]) << 32) | + (static_cast(fields[6]) << 16) | static_cast(fields[7]); + + x = static_cast(high) << 64 | low; + return true; + } + + static bool from_binary_string(vectorized::IPv6& x, std::string ipv6_binary_str) { + // Accepts a FixedString(16) value containing the IPv6 address in binary format + if (ipv6_binary_str.size() != 16) { + return false; + } + + uint64_t high = 0; + uint64_t low = 0; + + const uint8_t* ipv6_binary = reinterpret_cast(ipv6_binary_str.c_str()); + + for (int i = 0; i < 8; ++i) { + high |= (static_cast(ipv6_binary[i]) << (56 - i * 8)); + } + + for (int i = 8; i < 16; ++i) { + low |= (static_cast(ipv6_binary[i]) << (56 - (i - 8) * 8)); + } + + x = static_cast(high) << 64 | low; + return true; + } + + [[nodiscard]] std::string to_string() const { return to_string(_value); } + + static std::string to_string(vectorized::IPv6 x) { + // "0000:0000:0000:0000:0000:0000:0000:0000" + if (x == 0) { + return "::"; + } + + uint64_t low = static_cast(x); + uint64_t high = static_cast(x >> 64); + + uint16_t fields[8] = {static_cast((high >> 48) & 0xFFFF), + static_cast((high >> 32) & 0xFFFF), + static_cast((high >> 16) & 0xFFFF), + static_cast(high & 0xFFFF), + static_cast((low >> 48) & 0xFFFF), + static_cast((low >> 32) & 0xFFFF), + static_cast((low >> 16) & 0xFFFF), + static_cast(low & 0xFFFF)}; + + uint8_t zero_start = 0, zero_end = 0; + + while (zero_start < 8 && zero_end < 8) { + if (fields[zero_start] != 0) { + zero_start++; + zero_end = zero_start; + continue; + } + + while (zero_end < 7 && fields[zero_end + 1] == 0) { + zero_end++; + } + + if (zero_end > zero_start) { + break; + } + + zero_start++; + zero_end = zero_start; + } + + std::stringstream ss; + + if (zero_start == zero_end) { + for (uint8_t i = 0; i < 7; ++i) { + ss << std::hex << fields[i] << ":"; + } + ss << std::hex << fields[7]; + } else { + for (uint8_t i = 0; i < zero_start; ++i) { + ss << std::hex << fields[i] << ":"; + } + + if (zero_end == 7) { + ss << ":"; + } else { + for (uint8_t j = zero_end + 1; j < 8; ++j) { + ss << std::hex << ":" << fields[j]; + } + } + } + + return ss.str(); + } + + [[nodiscard]] std::string to_binary_string() const { return to_binary_string(_value); } + + static std::string to_binary_string(vectorized::IPv6 x) { + uint64_t low = static_cast(x); + uint64_t high = static_cast(x >> 64); + + uint8_t fields[16] = {static_cast((high >> 56) & 0xFF), + static_cast((high >> 48) & 0xFF), + static_cast((high >> 40) & 0xFF), + static_cast((high >> 32) & 0xFF), + static_cast((high >> 24) & 0xFF), + static_cast((high >> 16) & 0xFF), + static_cast((high >> 8) & 0xFF), + static_cast(high & 0xFF), + static_cast((low >> 56) & 0xFF), + static_cast((low >> 48) & 0xFF), + static_cast((low >> 40) & 0xFF), + static_cast((low >> 32) & 0xFF), + static_cast((low >> 24) & 0xFF), + static_cast((low >> 16) & 0xFF), + static_cast((low >> 8) & 0xFF), + static_cast(low & 0xFF)}; + + std::stringstream ss; + + for (int i = 0; i < 16; ++i) { + ss << (char)fields[i]; + } + + return ss.str(); + } + + static void remove_ipv6_space(std::string& ipv6) { + if (ipv6.empty()) { + return; + } + + std::string special_chars = "\r\n\t "; + + size_t pos = ipv6.find_first_not_of(special_chars); + if (pos != std::string::npos) { + ipv6.erase(0, pos); + } + + pos = ipv6.find_last_not_of(special_chars); + if (pos != std::string::npos) { + ipv6.erase(pos + 1); + } + } + + static bool is_valid_string(std::string ipv6) { + static std::regex IPV6_STD_REGEX("^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$"); + static std::regex IPV6_COMPRESS_REGEX( + "^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::((([0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4}" + ")?)$"); + + if (ipv6.size() > 39 || !(std::regex_match(ipv6, IPV6_STD_REGEX) || + std::regex_match(ipv6, IPV6_COMPRESS_REGEX))) { + return false; + } + return true; + } + +private: + vectorized::IPv6 _value; +}; + +} // namespace doris diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index cfae49e2b5352a..da38abe6577d37 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -43,6 +43,8 @@ public enum PrimitiveType { DOUBLE("DOUBLE", 8, TPrimitiveType.DOUBLE, true), DATE("DATE", 16, TPrimitiveType.DATE, true), DATETIME("DATETIME", 16, TPrimitiveType.DATETIME, true), + IPV4("IPV4", 4, TPrimitiveType.IPV4, true), + IPV6("IPV6", 16, TPrimitiveType.IPV6, true), // Fixed length char array. CHAR("CHAR", 16, TPrimitiveType.CHAR, true), // 8-byte pointer and 4-byte length indicator (12 bytes total). @@ -121,6 +123,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(NULL_TYPE, DATETIME); builder.put(NULL_TYPE, DATEV2); builder.put(NULL_TYPE, DATETIMEV2); + builder.put(NULL_TYPE, IPV4); + builder.put(NULL_TYPE, IPV6); builder.put(NULL_TYPE, DECIMALV2); builder.put(NULL_TYPE, DECIMAL32); builder.put(NULL_TYPE, DECIMAL64); @@ -147,6 +151,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(BOOLEAN, DATETIME); builder.put(BOOLEAN, DATEV2); builder.put(BOOLEAN, DATETIMEV2); + builder.put(BOOLEAN, IPV4); + builder.put(BOOLEAN, IPV6); builder.put(BOOLEAN, DECIMALV2); builder.put(BOOLEAN, DECIMAL32); builder.put(BOOLEAN, DECIMAL64); @@ -167,6 +173,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(TINYINT, DATETIME); builder.put(TINYINT, DATEV2); builder.put(TINYINT, DATETIMEV2); + builder.put(TINYINT, IPV4); + builder.put(TINYINT, IPV6); builder.put(TINYINT, DECIMALV2); builder.put(TINYINT, DECIMAL32); builder.put(TINYINT, DECIMAL64); @@ -189,6 +197,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(SMALLINT, DATETIME); builder.put(SMALLINT, DATEV2); builder.put(SMALLINT, DATETIMEV2); + builder.put(SMALLINT, IPV4); + builder.put(SMALLINT, IPV6); builder.put(SMALLINT, DECIMALV2); builder.put(SMALLINT, DECIMAL32); builder.put(SMALLINT, DECIMAL64); @@ -211,6 +221,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(INT, DATETIME); builder.put(INT, DATEV2); builder.put(INT, DATETIMEV2); + builder.put(INT, IPV4); + builder.put(INT, IPV6); builder.put(INT, DECIMALV2); builder.put(INT, DECIMAL32); builder.put(INT, DECIMAL64); @@ -233,6 +245,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(BIGINT, DATETIME); builder.put(BIGINT, DATEV2); builder.put(BIGINT, DATETIMEV2); + builder.put(BIGINT, IPV4); + builder.put(BIGINT, IPV6); builder.put(BIGINT, DECIMALV2); builder.put(BIGINT, DECIMAL32); builder.put(BIGINT, DECIMAL64); @@ -255,6 +269,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(LARGEINT, DATETIME); builder.put(LARGEINT, DATEV2); builder.put(LARGEINT, DATETIMEV2); + builder.put(LARGEINT, IPV4); + builder.put(LARGEINT, IPV6); builder.put(LARGEINT, DECIMALV2); builder.put(LARGEINT, DECIMAL32); builder.put(LARGEINT, DECIMAL64); @@ -277,6 +293,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(FLOAT, DATETIME); builder.put(FLOAT, DATEV2); builder.put(FLOAT, DATETIMEV2); + builder.put(FLOAT, IPV4); + builder.put(FLOAT, IPV6); builder.put(FLOAT, DECIMALV2); builder.put(FLOAT, DECIMAL32); builder.put(FLOAT, DECIMAL64); @@ -299,6 +317,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DOUBLE, DATETIME); builder.put(DOUBLE, DATEV2); builder.put(DOUBLE, DATETIMEV2); + builder.put(DOUBLE, IPV4); + builder.put(DOUBLE, IPV6); builder.put(DOUBLE, DECIMALV2); builder.put(DOUBLE, DECIMAL32); builder.put(DOUBLE, DECIMAL64); @@ -424,6 +444,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(VARCHAR, DATETIME); builder.put(VARCHAR, DATEV2); builder.put(VARCHAR, DATETIMEV2); + builder.put(VARCHAR, IPV4); + builder.put(VARCHAR, IPV6); builder.put(VARCHAR, DECIMALV2); builder.put(VARCHAR, DECIMAL32); builder.put(VARCHAR, DECIMAL64); @@ -449,6 +471,8 @@ public static ImmutableSetMultimap getImplicitCast builder.put(STRING, DATETIME); builder.put(STRING, DATEV2); builder.put(STRING, DATETIMEV2); + builder.put(STRING, IPV4); + builder.put(STRING, IPV6); builder.put(STRING, DECIMALV2); builder.put(STRING, DECIMAL32); builder.put(STRING, DECIMAL64); @@ -640,6 +664,8 @@ public static ImmutableSetMultimap getImplicitCast supportedTypes.add(DATEV2); supportedTypes.add(DATETIMEV2); supportedTypes.add(TIMEV2); + supportedTypes.add(IPV4); + supportedTypes.add(IPV6); supportedTypes.add(DECIMALV2); supportedTypes.add(DECIMAL32); supportedTypes.add(DECIMAL64); @@ -718,6 +744,10 @@ public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) { return DATEV2; case DATETIMEV2: return DATETIMEV2; + case IPV4: + return IPV4; + case IPV6: + return IPV6; case BINARY: return BINARY; case DECIMALV2: @@ -880,6 +910,18 @@ public boolean isIntegerType() { || this == INT || this == BIGINT); } + public boolean isIPType() { + return (this == IPV4 || this == IPV6); + } + + public boolean isIPv4Type() { + return (this == IPV4); + } + + public boolean isIPv6Type() { + return (this == IPV6); + } + // TODO(zhaochun): Add Mysql Type to it's private field public MysqlColType toMysqlType() { switch (this) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index 7271429b66d684..3eb13e5cf3c061 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -630,10 +630,14 @@ public String toSql(int depth) { return "smallint(6)"; case INT: return "int(11)"; + case IPV4: + return "ipv4"; case BIGINT: return "bigint(20)"; case LARGEINT: return "largeint(40)"; + case IPV6: + return "ipv6"; case FLOAT: case DOUBLE: case DATE: diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 64d14350ae915b..36d35f9e2a0fd6 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -64,6 +64,8 @@ public abstract class Type { public static final ScalarType LARGEINT = new ScalarType(PrimitiveType.LARGEINT); public static final ScalarType FLOAT = new ScalarType(PrimitiveType.FLOAT); public static final ScalarType DOUBLE = new ScalarType(PrimitiveType.DOUBLE); + public static final ScalarType IPV4 = new ScalarType(PrimitiveType.IPV4); + public static final ScalarType IPV6 = new ScalarType(PrimitiveType.IPV6); public static final ScalarType DATE = new ScalarType(PrimitiveType.DATE); public static final ScalarType DATETIME = new ScalarType(PrimitiveType.DATETIME); public static final ScalarType DATEV2 = new ScalarType(PrimitiveType.DATEV2); @@ -175,6 +177,8 @@ public abstract class Type { trivialTypes.add(DATETIME); trivialTypes.add(DATEV2); trivialTypes.add(DATETIMEV2); + trivialTypes.add(IPV4); + trivialTypes.add(IPV6); trivialTypes.add(TIME); trivialTypes.add(TIMEV2); trivialTypes.add(JSONB); @@ -198,6 +202,8 @@ public abstract class Type { arraySubTypes.add(DATETIME); arraySubTypes.add(DATEV2); arraySubTypes.add(DATETIMEV2); + arraySubTypes.add(IPV4); + arraySubTypes.add(IPV6); arraySubTypes.add(CHAR); arraySubTypes.add(VARCHAR); arraySubTypes.add(STRING); @@ -218,6 +224,8 @@ public abstract class Type { mapSubTypes.add(DATETIME); mapSubTypes.add(DATEV2); mapSubTypes.add(DATETIMEV2); + mapSubTypes.add(IPV4); + mapSubTypes.add(IPV6); mapSubTypes.add(CHAR); mapSubTypes.add(VARCHAR); mapSubTypes.add(STRING); @@ -236,6 +244,8 @@ public abstract class Type { structSubTypes.add(DATETIME); structSubTypes.add(DATEV2); structSubTypes.add(DATETIMEV2); + structSubTypes.add(IPV4); + structSubTypes.add(IPV6); structSubTypes.add(CHAR); structSubTypes.add(VARCHAR); structSubTypes.add(STRING); @@ -254,6 +264,7 @@ public abstract class Type { .put(PrimitiveType.FLOAT, Sets.newHashSet(Float.class, float.class)) .put(PrimitiveType.DOUBLE, Sets.newHashSet(Double.class, double.class)) .put(PrimitiveType.BIGINT, Sets.newHashSet(Long.class, long.class)) + .put(PrimitiveType.IPV4, Sets.newHashSet(Integer.class, int.class)) .put(PrimitiveType.CHAR, Sets.newHashSet(String.class)) .put(PrimitiveType.VARCHAR, Sets.newHashSet(String.class)) .put(PrimitiveType.STRING, Sets.newHashSet(String.class)) @@ -593,6 +604,18 @@ public boolean isDateV2OrDateTimeV2() { return isScalarType(PrimitiveType.DATEV2) || isScalarType(PrimitiveType.DATETIMEV2); } + public boolean isIP() { + return isScalarType(PrimitiveType.IPV4) || isScalarType(PrimitiveType.IPV6); + } + + public boolean isIPv4() { + return isScalarType(PrimitiveType.IPV4); + } + + public boolean isIPv6() { + return isScalarType(PrimitiveType.IPV6); + } + public boolean hasTemplateType() { return false; } @@ -871,6 +894,10 @@ public static Type fromPrimitiveType(PrimitiveType type) { return Type.FLOAT; case DOUBLE: return Type.DOUBLE; + case IPV4: + return Type.IPV4; + case IPV6: + return Type.IPV6; case DATE: return Type.DATE; case DATETIME: @@ -1277,6 +1304,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL32.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1307,6 +1336,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[TINYINT.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // SMALLINT @@ -1335,6 +1366,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[SMALLINT.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // INT @@ -1366,6 +1399,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[INT.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[INT.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[INT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[INT.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[INT.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[INT.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // BIGINT @@ -1398,6 +1433,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[BIGINT.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BIGINT.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BIGINT.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // LARGEINT @@ -1422,6 +1459,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[LARGEINT.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[LARGEINT.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[LARGEINT.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // FLOAT @@ -1445,6 +1484,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[FLOAT.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DOUBLE @@ -1467,6 +1508,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DOUBLE.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[DOUBLE.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; + compatibilityMatrix[DOUBLE.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DOUBLE.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DATE @@ -1488,6 +1531,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DATE.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATE.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATE.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DATEV2 @@ -1509,6 +1554,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DATEV2.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATEV2.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATEV2.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATEV2.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DATETIME @@ -1529,6 +1576,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DATETIME.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIME.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIME.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DATETIMEV2 @@ -1549,6 +1598,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DATETIMEV2.ordinal()][JSONB.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIMEV2.ordinal()][VARIANT.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIMEV2.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIMEV2.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // We can convert some but not all string values to timestamps. @@ -1569,6 +1620,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[CHAR.ordinal()][JSONB.ordinal()] = PrimitiveType.CHAR; compatibilityMatrix[CHAR.ordinal()][VARIANT.ordinal()] = PrimitiveType.CHAR; compatibilityMatrix[CHAR.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[CHAR.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[CHAR.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // VARCHAR @@ -1587,6 +1640,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[VARCHAR.ordinal()][JSONB.ordinal()] = PrimitiveType.VARCHAR; compatibilityMatrix[VARCHAR.ordinal()][VARIANT.ordinal()] = PrimitiveType.VARCHAR; compatibilityMatrix[VARCHAR.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[VARCHAR.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[VARCHAR.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; //String @@ -1603,6 +1658,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[STRING.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][JSONB.ordinal()] = PrimitiveType.STRING; compatibilityMatrix[STRING.ordinal()][VARIANT.ordinal()] = PrimitiveType.STRING; + compatibilityMatrix[STRING.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[STRING.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; //JSONB @@ -1621,6 +1678,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[JSONB.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; compatibilityMatrix[JSONB.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR; compatibilityMatrix[JSONB.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + compatibilityMatrix[JSONB.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[JSONB.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[JSONB.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // VARIANT compatibilityMatrix[VARIANT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1654,6 +1713,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL32.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMALV2.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMALV2.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1670,6 +1731,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DECIMAL32.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMAL32.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.DECIMAL64; compatibilityMatrix[DECIMAL32.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.DECIMAL128; + compatibilityMatrix[DECIMAL32.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMAL32.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMAL32.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.DECIMAL256; compatibilityMatrix[DECIMAL32.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1687,6 +1750,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DECIMAL64.ordinal()][DECIMAL32.ordinal()] = PrimitiveType.DECIMAL64; compatibilityMatrix[DECIMAL64.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.DECIMAL128; compatibilityMatrix[DECIMAL64.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.DECIMAL256; + compatibilityMatrix[DECIMAL64.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMAL64.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMAL64.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DECIMAL128 @@ -1703,6 +1768,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[DECIMAL128.ordinal()][DECIMAL32.ordinal()] = PrimitiveType.DECIMAL128; compatibilityMatrix[DECIMAL128.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.DECIMAL128; compatibilityMatrix[DECIMAL128.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.DECIMAL256; + compatibilityMatrix[DECIMAL128.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMAL128.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMAL128.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // DECIMAL256 @@ -1734,6 +1801,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[HLL.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[HLL.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[HLL.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1750,6 +1819,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[BITMAP.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BITMAP.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BITMAP.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BITMAP.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BITMAP.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BITMAP.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; //QUANTILE_STATE @@ -1761,6 +1832,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[QUANTILE_STATE.ordinal()][DECIMAL32.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[QUANTILE_STATE.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[QUANTILE_STATE.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[QUANTILE_STATE.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[QUANTILE_STATE.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[QUANTILE_STATE.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[QUANTILE_STATE.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1784,6 +1857,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[TIME.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIME.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIME.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIME.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIME.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIME.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIMEV2.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1792,6 +1867,8 @@ public Integer getNumPrecRadix() { compatibilityMatrix[TIMEV2.ordinal()][DECIMAL64.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIMEV2.ordinal()][DECIMAL128.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIMEV2.ordinal()][DECIMAL256.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIMEV2.ordinal()][IPV4.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIMEV2.ordinal()][IPV6.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TIMEV2.ordinal()][AGG_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; // Check all of the necessary entries that should be filled. @@ -1812,7 +1889,9 @@ public Integer getNumPrecRadix() { || t1 == PrimitiveType.TEMPLATE || t2 == PrimitiveType.TEMPLATE || t1 == PrimitiveType.UNSUPPORTED || t2 == PrimitiveType.UNSUPPORTED || t1 == PrimitiveType.VARIANT || t2 == PrimitiveType.VARIANT - || t1 == PrimitiveType.LAMBDA_FUNCTION || t2 == PrimitiveType.LAMBDA_FUNCTION) { + || t1 == PrimitiveType.LAMBDA_FUNCTION || t2 == PrimitiveType.LAMBDA_FUNCTION + || t1 == PrimitiveType.IPV4 || t2 == PrimitiveType.IPV4 + || t1 == PrimitiveType.IPV6 || t2 == PrimitiveType.IPV6) { continue; } Preconditions.checkNotNull(compatibilityMatrix[i][j]); @@ -1845,6 +1924,10 @@ public Type getResultType() { case BITMAP: case QUANTILE_STATE: return VARCHAR; + case IPV4: + return IPV4; + case IPV6: + return IPV6; case DECIMALV2: return DECIMALV2; case DECIMAL32: @@ -1957,6 +2040,18 @@ public static Type getCmpType(Type t1, Type t2) { || t2ResultType == PrimitiveType.LARGEINT)) { return Type.LARGEINT; } + if ((t1ResultType == PrimitiveType.IPV4 + || t1ResultType == PrimitiveType.VARCHAR) + && (t2ResultType == PrimitiveType.IPV4 + || t2ResultType == PrimitiveType.VARCHAR)) { + return Type.IPV4; + } + if ((t1ResultType == PrimitiveType.IPV6 + || t1ResultType == PrimitiveType.VARCHAR) + && (t2ResultType == PrimitiveType.IPV6 + || t2ResultType == PrimitiveType.VARCHAR)) { + return Type.IPV6; + } return Type.DOUBLE; } diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 010ae43dc3cbaf..7e64a79e30d5fc 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -293,6 +293,8 @@ INTERSECT: 'INTERSECT'; INTERVAL: 'INTERVAL'; INTO: 'INTO'; INVERTED: 'INVERTED'; +IPV4: 'IPV4'; +IPV6: 'IPV6'; IS: 'IS'; IS_NOT_NULL_PRED: 'IS_NOT_NULL_PRED'; IS_NULL_PRED: 'IS_NULL_PRED'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 4c74e424e60658..7c7fce72ce1f61 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -747,6 +747,8 @@ primitiveColType: | type=CHAR | type=DECIMAL | type=DECIMALV3 + | type=IPV4 + | type=IPV6 | type=ALL ; diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index b0d487a6ca32f3..c3e16e9b4c2d32 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -426,6 +426,8 @@ terminal String KW_INTERSECT, KW_INTERVAL, KW_INTO, + KW_IPV4, + KW_IPV6, KW_IS, KW_ISNULL, KW_ISOLATION, @@ -6370,6 +6372,10 @@ type ::= {: RESULT = ScalarType.createDatetimeV2Type(precision.intValue()); :} | KW_DATETIMEV2 {: RESULT = ScalarType.createDatetimeV2Type(0); :} + | KW_IPV4 + {: RESULT = Type.IPV4; :} + | KW_IPV6 + {: RESULT = Type.IPV6; :} | KW_BITMAP {: RESULT = Type.BITMAP; :} | KW_QUANTILE_STATE diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index 9c73d84857c3ba..f99bcacf441454 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -329,6 +329,19 @@ private boolean canCompareDate(PrimitiveType t1, PrimitiveType t2) { } } + private boolean canCompareIP(PrimitiveType t1, PrimitiveType t2) { + if (t1.isIPv4Type()) { + return t2.isIPv4Type() || t2.isStringType(); + } else if (t2.isIPv4Type()) { + return t1.isStringType(); + } else if (t1.isIPv6Type()) { + return t2.isIPv6Type() || t2.isStringType(); + } else if (t2.isIPv6Type()) { + return t1.isStringType(); + } + return false; + } + private Type dateV2ComparisonResultType(ScalarType t1, ScalarType t2) { if (!t1.isDatetimeV2() && !t2.isDatetimeV2()) { return Type.DATEV2; @@ -411,6 +424,19 @@ private Type getCmpType() throws AnalysisException { } } + if (canCompareIP(getChild(0).getType().getPrimitiveType(), getChild(1).getType().getPrimitiveType())) { + if ((getChild(0).getType().isIP() && getChild(1) instanceof StringLiteral) + || (getChild(1).getType().isIP() && getChild(0) instanceof StringLiteral) + || (getChild(0).getType().isIP() && getChild(1).getType().isIP())) { + if (getChild(0).getType().isIPv4() || getChild(1).getType().isIPv4()) { + return Type.IPV4; + } + if (getChild(0).getType().isIPv6() || getChild(1).getType().isIPv6()) { + return Type.IPV6; + } + } + } + // Following logical is compatible with MySQL: // Cast to DOUBLE by default, because DOUBLE has the largest range of values. if (t1 == PrimitiveType.VARCHAR && t2 == PrimitiveType.VARCHAR) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java new file mode 100644 index 00000000000000..7eee7fe1e6cc28 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TExprNode; +import org.apache.doris.thrift.TExprNodeType; +import org.apache.doris.thrift.TIPv4Literal; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.regex.Pattern; + +public class IPv4Literal extends LiteralExpr { + private static final Logger LOG = LogManager.getLogger(IPv4Literal.class); + + public static final long IPV4_MIN = 0L; // 0.0.0.0 + public static final long IPV4_MAX = (2L << 31) - 1; // 255.255.255.255 + private static final Pattern IPV4_STD_REGEX = + Pattern.compile("^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"); + + private long value; + + /** + * C'tor forcing type, e.g., due to implicit cast + */ + // for restore + private IPv4Literal() { + } + + public IPv4Literal(long value) { + super(); + this.value = value; + this.type = Type.IPV4; + analysisDone(); + } + + public IPv4Literal(String value) throws AnalysisException { + super(); + this.value = parseIPv4toLong(value); + this.type = Type.IPV4; + analysisDone(); + } + + protected IPv4Literal(IPv4Literal other) { + super(other); + this.value = other.value; + } + + private static long parseIPv4toLong(String ipv4) { + String[] parts = ipv4.split("\\."); + if (parts.length != 4) { + return 0L; + } + + long value = 0L; + for (int i = 0; i < 4; ++i) { + short octet; + try { + octet = Short.parseShort(parts[i]); + } catch (NumberFormatException e) { + return 0L; + } + if (octet < 0 || octet > 255) { + return 0L; + } + value = (value << 8) | octet; + } + + return value; + } + + private static String parseLongToIPv4(long ipv4) { + StringBuilder sb = new StringBuilder(); + for (int i = 3; i >= 0; i--) { + short octet = (short) ((ipv4 >> (i * 8)) & 0xFF); + sb.append(octet); + if (i > 0) { + sb.append("."); + } + } + return sb.toString(); + } + + private void checkValueValid(String ipv4) throws AnalysisException { + if (ipv4.length() > 15) { + throw new AnalysisException("The length of IPv4 must not exceed 15. type: " + Type.IPV4); + } else if (!IPV4_STD_REGEX.matcher(ipv4).matches()) { + throw new AnalysisException("Invalid IPv4 format: " + ipv4 + ". type: " + Type.IPV4); + } + } + + + @Override + public Expr clone() { + return new IPv4Literal(this); + } + + @Override + protected String toSqlImpl() { + return getStringValue(); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.IPV4_LITERAL; + msg.ipv4_literal = new TIPv4Literal(this.value); + } + + @Override + public boolean isMinValue() { + return this.value == IPV4_MIN; + } + + @Override + public int compareLiteral(LiteralExpr expr) { + return 0; + } + + @Override + public String getStringValue() { + return parseLongToIPv4(this.value); + } + + @Override + public String getStringValueForArray() { + return "\"" + getStringValue() + "\""; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java new file mode 100644 index 00000000000000..7597dedc7de67d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TExprNode; +import org.apache.doris.thrift.TExprNodeType; +import org.apache.doris.thrift.TIPv6Literal; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.regex.Pattern; + +public class IPv6Literal extends LiteralExpr { + private static final Logger LOG = LogManager.getLogger(IPv6Literal.class); + + public static final String IPV6_MIN = "0:0:0:0:0:0:0:0"; + public static final String IPV6_MAX = "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"; + private static final Pattern IPV6_STD_REGEX = + Pattern.compile("^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$"); + private static final Pattern IPV6_COMPRESS_REGEX = + Pattern.compile("^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::((([0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4})?)$"); + + private String value; + + /** + * C'tor forcing type, e.g., due to implicit cast + */ + // for restore + private IPv6Literal() { + } + + public IPv6Literal(String value) throws AnalysisException { + super(); + checkValueValid(value); + this.value = value; + this.type = Type.IPV6; + analysisDone(); + } + + protected IPv6Literal(IPv6Literal other) { + super(other); + this.value = other.value; + } + + @Override + public void checkValueValid() throws AnalysisException { + checkValueValid(this.value); + } + + private void checkValueValid(String ipv6) throws AnalysisException { + if (ipv6.length() > 39) { + throw new AnalysisException("The length of IPv6 must not exceed 39. type: " + Type.IPV6); + } else if (!IPV6_STD_REGEX.matcher(ipv6).matches() && !IPV6_COMPRESS_REGEX.matcher(ipv6).matches()) { + throw new AnalysisException("Invalid IPv6 format: " + ipv6 + ". type: " + Type.IPV6); + } + } + + @Override + protected String toSqlImpl() { + return getStringValue(); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.IPV6_LITERAL; + msg.ipv6_literal = new TIPv6Literal(this.value); + } + + @Override + public Expr clone() { + return new IPv6Literal(this); + } + + @Override + public boolean isMinValue() { + return IPV6_MIN.equals(this.value); + } + + @Override + public int compareLiteral(LiteralExpr expr) { + return 0; + } + + @Override + public String getStringValue() { + return this.value; + } + + @Override + public String getStringValueForArray() { + return "\"" + getStringValue() + "\""; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index bf24955970c7b2..bbeffe7366d379 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -94,6 +94,12 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti case DATETIMEV2: literalExpr = new DateLiteral(value, type); break; + case IPV4: + literalExpr = new IPv4Literal(value); + break; + case IPV6: + literalExpr = new IPv6Literal(value); + break; default: throw new AnalysisException("Type[" + type.toSql() + "] not supported."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 3119781c97cdb8..5b2836a318df0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -167,6 +167,40 @@ public String getRealValue() { return getStringValue(); } + /** + * Convert a string literal to a IPv4 literal + * + * @return new converted literal (not null) + * @throws AnalysisException when entire given string cannot be transformed into a date + */ + public LiteralExpr convertToIPv4() throws AnalysisException { + LiteralExpr newLiteral; + newLiteral = new IPv4Literal(value); + try { + newLiteral.checkValueValid(); + } catch (AnalysisException e) { + return NullLiteral.create(newLiteral.getType()); + } + return newLiteral; + } + + /** + * Convert a string literal to a IPv6 literal + * + * @return new converted literal (not null) + * @throws AnalysisException when entire given string cannot be transformed into a date + */ + public LiteralExpr convertToIPv6() throws AnalysisException { + LiteralExpr newLiteral; + newLiteral = new IPv6Literal(value); + try { + newLiteral.checkValueValid(); + } catch (AnalysisException e) { + return NullLiteral.create(newLiteral.getType()); + } + return newLiteral; + } + /** * Convert a string literal to a date literal * @@ -266,6 +300,10 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { } catch (AnalysisException e) { // pass; } + } else if (targetType.isIPv4()) { + return convertToIPv4(); + } else if (targetType.isIPv6()) { + return convertToIPv6(); } else if (targetType.equals(type)) { return this; } else if (targetType.isStringType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java new file mode 100644 index 00000000000000..e4e931738b02a2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.IPv4Type; + +import java.util.regex.Pattern; + +/** + * Represents IPv4 literal + */ +public class IPv4Literal extends Literal { + + private static final Pattern IPV4_STD_REGEX = + Pattern.compile("^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"); + + private long value; + + public IPv4Literal(String ipv4) throws AnalysisException { + super(IPv4Type.INSTANCE); + init(ipv4); + } + + protected IPv4Literal(long value) { + super(IPv4Type.INSTANCE); + this.value = value; + } + + @Override + public Long getValue() { + return value; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitIPv4Literal(this, context); + } + + @Override + public LiteralExpr toLegacyLiteral() { + return new org.apache.doris.analysis.IPv4Literal(value); + } + + void init(String ipv4) throws AnalysisException { + checkValueValid(ipv4); + + String[] parts = ipv4.split("\\."); + if (parts.length != 4) { + return; + } + + long value = 0L; + for (int i = 0; i < 4; ++i) { + short octet; + try { + octet = Short.parseShort(parts[i]); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid IPv4 format."); + } + if (octet < 0 || octet > 255) { + throw new AnalysisException("Invalid IPv4 format."); + } + value = (value << 8) | octet; + } + this.value = value; + } + + private void checkValueValid(String ipv4) throws AnalysisException { + if (ipv4.length() > 15) { + throw new AnalysisException("The length of IPv4 must not exceed 15."); + } else if (!IPV4_STD_REGEX.matcher(ipv4).matches()) { + throw new AnalysisException("Invalid IPv4 format."); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java new file mode 100644 index 00000000000000..0ac12ae85e477a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.IPv6Type; + +import java.util.regex.Pattern; + +/** + * Represents IPv6 literal + */ +public class IPv6Literal extends Literal { + + private static final Pattern IPV6_STD_REGEX = + Pattern.compile("^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$"); + private static final Pattern IPV6_COMPRESS_REGEX = + Pattern.compile("^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::((([0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4})?)$"); + + private final String value; + + public IPv6Literal(String ipv6) throws AnalysisException { + super(IPv6Type.INSTANCE); + checkValueValid(ipv6); + this.value = ipv6; + } + + @Override + public String getValue() { + return value; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitIPv6Literal(this, context); + } + + @Override + public LiteralExpr toLegacyLiteral() { + try { + return new org.apache.doris.analysis.IPv6Literal(value); + } catch (Exception e) { + throw new AnalysisException("Invalid IPv6 format."); + } + } + + public void checkValueValid(String ipv6) throws AnalysisException { + if (ipv6.length() > 39) { + throw new AnalysisException("The length of IPv6 must not exceed 39."); + } else if (!IPV6_STD_REGEX.matcher(ipv6).matches() && !IPV6_COMPRESS_REGEX.matcher(ipv6).matches()) { + throw new AnalysisException("Invalid IPv6 format."); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 1a28817ce83b32..e716d66994c69a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -249,6 +249,10 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return new DateTimeV2Literal((DateTimeV2Type) targetType, desc); } else if (targetType.isJsonType()) { return new JsonLiteral(desc); + } else if (targetType.isIPv4Type()) { + return new IPv4Literal(desc); + } else if (targetType.isIPv6Type()) { + return new IPv6Literal(desc); } throw new AnalysisException("cannot cast " + desc + " from type " + this.dataType + " to type " + targetType); } @@ -296,6 +300,10 @@ public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) { return new DateTimeV2Literal(stringValue); } else if (dataType.isJsonType()) { return new JsonLiteral(stringValue); + } else if (dataType.isIPv4Type()) { + return new IPv4Literal(stringValue); + } else if (dataType.isIPv6Type()) { + return new IPv6Literal(stringValue); } else { throw new AnalysisException("Unsupported convert the " + literalExpr.getType() + " of legacy literal to nereids literal"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index 01853926ce3e5f..308a4387ce9cd5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -100,6 +100,8 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IPv4Literal; +import org.apache.doris.nereids.trees.expressions.literal.IPv6Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Interval; import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; @@ -303,6 +305,14 @@ public R visitDateTimeV2Literal(DateTimeV2Literal dateTimeV2Literal, C context) return visitLiteral(dateTimeV2Literal, context); } + public R visitIPv4Literal(IPv4Literal ipv4Literal, C context) { + return visitLiteral(ipv4Literal, context); + } + + public R visitIPv6Literal(IPv6Literal ipv6Literal, C context) { + return visitLiteral(ipv6Literal, context); + } + public R visitArrayLiteral(ArrayLiteral arrayLiteral, C context) { return visitLiteral(arrayLiteral, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index c55e8dfc66f7ad..c939786237a34c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -259,6 +259,12 @@ public static DataType convertPrimitiveFromStrings(List types, boolean u case "jsonb": dataType = JsonType.INSTANCE; break; + case "ipv4": + dataType = IPv4Type.INSTANCE; + break; + case "ipv6": + dataType = IPv6Type.INSTANCE; + break; default: throw new AnalysisException("Nereids do not support type: " + type); } @@ -358,6 +364,10 @@ public static DataType fromCatalogType(Type type) { List types = catalogType.getSubTypes().stream().map(DataType::fromCatalogType) .collect(Collectors.toList()); return new AggStateType(catalogType.getFunctionName(), types, catalogType.getSubTypeNullables()); + } else if (type.isIPv4()) { + return IPv4Type.INSTANCE; + } else if (type.isIPv6()) { + return IPv6Type.INSTANCE; } else { return UnsupportedType.INSTANCE; } @@ -542,6 +552,14 @@ public boolean isDateTimeV2Type() { return this instanceof DateTimeV2Type; } + public boolean isIPv4Type() { + return this instanceof IPv4Type; + } + + public boolean isIPv6Type() { + return this instanceof IPv6Type; + } + public boolean isBitmapType() { return this instanceof BitmapType; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/IPv4Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/IPv4Type.java new file mode 100644 index 00000000000000..c86377b1010ab4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/IPv4Type.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.types.coercion.PrimitiveType; + +/** + * IPv4 type in Nereids. + */ +public class IPv4Type extends PrimitiveType { + + public static final IPv4Type INSTANCE = new IPv4Type(); + + public static final int WIDTH = 4; + + private IPv4Type() { + } + + @Override + public Type toCatalogDataType() { + return Type.IPV4; + } + + @Override + public boolean acceptsType(DataType other) { + return other instanceof IPv4Type; + } + + @Override + public String simpleString() { + return "ipv4"; + } + + @Override + public DataType defaultConcreteType() { + return INSTANCE; + } + + @Override + public boolean equals(Object o) { + return o instanceof IPv4Type; + } + + @Override + public int width() { + return WIDTH; + } + + @Override + public String toSql() { + return "IPV4"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/IPv6Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/IPv6Type.java new file mode 100644 index 00000000000000..ece20835a5dbdc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/IPv6Type.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.types.coercion.PrimitiveType; + +/** + * IPv6 type in Nereids. + */ +public class IPv6Type extends PrimitiveType { + + public static final IPv6Type INSTANCE = new IPv6Type(); + + public static final int WIDTH = 16; + + private IPv6Type() { + } + + @Override + public Type toCatalogDataType() { + return Type.IPV6; + } + + @Override + public boolean acceptsType(DataType other) { + return other instanceof IPv6Type; + } + + @Override + public String simpleString() { + return "ipv6"; + } + + @Override + public DataType defaultConcreteType() { + return INSTANCE; + } + + @Override + public boolean equals(Object o) { + return o instanceof IPv6Type; + } + + @Override + public int width() { + return WIDTH; + } + + @Override + public String toSql() { + return "IPV6"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 3adda07c3fc27b..490c97a47fc49e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -82,6 +82,8 @@ import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.FloatType; +import org.apache.doris.nereids.types.IPv4Type; +import org.apache.doris.nereids.types.IPv6Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.JsonType; import org.apache.doris.nereids.types.LargeIntType; @@ -227,6 +229,8 @@ private static Optional implicitCastPrimitiveInternal(DataType input, } else if (expected instanceof NumericType) { // For any other numeric types, implicitly cast to each other, e.g. bigint -> int, int -> bigint returnType = expected.defaultConcreteType(); + } else if (expected instanceof IPv4Type) { + returnType = IPv4Type.INSTANCE; } } else if (input instanceof CharacterType) { if (expected instanceof DecimalV2Type) { @@ -239,6 +243,10 @@ private static Optional implicitCastPrimitiveInternal(DataType input, returnType = DateTimeType.INSTANCE; } else if (expected instanceof JsonType) { returnType = JsonType.INSTANCE; + } else if (expected instanceof IPv4Type) { + returnType = IPv4Type.INSTANCE; + } else if (expected instanceof IPv6Type) { + returnType = IPv6Type.INSTANCE; } } else if (input.isDateType()) { if (expected instanceof DateTimeType) { @@ -1205,6 +1213,16 @@ private static Optional findCommonPrimitiveTypeForComparison( return Optional.of(commonType); } + // ip type + if ((leftType.isIPv4Type() && rightType.isStringLikeType()) + || (rightType.isIPv4Type() && leftType.isStringLikeType())) { + return Optional.of(IPv4Type.INSTANCE); + } + if ((leftType.isIPv6Type() && rightType.isStringLikeType()) + || (rightType.isIPv6Type() && leftType.isStringLikeType())) { + return Optional.of(IPv6Type.INSTANCE); + } + return Optional.of(DoubleType.INSTANCE); } diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index c360d63701de79..6d9db3a1e28444 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -279,6 +279,8 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("intersect", new Integer(SqlParserSymbols.KW_INTERSECT)); keywordMap.put("interval", new Integer(SqlParserSymbols.KW_INTERVAL)); keywordMap.put("into", new Integer(SqlParserSymbols.KW_INTO)); + keywordMap.put("ipv4", new Integer(SqlParserSymbols.KW_IPV4)); + keywordMap.put("ipv6", new Integer(SqlParserSymbols.KW_IPV6)); keywordMap.put("is", new Integer(SqlParserSymbols.KW_IS)); keywordMap.put("isnull", new Integer(SqlParserSymbols.KW_ISNULL)); keywordMap.put("isolation", new Integer(SqlParserSymbols.KW_ISOLATION)); diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 240b68c89d8cd3..b7a4b6ee53a637 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -113,6 +113,8 @@ message PGenericType { AGG_STATE = 36; TIMEV2 = 37; DECIMAL256 = 38; + IPV4 = 39; + IPV6 = 40; UNKNOWN = 999; } required TypeId id = 2; diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index e102babca21a58..4311bd44b23320 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -74,6 +74,9 @@ enum TExprNodeType { LAMBDA_FUNCTION_CALL_EXPR, // for column_ref expr COLUMN_REF, + + IPV4_LITERAL, + IPV6_LITERAL } //enum TAggregationOp { @@ -127,6 +130,14 @@ struct TLargeIntLiteral { 1: required string value } +struct TIPv4Literal { + 1: required i64 value +} + +struct TIPv6Literal { + 1: required string value +} + struct TInPredicate { 1: required bool is_not_in } @@ -242,6 +253,8 @@ struct TExprNode { 32: optional TColumnRef column_ref 33: optional TMatchPredicate match_predicate + 34: optional TIPv4Literal ipv4_literal + 35: optional TIPv6Literal ipv6_literal } // A flattened representation of a tree of Expr nodes, obtained by depth-first diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 75266e3ceda74e..7da21f30bc17d9 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -95,7 +95,9 @@ enum TPrimitiveType { VARIANT, LAMBDA_FUNCTION, AGG_STATE, - DECIMAL256 + DECIMAL256, + IPV4, + IPV6 } enum TTypeNodeType { diff --git a/regression-test/data/datatype_p0/ip/test_data/test.csv b/regression-test/data/datatype_p0/ip/test_data/test.csv new file mode 100644 index 00000000000000..d8672b42205278 --- /dev/null +++ b/regression-test/data/datatype_p0/ip/test_data/test.csv @@ -0,0 +1,100 @@ +1,116.253.40.133,2606:2b00::1 +2,183.247.232.58,2001:2000:3080:1351::2 +3,116.106.34.242,2a01:8840:16::1 +4,111.56.27.171,2001:550:0:1000::9a36:2a61 +5,183.245.137.140,2001:578:400:4:2000::19 +6,183.212.25.70,2607:f290::1 +7,162.144.2.57,2a02:23f0:ffff:8::5 +8,111.4.229.190,2400:c700:0:158:: +9,59.52.3.168,2001:67c:24e4:4::250 +10,115.11.21.200,2a02:2a38:37:5::2 +11,121.28.97.113,2001:41a8:400:2::13a +12,111.46.39.248,2405:9800:9800:66::2 +13,120.192.122.34,2a07:a343:f210::1 +14,113.56.44.105,2403:5000:171:46::2 +15,116.66.238.92,2800:c20:1141::8 +16,67.22.254.206,2402:7800:40:2::62 +17,115.0.24.191,2a00:de00::1 +18,182.30.107.86,2001:688:0:2:1::9e +19,223.73.153.243,2001:2000:3080:80::2 +20,115.159.103.38,2001:428::205:171:200:230 +21,36.186.75.121,2001:fb1:fe0:9::8 +22,111.56.188.125,2001:2000:3080:10ca::2 +23,115.14.93.25,2400:dd0b:1003::2 +24,211.97.110.141,2001:1a98:6677::9d9d:140a +25,61.58.96.173,2001:2000:3018:3b::1 +26,203.126.212.37,2607:fa70:3:33::2 +27,192.220.125.142,2001:5b0:23ff:fffa::113 +28,115.22.20.223,2001:450:2001:1000:0:40:6924:23 +29,121.25.160.80,2001:418:0:5000::c2d +30,117.150.98.199,2a01:b740:a09::1 +31,183.211.172.143,2607:f0d0:2:2::243 +32,180.244.18.143,2a01:348::e:1:1 +33,209.131.3.252,2405:4800::3221:3621:2 +34,220.200.1.22,2a02:aa08:e000:3100::2 +35,171.225.130.45,2001:44c8:129:2632:33:0:252:2 +36,115.4.78.200,2a02:e980:1e::1 +37,36.183.59.29,2a0a:6f40:2::1 +38,218.42.159.17,2001:550:2:29::2c9:1 +39,115.13.39.164,2001:c20:4800::175 +40,142.254.161.133,2c0f:feb0:1:2::d1 +41,116.2.211.43,2a0b:7086:fff0::1 +42,36.183.126.25,2a04:2dc0::16d +43,66.150.171.196,2604:7e00::105d +44,104.149.148.137,2001:470:1:946::2 +45,120.239.82.212,2a0c:3240::1 +46,111.14.182.156,2800:630:4010:8::2 +47,115.6.63.224,2001:1af8:4040::12 +48,153.35.83.233,2c0f:fc98:1200::2 +49,113.142.1.1,2001:470:1:248::2 +50,121.25.82.29,2620:44:a000::1 +51,62.151.203.189,2402:800:63ff:40::1 +52,104.27.46.146,2a02:b000:fff::524 +53,36.189.46.88,2001:470:0:327::1 +54,116.252.54.207,2401:7400:8888:2::8 +55,64.77.240.1,2001:500:55::1 +56,142.252.102.78,2001:668:0:3::f000:c2 +57,36.82.224.170,2400:bf40::1 +58,117.33.191.217,2001:67c:754::1 +59,144.12.164.251,2402:28c0:100:ffff:ffff:ffff:ffff:ffff +60,122.10.93.66,2001:470:0:1fa::2 +61,104.25.84.59,2001:550:0:1000::9a18:292a +62,111.4.242.106,2001:470:1:89e::2 +63,222.216.51.186,2001:579:6f05:500:9934:5b3e:b7fe:1447 +64,112.33.13.212,2804:158c::1 +65,115.9.240.116,2600:140e:6::1 +66,171.228.0.153,2a00:18e0:0:bb04::82 +67,45.3.47.158,2a02:2698:5000::1e06 +68,69.57.193.230,2402:800:63ff:10::7:2 +69,115.6.104.199,2a02:e980:19::1 +70,104.24.237.140,2001:4888::342:1:0:0 +71,199.17.84.108,2607:fc68:0:4:0:2:2711:21 +72,120.193.17.57,2606:2800:602a::1 +73,112.40.38.145,2404:c600:1000:2::1d1 +74,67.55.90.43,2001:578:1400:4::9d +75,180.253.57.249,2804:64:0:25::1 +76,14.204.253.158,2605:3e00::1:2:2 +77,1.83.241.116,2c0f:fa18:0:4::b +78,202.198.37.147,2606:2800:602c:b::d004 +79,115.6.31.95,2610:18:181:4000::66 +80,117.32.14.179,2001:48f8:1000:1::16 +81,23.238.237.26,2408:8000:c000::1 +82,116.97.76.104,2a03:4200:441:2::4e +83,1.80.2.248,2400:dd00:1:200a::2 +84,59.50.185.152,2a02:e980:83:5b09:ecb8:c669:b336:650e +85,42.117.228.166,2001:16a0:2:200a::2 +86,119.36.22.147,2001:4888:1f:e891:161:26:: +87,210.66.18.184,2a0c:f743::1 +88,115.19.192.159,2a02:e980:b::1 +89,112.15.128.113,2001:578:201:1::601:9 +90,1.55.138.211,2001:438:ffff::407d:1bc1 +91,210.183.19.113,2001:920:1833::1 +92,42.115.43.114,2001:1b70:a1:610::b102:2 +93,58.16.171.31,2001:13c7:6014::1 +94,171.234.78.185,2003:0:1203:4001::1 +95,113.56.43.134,2804:a8:2:c8::d6 +96,111.53.182.225,2a02:2e00:2080:f000:0:261:1:11 +97,107.160.215.141,2001:578:20::d +98,171.229.231.90,2001:550:2:48::34:1 +99,58.19.84.138,2a03:9d40:fe00:5:: +100,36.79.88.107,2403:e800:200:102::2 diff --git a/regression-test/data/datatype_p0/ip/test_ip_crud.out b/regression-test/data/datatype_p0/ip/test_ip_crud.out new file mode 100644 index 00000000000000..07323da29e5bef --- /dev/null +++ b/regression-test/data/datatype_p0/ip/test_ip_crud.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +2 42.117.228.166 2001:16a0:2:200a::2 +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql2 -- +2 42.117.228.166 2001:16a0:2:200a::2 + +-- !sql3 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e + +-- !sql4 -- +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql5 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +2 42.117.228.166 2001:16a0:2:200a::2 +3 0.0.0.0 2804:64:0:25::1 + +-- !sql6 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +2 42.117.228.166 2001:16a0:2:200a::2 +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql7 -- +2 42.117.228.166 2001:16a0:2:200a::2 + +-- !sql8 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e + +-- !sql9 -- +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql10 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +2 42.117.228.166 2001:16a0:2:200a::2 +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql11 -- +2 42.117.228.166 2001:16a0:2:200a::2 + +-- !sql12 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e + +-- !sql13 -- +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql14 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql15 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e diff --git a/regression-test/data/datatype_p0/ip/test_ip_implicit_cast.out b/regression-test/data/datatype_p0/ip/test_ip_implicit_cast.out new file mode 100644 index 00000000000000..7e7f571ef5a3e2 --- /dev/null +++ b/regression-test/data/datatype_p0/ip/test_ip_implicit_cast.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +-1 \N \N +0 0.0.0.0 :: +1 0.0.0.1 ::1 +2130706433 127.0.0.1 2001:1b70:a1:610::b102:2 +4294967295 255.255.255.255 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff diff --git a/regression-test/data/datatype_p0/ip/test_ip_in_predicate.out b/regression-test/data/datatype_p0/ip/test_ip_in_predicate.out new file mode 100644 index 00000000000000..d94b90c0bebedd --- /dev/null +++ b/regression-test/data/datatype_p0/ip/test_ip_in_predicate.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +2 42.117.228.166 2001:16a0:2:200a::2 +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql2 -- +2 42.117.228.166 2001:16a0:2:200a::2 +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql3 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e + +-- !sql4 -- +1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +3 119.36.22.147 2001:4888:1f:e891:161:26:: + +-- !sql5 -- +2 42.117.228.166 2001:16a0:2:200a::2 diff --git a/regression-test/data/datatype_p0/ip/test_ip_stream_load.out b/regression-test/data/datatype_p0/ip/test_ip_stream_load.out new file mode 100644 index 00000000000000..3030b07a3d04b0 --- /dev/null +++ b/regression-test/data/datatype_p0/ip/test_ip_stream_load.out @@ -0,0 +1,102 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 116.253.40.133 2606:2b00::1 +2 183.247.232.58 2001:2000:3080:1351::2 +3 116.106.34.242 2a01:8840:16::1 +4 111.56.27.171 2001:550:0:1000::9a36:2a61 +5 183.245.137.140 2001:578:400:4:2000::19 +6 183.212.25.70 2607:f290::1 +7 162.144.2.57 2a02:23f0:ffff:8::5 +8 111.4.229.190 2400:c700:0:158:: +9 59.52.3.168 2001:67c:24e4:4::250 +10 115.11.21.200 2a02:2a38:37:5::2 +11 121.28.97.113 2001:41a8:400:2::13a +12 111.46.39.248 2405:9800:9800:66::2 +13 120.192.122.34 2a07:a343:f210::1 +14 113.56.44.105 2403:5000:171:46::2 +15 116.66.238.92 2800:c20:1141::8 +16 67.22.254.206 2402:7800:40:2::62 +17 115.0.24.191 2a00:de00::1 +18 182.30.107.86 2001:688:0:2:1::9e +19 223.73.153.243 2001:2000:3080:80::2 +20 115.159.103.38 2001:428::205:171:200:230 +21 36.186.75.121 2001:fb1:fe0:9::8 +22 111.56.188.125 2001:2000:3080:10ca::2 +23 115.14.93.25 2400:dd0b:1003::2 +24 211.97.110.141 2001:1a98:6677::9d9d:140a +25 61.58.96.173 2001:2000:3018:3b::1 +26 203.126.212.37 2607:fa70:3:33::2 +27 192.220.125.142 2001:5b0:23ff:fffa::113 +28 115.22.20.223 2001:450:2001:1000:0:40:6924:23 +29 121.25.160.80 2001:418:0:5000::c2d +30 117.150.98.199 2a01:b740:a09::1 +31 183.211.172.143 2607:f0d0:2:2::243 +32 180.244.18.143 2a01:348::e:1:1 +33 209.131.3.252 2405:4800::3221:3621:2 +34 220.200.1.22 2a02:aa08:e000:3100::2 +35 171.225.130.45 2001:44c8:129:2632:33:0:252:2 +36 115.4.78.200 2a02:e980:1e::1 +37 36.183.59.29 2a0a:6f40:2::1 +38 218.42.159.17 2001:550:2:29::2c9:1 +39 115.13.39.164 2001:c20:4800::175 +40 142.254.161.133 2c0f:feb0:1:2::d1 +41 116.2.211.43 2a0b:7086:fff0::1 +42 36.183.126.25 2a04:2dc0::16d +43 66.150.171.196 2604:7e00::105d +44 104.149.148.137 2001:470:1:946::2 +45 120.239.82.212 2a0c:3240::1 +46 111.14.182.156 2800:630:4010:8::2 +47 115.6.63.224 2001:1af8:4040::12 +48 153.35.83.233 2c0f:fc98:1200::2 +49 113.142.1.1 2001:470:1:248::2 +50 121.25.82.29 2620:44:a000::1 +51 62.151.203.189 2402:800:63ff:40::1 +52 104.27.46.146 2a02:b000:fff::524 +53 36.189.46.88 2001:470:0:327::1 +54 116.252.54.207 2401:7400:8888:2::8 +55 64.77.240.1 2001:500:55::1 +56 142.252.102.78 2001:668:0:3::f000:c2 +57 36.82.224.170 2400:bf40::1 +58 117.33.191.217 2001:67c:754::1 +59 144.12.164.251 2402:28c0:100:ffff:ffff:ffff:ffff:ffff +60 122.10.93.66 2001:470:0:1fa::2 +61 104.25.84.59 2001:550:0:1000::9a18:292a +62 111.4.242.106 2001:470:1:89e::2 +63 222.216.51.186 2001:579:6f05:500:9934:5b3e:b7fe:1447 +64 112.33.13.212 2804:158c::1 +65 115.9.240.116 2600:140e:6::1 +66 171.228.0.153 2a00:18e0:0:bb04::82 +67 45.3.47.158 2a02:2698:5000::1e06 +68 69.57.193.230 2402:800:63ff:10::7:2 +69 115.6.104.199 2a02:e980:19::1 +70 104.24.237.140 2001:4888::342:1:0:0 +71 199.17.84.108 2607:fc68:0:4:0:2:2711:21 +72 120.193.17.57 2606:2800:602a::1 +73 112.40.38.145 2404:c600:1000:2::1d1 +74 67.55.90.43 2001:578:1400:4::9d +75 180.253.57.249 2804:64:0:25::1 +76 14.204.253.158 2605:3e00::1:2:2 +77 1.83.241.116 2c0f:fa18:0:4::b +78 202.198.37.147 2606:2800:602c:b::d004 +79 115.6.31.95 2610:18:181:4000::66 +80 117.32.14.179 2001:48f8:1000:1::16 +81 23.238.237.26 2408:8000:c000::1 +82 116.97.76.104 2a03:4200:441:2::4e +83 1.80.2.248 2400:dd00:1:200a::2 +84 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e +85 42.117.228.166 2001:16a0:2:200a::2 +86 119.36.22.147 2001:4888:1f:e891:161:26:: +87 210.66.18.184 2a0c:f743::1 +88 115.19.192.159 2a02:e980:b::1 +89 112.15.128.113 2001:578:201:1::601:9 +90 1.55.138.211 2001:438:ffff::407d:1bc1 +91 210.183.19.113 2001:920:1833::1 +92 42.115.43.114 2001:1b70:a1:610::b102:2 +93 58.16.171.31 2001:13c7:6014::1 +94 171.234.78.185 2003:0:1203:4001::1 +95 113.56.43.134 2804:a8:2:c8::d6 +96 111.53.182.225 2a02:2e00:2080:f000:0:261:1:11 +97 107.160.215.141 2001:578:20::d +98 171.229.231.90 2001:550:2:48::34:1 +99 58.19.84.138 2a03:9d40:fe00:5:: +100 36.79.88.107 2403:e800:200:102::2 diff --git a/regression-test/data/types/test_show_data_types.out b/regression-test/data/types/test_show_data_types.out index fe820a5c7f083b..e6f69c6718123e 100644 --- a/regression-test/data/types/test_show_data_types.out +++ b/regression-test/data/types/test_show_data_types.out @@ -18,6 +18,8 @@ DOUBLE 8 FLOAT 4 HLL 16 INT 4 +IPV4 4 +IPV6 16 JSON 16 LARGEINT 16 MAP 24 diff --git a/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy b/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy new file mode 100644 index 00000000000000..0412f708d478c0 --- /dev/null +++ b/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy @@ -0,0 +1,105 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ip_crud") { + sql """ DROP TABLE IF EXISTS test_unique_ip_crud """ + sql """ DROP TABLE IF EXISTS test_dup_ip_crud """ + + sql """ SET enable_nereids_planner=true """ + sql """ SET enable_fallback_to_original_planner=false """ + + sql """ + CREATE TABLE test_unique_ip_crud ( + `id` int, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + UNIQUE KEY (`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql "insert into test_unique_ip_crud values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e')" + sql "insert into test_unique_ip_crud values(2, '42.117.228.166', '2001:16a0:2:200a::2')" + sql "insert into test_unique_ip_crud values(3, '119.36.22.147', '2001:4888:1f:e891:161:26::')" + + qt_sql1 "select * from test_unique_ip_crud order by id" + qt_sql2 "select * from test_unique_ip_crud where ip_v4='42.117.228.166'" + qt_sql3 "select * from test_unique_ip_crud where ip_v6='2a02:e980:83:5b09:ecb8:c669:b336:650e'" + qt_sql4 "select * from test_unique_ip_crud where ip_v4='119.36.22.147' and ip_v6='2001:4888:1f:e891:161:26::'" + + // Only unique table could be updated + sql "update test_unique_ip_crud set ip_v4=0, ip_v6='2804:64:0:25::1' where id=3" + qt_sql5 "select * from test_unique_ip_crud order by id" + + // test ip datatype in aggregate table + sql """ + CREATE TABLE test_agg_ip_crud ( + `id` int, + `ip_v4` ipv4 REPLACE, + `ip_v6` ipv6 REPLACE + ) ENGINE=OLAP + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql "insert into test_agg_ip_crud values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e')" + sql "insert into test_agg_ip_crud values(2, '42.117.228.166', '2001:16a0:2:200a::2')" + sql "insert into test_agg_ip_crud values(3, '119.36.22.147', '2001:4888:1f:e891:161:26::')" + + qt_sql6 "select * from test_agg_ip_crud order by id" + qt_sql7 "select * from test_agg_ip_crud where ip_v4='42.117.228.166'" + qt_sql8 "select * from test_agg_ip_crud where ip_v6='2a02:e980:83:5b09:ecb8:c669:b336:650e'" + qt_sql9 "select * from test_agg_ip_crud where ip_v4='119.36.22.147' and ip_v6='2001:4888:1f:e891:161:26::'" + + // test ip datatype in duplicate table + sql """ + CREATE TABLE test_dup_ip_crud ( + `id` int, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql "insert into test_dup_ip_crud values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e')" + sql "insert into test_dup_ip_crud values(2, '42.117.228.166', '2001:16a0:2:200a::2')" + sql "insert into test_dup_ip_crud values(3, '119.36.22.147', '2001:4888:1f:e891:161:26::')" + + qt_sql10 "select * from test_dup_ip_crud order by id" + qt_sql11 "select * from test_dup_ip_crud where ip_v4='42.117.228.166'" + qt_sql12 "select * from test_dup_ip_crud where ip_v6='2a02:e980:83:5b09:ecb8:c669:b336:650e'" + qt_sql13 "select * from test_dup_ip_crud where ip_v4='119.36.22.147' and ip_v6='2001:4888:1f:e891:161:26::'" + + sql "delete from test_dup_ip_crud where ip_v4='42.117.228.166'" + qt_sql14 "select * from test_dup_ip_crud order by id" + sql "delete from test_dup_ip_crud where ip_v6='2001:4888:1f:e891:161:26::'" + qt_sql15 "select * from test_dup_ip_crud order by id" + + sql "DROP TABLE test_unique_ip_crud" + sql "DROP TABLE test_agg_ip_crud" + sql "DROP TABLE test_dup_ip_crud" +} diff --git a/regression-test/suites/datatype_p0/ip/test_ip_implicit_cast.groovy b/regression-test/suites/datatype_p0/ip/test_ip_implicit_cast.groovy new file mode 100644 index 00000000000000..96ef6e8134aca6 --- /dev/null +++ b/regression-test/suites/datatype_p0/ip/test_ip_implicit_cast.groovy @@ -0,0 +1,46 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ip_implicit_cast") { + def tableName = "test_ip_implicit_cast" + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ SET enable_nereids_planner=true """ + sql """ SET enable_fallback_to_original_planner=false """ + + sql """ + CREATE TABLE ${tableName} ( + `id` bigint, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql "insert into ${tableName} values(-1, NULL, NULL)" + sql "insert into ${tableName} values(0, 0, '::')" + sql "insert into ${tableName} values(1, 1, '::1')" + sql "insert into ${tableName} values(2130706433, 2130706433, '2001:1b70:a1:610::b102:2')" + sql "insert into ${tableName} values(4294967295, 4294967295, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')" + + qt_sql1 "select id, ip_v4 from ${tableName} order by id" + + sql "DROP TABLE ${tableName}" +} diff --git a/regression-test/suites/datatype_p0/ip/test_ip_in_predicate.groovy b/regression-test/suites/datatype_p0/ip/test_ip_in_predicate.groovy new file mode 100644 index 00000000000000..654a5d9f632d7e --- /dev/null +++ b/regression-test/suites/datatype_p0/ip/test_ip_in_predicate.groovy @@ -0,0 +1,47 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ip_in_predicate") { + def tbName = "test_ip_in_predicate" + sql """ DROP TABLE IF EXISTS ${tbName} """ + + sql """ SET enable_nereids_planner=true """ + sql """ SET enable_fallback_to_original_planner=false """ + + sql """ + CREATE TABLE ${tbName} ( + `id` int, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql "insert into ${tbName} values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e')" + sql "insert into ${tbName} values(2, '42.117.228.166', '2001:16a0:2:200a::2')" + sql "insert into ${tbName} values(3, '119.36.22.147', '2001:4888:1f:e891:161:26::')" + + qt_sql1 "select * from ${tbName} order by id" + qt_sql2 "select * from ${tbName} where ip_v4 in ('42.117.228.166', '119.36.22.147') order by id" + qt_sql3 "select * from ${tbName} where ip_v4 not in ('42.117.228.166', '119.36.22.147') order by id" + qt_sql4 "select * from ${tbName} where ip_v6 in ('2a02:e980:83:5b09:ecb8:c669:b336:650e', '2001:4888:1f:e891:161:26::') order by id" + qt_sql5 "select * from ${tbName} where ip_v6 not in ('2a02:e980:83:5b09:ecb8:c669:b336:650e', '2001:4888:1f:e891:161:26::') order by id" + sql "DROP TABLE ${tbName}" +} diff --git a/regression-test/suites/datatype_p0/ip/test_ip_stream_load.groovy b/regression-test/suites/datatype_p0/ip/test_ip_stream_load.groovy new file mode 100644 index 00000000000000..166ef9ea44a7b0 --- /dev/null +++ b/regression-test/suites/datatype_p0/ip/test_ip_stream_load.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_ip_stream_load") { + def dbName = "test_stream_load" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "USE $dbName" + + sql """ SET enable_nereids_planner=true """ + sql """ SET enable_fallback_to_original_planner=false """ + + def tableName = "test_ip_stream_load" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `id` int, + `ip_v4` ipv4, + `ip_v6` ipv6 + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + streamLoad { + db dbName + table tableName + + set 'column_separator', ',' + + file 'test_data/test.csv' + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + } + + sql """sync""" + qt_select """ SELECT * FROM ${tableName} ORDER BY id; """ + + try_sql("DROP TABLE IF EXISTS ${tableName}") +}